diff --git a/.env.example b/.env.example index 581461142..b8c0a55d7 100644 --- a/.env.example +++ b/.env.example @@ -4,7 +4,8 @@ ENV=dev # options: dev|s # metadata, and default-model lookups. Leave unset to default to nv_build. # Options: openai | anthropic | anthropic_proxy | bedrock | nv_build | # ollama | azure_openai | openai_compatible | claude_cli | -# codex_cli | gemini_cli | opencode_cli +# gemini_cli | opencode_cli +# codex_cli is registered but disabled: its read-only sandbox permits host-file reads. SKILLSPECTOR_PROVIDER= # Aggregate deadline for one complete scan workflow. Defaults to 600 seconds; @@ -28,7 +29,8 @@ NVIDIA_INFERENCE_KEY= OPENAI_API_KEY= OPENAI_BASE_URL= # Optional provider- and model-dependent reasoning-effort setting. Non-empty values -# are trimmed and passed through unchanged; unset or blank uses the provider default. +# are trimmed and passed through unchanged. Unset or blank sends high for nv_build +# with z-ai/glm-5.3; other provider/model combinations keep their endpoint defaults. SKILLSPECTOR_REASONING_EFFORT= # Optional language for human-readable LLM finding text. Machine-readable values # such as rule IDs and severity values remain unchanged. @@ -67,7 +69,7 @@ AZURE_OPENAI_ENDPOINT= SKILLSPECTOR_COMPAT_API_KEY= SKILLSPECTOR_COMPAT_BASE_URL= -# claude_cli, codex_cli, gemini_cli, and opencode_cli use their CLI's existing local +# claude_cli, gemini_cli, and opencode_cli use their CLI's existing local # authentication session and do not need an API key here. # SkillSpector config diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 462f5d59e..ec75cf5ac 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,9 +17,9 @@ name: CI on: pull_request: - branches: ["main"] + branches: ["main", "yashraj/v2.13.0-base"] push: - branches: ["main"] + branches: ["main", "yashraj/v2.13.0-base"] # Least privilege: these jobs only read the repo; no write scopes are needed. permissions: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 875d75729..3ef90b24d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -5,7 +5,7 @@ name: Publish GitHub Release on: pull_request: - branches: ["main"] + branches: ["main", "yashraj/v2.13.0-base"] types: [closed] permissions: @@ -30,12 +30,13 @@ jobs: permissions: contents: write steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: ref: ${{ github.event.pull_request.merge_commit_sha }} + persist-credentials: false - name: Set up uv # Pinned to a full commit SHA (third-party action); comment tracks the tag. - uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v5 + uses: astral-sh/setup-uv@d4b2f3b6ecc6e67c4457f6d3e41ec42d3d0fcb86 # v5.4.2 with: version: "${{ env.UV_VERSION }}" enable-cache: true diff --git a/.opencode/tools/skillspector_scan_lib.ts b/.opencode/tools/skillspector_scan_lib.ts index b562042de..6a71f5d53 100644 --- a/.opencode/tools/skillspector_scan_lib.ts +++ b/.opencode/tools/skillspector_scan_lib.ts @@ -44,25 +44,94 @@ export function truncate(text: string, max: number): string { export function redact(text: string, env: Env = process.env): string { const credentialValues = [...new Set( CREDENTIAL_ENV_NAMES.map((name) => env[name]?.trim()).filter( - (value): value is string => Boolean(value && value.length >= 4), + (value): value is string => Boolean(value), ), )].sort((left, right) => right.length - left.length) - const credentialPattern = credentialValues.length - ? new RegExp( - credentialValues - .map((value) => value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")) - .join("|"), - "g", - ) + const valuePatterns = credentialValues.map((value) => { + const literal = value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&") + // Short values are usually local-provider placeholders. Treat them as + // tokens so they cannot change words, JSON fields, or longer credentials. + return value.length < 4 ? `(? credentialValues[index].length >= 4) + const keyBoundary = longValues.length ? `(?:\\b|(?<=${longValues.join("|")}))` : "\\b" + const keyPattern = new RegExp([ + "sk-ant-[A-Za-z0-9_-]+", + `${keyBoundary}sk-[A-Za-z0-9_-]{6,}\\b`, + // Consume each uppercase name once, even when it is not an assignment. + // Otherwise a known value like AAAA permits a costly retry at every A. + `${keyBoundary}([A-Z][A-Z0-9_]*)`, + ].join("|"), "g") + const assignmentName = /[A-Z0-9_](?:API_KEY|TOKEN|ACCESS_KEY_ID|SECRET_ACCESS_KEY|INFERENCE_KEY)$/ + const assignmentValue = /(\s*[:=]\s*["']?)[^"'\s,}]+/y + const scrub = (value: string): string => { + function* spans(pattern: RegExp | undefined, literal: boolean): Generator<[number, number]> { + if (!pattern) return + for (const match of value.matchAll(pattern)) { + if (!literal && match[1]) { + if (!assignmentName.test(match[1])) continue + assignmentValue.lastIndex = match.index + match[0].length + const assigned = assignmentValue.exec(value) + if (assigned) { + yield [assigned.index + assigned[1].length, assigned.index + assigned[0].length] + } + } else { + yield [match.index, match.index + (literal ? match[1].length : match[0].length)] + } + } + } + const known = spans(credentialPattern, true) + const shaped = spans(keyPattern, false) + let nextKnown = known.next().value + let nextShaped = shaped.next().value + let cursor = 0 + let start = -1 + let end = -1 + const parts: string[] = [] + // Merge two ordered span streams in linear time. A shorter key/assignment + // match cannot expose the tail of a longer configured credential. + while (nextKnown || nextShaped) { + let span: [number, number] + if (nextKnown && (!nextShaped || nextKnown[0] <= nextShaped[0])) { + span = nextKnown + nextKnown = known.next().value + } else { + span = nextShaped! + nextShaped = shaped.next().value + } + if (span[0] > end) { + if (start >= 0) { + parts.push(value.slice(cursor, start), "[REDACTED]") + cursor = end + } + start = span[0] + } + end = Math.max(end, span[1]) + } + if (start >= 0) { + parts.push(value.slice(cursor, start), "[REDACTED]") + cursor = end + } + parts.push(value.slice(cursor)) + return parts.join("") + } + try { + const report = JSON.parse(text) + if (report === null || typeof report !== "object") return scrub(text) + // Preserve JSON numbers and booleans, including a score equal to a short + // placeholder. Credentials inside JSON strings still receive full scrubbing. + return text.replace(/"(?:[^"\\]|\\.)*"/g, (token) => JSON.stringify(scrub(JSON.parse(token)))) + } catch { + return scrub(text) + } } export function isScpGitTarget(target: string): boolean { diff --git a/CHANGELOG.md b/CHANGELOG.md index b924b4113..3db028dda 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,66 @@ +### 2.13.0 (Unreleased) +### Security and blocking fixes +* Selected release based on v2.12.0; includes 55 PRs, with #743 included through #734. +* Reads SKILL.md metadata correctly when the file starts with an invisible encoding marker. (#471) +* Detects text-direction control characters that can disguise code in bundled scripts, not just Markdown. (#473) +* Marks recursive scan coverage as incomplete when linked skill folders are deliberately skipped. (#499) +* Detects access to sensitive files even when their paths are assembled from fixed strings. (#545) +* Marks analysis incomplete when custom YARA rules cannot be loaded or compiled. (#557) +* Preserves dangerous shell-command findings when deeper Python analysis cannot confidently replace them. (#577) +* Keeps distinct shell-command findings separate, even when their displayed snippets look identical. (#578) +* Recognizes more executable Python files, including .pyw files and extensionless launchers. (#579) +* Adds lower-confidence warnings for multiline scripts that appear to open a remote shell, while preserving existing one-line detections. (#593) +* Shows how many linked skill folders were skipped and includes that count in recursive scan coverage. (#595) +* Checks complete TypeScript declaration files before treating them as non-executable, so partial or disguised runtime code is not skipped. (#601) +* Flags bundled settings that run commands, redirect AI traffic, or automatically approve connected tool servers. (#604) +* Finds Python flows from secret reads to risky uses even when the assignments and uses appear at different nesting depths. (#611) +* Keeps Python findings when parser location columns are missing, instead of crashing or merging distinct findings together. (#623) +* Recognizes more ordinary Markdown sentences and lists so their apostrophes do not cause false shell-parsing and incomplete-scan warnings. (#627) +* Inspects hooks declared in Claude plugin manifests and reports incomplete coverage when those declarations cannot be fully checked. (#632) +* Parses quoted shell assignments correctly so harmless trailing text does not falsely make a scan incomplete. (#634) +* Makes saved baselines cover every occurrence of a repeated finding, so later scans suppress them consistently. (#637) +* Detects bundled Node settings that use the short -r flag to load and run extra code. (#655) +* Preserves exact findings in baselines and protects existing baseline files during regeneration. (#657) +* Avoids misleading high-severity warnings on SKILL.md self-references while still reporting missing or ambiguous files. (#661) +* Accepts ordinary shell variable defaults and assignment values without falsely marking their files as partly scanned. (#686) +* Improves TP4 responses for the specific OpenAI-provider Opus 5 route and rejects missing, malformed, or contradictory tool results. (#691) +* Lowers alerts for simple download pipelines that demonstrably parse incoming data rather than execute it. (#700) +* Attaches AI review decisions to the correct findings and distinguishes disagreement from missing or failed reviews. (#702) +* Recognizes safely decodable shell printf escapes without incorrectly marking analysis incomplete. (#704) +* Detects sensitive data passed to urllib opener requests, including inline opener calls. (#721) +* Displays untrusted names, filenames and findings literally in terminal reports, preventing markup crashes and spoofing. (#725) +* Rejects scan output that points to the input file, including symbolic or hard links, preventing accidental overwrites. (#729) +* Prevents untrusted names and findings from inserting fake headings, rows or HTML into batch Markdown reports. (#733) +* Displays untrusted skill names and errors literally in batch terminals so markup and control characters cannot spoof the output. (#734) +* Prevents untrusted finding text and filenames from changing Markdown report structure, using shared protection for core and batch reports. (#735) +* Stops remote MCP scans from using the server’s Git credentials to clone private repositories while preserving trusted local Git access. (#736) +* Disables the Codex CLI analysis provider because its read-only sandbox still permits host-secret reads, directing users to another provider. (#737) +* Limits environment variables passed to agent CLI providers so unexpected secret variables are not forwarded to child processes. (#738) +* Prevents long scanner output from blocking Pi’s secret redaction while still hiding secrets before the displayed output is shortened. (#739) +* Prevents crafted Python literals and unfinished function headers from causing extremely slow scans. (#740) +* Enforces per-skill batch timeouts by stopping stuck workers while other scans continue. (#743) +* Limits unauthenticated MCP HTTP access to the local machine; remote access requires an authenticating proxy. (#744) +* Redacts short configured secrets from OpenCode’s returned output without corrupting JSON values; separately written report files remain outside this redactor. (#745) +* Pins the release workflow’s checkout action to a fixed commit so a moving version tag cannot silently change the code it runs. (#746) +* Switches NVIDIA Build’s default from retired GLM-5.2 to GLM-5.3 while preserving explicit model overrides. (#748) +* Prevents registry reports from overwriting their input files. (#758) +* Keeps the release GitHub token out of Git configuration during dependency installation and builds. (#767) +* Detects encoded shell payloads that are decoded and executed. (#786) +* Requires approval for external Pi scan inputs and blocks file-reading YARA includes. (#791) +* Separates local and referenced-content caches so baseline suppression cannot use the wrong file. (#792) +* Keeps real disk content separate from archive members with colliding paths and reports failed coverage for ambiguous paths. (#793) +* Inspects readable instructions hidden behind binary prefixes while preserving benign PDF/GIF and XMP handling and explicit limits for referenced or executable binaries. (#794) +* Inspects untrusted OMS signing metadata and decoded payloads, preserving raw evidence and incomplete coverage on parsing failures. (#795) +* Reports malformed LLM responses as incomplete analysis instead of successful empty results. (#796) +* Tracks sensitive-data flow by function scope, avoiding collisions between unrelated variable names. (#799) +* Keeps intentionally excluded files separate from failed analyzer work in coverage reports. (#802) +* Preserve atomic baseline updates and refuse updates that cannot retain ownership or group safely. +* Keep static-only scans independent of inactive LLM provider settings while retaining strict provider checks for LLM-enabled scans. +* Bound TypeScript declaration matching so malformed files cannot stall scans; timed-out checks leave the files eligible for executable analysis. +* Counts only the bytecode walk’s own CPU time toward its processing limit, while retaining the overall scan deadline. Waiting for other analyzers no longer makes clean scans incomplete. (#828) +* Avoids execution warnings for simple decode-to-checksum pipelines and preserves disk-versus-archive error locations when paths collide. (#844) +* Enable CI and label-gated publishing for the selected v2.13.0 release branch. +--- ### 2.12.0 (Unreleased) ### Features/Bug Fixes * fix(security): retain incomplete coverage for runtime-selected commands and remeasure active Git clones strictly (#514) diff --git a/README.md b/README.md index 9a32690ad..0c1f82010 100644 --- a/README.md +++ b/README.md @@ -235,15 +235,32 @@ inference gateways. | `anthropic` | `ANTHROPIC_API_KEY` | api.anthropic.com | `claude-opus-4-6` | | `anthropic_proxy` | `ANTHROPIC_PROXY_API_KEY` + `ANTHROPIC_PROXY_ENDPOINT_URL` | Any Vertex-style raw-predict proxy | `claude-sonnet-4-6` | | `bedrock` | `AWS_PROFILE` (optional) + `AWS_REGION` — SigV4 via boto3 | AWS Bedrock Runtime | `us.anthropic.claude-sonnet-4-6-20250915-v1:0` | -| `nv_build` | `NVIDIA_INFERENCE_KEY` | build.nvidia.com | `z-ai/glm-5.2` | +| `nv_build` | `NVIDIA_INFERENCE_KEY` | build.nvidia.com | `z-ai/glm-5.3` | | `ollama` | _(none)_ | `OLLAMA_BASE_URL` (default `http://localhost:11434/v1`) | `llama3.1:8b` | | `azure_openai` | `AZURE_OPENAI_API_KEY` + `AZURE_OPENAI_ENDPOINT` | Azure OpenAI Service | `gpt-4o` (deployment defaults to the model label) | | `openai_compatible` | `SKILLSPECTOR_COMPAT_API_KEY` + `SKILLSPECTOR_COMPAT_BASE_URL` | Any OpenAI-compatible endpoint | `llama-3.1-70b-versatile` | | `claude_cli` | _(none — uses local CLI auth)_ | local `claude` binary | local Claude runtime fallback, or `SKILLSPECTOR_MODEL` | -| `codex_cli` | _(none — uses local CLI auth)_ | local `codex` binary | local Codex runtime fallback, or `SKILLSPECTOR_MODEL` | +| `codex_cli` | Disabled | Registered for compatibility; its read-only sandbox permits host-file reads | Use an HTTP API provider or another supported CLI provider | | `gemini_cli` | _(none — uses local CLI auth)_ | local `gemini` binary | local Gemini runtime fallback, or `SKILLSPECTOR_MODEL` | | `opencode_cli` | _(none — uses local CLI auth)_ | local `opencode` 1.18.31 binary | local OpenCode runtime fallback, or `SKILLSPECTOR_MODEL` | +For NVIDIA Build's `z-ai/glm-5.3`, SkillSpector requests `high` reasoning effort. +Set `SKILLSPECTOR_REASONING_EFFORT` to override it with `low`, `high`, or `max`. +The bundled 128,000-token context and 32,000-token output budgets are conservative +application limits; they do not claim the hosted endpoint's maximum capacity. + +Structured output is requested through LangChain's `with_structured_output`. +The default is client-specific: `ChatOpenAI` uses `json_schema`, while other +clients may use tool calling. +`SKILLSPECTOR_STRUCTURED_OUTPUT_METHOD=json_schema|function_calling` +overrides the method for any provider. +The method precedence is `SKILLSPECTOR_STRUCTURED_OUTPUT_METHOD`, the provider's +method hint, an analyzer preference, then the client default. TP4 prefers +`function_calling` only for provider `openai` and the exact model label +`azure/anthropic/claude-opus-5`, with reasoning/thinking controls unset; dated +or suffixed labels and other providers keep their existing behavior. An explicit +environment override or provider hint still wins when reasoning is configured. + ```bash # Stock OpenAI export SKILLSPECTOR_PROVIDER=openai @@ -286,11 +303,6 @@ export SKILLSPECTOR_PROVIDER=claude_cli # export SKILLSPECTOR_MODEL=claude-sonnet-4-6 skillspector scan ./my-skill/ -# Local Codex CLI — no API key; uses your existing `codex login` session -# Requires: codex CLI installed and authenticated -export SKILLSPECTOR_PROVIDER=codex_cli -skillspector scan ./my-skill/ - # Gemini (via OpenAI compatibility layer) export SKILLSPECTOR_PROVIDER=openai export OPENAI_API_KEY="YOUR_GEMINI_API_KEY" @@ -329,8 +341,8 @@ skillspector scan ./my-skill/ --no-llm ### MCP Server Run SkillSpector as a [Model Context Protocol](https://modelcontextprotocol.io) -server so any MCP-capable agent (Claude Code, Codex CLI, Gemini CLI) or remote -runtime can call scanning as a tool and **gate skill/MCP installs on the +server so local MCP-capable agents (Claude Code, Codex CLI, Gemini CLI) +can call scanning as a tool and **gate skill/MCP installs on the result** — turning SkillSpector into a runtime guardrail instead of an out-of-band audit step. @@ -343,7 +355,7 @@ uv tool install --force 'skillspector[mcp] @ git+https://github.com/NVIDIA/skill # FastMCP stdio transport for local CLI agents skillspector mcp -# streamable HTTP/SSE transport for remote / A2A callers +# Streamable HTTP transport on a local loopback interface skillspector mcp --transport http --host 127.0.0.1 --port 8000 ``` @@ -368,14 +380,21 @@ claude mcp add skillspector -- skillspector mcp > **Security — HTTP transport trust model** > > The HTTP transport ships **without authentication**. Any caller that can -> reach the port can invoke `scan_skill`. Over stdio or `127.0.0.1` this is -> the same trust boundary as the CLI. If you bind to a routable interface: +> reach the port can invoke `scan_skill`. HTTP bindings are restricted to +> loopback IPs (`127.0.0.1` or `::1`); `localhost` binds to `127.0.0.1` without +> DNS resolution. Wildcard, routable and other hostname bindings are rejected. > -> - Sit the server behind an authenticating reverse proxy (e.g. nginx + mTLS) -> before exposing it externally. +> - For remote access, put an authenticating reverse proxy (e.g. nginx + mTLS) +> in front of the loopback listener. > - Local paths and `file://` URLs are **automatically rejected** over HTTP to > prevent unauthenticated callers from reading arbitrary host files. Only > remote Git and `.zip` URLs are accepted. +> +> HTTP scans accept only credential-free `https://` targets. Git scans never use +> the server's Git login or Git config. If a proxy or custom CA is needed, set +> `HTTPS_PROXY`, `NO_PROXY` or `GIT_SSL_CAINFO` in the server environment. +> Use stdio or the CLI for private repositories. Git redirects are rejected; +> use the current repository URL after a rename or transfer. ## Vulnerability Patterns @@ -606,11 +625,11 @@ Issues (2) | Variable | Description | Required | |----------|-------------|----------| -| `SKILLSPECTOR_PROVIDER` | Active LLM provider: `openai`, `anthropic`, `anthropic_proxy`, `bedrock`, `nv_build`, `ollama`, `azure_openai`, `openai_compatible`, `claude_cli`, `codex_cli`, `gemini_cli`, or `opencode_cli`. Hosted providers use bundled `model_registry.yaml` defaults; CLI providers fall back to the local runtime's default model unless `SKILLSPECTOR_MODEL` is set. Defaults to `nv_build`. | Optional | +| `SKILLSPECTOR_PROVIDER` | Active LLM provider: `openai`, `anthropic`, `anthropic_proxy`, `bedrock`, `nv_build`, `ollama`, `azure_openai`, `openai_compatible`, `claude_cli`, `gemini_cli`, or `opencode_cli`. Hosted providers use bundled `model_registry.yaml` defaults; CLI providers fall back to the local runtime's default model unless `SKILLSPECTOR_MODEL` is set. Defaults to `nv_build`. | Optional | | `NVIDIA_INFERENCE_KEY` | Credential for the `nv_build` provider (build.nvidia.com). | Required for LLM analysis when `SKILLSPECTOR_PROVIDER=nv_build` | | `OPENAI_API_KEY` | Credential for the OpenAI provider (`SKILLSPECTOR_PROVIDER=openai`). Also serves as the tier-2 fallback in the credential waterfall when the active provider returns no credentials. | Required for LLM analysis when `SKILLSPECTOR_PROVIDER=openai` | | `OPENAI_BASE_URL` | Override the OpenAI endpoint (e.g. point at Ollama). | Optional | -| `SKILLSPECTOR_REASONING_EFFORT` | Optional provider- and model-dependent reasoning-effort setting. Non-empty values are trimmed and passed through unchanged; unset or blank preserves provider-default behavior. | Optional | +| `SKILLSPECTOR_REASONING_EFFORT` | Optional provider- and model-dependent reasoning-effort setting. Non-empty values are trimmed and passed through unchanged. When unset or blank, SkillSpector sends `high` for `nv_build` with `z-ai/glm-5.3`; other provider/model combinations keep their endpoint defaults. | Optional | | `SKILLSPECTOR_OUTPUT_LANGUAGE` | Short, single-line language label (letters, numbers, spaces, `_`, or `-`; maximum 64 characters) for human-readable LLM finding text such as messages, explanations, and remediation. Rule IDs, severity values, paths, code, and other machine-readable values remain unchanged. Unset, blank, or invalid values preserve the default output language. | Optional | | `SKILLSPECTOR_TEMPERATURE` | Optional sampling temperature from `0` to `1` for hosted providers. Unset or blank preserves the provider default. Lower values can reduce run-to-run variation but do not guarantee identical output. | Optional | | `SKILLSPECTOR_SEED` | Optional integer sampling seed for OpenAI-compatible and Azure OpenAI providers. Other hosted providers and CLI providers do not receive it. Provider support remains model-dependent. | Optional | @@ -633,7 +652,9 @@ Issues (2) | `SKILLSPECTOR_MODEL_REGISTRY` | Override the bundled per-provider YAML registry (`src/skillspector/providers//model_registry.yaml`) with a custom path. | Optional | | `SKILLSPECTOR_LOG_LEVEL` | Log level: `DEBUG`, `INFO`, `WARNING`, `ERROR` (default: `WARNING`). | Optional | -> **CLI providers** (`claude_cli`, `codex_cli`, `gemini_cli`, `opencode_cli`): No API key is needed. Authentication is managed entirely by the agent CLI's own login session. SkillSpector never reads or forwards API keys when these providers are active. The subprocess is run with capabilities restricted, and untrusted skill content is delivered only via stdin. +> **Disabled provider:** `codex_cli` remains registered but cannot run LLM analysis because its read-only sandbox allows host-file reads. Existing users should select an HTTP API provider or another supported CLI provider. + +> **CLI providers** (`claude_cli`, `gemini_cli`, `opencode_cli`): No API key is needed. Authentication is managed entirely by the agent CLI's own login session. SkillSpector never reads or forwards API keys when these providers are active. The subprocess is run with capabilities restricted, and untrusted skill content is delivered only via stdin. > > `opencode_cli` currently fails closed unless the installed OpenCode version is exactly `1.18.31`, the version whose configuration precedence and deny-all semantics are verified by this release. @@ -789,13 +810,15 @@ SkillSpector uses a two-stage detection pipeline: - High recall (catches most issues) - Moderate precision (some false positives) -A valid, root-level OpenSSF Model Signing signature (`skill.oms.sig`) is retained in the -component inventory as type `oms_signature`, but excluded from static and LLM content analysis. -OMS bundles necessarily contain long base64-encoded payload, signature, and certificate fields; -generic obfuscated-code checks can otherwise misclassify those fields as hidden executable content. -The recognizer checks the minimal OMS DSSE/in-toto structure; it does not verify the signature, -certificate chain, transparency-log entry, or signer identity. Invalid or unrecognized signature -files are scanned normally. +OpenSSF Model Signing / DSSE bundles remain in content analysis, regardless of +filename. Static and LLM analyzers inspect decoded JSON payloads, readable DER +certificate/signature content, and all unknown wrapper fields. Strictly parsed +binary fields are projected rather than treated as executable base64; canonical +bytes remain available to YARA and fingerprints. Unsupported encodings (including +environments without the optional `cryptography` dependency) retain raw fields +and produce an explicit incomplete-analysis warning. Findings in decoded content +point to the original bundle and label their projection in evidence. None of this +verifies signatures, certificate chains, transparency logs, or signer identity. ### Stage 2: LLM Semantic Analysis (Optional) - Evaluates context and intent @@ -821,7 +844,7 @@ The tool requires outbound HTTPS access to `api.osv.dev` for live vulnerability SkillSpector is defense-in-depth, not a sandbox. Know what it does and does not do before relying on it: - **It never executes the scanned skill.** All analysis is static (regex, Python AST, YARA) plus optional LLM evaluation of file *contents* — the skill's code is never run. -- **LLM analysis sends analyzer-eligible file contents to the configured provider.** When LLM analysis is enabled (the default), file contents are sent to the active `SKILLSPECTOR_PROVIDER` endpoint. Recognized OMS signature files are excluded. Use `--no-llm` to keep contents local (static analysis only). +- **LLM analysis sends analyzer-eligible file contents to the configured provider.** When LLM analysis is enabled (the default), file contents are sent to the active `SKILLSPECTOR_PROVIDER` endpoint. DSSE payloads and readable signing metadata are included. Use `--no-llm` to keep contents local (static analysis only). - **SC4 sends dependency names to OSV.dev.** The supply-chain check queries [OSV.dev](https://osv.dev) with the package names and versions the skill declares, to look up known CVEs. This is fundamental to the check and runs even with `--no-llm`. It sends dependency coordinates (not file contents), requires no API key, and falls back to a bundled list when OSV.dev is unreachable. - **It does not sandbox the host.** SkillSpector flags risky patterns *before* you install a skill; it does not contain or isolate a skill you choose to install anyway. diff --git a/contrib/batch_scan/CONTRIBUTING.md b/contrib/batch_scan/CONTRIBUTING.md index ea14f0167..01415c700 100644 --- a/contrib/batch_scan/CONTRIBUTING.md +++ b/contrib/batch_scan/CONTRIBUTING.md @@ -24,7 +24,7 @@ python -m contrib.batch_scan.batch_scan ./tests/fixtures/ -f terminal --workers ``` contrib/batch_scan/ -├── batch_scan.py # CLI entry + ThreadPoolExecutor (start here) +├── batch_scan.py # CLI entry + bounded process supervision (start here) ├── runner.py # graph.invoke() wrapper + 7 patches + pool wiring (core) ├── gap_fill.py # GapFillAnalyzer — LLM pass for 8 uncovered rules ├── api_pool.py # ApiKeyPool — multi-key scheduler + 429 backoff diff --git a/contrib/batch_scan/api_pool.py b/contrib/batch_scan/api_pool.py index 6960eab9b..b068f8b06 100644 --- a/contrib/batch_scan/api_pool.py +++ b/contrib/batch_scan/api_pool.py @@ -159,10 +159,12 @@ def __init__(self, keys: list[ApiKey]) -> None: self._retry_successes: int = 0 self._total_requests_served: int = 0 self._peak_active_requests: int = 0 + self._leases: dict[str, list[ApiKey]] = {} + self._cancelled_owners: set[str] = set() # -- Public API ----------------------------------------------------------- - def acquire(self, timeout: float | None = None) -> ApiKey: + def acquire(self, timeout: float | None = None, *, owner: str | None = None) -> ApiKey: """Acquire a slot on the least-loaded available key. Scheduling priority: @@ -193,6 +195,8 @@ def acquire(self, timeout: float | None = None) -> ApiKey: with self._condition: while True: + if owner in self._cancelled_owners: + raise RuntimeError("Scan worker has stopped") now = time.monotonic() # Step 1: recover rate-limited keys whose backoff has expired @@ -208,6 +212,8 @@ def acquire(self, timeout: float | None = None) -> ApiKey: _now_active = sum(k.active_requests for k in self._keys) if _now_active > self._peak_active_requests: self._peak_active_requests = _now_active + if owner is not None: + self._leases.setdefault(owner, []).append(key) logger.debug( "Pool: slot on key …%s (%d/%d active)", key.key[-8:], @@ -236,7 +242,7 @@ def acquire(self, timeout: float | None = None) -> ApiKey: ) self._condition.wait(timeout=wait) - def try_acquire(self) -> ApiKey | None: + def try_acquire(self, *, owner: str | None = None) -> ApiKey | None: """Non-blocking acquire — returns a key immediately or ``None``. Unlike :meth:`acquire`, this never blocks. If a slot is available @@ -244,6 +250,8 @@ def try_acquire(self) -> ApiKey | None: Useful in async contexts where blocking would stall the event loop. """ with self._lock: + if owner in self._cancelled_owners: + raise RuntimeError("Scan worker has stopped") self._recover_expired_keys(time.monotonic()) available = [k for k in self._keys if k.available] if not available: @@ -255,9 +263,11 @@ def try_acquire(self) -> ApiKey | None: _now_active = sum(k.active_requests for k in self._keys) if _now_active > self._peak_active_requests: self._peak_active_requests = _now_active + if owner is not None: + self._leases.setdefault(owner, []).append(key) return key - def release(self, key: ApiKey, *, success: bool = True) -> None: + def release(self, key: ApiKey, *, success: bool = True, owner: str | None = None) -> None: """Release a slot on *key* back to the pool. Parameters @@ -270,6 +280,15 @@ def release(self, key: ApiKey, *, success: bool = True) -> None: rate-limited with exponential backoff. """ with self._condition: + if owner is not None: + # Manager RPC returns a copy: release the server's original lease. + leases = self._leases.get(owner, []) + for index, leased in enumerate(leases): + if (leased.key, leased.base_url, leased.model) == (key.key, key.base_url, key.model): + key = leases.pop(index) + break + else: + return key.active_requests = max(0, key.active_requests - 1) if success: @@ -299,6 +318,14 @@ def release(self, key: ApiKey, *, success: bool = True) -> None: self._condition.notify_all() + def release_owner(self, owner: str) -> None: + """Reclaim slots and wake waiters after a scan process exits or is killed.""" + with self._condition: + self._cancelled_owners.add(owner) + for key in self._leases.pop(owner, []): + key.active_requests = max(0, key.active_requests - 1) + self._condition.notify_all() + def record_retry_success(self) -> None: """Increment the retry-success counter for reporting. @@ -448,99 +475,87 @@ async def ainvoke_with_usage(self, prompt: str, collector: object) -> object: # -- Internal ------------------------------------------------------------- + @staticmethod + def _remaining(deadline: float) -> float: + from skillspector.llm_analyzer_base import LLMRuntimeLimitError + + remaining = deadline - time.monotonic() + if remaining <= 0: + raise LLMRuntimeLimitError("pooled request runtime limit reached") + return remaining + def _invoke_with_retry( - self, - prompt: str, - *, - callbacks: list[object] | None = None, + self, prompt: str, *, callbacks: list[object] | None = None, ) -> object: - """Sync retry loop — acquire slot, call LLM, release, retry on 429.""" - last_exception: Exception | None = None - + """Share one deadline across slot waiting, requests, and key retries.""" + deadline = time.monotonic() + self._timeout for attempt in range(self._max_retries + 1): - key = self._pool.acquire() - llm = self._build_llm(key) try: - if callbacks is None: - result = llm.invoke(prompt) - else: - result = llm.invoke(prompt, config={"callbacks": callbacks}) - self._pool.release(key, success=True) - if attempt > 0: - self._pool.record_retry_success() - return result + key = self._pool.acquire(timeout=self._remaining(deadline)) + except Exception: + self._remaining(deadline) + raise + rate_limited = False + try: + llm = self._build_llm(key, timeout=self._remaining(deadline)) + result = llm.invoke(prompt) if callbacks is None else llm.invoke( + prompt, config={"callbacks": callbacks} + ) except Exception as exc: - if self._is_rate_limit(exc) and attempt < self._max_retries: - self._pool.release(key, success=False) - logger.debug( - "PooledChatModel: rate-limited, retrying " - "(attempt %d/%d)", - attempt + 1, - self._max_retries, - ) + rate_limited = self._is_rate_limit(exc) + if rate_limited and attempt < self._max_retries: continue - self._pool.release(key, success=True) - last_exception = exc raise - - raise RuntimeError( - f"PooledChatModel: exhausted {self._max_retries} retries " - "due to rate-limit errors" - ) from last_exception + finally: + self._pool.release(key, success=not rate_limited) + if attempt > 0: + self._pool.record_retry_success() + return result + raise RuntimeError("PooledChatModel exhausted key retries") async def _ainvoke_with_retry( - self, - prompt: str, - *, - callbacks: list[object] | None = None, + self, prompt: str, *, callbacks: list[object] | None = None, ) -> object: - """Async retry loop — non-blocking acquire first, block only if full.""" + """Bound asynchronous slot waiting without leaving an acquiring thread.""" import asyncio - last_exception: Exception | None = None + deadline = time.monotonic() + self._timeout for attempt in range(self._max_retries + 1): + self._remaining(deadline) key = self._pool.try_acquire() - if key is None: - key = await asyncio.to_thread(self._pool.acquire) - llm = self._build_llm(key) + while key is None: + await asyncio.sleep(min(0.05, self._remaining(deadline))) + self._remaining(deadline) + key = self._pool.try_acquire() + rate_limited = False try: - if callbacks is None: - result = await llm.ainvoke(prompt) - else: - result = await llm.ainvoke(prompt, config={"callbacks": callbacks}) - self._pool.release(key, success=True) - if attempt > 0: - self._pool.record_retry_success() - return result + llm = self._build_llm(key, timeout=self._remaining(deadline)) + result = await llm.ainvoke(prompt) if callbacks is None else await llm.ainvoke( + prompt, config={"callbacks": callbacks} + ) except Exception as exc: - if self._is_rate_limit(exc) and attempt < self._max_retries: - self._pool.release(key, success=False) - logger.debug( - "PooledChatModel: rate-limited, retrying " - "(attempt %d/%d)", - attempt + 1, - self._max_retries, - ) + rate_limited = self._is_rate_limit(exc) + if rate_limited and attempt < self._max_retries: continue - self._pool.release(key, success=True) - last_exception = exc raise - - raise RuntimeError( - f"PooledChatModel: exhausted {self._max_retries} retries " - "due to rate-limit errors" - ) from last_exception - - def _build_llm(self, key: ApiKey): + finally: + self._pool.release(key, success=not rate_limited) + if attempt > 0: + self._pool.record_retry_success() + return result + raise RuntimeError("PooledChatModel exhausted key retries") + + def _build_llm(self, key: ApiKey, *, timeout: float | None = None): """Build a fresh :class:`~langchain_openai.ChatOpenAI` for *key*.""" from langchain_openai import ChatOpenAI from pydantic import SecretStr + timeout = self._timeout if timeout is None else timeout try: import httpx - _timeout = httpx.Timeout(self._timeout, connect=8.0) + _timeout = httpx.Timeout(timeout, connect=min(8.0, timeout)) except ImportError: - _timeout = self._timeout + _timeout = timeout return ChatOpenAI( model=key.model, @@ -548,6 +563,7 @@ def _build_llm(self, key: ApiKey): api_key=SecretStr(key.key), max_completion_tokens=self._max_tokens, timeout=_timeout, + max_retries=0, ) @staticmethod diff --git a/contrib/batch_scan/batch_scan.py b/contrib/batch_scan/batch_scan.py index 272c5f7bf..50e82a65a 100644 --- a/contrib/batch_scan/batch_scan.py +++ b/contrib/batch_scan/batch_scan.py @@ -23,9 +23,10 @@ Concurrency model ----------------- Each skill runs the full ``graph.invoke(state)`` pipeline in a dedicated -thread via :class:`~concurrent.futures.ThreadPoolExecutor`. The number of +process supervised by a :class:`~concurrent.futures.ThreadPoolExecutor`. The number of parallel workers is controlled by ``--workers`` (default 4). A 90-second -per-skill timeout prevents stalled workers from blocking the batch. This +per-skill work timeout prevents stalled workers from blocking the batch. Worker +startup has a separate 90-second bound. Failed skills remain in reports. This sits on top of two built-in parallelism layers: * **Layer 1** — 20 analyzers fan-out inside the LangGraph (per-skill) @@ -57,10 +58,26 @@ _dotenv.load_dotenv(_dotenv.find_dotenv(usecwd=True), override=True) import argparse +import json +import multiprocessing +import os +import shutil +import signal +import subprocess import sys +import tempfile import threading +import time from concurrent.futures import ThreadPoolExecutor, TimeoutError, as_completed +from contextlib import ExitStack +from functools import partial +from multiprocessing.connection import wait +from multiprocessing.managers import BaseManager from pathlib import Path +from tempfile import TemporaryDirectory +from types import SimpleNamespace +from uuid import uuid4 + from skillspector.logging_config import set_level from .api_pool import create_api_key_pool_from_env @@ -68,11 +85,21 @@ from .reports import _format_json as format_json from .reports import _format_markdown as format_markdown from .reports import _format_terminal as format_terminal -from .runner import run_one +from .reports import _terminal_text +from .runner import entry_from_error, run_one # Progress-print lock — Rich consoles are not thread-safe; serialize output # from the main thread via this lock. _print_lock = threading.Lock() +# Process.start() reaps finished siblings; serialize it with live-worker cleanup. +_process_lock = threading.Lock() + + +class _PoolManager(BaseManager): + """Keep rate limits shared across independently killable scan processes.""" + + +_PoolManager.register("create_pool", create_api_key_pool_from_env) # --------------------------------------------------------------------------- @@ -88,6 +115,7 @@ def _scan_skill( lang: str, require_llm: bool, api_pool=None, + verbose: bool = False, ) -> tuple[dict[str, object], str | None, str]: """Scan a single skill through the full pipeline. @@ -100,6 +128,9 @@ def _scan_skill( except ValueError: rel_name = skill_dir.name + if verbose: + set_level("DEBUG") + # Core scan and optional gap-fill share the graph's validated file cache. entry, error_msg = run_one( skill_dir, @@ -113,6 +144,113 @@ def _scan_skill( return entry, error_msg, rel_name +def _kill_worker_group(pid: int) -> None: + """Stop a worker and any descendants that share its process group.""" + if os.name == "posix": + try: + os.killpg(pid, signal.SIGKILL) + except ProcessLookupError: + pass + else: + try: + subprocess.run( + ["taskkill", "/PID", str(pid), "/T", "/F"], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + timeout=5, check=False, + ) + except (OSError, subprocess.TimeoutExpired): + pass + + +def _watch_parent(scratch: Path | None = None) -> None: + parent = multiprocessing.parent_process() + if parent is None: + return + wait([parent.sentinel]) + if scratch is not None: + # A dead supervisor cannot clean its scratch or stop the worker group. + shutil.rmtree(scratch, ignore_errors=True) + _kill_worker_group(os.getpid()) + # The pool manager has no worker group, but must not retain API keys after + # its supervisor disappears. + os._exit(0) + + +def _start_parent_watch(scratch: Path | None = None) -> None: + threading.Thread(target=_watch_parent, args=(scratch,), daemon=True).start() + + +def _scan_skill_process( + skill_dir: Path, root: Path, result_path: Path, options: dict, started +) -> None: + if os.name == "posix": + os.setsid() + _start_parent_watch(result_path.parent) + # A killed worker cannot run finally blocks. Its supervisor owns all scratch. + tempfile.tempdir = str(result_path.parent) + for name in ("TMPDIR", "TEMP", "TMP"): + os.environ[name] = str(result_path.parent) + from .runner import deepseek_compat, set_api_pool + + if options.get("api_pool") is not None: + set_api_pool(options["api_pool"]) + with deepseek_compat(): + started.set() + result = _scan_skill(skill_dir, root, **options) + result_path.write_text(json.dumps(result), encoding="utf-8") + + +def _scan_skill_bounded( + skill_dir: Path, root: Path, *, api_pool=None, timeout: float = 90, + startup_timeout: float = 90, **options +) -> tuple[dict[str, object], str | None, str]: + """Enforce the wall-clock limit on actual work, including local analysis.""" + owner = uuid4().hex + if api_pool is not None: + options["api_pool"] = SimpleNamespace( + acquire=partial(api_pool.acquire, owner=owner), + try_acquire=partial(api_pool.try_acquire, owner=owner), + release=partial(api_pool.release, owner=owner), + record_retry_success=api_pool.record_retry_success, + ) + with TemporaryDirectory(prefix="skillspector-batch-") as scratch: + result_path = Path(scratch) / "result.json" + context = multiprocessing.get_context("spawn") + started = context.Event() + process = context.Process( + target=_scan_skill_process, args=(skill_dir, root, result_path, options, started) + ) + try: + with _process_lock: + process.start() + startup_deadline = time.monotonic() + startup_timeout + while not started.is_set() and process.is_alive(): + remaining = startup_deadline - time.monotonic() + if remaining <= 0: + raise TimeoutError(f"scan worker startup timed out after {startup_timeout:g}s") + started.wait(min(0.1, remaining)) + process.join(timeout if started.is_set() else 0) + if process.is_alive(): + raise TimeoutError(f"scan timed out after {timeout:g}s") + if process.exitcode != 0: + raise RuntimeError(f"Skill scan worker exited with code {process.exitcode}") + entry, error, name = json.loads(result_path.read_text(encoding="utf-8")) + return entry, error, name + finally: + with _process_lock: + if process.pid is not None: + # Only signal a live, unreaped worker while sibling starts + # cannot recycle its PID. Providers own normal-exit cleanup. + if process.exitcode is None: + _kill_worker_group(process.pid) + if process.is_alive(): + process.kill() + process.join() + process.close() + if api_pool is not None: + api_pool.release_owner(owner) + + # --------------------------------------------------------------------------- # Main # --------------------------------------------------------------------------- @@ -138,10 +276,16 @@ def _main_impl() -> None: # -- Rich detection ------------------------------------------------------- try: from rich.console import Console + from rich.markup import escape + from rich.text import Text except ImportError: Console = None # type: ignore[assignment] # noqa: N806 - c = Console() if Console is not None else None + c = Console(emoji=False) if Console is not None else None + + def display(value: object) -> str: + text = _terminal_text(value) + return escape(text) if c else text def _print(*args: object, **kwargs: object) -> None: """Print through Rich when available, falling back to plain text.""" @@ -191,7 +335,8 @@ def _print(*args: object, **kwargs: object) -> None: metavar="N", help="Number of parallel scan workers (default: 4). " "Reduce to 1 for free-tier API keys, increase for enterprise tiers. " - "Skills that time out (90s) are skipped; other workers continue.", + "Skills that time out (90s of scanning, plus up to 90s for startup) " + "are reported as errors; other workers continue.", ) parser.add_argument( "-V", @@ -226,7 +371,7 @@ def _print(*args: object, **kwargs: object) -> None: # -- Validation ---------------------------------------------------------- root = args.input_dir.resolve() if not root.is_dir(): - _print(f"[red]Error:[/red] {root} is not a directory", file=sys.stderr) + _print(f"[red]Error:[/red] {display(root)} is not a directory", file=sys.stderr) sys.exit(2) skill_dirs = discover_skills(root) @@ -240,9 +385,6 @@ def _print(*args: object, **kwargs: object) -> None: # -- API Pool (optional — returns None if single-key) -------------------- api_pool = create_api_key_pool_from_env() - if api_pool: - from .runner import set_api_pool - set_api_pool(api_pool) use_llm = not args.no_llm # -- Header -------------------------------------------------------------- @@ -254,7 +396,7 @@ def _print(*args: object, **kwargs: object) -> None: ) _print( f"\n[bold]SkillSpector Batch Scan[/bold] — " - f"{len(skill_dirs)} skill(s) in [dim]{root}[/dim]" + f"{len(skill_dirs)} skill(s) in [dim]{display(root)}[/dim]" f" ([cyan]{args.workers} workers[/cyan]{pool_note})\n" ) @@ -273,16 +415,23 @@ def _print(*args: object, **kwargs: object) -> None: total = len(skill_dirs) - with ThreadPoolExecutor(max_workers=args.workers) as executor: + pool_snapshot = None + with ExitStack() as cleanup, ThreadPoolExecutor(max_workers=args.workers) as executor: + if api_pool is not None: + pool_manager = _PoolManager(ctx=multiprocessing.get_context("spawn")) + pool_manager.start(_start_parent_watch) + cleanup.callback(pool_manager.shutdown) + api_pool = pool_manager.create_pool() future_map = { executor.submit( - _scan_skill, + _scan_skill_bounded, skill_dir, root, use_llm=use_llm, lang=args.lang, require_llm=args.require_llm, api_pool=api_pool, + verbose=args.verbose, ): idx for idx, skill_dir in enumerate(skill_dirs, 1) } @@ -291,49 +440,36 @@ def _print(*args: object, **kwargs: object) -> None: idx = future_map[future] rel_name = str(skill_dirs[idx - 1].relative_to(root)) if idx <= len(skill_dirs) else "?" try: - entry, error_msg, rel_name = future.result(timeout=90) - except TimeoutError: - errors += 1 - with _print_lock: - _print( - f" [{idx}/{total}] [cyan]{rel_name}[/cyan] → " - f"[red]TIMEOUT (90s)[/red]" - ) - # Don't retry — the worker thread is still stuck and a - # retry would consume another slot. HTTP-level timeouts - # (runner.py Patch 6) prevent most hangs from happening. - continue - except Exception: - # Unexpected crash (e.g. asyncio event-loop failure). - # Don't retry — log and continue. - errors += 1 - with _print_lock: - _print( - f" [{idx}/{total}] [cyan]{rel_name}[/cyan] → " - f"[red]CRASH[/red]" - ) - continue + entry, error_msg, rel_name = future.result() + except Exception as exc: + error_msg = str(exc) or type(exc).__name__ + entry = entry_from_error(skill_dirs[idx - 1], root, error_msg, args.lang) lang = entry["skill"]["language"] results.append(entry) # -- Progress (main thread via lock — safe for Rich) --------- with _print_lock: # Non-English LLM guard warning - if lang != "en" and not use_llm and args.require_llm: + if not error_msg and lang != "en" and not use_llm and args.require_llm: + warning = ( + f"non-English skill '{_terminal_text(rel_name)}' " + f"({_terminal_text(lang)}) scanned with --no-llm. " + "Static pattern recall is reduced for this language. " + "Re-run without --no-llm for full coverage, or use " + "--no-require-llm to suppress this warning." + ) _print( - f"[yellow]WARNING:[/yellow] non-English skill " - f"'{rel_name}' ({lang}) scanned with --no-llm. " - f"Static pattern recall is reduced for this language. " - f"Re-run without --no-llm for full coverage, or use " - f"--no-require-llm to suppress this warning.", + Text.assemble(("WARNING:", "yellow"), " ", warning) + if c + else f"WARNING: {warning}", file=sys.stderr, ) if error_msg: errors += 1 _print( - f" [{idx}/{total}] [cyan]{rel_name}[/cyan] → " - f"[red]ERROR: {error_msg}[/red]" + f" [{idx}/{total}] [cyan]{display(rel_name)}[/cyan] → " + f"[red]ERROR: {display(error_msg)}[/red]" ) else: risk = entry.get("risk_assessment", {}) @@ -344,11 +480,14 @@ def _print(*args: object, **kwargs: object) -> None: has_high_risk = True color = _sev_colors.get(severity, "") _print( - f" [{idx}/{total}] [cyan]{rel_name}[/cyan] → " - f"[{color}]{score}/100 {severity}[/{color}] " + f" [{idx}/{total}] [cyan]{display(rel_name)}[/cyan] → " + f"[{color}]{score}/100 {display(severity)}[/{color}] " f"({n_issues} issue(s))" ) + if api_pool is not None: + pool_snapshot = api_pool.snapshot() + # -- Sort results by risk score descending ------------------------------- results.sort( key=lambda x: x.get("risk_assessment", {}).get("score", 0), # type: ignore[no-any-return] @@ -356,8 +495,8 @@ def _print(*args: object, **kwargs: object) -> None: ) # -- API Pool summary (if active) ---------------------------------------- - if api_pool: - snap = api_pool.snapshot() + if pool_snapshot: + snap = pool_snapshot _parts = [ f"{snap['total_requests_served']} requests served", ] @@ -384,10 +523,10 @@ def _print(*args: object, **kwargs: object) -> None: if args.output: args.output.write_text(report_body, encoding="utf-8") - _print(f"\n[green]Batch report saved to:[/green] {args.output}") + _print(f"\n[green]Batch report saved to:[/green] {display(args.output)}") else: if fmt == "terminal": - _print(report_body) + _print(report_body, markup=False, highlight=False) else: sys.stdout.write(report_body + "\n") diff --git a/contrib/batch_scan/docs/DESIGN.md b/contrib/batch_scan/docs/DESIGN.md index bb44ca0b1..2cdd8d464 100644 --- a/contrib/batch_scan/docs/DESIGN.md +++ b/contrib/batch_scan/docs/DESIGN.md @@ -16,10 +16,10 @@ batch_scan.py :: main() ├─ detect language (Unicode script-ratio, per skill) ├─ create API pool (optional, 10-key scheduler) ├─ ThreadPoolExecutor(max_workers=N) - │ ├─ Thread A: skill_1 → graph.invoke() + gap-fill - │ ├─ Thread B: skill_2 → graph.invoke() + gap-fill + │ ├─ Thread A: supervise process → skill_1 → graph.invoke() + gap-fill + │ ├─ Thread B: supervise process → skill_2 → graph.invoke() + gap-fill │ └─ ... - ├─ collect results, sort by risk score + ├─ retain successful, timed-out and crashed scans; sort by risk score └─ report (terminal / JSON / Markdown) ``` @@ -39,7 +39,7 @@ run_one(skill_dir) ## Three-layer concurrency ``` -Layer 3 — batch_scan.py: ThreadPoolExecutor(max_workers=N) [CONTRIB] +Layer 3 — batch_scan.py: ThreadPoolExecutor → per-skill processes [CONTRIB] Layer 2 — llm_analyzer_base: asyncio.Semaphore(10) [UPSTREAM] Layer 1 — graph.py: 20 analyzers fan-out [UPSTREAM] ``` @@ -47,11 +47,20 @@ Layer 1 — graph.py: 20 analyzers fan-out [UPSTREAM] Each layer is unaware of the others. The graph doesn't know it's being called concurrently; the workers don't know the graph fans out internally. -## Why ThreadPoolExecutor +## Why supervised scan processes -- ProcessPoolExecutor hangs on macOS (spawn mode reimports LangGraph per child) -- `graph.invoke()` is a pure function — same state → same result, no shared state -- Each thread operates on its own state dict, isolated from other threads +Threads supervise independently killable processes; they do not run scans themselves. +A worker has up to 90 seconds to start its interpreter and import the graph. It then +signals that scanning has begun, starting a separate 90-second work deadline. +Timeouts and crashes remain ERROR entries in every saved report, and other skills +continue. The work deadline includes provider-pool waits and gap-fill. + +Each worker owns a process group. The supervisor kills that group on timeout, +reclaims API-pool leases, and removes scratch files. A worker also watches its +parent's process sentinel so parent termination stops the group and removes its +scratch. Descendants that deliberately start a new session are outside this group. +The API-pool manager also watches its parent and exits if the supervisor dies. +Verbose logging is forwarded explicitly to spawned workers. ## DeepSeek compatibility patches @@ -173,7 +182,8 @@ Windows. ## Per-skill timeout (90s) -A skill that takes >90s is marked TIMEOUT and skipped. Other workers continue. +A skill that takes more than 90 seconds of scan time is retained as an ERROR +entry in the report. Startup has a separate 90-second bound. Other workers continue. HTTP-level timeouts (Patch 6) prevent most hangs from reaching the 90s ceiling. ## Exit codes @@ -189,7 +199,7 @@ HTTP-level timeouts (Patch 6) prevent most hangs from reaching the 90s ceiling. ``` contrib/batch_scan/ ├── __init__.py # package init + dotenv preload -├── batch_scan.py # CLI + ThreadPoolExecutor +├── batch_scan.py # CLI + supervised scan processes ├── runner.py # graph wrapper + setup_deepseek_compat() ├── discovery.py # SKILL.md finder ├── detection.py # language detection @@ -228,9 +238,10 @@ requirement. It would also require all batch orchestration code to be async, complicating the CLI layer (`argparse`, Rich console output) with no throughput gain. -`ProcessPoolExecutor` was tested and rejected: macOS Python 3.13 `spawn` mode -reimports LangGraph + LangChain per child process, causing 30+ second startup -timeouts. `fork` mode is unavailable on macOS since Python 3.8. +The earlier `ProcessPoolExecutor` experiment had 30+ second startup delays on +macOS because `spawn` reimports LangGraph and LangChain. The current supervisor +still pays this import cost per skill, but accounts for it separately from scan +time. It uses `spawn` consistently rather than forking a threaded scanner. ### Why monkey-patch, not fork upstream? @@ -273,22 +284,22 @@ raw LLM string → _strip_markdown_fences() → json.loads() → model_validate( The two-step parse (stdlib `json.loads` then Pydantic `model_validate`) exists because: -1. `json.loads` is fast, deterministic, and raises clear `JSONDecodeError` on - malformed output — we catch this and return `[]` (empty findings). -2. `model_validate` enforces the schema: required fields, literal enums, - confidence range, string length. Schema violations are caught and returned - as `[]` with a warning log. - -**Error propagation:** If the LLM returns invalid JSON or schema-mismatched -output, the analyzer returns `[]` (no findings for that file). The scan -continues — a single malformed LLM response never blocks the pipeline. -The warning is logged at `WARNING` level so operators can monitor parse-failure -rates without sifting through debug logs. - -Patch 3 adds a `_sanitize_meta_finding()` pass after validation to handle -known LLM quirks: `null` string fields → `""`, unrecognized enum values -(e.g., `"none"`) → `"low"`. These are applied post-validation because they -represent recoverable soft errors, not hard schema violations. +1. `json.loads` parses the response. Invalid JSON raises the core structured-response + validation error rather than producing an empty findings list. +2. `model_validate` checks the required fields and value types. Schema failures + use the same retry and reporting path. + +**Error propagation:** Invalid JSON and schema failures are retried up to four +attempts. If they persist, the ledger records `skipped` with reason +`llm_structured_response_invalid`; the analyzer is degraded and the skill counts +as incomplete. The scan continues and keeps findings from completed work. +Core logs `LLM structured response validation failed for ... retrying` and, +on exhaustion, `... after 4 attempts`, without logging the raw response. + +Patch 3 sanitizes known soft quirks before validation: null explanation and +remediation become empty strings, impact labels are case-folded, and unknown +impact labels use `low`. Invalid findings still fail validation. Optional prose +in `overall_assessment` is ignored so valid finding verdicts are retained. ## Gap-Fill Rule Selection Criteria diff --git a/contrib/batch_scan/docs/README.md b/contrib/batch_scan/docs/README.md index 87c5dc577..ad012f42e 100644 --- a/contrib/batch_scan/docs/README.md +++ b/contrib/batch_scan/docs/README.md @@ -330,7 +330,7 @@ fi |---------|-----| | "No LLM API key configured" | Set up `.env` or use `--no-llm` | | Connection errors / 429 | Reduce `--workers` | -| Skills timing out (90s) | Check network; the scanner skips and continues | +| Skills timing out (90s of scanning) | Check network and API-pool capacity; the saved report retains an ERROR entry and other skills continue. Worker startup has its own 90s bound. | | "Event loop is closed" | Harmless, suppressed | | model_info token limit warning | Harmless, 128K default used | @@ -341,10 +341,11 @@ fi classified as English and lose gap-fill coverage. 3. **No SARIF output.** Upstream supports it; this contrib adds terminal/JSON/Markdown. 4. **Gap-fill quality not benchmarked for non-English.** No ground-truth comparison exists. -5. **`parse_response` JSON recovery is best-effort.** When the LLM returns - malformed JSON, the analyzer returns empty findings (no crash). This is a - graceful-degradation choice: a single malformed response won't block the - pipeline, but the user won't know which findings were lost. +5. **Malformed provider responses reduce coverage.** Compatibility-mode discovery + and meta responses are retried up to four attempts, then recorded as + `llm_structured_response_invalid` and counted as incomplete. Valid findings + from completed batches are kept. Gap-fill parsing still drops malformed + responses until the separate gap-fill failure fix is included. See `DESIGN.md` for architecture details and `docs/archive/FUTURE_WORK.md` for suggested directions. diff --git a/contrib/batch_scan/reports.py b/contrib/batch_scan/reports.py index 880869d7f..cb6de30fd 100644 --- a/contrib/batch_scan/reports.py +++ b/contrib/batch_scan/reports.py @@ -28,6 +28,9 @@ from io import StringIO from skillspector import __version__ as _skillspector_version +from skillspector.nodes.report import _markdown_cell as _markdown_text +from skillspector.nodes.report import _markdown_code +from skillspector.nodes.report import _markdown_plain_text as _markdown_plain_text def sorted_results(results: list[dict[str, object]]) -> list[dict[str, object]]: @@ -85,15 +88,27 @@ def _exception_groups(results: list[dict[str, object]]) -> list[tuple[str, list[ # ═══════════════════════════════════════════════════════════════════ +def _language_counts(results: list[dict[str, object]]) -> dict[str, int]: + """Count detected languages only for scans that produced a usable result.""" + counts: dict[str, int] = defaultdict(int) + for result in results: + language = result.get("skill", {}).get("language", "en") + if "error" not in result and language not in (None, "", "auto", "unknown"): + counts[language] += 1 + return counts + + def _format_terminal(results: list[dict[str, object]]) -> str: try: from rich.console import Console + from rich.markup import escape from rich.panel import Panel from rich.table import Table + from rich.text import Text except ImportError: return _format_terminal_plain(results) - capture = Console(record=True, force_terminal=True, width=80, file=StringIO()) + capture = Console(record=True, force_terminal=True, width=80, file=StringIO(), emoji=False) total = len(results) critical = _count_sev(results, "CRITICAL") @@ -104,7 +119,7 @@ def _format_terminal(results: list[dict[str, object]]) -> str: completed = total - errs # ── Enhancement summary (for multilingual-enhanced mode) ──── - non_en = sum(1 for r in results if r.get("skill", {}).get("language", "en") != "en") + non_en = sum(count for lang, count in _language_counts(results).items() if lang != "en") gap_fill_total = sum( r.get("enhancements", {}).get("gap_fill_findings", 0) for r in results ) @@ -179,15 +194,15 @@ def _format_terminal(results: list[dict[str, object]]) -> str: lr = _lr_icon(sev, lang) if r.get("error"): - table.add_row(str(name), "-", "ERR", "[red]ERROR[/red]", "—", lang) + table.add_row(Text(_terminal_text(name)), "-", "ERR", "[red]ERROR[/red]", "—", Text(_terminal_text(lang))) else: table.add_row( - str(name), + Text(_terminal_text(name)), lr, f"[{color}]{score}/100[/{color}]", - f"[{color}]{sev}[/{color}]", + f"[{color}]{escape(_terminal_text(sev))}[/{color}]", str(issues), - lang, + Text(_terminal_text(lang)), ) capture.print(table) capture.print() @@ -207,12 +222,13 @@ def _format_terminal(results: list[dict[str, object]]) -> str: f"[green]{low_count} skill(s)[/green] with LOW risk — likely safe" ) for skill_name, exceptions in _exception_groups(results): - capture.print(f"[bold]Ledger exceptions — {skill_name}[/bold]") + capture.print(f"[bold]Ledger exceptions — {escape(_terminal_text(skill_name))}[/bold]") for exception in exceptions: capture.print( " - " - f"{exception.get('reason_code', 'unknown')} " - f"{exception.get('path', '')}: {exception.get('message', '')}" + f"{_terminal_text(exception.get('reason_code', 'unknown'))} " + f"{_terminal_text(exception.get('path', ''))}: {_terminal_text(exception.get('message', ''))}", + markup=False, ) capture.print() @@ -235,6 +251,8 @@ def _lr_icon(severity: str, language: str) -> str: def _print_source_breakdown(c, results: list[dict[str, object]]) -> None: + from rich.text import Text + group_stats: dict[str, dict[str, int]] = defaultdict( lambda: {"total": 0, "CRITICAL": 0, "HIGH": 0, "MEDIUM": 0, "LOW": 0} ) @@ -249,25 +267,22 @@ def _print_source_breakdown(c, results: list[dict[str, object]]) -> None: c.print("[bold]Source Breakdown:[/bold]") for group in sorted(group_stats): st = group_stats[group] - parts = [f" {group:<30s} {st['total']:>4d} skills"] + prefix = Text(f" {_terminal_text(group):<30s} {st['total']:>4d} skills") + parts = [] if st["CRITICAL"]: parts.append(f"[bold red]{st['CRITICAL']} CRITICAL[/bold red]") if st["HIGH"]: parts.append(f"[red]{st['HIGH']} HIGH[/red]") if st["MEDIUM"]: parts.append(f"[yellow]{st['MEDIUM']} MEDIUM[/yellow]") - c.print(", ".join(parts)) + c.print(prefix + Text.from_markup((", " + ", ".join(parts)) if parts else "")) c.print() def _print_language_breakdown(c, results: list[dict[str, object]]) -> None: - lang_stats: dict[str, int] = defaultdict(int) - lang_non_en: set[str] = set() - for r in results: - lang = r.get("skill", {}).get("language", "en") - lang_stats[lang] = lang_stats.get(lang, 0) + 1 - if lang != "en": - lang_non_en.add(lang) + from rich.text import Text + + lang_stats = _language_counts(results) if len(lang_stats) > 1: c.print("[bold]Language Breakdown:[/bold]") @@ -277,27 +292,32 @@ def _print_language_breakdown(c, results: list[dict[str, object]]) -> None: c.print(f" {lang:<6s} {count:>4d} skills (static + LLM coverage: full)") else: c.print( - f" {lang:<6s} {count:>4d} skills " - f"[yellow](static: partial, LLM: full)[/yellow]" + Text(f" {_terminal_text(lang):<6s} {count:>4d} skills ") + + Text("(static: partial, LLM: full)", style="yellow") ) c.print() +def _terminal_text(value: object) -> str: + """Keep an untrusted console field on one line without terminal controls.""" + return " ".join("".join(c for c in str(value) if c.isprintable() or c.isspace()).split()) + + def _format_terminal_plain(results: list[dict[str, object]]) -> str: lines: list[str] = [] for r in sorted_results(results): risk = r.get("risk_assessment", {}) skill = r.get("skill", {}) lines.append( - f" {skill.get('name', '?'):40s} " - f"{risk.get('score', 0):>3}/100 {risk.get('severity', 'LOW'):<8s}" + f" {_terminal_text(skill.get('name', '?')):40s} " + f"{risk.get('score', 0):>3}/100 {_terminal_text(risk.get('severity', 'LOW')):<8s}" ) for skill_name, exceptions in _exception_groups(results): - lines.append(f"Ledger exceptions — {skill_name}") + lines.append(f"Ledger exceptions — {_terminal_text(skill_name)}") for exception in exceptions: lines.append( - f" - {exception.get('reason_code', 'unknown')} " - f"{exception.get('path', '')}: {exception.get('message', '')}" + f" - {_terminal_text(exception.get('reason_code', 'unknown'))} " + f"{_terminal_text(exception.get('path', ''))}: {_terminal_text(exception.get('message', ''))}" ) return "\n".join(lines) @@ -332,13 +352,10 @@ def _format_json(results: list[dict[str, object]]) -> str: entries.append(entry) # Aggregate enhancement stats for the batch envelope - non_en_langs: set[str] = set() + languages = _language_counts(results) gap_fill_total = 0 gap_fill_skills = 0 for r in results: - lang = r.get("skill", {}).get("language", "en") - if lang != "en": - non_en_langs.add(lang) enhancements = r.get("enhancements", {}) gap_fill_total += enhancements.get("gap_fill_findings", 0) if enhancements.get("gap_fill_applied"): @@ -351,10 +368,9 @@ def _format_json(results: list[dict[str, object]]) -> str: "scan_mode": "multilingual-enhanced", "enhancements": { "language_detection": "unicode-script-ratio", - "languages_detected": {lang: sum( - 1 for r in results - if r.get("skill", {}).get("language") == lang - ) for lang in sorted(non_en_langs)}, + "languages_detected": { + lang: languages[lang] for lang in sorted(languages) if lang != "en" + }, "gap_fill_applied": gap_fill_skills, "gap_fill_findings": gap_fill_total, }, @@ -378,7 +394,7 @@ def _format_markdown(results: list[dict[str, object]]) -> str: total = len(results) # ── Enhancement summary ───────────────────────────────────── - non_en = sum(1 for r in results if r.get("skill", {}).get("language", "en") != "en") + non_en = sum(count for lang, count in _language_counts(results).items() if lang != "en") gap_fill_total = sum( r.get("enhancements", {}).get("gap_fill_findings", 0) for r in results ) @@ -438,9 +454,12 @@ def _format_markdown(results: list[dict[str, object]]) -> str: lang = skill.get("language", "en") if r.get("error"): - lines.append(f"| `{name}` | ERR | ERROR | — | {lang} |") + lines.append(f"| {_markdown_code(name, table_cell=True)} | ERR | ERROR | — | {_markdown_text(lang)} |") else: - lines.append(f"| `{name}` | {score}/100 | {sev} | {issues} | {lang} |") + lines.append( + f"| {_markdown_code(name, table_cell=True)} | {score}/100 | {_markdown_text(sev)} | " + f"{issues} | {_markdown_text(lang)} |" + ) lines.append("") # ── Issue details for HIGH / CRITICAL ──────────────────────── @@ -458,8 +477,8 @@ def _format_markdown(results: list[dict[str, object]]) -> str: risk = r.get("risk_assessment", {}) name = skill.get("name", "?") lines.append( - f"### {name} — {risk.get('score', 0)}/100 " - f"{risk.get('severity', 'HIGH')}\n" + f"### {_markdown_text(name)} — {risk.get('score', 0)}/100 " + f"{_markdown_text(risk.get('severity', 'HIGH'))}\n" ) for issue in r.get("issues", []): sev = str(issue.get("severity", "LOW")).upper() @@ -469,14 +488,14 @@ def _format_markdown(results: list[dict[str, object]]) -> str: loc_file = loc.get("file", "") if isinstance(loc, dict) else "" rule_id = issue.get("id", "?") explanation = issue.get("explanation", issue.get("message", "")) - lines.append(f"- **{emoji} {rule_id}**: {explanation}") + lines.append(f"- **{emoji} {_markdown_text(rule_id)}**: {_markdown_text(explanation)}") if loc_file: - lines.append(f" - Location: `{loc_file}:{loc_start}`") + lines.append(f" - Location: {_markdown_code(f'{loc_file}:{loc_start}')}") conf = issue.get("confidence", 0) lines.append(f" - Confidence: {float(conf):.0%}") rem = issue.get("remediation") if rem: - lines.append(f" - Remediation: {rem}") + lines.append(f" - Remediation: {_markdown_text(rem)}") lines.append("") lines.append("") @@ -484,11 +503,12 @@ def _format_markdown(results: list[dict[str, object]]) -> str: if exception_groups: lines.append("## Ledger Exceptions\n") for skill_name, exceptions in exception_groups: - lines.append(f"### {skill_name}\n") + lines.append(f"### {_markdown_text(skill_name)}\n") for exception in exceptions: lines.append( - f"- **{exception.get('reason_code', 'unknown')}** " - f"`{exception.get('path', '')}`: {exception.get('message', '')}" + f"- **{_markdown_text(exception.get('reason_code', 'unknown'))}** " + f"{_markdown_code(exception.get('path', ''))}: " + f"{_markdown_text(exception.get('message', ''))}" ) lines.append("") diff --git a/contrib/batch_scan/runner.py b/contrib/batch_scan/runner.py index 30a51c146..42f4945b0 100644 --- a/contrib/batch_scan/runner.py +++ b/contrib/batch_scan/runner.py @@ -37,8 +37,14 @@ from datetime import UTC, datetime from pathlib import Path +from pydantic import ValidationError + from skillspector.graph import graph -from skillspector.llm_analyzer_base import LLMAnalyzerBase, LLMAnalysisResult +from skillspector.llm_analyzer_base import ( + LLMAnalyzerBase, + LLMAnalysisResult, + _StructuredResponseValidationError, +) from skillspector.logging_config import get_logger from skillspector.nodes.meta_analyzer import LLMMetaAnalyzer, MetaAnalyzerResult @@ -85,13 +91,16 @@ def set_api_pool(pool: "ApiKeyPool | None") -> None: if _original_get_chat_model is None: _original_get_chat_model = _llm_utils.get_chat_model - def _pooled_get_chat_model(model=None): + def _pooled_get_chat_model(model=None, *, timeout=None): if _api_pool: from .api_pool import PooledChatModel - pooled_model = PooledChatModel(_api_pool) + pooled_model = PooledChatModel( + _api_pool, + timeout=_DEFAULT_REQUEST_TIMEOUT if timeout is None else timeout, + ) _llm_utils.register_chat_model_provider(pooled_model, "openai") return pooled_model - return _original_get_chat_model(model) + return _original_get_chat_model(model=model, timeout=timeout) _llm_utils.get_chat_model = _pooled_get_chat_model _llm_analyzer_base.get_chat_model = _pooled_get_chat_model @@ -124,7 +133,7 @@ def _pooled_get_chat_model(model=None): _original_base_init = LLMAnalyzerBase.__init__ -def _patched_base_init(self, base_prompt, model, *, node="llm_analyzer"): +def _patched_base_init(self, base_prompt, model, *, node="llm_analyzer", timeout=None): """Set response_schema=None on the instance dict BEFORE original init. Relies on Python MRO guarantee: instance.__dict__ is always checked @@ -132,7 +141,7 @@ def _patched_base_init(self, base_prompt, model, *, node="llm_analyzer"): a library internal. """ self.response_schema = None - _original_base_init(self, base_prompt, model, node=node) + _original_base_init(self, base_prompt, model, node=node, timeout=timeout) # -- Patch 2: LLMAnalyzerBase.parse_response handles raw JSON -------------- @@ -146,23 +155,11 @@ def _patched_base_parse(self, response, batch): text = _strip_markdown_fences(str(response)) try: data = json.loads(text) - except json.JSONDecodeError as exc: - logger.warning( - "LLMAnalyzerBase.parse_response: invalid JSON for %s: %s", - batch.file_label, - exc, - ) - return [] - try: result = LLMAnalysisResult.model_validate(data) - return [f.to_finding(batch.file_path) for f in result.findings] - except Exception as exc: - logger.warning( - "LLMAnalyzerBase.parse_response: schema validation failed for %s: %s", - batch.file_label, - exc, - ) - return [] + except (json.JSONDecodeError, ValidationError) as exc: + # Keep raw-response providers on the core retry and failure-ledger path. + raise _StructuredResponseValidationError from exc + return [f.to_finding(batch.file_path) for f in result.findings] # -- Patch 3: LLMMetaAnalyzer.parse_response handles raw JSON --------------- @@ -174,7 +171,9 @@ def _sanitize_meta_finding(d: dict) -> dict: for key in ("remediation", "explanation"): if d.get(key) is None: d[key] = "" - if d.get("impact") not in ("critical", "high", "medium", "low"): + impact = d.get("impact") + d["impact"] = impact.casefold() if isinstance(impact, str) else "low" + if d["impact"] not in ("critical", "high", "medium", "low"): d["impact"] = "low" return d @@ -186,28 +185,20 @@ def _patched_meta_parse(self, response, batch): text = _strip_markdown_fences(str(response)) try: data = json.loads(text) - except json.JSONDecodeError as exc: - logger.warning( - "LLMMetaAnalyzer.parse_response: invalid JSON for %s: %s", - batch.file_label, - exc, - ) - return [] - try: + if isinstance(data, dict) and isinstance(data.get("findings"), list): + data["findings"] = [ + _sanitize_meta_finding(dict(item)) if isinstance(item, dict) else item + for item in data["findings"] + ] result = MetaAnalyzerResult.model_validate(data) - items = [] - for f in result.findings: - d = _sanitize_meta_finding(f.model_dump()) - d["_file"] = batch.file_path - items.append(d) - return items - except Exception as exc: - logger.warning( - "LLMMetaAnalyzer.parse_response: schema validation failed for %s: %s", - batch.file_label, - exc, - ) - return [] + except (json.JSONDecodeError, ValidationError) as exc: + raise _StructuredResponseValidationError from exc + items = [] + for f in result.findings: + d = f.model_dump() + d["_file"] = batch.file_path + items.append(d) + return items # -- Patch 4: append JSON output format to base prompt --------------------- @@ -264,10 +255,14 @@ def _patched_meta_build_prompt(self, batch, **kwargs): def _patched_chatopenai_init(self, **kwargs): import httpx - _to = httpx.Timeout( - _DEFAULT_REQUEST_TIMEOUT, - connect=_DEFAULT_CONNECT_TIMEOUT, - ) + requested = kwargs.get("request_timeout", kwargs.get("timeout")) + _to = httpx.Timeout(_DEFAULT_REQUEST_TIMEOUT, connect=_DEFAULT_CONNECT_TIMEOUT) + if requested is not None: + requested = httpx.Timeout(requested).as_dict() + _to = httpx.Timeout(**{ + key: min(cap, requested[key]) if requested[key] is not None else cap + for key, cap in _to.as_dict().items() + }) # Set both the Pydantic alias AND the canonical field name so we don't # depend on alias-precedence behaviour (which is a Pydantic v2 internal). kwargs["timeout"] = _to @@ -327,12 +322,13 @@ def _verify_patch_targets() -> None: "LLMAnalyzerBase.__init__", 1, ) - _node_param = inspect.signature(LLMAnalyzerBase.__init__).parameters.get("node") - if _node_param is None or _node_param.kind != inspect.Parameter.KEYWORD_ONLY: - raise RuntimeError( - "Patch 1 target changed: LLMAnalyzerBase.__init__ must retain its " - "keyword-only 'node' parameter." - ) + for name in ("node", "timeout"): + parameter = inspect.signature(LLMAnalyzerBase.__init__).parameters.get(name) + if parameter is None or parameter.kind != inspect.Parameter.KEYWORD_ONLY: + raise RuntimeError( + "Patch 1 target changed: LLMAnalyzerBase.__init__ must retain its " + f"keyword-only '{name}' parameter." + ) if not hasattr(LLMAnalyzerBase, "response_schema"): raise RuntimeError( "Patch 1 target lost: LLMAnalyzerBase no longer has " @@ -347,7 +343,7 @@ def _verify_patch_targets() -> None: "LLMAnalyzerBase.parse_response", 2, ) - # Deep deps (called inside try/except — silent degradation if broken): + # Validate dependencies before patching rather than failing during a scan: if not hasattr(LLMAnalysisResult, "model_validate"): raise RuntimeError( "Patch 2 deep dependency lost: LLMAnalysisResult.model_validate " @@ -783,36 +779,42 @@ def run_one( entry["enhancements"]["gap_fill_findings"] = len(gap_findings) return entry, None except Exception as exc: - rel_name = _rel_name(skill_dir, root) - error_entry: dict[str, object] = { - "skill": { - "name": rel_name, - "source": str(skill_dir), - "source_group": rel_name.split("/")[0] if "/" in rel_name else ".", - "language": detected_language, - "scanned_at": datetime.now(UTC).isoformat(), - }, - "risk_assessment": { - "score": 0, - "severity": "ERROR", - "recommendation": "ERROR", - }, - "components": [], - "issues": [], - "scan_mode": "multilingual-enhanced", - "enhancements": { - "gap_fill_applied": False, - "gap_fill_findings": 0, - "english_keyword_rules_skipped": 0, - }, - "error": str(exc), - } - return error_entry, str(exc) + return entry_from_error(skill_dir, root, str(exc), detected_language), str(exc) finally: if result is not None: cleanup_result(result) +def entry_from_error( + skill_dir: Path, root: Path, error: str, detected_language: str +) -> dict[str, object]: + """Keep failed and interrupted scans visible in every batch report.""" + rel_name = _rel_name(skill_dir, root) + return { + "skill": { + "name": rel_name, + "source": str(skill_dir), + "source_group": rel_name.split("/")[0] if "/" in rel_name else ".", + "language": "unknown" if detected_language == "auto" else detected_language, + "scanned_at": datetime.now(UTC).isoformat(), + }, + "risk_assessment": { + "score": 0, + "severity": "ERROR", + "recommendation": "ERROR", + }, + "components": [], + "issues": [], + "scan_mode": "multilingual-enhanced", + "enhancements": { + "gap_fill_applied": False, + "gap_fill_findings": 0, + "english_keyword_rules_skipped": 0, + }, + "error": error, + } + + def _rel_name(skill_dir: Path, root: Path) -> str: """Best-effort relative name for display in progress lines.""" try: diff --git a/contrib/batch_scan/tests/docs/TEST_DESIGN.md b/contrib/batch_scan/tests/docs/TEST_DESIGN.md index 372c0b06b..e01d31170 100644 --- a/contrib/batch_scan/tests/docs/TEST_DESIGN.md +++ b/contrib/batch_scan/tests/docs/TEST_DESIGN.md @@ -95,7 +95,7 @@ It checks every assumption our patches depend on. If anything changed, it raises |------|-----------------| | **Guard passes current upstream** | Verifies no false positive. Tested against NVIDIA/SkillSpector@ab0431f (130+ commits, 89 files) — guard must not raise on the currently-installed upstream. Also tested after apply+restore cycle (state corruption check). | | **Each of 7 patches individually verified** | For each patch, we temporarily break its specific target and verify the guard catches it with the correct patch number in the error message. This proves every guard check is unique and distinguishable — an operator seeing "Patch 3" in the error knows exactly what broke. | -| **Deep dependency detection** | Beyond function signatures, our patches call `model_validate()`, `to_finding()`, `Batch.file_path`, `MetaAnalyzerResult.findings`, `asyncio.new_event_loop`. These are inside `try/except` blocks — if they silently disappear, the patch catches the exception and returns `[]`, masking the problem. The guard checks these BEFORE patching. | +| **Deep dependency detection** | Beyond function signatures, our patches call `model_validate()`, `to_finding()`, `Batch.file_path`, `MetaAnalyzerResult.findings`, `asyncio.new_event_loop`. Missing dependencies would fail during scanning; the guard checks them BEFORE patching so installed-core API drift is reported immediately. Malformed provider data follows the retry and incomplete-ledger path. | | **Keyword-only migration** | Python 3.x can change positional params to keyword-only. `_check_signature` detects `Parameter.KEYWORD_ONLY` kind and raises — our call sites pass these positionally. | | **Atomicity** | Guard failure must leave the process in its original state. We break a target, call `_apply_patches()`, and verify all 5 methods are still originals — the guard raised before any assignment happened. | diff --git a/contrib/batch_scan/tests/test_inspection_reporting.py b/contrib/batch_scan/tests/test_inspection_reporting.py index bcc00603c..db3da0ac7 100644 --- a/contrib/batch_scan/tests/test_inspection_reporting.py +++ b/contrib/batch_scan/tests/test_inspection_reporting.py @@ -8,6 +8,8 @@ import json from pathlib import Path +from markdown_it import MarkdownIt + from contrib.batch_scan.reports import _format_json, _format_markdown, _format_terminal from contrib.batch_scan.runner import entry_from_result @@ -61,6 +63,9 @@ def test_batch_formats_preserve_every_child_ledger_exception() -> None: assert [item["reason_code"] for item in exceptions] == ["read_error", "syntax_error"] assert payload["skills"][0]["issues"][0]["finding_id"] == "finding-batch-1" - for rendered in (_format_terminal([entry]), _format_markdown([entry])): + for rendered in ( + _format_terminal([entry]), + MarkdownIt().enable("table").render(_format_markdown([entry])), + ): assert "read_error" in rendered assert "syntax_error" in rendered diff --git a/contrib/batch_scan/tests/test_monkeypatch_fragility.py b/contrib/batch_scan/tests/test_monkeypatch_fragility.py index 950cd0778..77cc33744 100644 --- a/contrib/batch_scan/tests/test_monkeypatch_fragility.py +++ b/contrib/batch_scan/tests/test_monkeypatch_fragility.py @@ -263,22 +263,28 @@ def _broken_init(self, base_prompt, model): LLMAnalyzerBase.__init__ = original def test_patched_init_forwards_keyword_only_node(self) -> None: - instance = SimpleNamespace() - with patch("contrib.batch_scan.runner._original_base_init") as original_init: - _patched_base_init( - instance, - "prompt", - "model", - node="semantic_security_discovery", - ) - - original_init.assert_called_once_with( - instance, - "prompt", - "model", - node="semantic_security_discovery", - ) - self.assertIsNone(instance.response_schema) + for timeout in (None, 7, lambda: 7): + with self.subTest(timeout=timeout): + instance = SimpleNamespace() + with patch("contrib.batch_scan.runner._original_base_init") as original_init: + _patched_base_init( + instance, "prompt", "model", + node="semantic_security_discovery", timeout=timeout, + ) + original_init.assert_called_once_with( + instance, "prompt", "model", + node="semantic_security_discovery", timeout=timeout, + ) + self.assertIs(original_init.call_args.kwargs["timeout"], timeout) + self.assertIsNone(instance.response_schema) + + def test_guard_catches_missing_timeout_param(self) -> None: + def without_timeout(self, base_prompt, model, *, node="llm_analyzer"): + pass + + with patch.object(LLMAnalyzerBase, "__init__", without_timeout): + with self.assertRaisesRegex(RuntimeError, "timeout"): + _verify_patch_targets() def test_guard_catches_missing_response_schema_attr(self) -> None: """If upstream removes response_schema class attr, guard must raise.""" diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md index 4b2022c53..f7f3358d0 100644 --- a/docs/DEVELOPMENT.md +++ b/docs/DEVELOPMENT.md @@ -34,7 +34,7 @@ make install-dev - **Python**: 3.12+ (see [pyproject.toml](../pyproject.toml)). `make install` and `make install-dev` use **uv** if available (`uv sync` / `uv sync --all-extras`), otherwise **pip** (`pip install -e .` / `pip install -e ".[dev]"`). You must create and activate the virtual environment yourself before running any make target. - **Environment**: Optional `.env` in the project root. The LangGraph dev server loads it (see [langgraph.json](../langgraph.json) `"env": ".env"`). Key variables: - - **`SKILLSPECTOR_PROVIDER`**: Selects the active LLM provider — `openai`, `anthropic`, `anthropic_proxy`, `bedrock`, `nv_build`, `ollama`, `azure_openai`, `openai_compatible`, `claude_cli`, `codex_cli`, `gemini_cli`, or `opencode_cli`. Defaults to `nv_build` when unset. + - **`SKILLSPECTOR_PROVIDER`**: Selects the active LLM provider — `openai`, `anthropic`, `anthropic_proxy`, `bedrock`, `nv_build`, `ollama`, `azure_openai`, `openai_compatible`, `claude_cli`, `gemini_cli`, or `opencode_cli`. Defaults to `nv_build` when unset. - **Provider credential**: depends on the active provider. Hosted providers use the matching variables in [.env.example](../.env.example); Ollama and CLI providers do not require an API key. See [providers/](../src/skillspector/providers/). - **`OPENAI_BASE_URL`**: Override the OpenAI endpoint (e.g. point at Ollama). - **`SKILLSPECTOR_MODEL`**: Override default model; see [constants.py](../src/skillspector/constants.py). @@ -81,7 +81,7 @@ All targets assume the virtual environment is **already created and activated**. | `zip_bytes`, `mode` | Optional zip input and scan mode | | `components` | List of relative file paths in the skill | | `file_cache` | Map of path → file contents | -| `inspection_ledger` | Structured evidence for files excluded, skipped, or failed during analysis; a recognized OMS signature is recorded as an `oms_signature` scope exclusion. | +| `inspection_ledger` | Structured evidence for files excluded, skipped, or failed during analysis; unsupported DSSE decoding is recorded as an `oms_signature` partial event; successfully projected bundles remain in analysis without a scope exclusion. | | `ast_cache` | Map of path → AST representation (for future use) | | `manifest`, `previous_manifest` | Parsed skill metadata (e.g. from SKILL.md) | | `component_metadata` | List of dicts: path, type, lines, executable, size_bytes (from build_context) | @@ -296,11 +296,11 @@ Copy [.env.example](../.env.example) to `.env` in the project root and set value | Variable | Description | Example | |----------|-------------|---------| -| `SKILLSPECTOR_PROVIDER` | Active LLM provider: `openai` \| `anthropic` \| `anthropic_proxy` \| `bedrock` \| `nv_build` \| `ollama` \| `azure_openai` \| `openai_compatible` \| `claude_cli` \| `codex_cli` \| `gemini_cli` \| `opencode_cli`. Defaults to `nv_build`. | `claude_cli` | +| `SKILLSPECTOR_PROVIDER` | Active LLM provider: `openai` \| `anthropic` \| `anthropic_proxy` \| `bedrock` \| `nv_build` \| `ollama` \| `azure_openai` \| `openai_compatible` \| `claude_cli` \| `gemini_cli` \| `opencode_cli`. Defaults to `nv_build`. | `claude_cli` | | `NVIDIA_INFERENCE_KEY` | Credential for `nv_build`. | `nvapi-...` | | `OPENAI_API_KEY` | Credential for `SKILLSPECTOR_PROVIDER=openai`. Also tier-2 fallback for non-OpenAI providers. | `sk-...` | | `OPENAI_BASE_URL` | Override the OpenAI endpoint (e.g. point at Ollama). | `http://localhost:11434/v1` | -| `SKILLSPECTOR_REASONING_EFFORT` | Optional provider- and model-dependent reasoning-effort setting. Non-empty values are trimmed and passed through unchanged; unset or blank preserves provider-default behavior. | `high` | +| `SKILLSPECTOR_REASONING_EFFORT` | Optional provider- and model-dependent reasoning-effort setting. Non-empty values are trimmed and passed through unchanged. When unset or blank, SkillSpector sends `high` for `nv_build` with `z-ai/glm-5.3`; other provider/model combinations keep their endpoint defaults. | `high` | | `SKILLSPECTOR_OUTPUT_LANGUAGE` | Optional short, single-line language label (letters, numbers, spaces, `_`, or `-`; maximum 64 characters) for human-readable LLM finding text. Rule IDs, severity values, paths, code, and other machine-readable values remain unchanged. Unset, blank, or invalid values preserve the default output language. | `Japanese` | | `SKILLSPECTOR_TEMPERATURE` | Optional sampling temperature from `0` to `1` for hosted providers. Unset or blank preserves provider defaults. Lower values reduce variation but do not guarantee identical output. | `0` | | `SKILLSPECTOR_SEED` | Optional integer sampling seed for OpenAI-compatible and Azure OpenAI providers. Provider/model support is best-effort; CLI providers ignore it. | `42` | @@ -312,7 +312,9 @@ Copy [.env.example](../.env.example) to `.env` in the project root and set value | `SKILLSPECTOR_COMPAT_BASE_URL` | OpenAI-compatible endpoint base URL. | `https://api.groq.com/openai/v1` | | `SKILLSPECTOR_MODEL` | Override the active provider's bundled default model (see [README.md](../README.md) for per-provider defaults). CLI providers forward it as `--model`. | `gpt-5.2` | -> **CLI providers** (`claude_cli`, `codex_cli`, `gemini_cli`, `opencode_cli`): no credential env var is needed. Authentication is managed by the agent CLI's own session. The subprocess is heavily sandboxed — see [providers/_agent_cli.py](../src/skillspector/providers/_agent_cli.py). +> **Disabled provider:** `codex_cli` remains registered for compatibility but refuses inference until a complete no-tools policy is verified. + +> **CLI providers** (`claude_cli`, `gemini_cli`, `opencode_cli`): no credential env var is needed. Authentication is managed by the agent CLI's own session. The subprocess is heavily sandboxed — see [providers/_agent_cli.py](../src/skillspector/providers/_agent_cli.py). ### Live provider tests @@ -343,15 +345,15 @@ Base URL env vars are not needed for live provider tests; the tests intentionall - `azure_openai/` — Azure OpenAI Service (`AZURE_OPENAI_API_KEY`, `AZURE_OPENAI_ENDPOINT`) - `openai_compatible/` — generic compatible endpoint (`SKILLSPECTOR_COMPAT_API_KEY`, `SKILLSPECTOR_COMPAT_BASE_URL`) - `claude_cli/` — **local `claude` binary; no API key**. Uses the CLI's own auth session (`claude auth login`). Set `SKILLSPECTOR_PROVIDER=claude_cli`. - - `codex_cli/` — **local `codex` binary; no API key**. Uses the CLI's own auth session (`codex login`). Set `SKILLSPECTOR_PROVIDER=codex_cli`. + - `codex_cli/` — **registered but disabled**. Its read-only sandbox permits host-file reads. Select an HTTP API provider or another supported CLI provider for LLM analysis. - `gemini_cli/` — **local `gemini` binary; no API key**. Uses the CLI's own auth session. Set `SKILLSPECTOR_PROVIDER=gemini_cli`. - `opencode_cli/` — **local `opencode` 1.18.31 binary; no API key**. Uses the CLI's own auth session (`opencode auth login`) and fails closed on every other runtime version because the deny-all policy is verified against that exact release. Set `SKILLSPECTOR_PROVIDER=opencode_cli`. - CLI providers (`claude_cli`, `codex_cli`, `gemini_cli`, `opencode_cli`) implement the optional `AgentCLICapable` interface (`is_available()` + `complete()`) defined in [providers/base.py](../src/skillspector/providers/base.py). `has_cli_capability(provider)` detects this at runtime. All subprocess calls go through the hardened helper [providers/_agent_cli.py](../src/skillspector/providers/_agent_cli.py) which enforces: no shell (`shell=False`), untrusted content via stdin only, capability stripping (tools disabled / sandboxed), environment scrubbing (no API keys forwarded), per-call timeout, and fail-closed error handling. + CLI providers (`claude_cli`, `gemini_cli`, `opencode_cli`) implement the optional `AgentCLICapable` interface (`is_available()` + `complete()`) defined in [providers/base.py](../src/skillspector/providers/base.py). `has_cli_capability(provider)` detects this at runtime. All subprocess calls go through the hardened helper [providers/_agent_cli.py](../src/skillspector/providers/_agent_cli.py) which enforces: no shell (`shell=False`), untrusted content via stdin only, capability stripping (tools disabled / sandboxed), environment scrubbing (no API keys forwarded), per-call timeout, and fail-closed error handling. - **LLM calls** ([llm_utils.py](../src/skillspector/llm_utils.py)): **`get_chat_model()`** and **`chat_completion()`** dispatch based on the active provider: - **HTTP providers**: resolve credentials in two tiers — active provider (`NVIDIA_INFERENCE_KEY` / `ANTHROPIC_API_KEY` / `OPENAI_API_KEY` → endpoint) — against any OpenAI-compatible endpoint. `max_tokens` is auto-bound to `get_max_output_tokens(model)` from `model_info`. - - **CLI providers** (`claude_cli`, `codex_cli`): `get_chat_model()` returns an `AgentCLIChatModel` adapter backed by `provider.complete()`, so the analyzers' `.invoke()` / `.with_structured_output(schema).invoke()` calls work with no API key (structured output is produced by prompting for JSON, then Pydantic-validating). `chat_completion()` routes through `get_chat_model()` as well. `is_llm_available()` calls `provider.is_available()` instead of credential resolution. + - **CLI providers** (`claude_cli`, `gemini_cli`, `opencode_cli`): `get_chat_model()` returns an `AgentCLIChatModel` adapter backed by `provider.complete()`, so the analyzers' `.invoke()` / `.with_structured_output(schema).invoke()` calls work with no API key (structured output is produced by prompting for JSON, then Pydantic-validating). `chat_completion()` routes through `get_chat_model()` as well. `is_llm_available()` calls `provider.is_available()` instead of credential resolution. - **LLM analyzer base** ([llm_analyzer_base.py](../src/skillspector/nodes/llm_analyzer_base.py)): `LLMAnalyzerBase` provides per-file/per-chunk batching, token-budget-aware chunking, and a run loop for all LLM-based analyzers. `LLMMetaAnalyzer` extends it for filter/enrich (meta_analyzer node). Future semantic analyzers extend `LLMAnalyzerBase` for discovery mode. --- diff --git a/docs/NESTED_ARTIFACT_INSPECTION.md b/docs/NESTED_ARTIFACT_INSPECTION.md index e6ff04efa..620a3440a 100644 --- a/docs/NESTED_ARTIFACT_INSPECTION.md +++ b/docs/NESTED_ARTIFACT_INSPECTION.md @@ -63,6 +63,15 @@ makes the analysis incomplete. A result that would otherwise be `SAFE` is report a failing gate with `--fail-on-incomplete`. Inspection exceptions and their affected outer or nested paths are surfaced in terminal, JSON, Markdown, and SARIF output. +If a real disk path containing `!/` has the same identity as an archive member, the scan +records `artifact_path_collision`, retains the disk bytes, and withholds the archive member. +This fails the scan (CLI exit 2 and MCP `safe_to_install=false`). Rename the disk directory +ending in `!` to remove the ambiguity. Non-colliding directory names ending in `!` remain valid. +The colliding member file is withheld; distinct descendant members still receive analysis. +Exceptions for that withheld file or a skipped duplicate descendant are reported against its +archive container. Disk format and malformed-container errors stay on the disk file, and an +analyzed disk descendant is not marked partial merely because the archive copy was skipped. + ## SC9: Concealed Executable Artifact SC9 is a deterministic HIGH finding when executable content is concealed inside an Office document diff --git a/docs/OPENCODE_EXTENSION.md b/docs/OPENCODE_EXTENSION.md index 957891767..24ca2c598 100644 --- a/docs/OPENCODE_EXTENSION.md +++ b/docs/OPENCODE_EXTENSION.md @@ -70,7 +70,7 @@ Use skillspector_scan on ./my-skill with noLlm=false. export SKILLSPECTOR_PROVIDER=nv_build export NVIDIA_INFERENCE_KEY=nvapi-... # Optional; omit to use nv_build's bundled default model. -# export SKILLSPECTOR_MODEL=z-ai/glm-5.2 +# export SKILLSPECTOR_MODEL=z-ai/glm-5.3 ``` Other valid providers and their credential variables are listed in the main diff --git a/docs/PI_EXTENSION.md b/docs/PI_EXTENSION.md index d889807ea..ce3e5f422 100644 --- a/docs/PI_EXTENSION.md +++ b/docs/PI_EXTENSION.md @@ -48,6 +48,14 @@ Equivalent CLI: - `yaraRulesDir`: optional directory of extra YARA rules. - `verbose`: optional detailed progress. +Inputs inside the session's working directory run without a prompt. Remote targets +and external paths require confirmation; redirected aliases show their resolved +destination too. Print and JSON sessions reject these requests because they cannot +show a dialog. Use TUI or RPC mode, move the skill into the working directory, or +run the CLI directly. Local targets retain the CLI's refusal of symlinked paths. +Missing rule directories fail before scanning, and YARA `include` directives are +disabled: put self-contained rule files in the selected directory. + ## LLM-backed analysis Static scan is default. To use semantic LLM analysis, configure provider credentials in your shell before launching Pi, then call the tool with `noLlm=false` and a provider. diff --git a/docs/SUPPRESSION.md b/docs/SUPPRESSION.md index 9a7065ca2..9d7ae0aac 100644 --- a/docs/SUPPRESSION.md +++ b/docs/SUPPRESSION.md @@ -39,6 +39,16 @@ skillspector scan ./my-skill/ --baseline .skillspector-baseline.yaml --show-supp | `skillspector scan --baseline FILE --show-suppressed` | Also list the suppressed findings (they still don't affect the score). | A missing, malformed, or unsupported baseline file exits with code 2. +Generation also exits with code 2 without creating or replacing the output when +scan execution fails, including failed semantic analysis or unreadable primary +content. Resolve the failure and rerun the scan before accepting its findings. +Use `--no-llm` when you intentionally want to accept only static findings. + +A successful but incomplete scan can generate a baseline for its observed +findings, with a warning on stderr. This does not accept uninspected content or +clear coverage gaps: subsequent reports retain their incomplete status and +recommendation even if every observed finding is suppressed. + When a selected baseline or baseline output is stored inside the scan target, SkillSpector treats that exact file as an explicit scope exclusion. This prevents sensitive rule text from creating a finding against itself or entering @@ -103,6 +113,27 @@ Generated by `skillspector baseline`, it is intentionally exact: editing the source or upgrading SkillSpector keeps the finding active until it is reviewed and the baseline is regenerated. +LLM and meta-analysis can change explanations, remediation, confidence, or other +evidence between scans of unchanged files. Those results remain active when +their exact fingerprints differ. For repeatable static acceptance, use +`--no-llm` for both baseline generation and subsequent scans. + +Generation fingerprints each original finding before report compaction, including +repeated matches on different lines and identical matches in different files. +Only identical exact fingerprints share an entry. A baseline generated by an +affected version may omit these occurrences; regenerate it to include them. +If the complete baseline exceeds the loader's size or record limits, generation +fails before replacing the output file instead of writing a partial baseline. +After validation, generation normally writes a complete temporary file beside the +destination and atomically replaces the output. A failed write or replacement +preserves the existing baseline. Existing destinations must be writable regular +files; symlinks and special files are rejected. Replacement requires a writable +parent directory. On POSIX, new files grant access only to their owner; existing +ordinary permission bits and ownership are preserved, including group-write access. +When a non-owner has write access to an existing file, or the owner cannot +preserve the file's group, generation refuses the update without changing the +existing file. Write to a new file so publication remains atomic. + Every v2 entry must be a mapping with a 64-hex-character `sha256:` hash and a non-empty `reason`. `rule_id` and `file` are informational fields for reviewers. If source content is unavailable or `scanner_version` does not match, exact diff --git a/docs/release/skillspector-2.13.0.md b/docs/release/skillspector-2.13.0.md new file mode 100644 index 000000000..bc242dbac --- /dev/null +++ b/docs/release/skillspector-2.13.0.md @@ -0,0 +1,99 @@ +# SkillSpector v2.13.0 + +## Summary + +This release selectively backports security hardening, blocking scan fixes, and executable Python support onto v2.12.0. It includes 55 selected PRs; #743 is included through #734. It does not include every change merged into main since v2.12.0. + +## Compatibility notes + +- NVIDIA Build defaults to GLM-5.3 after retirement of the previous default. Explicit model overrides remain supported. +- The Codex CLI analysis provider is disabled because its read-only sandbox permits access to host secrets. Use another supported provider. +- MCP HTTP listeners are restricted to loopback. Remote deployments require an authenticating proxy. +- Baseline generation uses atomic replacement. Updates that cannot preserve ownership or group safely are refused; write a new baseline file. +- If a real directory ending in `!` collides with an archive-member path, rename that directory to allow both files to be fully inspected. +- Exact-fingerprint baselines are version-bound and must be regenerated after upgrading. +- Static-only scans do not require an active LLM provider. LLM-enabled scans still reject unknown provider settings. +- Other provider defaults remain as in v2.12.0. The newer-model default change from #806 and its rollback in #824 are not included. + +## Included changes + +### Security and scan integrity + +- [#471](https://github.com/NVIDIA/SkillSpector/pull/471): Reads SKILL.md metadata correctly when the file starts with an invisible encoding marker. +- [#473](https://github.com/NVIDIA/SkillSpector/pull/473): Detects text-direction control characters that can disguise code in bundled scripts, not just Markdown. +- [#499](https://github.com/NVIDIA/SkillSpector/pull/499): Marks recursive scan coverage as incomplete when linked skill folders are deliberately skipped. +- [#545](https://github.com/NVIDIA/SkillSpector/pull/545): Detects access to sensitive files even when their paths are assembled from fixed strings. +- [#557](https://github.com/NVIDIA/SkillSpector/pull/557): Marks analysis incomplete when custom YARA rules cannot be loaded or compiled. +- [#577](https://github.com/NVIDIA/SkillSpector/pull/577): Preserves dangerous shell-command findings when deeper Python analysis cannot confidently replace them. +- [#578](https://github.com/NVIDIA/SkillSpector/pull/578): Keeps distinct shell-command findings separate, even when their displayed snippets look identical. +- [#593](https://github.com/NVIDIA/SkillSpector/pull/593): Adds lower-confidence warnings for multiline scripts that appear to open a remote shell, while preserving existing one-line detections. +- [#595](https://github.com/NVIDIA/SkillSpector/pull/595): Shows how many linked skill folders were skipped and includes that count in recursive scan coverage. +- [#601](https://github.com/NVIDIA/SkillSpector/pull/601): Checks complete TypeScript declaration files before treating them as non-executable, so partial or disguised runtime code is not skipped. +- [#604](https://github.com/NVIDIA/SkillSpector/pull/604): Flags bundled settings that run commands, redirect AI traffic, or automatically approve connected tool servers. +- [#611](https://github.com/NVIDIA/SkillSpector/pull/611): Finds Python flows from secret reads to risky uses even when the assignments and uses appear at different nesting depths. +- [#632](https://github.com/NVIDIA/SkillSpector/pull/632): Inspects hooks declared in Claude plugin manifests and reports incomplete coverage when those declarations cannot be fully checked. +- [#637](https://github.com/NVIDIA/SkillSpector/pull/637): Makes saved baselines cover every occurrence of a repeated finding, so later scans suppress them consistently. +- [#655](https://github.com/NVIDIA/SkillSpector/pull/655): Detects bundled Node settings that use the short -r flag to load and run extra code. +- [#657](https://github.com/NVIDIA/SkillSpector/pull/657): Preserves exact findings in baselines and protects existing baseline files during regeneration. +- [#691](https://github.com/NVIDIA/SkillSpector/pull/691): Improves TP4 responses for the specific OpenAI-provider Opus 5 route and rejects missing, malformed, or contradictory tool results. +- [#702](https://github.com/NVIDIA/SkillSpector/pull/702): Attaches AI review decisions to the correct findings and distinguishes disagreement from missing or failed reviews. +- [#721](https://github.com/NVIDIA/SkillSpector/pull/721): Detects sensitive data passed to urllib opener requests, including inline opener calls. +- [#725](https://github.com/NVIDIA/SkillSpector/pull/725): Displays untrusted names, filenames and findings literally in terminal reports, preventing markup crashes and spoofing. +- [#729](https://github.com/NVIDIA/SkillSpector/pull/729): Rejects scan output that points to the input file, including symbolic or hard links, preventing accidental overwrites. +- [#733](https://github.com/NVIDIA/SkillSpector/pull/733): Prevents untrusted names and findings from inserting fake headings, rows or HTML into batch Markdown reports. +- [#734](https://github.com/NVIDIA/SkillSpector/pull/734): Displays untrusted skill names and errors literally in batch terminals so markup and control characters cannot spoof the output. +- [#735](https://github.com/NVIDIA/SkillSpector/pull/735): Prevents untrusted finding text and filenames from changing Markdown report structure, using shared protection for core and batch reports. +- [#736](https://github.com/NVIDIA/SkillSpector/pull/736): Stops remote MCP scans from using the server’s Git credentials to clone private repositories while preserving trusted local Git access. +- [#737](https://github.com/NVIDIA/SkillSpector/pull/737): Disables the Codex CLI analysis provider because its read-only sandbox still permits host-secret reads, directing users to another provider. +- [#738](https://github.com/NVIDIA/SkillSpector/pull/738): Limits environment variables passed to agent CLI providers so unexpected secret variables are not forwarded to child processes. +- [#739](https://github.com/NVIDIA/SkillSpector/pull/739): Prevents long scanner output from blocking Pi’s secret redaction while still hiding secrets before the displayed output is shortened. +- [#740](https://github.com/NVIDIA/SkillSpector/pull/740): Prevents crafted Python literals and unfinished function headers from causing extremely slow scans. +- [#743](https://github.com/NVIDIA/SkillSpector/pull/743): Enforces per-skill batch timeouts by stopping stuck workers while other scans continue. +- [#744](https://github.com/NVIDIA/SkillSpector/pull/744): Limits unauthenticated MCP HTTP access to the local machine; remote access requires an authenticating proxy. +- [#745](https://github.com/NVIDIA/SkillSpector/pull/745): Redacts short configured secrets from OpenCode’s returned output without corrupting JSON values; separately written report files remain outside this redactor. +- [#746](https://github.com/NVIDIA/SkillSpector/pull/746): Pins the release workflow’s checkout action to a fixed commit so a moving version tag cannot silently change the code it runs. +- [#758](https://github.com/NVIDIA/SkillSpector/pull/758): Prevents registry reports from overwriting their input files. +- [#767](https://github.com/NVIDIA/SkillSpector/pull/767): Keeps the release GitHub token out of Git configuration during dependency installation and builds. +- [#786](https://github.com/NVIDIA/SkillSpector/pull/786): Detects encoded shell payloads that are decoded and executed. +- [#791](https://github.com/NVIDIA/SkillSpector/pull/791): Requires approval for external Pi scan inputs and blocks file-reading YARA includes. +- [#792](https://github.com/NVIDIA/SkillSpector/pull/792): Separates local and referenced-content caches so baseline suppression cannot use the wrong file. +- [#793](https://github.com/NVIDIA/SkillSpector/pull/793): Prevents real disk paths from being overwritten by archive members with the same virtual path. Collisions retain the real content and report failed coverage. +- [#794](https://github.com/NVIDIA/SkillSpector/pull/794): Inspects readable instructions hidden behind binary file prefixes while keeping ordinary PDF/GIF assets and XMP metadata from producing false alarms. Referenced or executable binaries still report their inspection limits. +- [#795](https://github.com/NVIDIA/SkillSpector/pull/795): Treats OMS signing bundles as untrusted input, inspecting decoded metadata and preserving raw evidence and incomplete coverage when parsing fails. +- [#796](https://github.com/NVIDIA/SkillSpector/pull/796): Reports malformed LLM responses as incomplete analysis instead of successful empty results. + +### Blocking and regression fixes + +- [#623](https://github.com/NVIDIA/SkillSpector/pull/623): Keeps Python findings when parser location columns are missing, instead of crashing or merging distinct findings together. +- [#627](https://github.com/NVIDIA/SkillSpector/pull/627): Recognizes more ordinary Markdown sentences and lists so their apostrophes do not cause false shell-parsing and incomplete-scan warnings. +- [#634](https://github.com/NVIDIA/SkillSpector/pull/634): Parses quoted shell assignments correctly so harmless trailing text does not falsely make a scan incomplete. +- [#661](https://github.com/NVIDIA/SkillSpector/pull/661): Avoids misleading high-severity warnings on SKILL.md self-references while still reporting missing or ambiguous files. +- [#686](https://github.com/NVIDIA/SkillSpector/pull/686): Accepts ordinary shell variable defaults and assignment values without falsely marking their files as partly scanned. +- [#700](https://github.com/NVIDIA/SkillSpector/pull/700): Lowers alerts for simple download pipelines that demonstrably parse incoming data rather than execute it. +- [#704](https://github.com/NVIDIA/SkillSpector/pull/704): Recognizes safely decodable shell printf escapes without incorrectly marking analysis incomplete. +- [#748](https://github.com/NVIDIA/SkillSpector/pull/748): Switches NVIDIA Build’s default from retired GLM-5.2 to GLM-5.3 while preserving explicit model overrides. +- [#799](https://github.com/NVIDIA/SkillSpector/pull/799): Tracks sensitive-data flow by function scope, avoiding collisions between unrelated variable names. +- [#802](https://github.com/NVIDIA/SkillSpector/pull/802): Keeps intentionally excluded files separate from failed analyzer work in coverage reports. + +- [#828](https://github.com/NVIDIA/SkillSpector/pull/828): Counts the bytecode walk’s own CPU time toward its processing limit, so waiting behind concurrent analyzers does not make clean scans incomplete. The overall scan deadline still applies. +- [#844](https://github.com/NVIDIA/SkillSpector/pull/844): Avoids execution warnings for simple decode-to-checksum pipelines while retaining warnings for executable or redirected payloads. Keeps disk and archive error locations distinct when paths collide. + +### Executable Python support + +- [#579](https://github.com/NVIDIA/SkillSpector/pull/579): Recognizes more executable Python files, including .pyw files and extensionless launchers. + +## Integration notes + +Selected changes are applied as their merge deltas against v2.12.0. Required shared response-handling, analyzer support, and provider-neutral static-mode helpers are adapted without importing the deferred features. The release PR records the exact adaptations and validation results. + +Release review also identified excessive backtracking in TypeScript declaration recognition. Matching now has a time limit; files that cannot be proven inert within it remain eligible for executable analysis. + +## Validation + +The [release PR](https://github.com/NVIDIA/SkillSpector/pull/833) records the tested commits, wheel identities, source and installed-wheel test results, sample scans, live NVIDIA Build checks, and remaining review gaps. Each result is scoped to its tested revision. + +## Known limitations + +- Existing limits on analysis, unsupported inputs, and unavailable requested providers remain visible as incomplete coverage. A completed process is not proof of a clean skill. +- Short-secret redaction applies to the OpenCode extension response; separately written report files are outside that redactor. +- This selected release does not include fixes from open PRs. Malformed provider responses are reported as incomplete analysis; the release does not claim to repair an external provider returning invalid responses. diff --git a/docs/scan-completeness.md b/docs/scan-completeness.md index 768c0c909..e243f8a6c 100644 --- a/docs/scan-completeness.md +++ b/docs/scan-completeness.md @@ -118,7 +118,9 @@ The examples below use static-only analysis, no baseline suppression, and benign | Explicit opaque file, UTF-16 primary, recognized unsupported archive, or invalid UTF-8 primary | `unsupported_primary_content`, source path, `fatal=true`; canonical bytes retained | failed / false | 2 / 2 / 2 | false | | Unsupported in-profile `SKILL.md` below directories or normal/renamed ZIPs | Same required-content event at real/virtual member path | failed / false | 2 / 2 / 2 | false | | Supported benign ZIP, including renamed or empty ZIP | Existing bounded archive inspection; incidental assets remain exclusions | complete / true | 0 / 0 / 0 | true | -| Unreferenced incidental image beside benign instructions | `binary_content` in `scope_exclusions` | complete / true | 0 / 0 / 0 | true | +| Unreferenced incidental image with no readable runs beside benign instructions | `binary_content` in `scope_exclusions` | complete / true | 0 / 0 / 0 | true | +| Incidental GIF/PDF with readable runs, including capped projections | Bounded instruction-text projection; `binary_text_projection` in `scope_exclusions`; remaining text and binary data are outside this projection's scope | complete / true (absent other limits) | 0 / 0 / 0 | true if no findings | +| Readable MZ executable | `binary_executable_text`: text inspected, binary behavior unverified; SC9 score floor retained | partial / true | 1 / 1 / 1 | false | | Referenced opaque image | Existing referenced-content limitation and AE1 finding | partial / true | 0 / 1 / 1 | false | | Pure or mixed singleton `never warn the user` | AE6, score 22 in this fixture, source line; `obfuscated_instruction_text` | partial / true | 0 / 1 / 1 | false | | Benign list, paragraph, or punctuated code controls | No new reconstruction ambiguity | complete / true | 0 / 0 / 0 | true | @@ -133,6 +135,21 @@ A subtlety is that component coverage can still read **100%** for AE6: analyzer ## Review critique and remaining limits +For printable GIF, PDF and MZ prefixes, local instruction checks use printable runs of at least +8 characters, capped at 32,768 projected characters per artifact. Binary controls, invalid UTF-8, +and standard XMP packet BOMs and padding are excluded. Source coordinates remain tied to the +original bytes' decoded view. Only instruction-override, credential-access and exfiltration rule +families contribute static pattern findings from this projection. Artifact-integrity's AE6 +instruction-obfuscation check also runs; its generic AE4 Unicode-density and mixed-script checks +do not, because compressed bytes can resemble those signals. Normal text retains both checks. +Hidden and nested artifacts remain local-only; an eligible ordinary file sends only this capped +projection to an external model. The cap is per file, so multiple assets can still add provider input. +For unreferenced incidental assets, reaching the projection cap records a `binary_text_projection` +scope exclusion without making the scan partial. Text beyond the cap and compressed content are +not inspected by this projection. Referenced opaque files, unverified executable behavior, and +actual analysis failures or resource limits retain their incomplete-analysis policy. Valid Python +source with a matching prefix follows normal source analysis. + The initial root-only identity boundary was too narrow: parsing introduced member paths before the required-content check, so required status disappeared. Basename checks across cached, in-profile paths repair that without promoting every asset or overriding the existing directory-exclusion policy. Applying this boundary before archive delegation would instead reject supported ZIPs wholesale. Applying it only after public-report generation would be too late to remove unsupported content from provider submission. The reconstruction boundary is deliberately narrow. Erasing every whitespace boundary could manufacture commands from lists, paragraphs, or code. Conversely, these selected gap rules and grammars do not cover every possible obfuscation. Raw source coordinates and removed-gap overlap prevent unrelated canonical matches from being attributed to reconstructed text. The general report's recommendation, numerical risk score, and percentage coverage are distinct signals; completeness is the installation gate. diff --git a/extensions/skillspector.ts b/extensions/skillspector.ts index bec77df11..7b3bdeb46 100644 --- a/extensions/skillspector.ts +++ b/extensions/skillspector.ts @@ -1,4 +1,4 @@ -import type { ExtensionAPI } from "@earendil-works/pi-coding-agent"; +import type { ExtensionAPI, ExtensionContext } from "@earendil-works/pi-coding-agent"; import { StringEnum } from "@earendil-works/pi-ai"; import { Type, type Static } from "typebox"; import { chmodSync, constants, copyFileSync, existsSync, lstatSync, mkdtempSync, realpathSync, renameSync, rmSync } from "node:fs"; @@ -7,7 +7,7 @@ import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "nod import { fileURLToPath } from "node:url"; const scanSchema = Type.Object({ - target: Type.String({ description: "Path, URL, zip, Git repo, or SKILL.md to scan." }), + target: Type.String({ description: "Path, URL, zip, Git repo, or SKILL.md to scan. External paths and remote targets require user confirmation." }), format: Type.Optional( StringEnum(["terminal", "json", "markdown", "sarif"] as const, { description: "SkillSpector output format. Defaults to terminal.", @@ -21,28 +21,18 @@ const scanSchema = Type.Object({ }), ), model: Type.Optional(Type.String({ description: "Optional model override." })), - yaraRulesDir: Type.Optional(Type.String({ description: "Optional extra YARA rules directory." })), + yaraRulesDir: Type.Optional(Type.String({ description: "Optional extra YARA rules directory. External paths require user confirmation." })), verbose: Type.Optional(Type.Boolean({ description: "Show detailed progress." })), }); type SkillSpectorScanParams = Static; -function isLikelyUrl(value: string): boolean { - return /^[a-z][a-z0-9+.-]*:\/\//i.test(value) || /^[\w.-]+\/[\w.-]+(?:\.git)?(?:@.+)?$/i.test(value); -} - -function resolveMaybePath(ctxCwd: string, value?: string): string | undefined { - if (!value) return undefined; - if (isLikelyUrl(value)) return value; - return isAbsolute(value) ? value : resolve(ctxCwd, value); -} - function redactSecrets(value: string): string { return value .replace(/(sk-ant-[A-Za-z0-9_-]{12,})/g, "[REDACTED_ANTHROPIC_KEY]") .replace(/(sk-[A-Za-z0-9_-]{20,})/g, "[REDACTED_OPENAI_KEY]") - .replace(/([A-Za-z0-9_]*API_KEY[=:]\s*)[^\s]+/gi, "$1[REDACTED]") - .replace(/([A-Za-z0-9_]*TOKEN[=:]\s*)[^\s]+/gi, "$1[REDACTED]"); + // Start only at a word boundary; retrying at every character is quadratic. + .replace(/\b([A-Za-z0-9_]*(?:API_KEY|TOKEN)[=:]\s*)[^\s]+/gi, "$1[REDACTED]"); } function truncateText(value: string, maxChars = 12000): { text: string; truncated: boolean } { @@ -71,6 +61,85 @@ function isWithin(root: string, path: string): boolean { return rel !== ".." && !rel.startsWith(`..${sep}`) && !isAbsolute(rel); } +async function approveScanInputs( + params: SkillSpectorScanParams, + ctx: ExtensionContext, + signal?: AbortSignal, +): Promise { + signal?.throwIfAborted(); + const workspace = realpathSync(ctx.cwd); + const prepared = { ...params }; + const localPaths: Array<{ field: "target" | "yaraRulesDir"; input: string; resolved?: string }> = []; + const requests: string[] = []; + const readRequest = (field: string, path: string) => + `Read external ${field === "target" ? "scan target" : "YARA rules"}: ${JSON.stringify(path)}`; + async function approve(requests: string[]): Promise { + if (!requests.length) return; + signal?.throwIfAborted(); + if (!ctx.hasUI) throw new Error("External scan inputs require user confirmation in an interactive or RPC session."); + const approved = await ctx.ui.confirm( + "Allow SkillSpector external access?", + `${requests.join("\n")}\n\nScanned content and matching rule text can appear in the agent conversation.`, + { signal }, + ); + signal?.throwIfAborted(); + if (!approved) throw new Error("SkillSpector external access was not approved."); + } + for (const field of ["target", "yaraRulesDir"] as const) { + const value = params[field]?.trim(); + if (field === "yaraRulesDir" && !value) { + prepared[field] = undefined; + continue; + } + if (!value) throw new Error("A scan target is required."); + // Match the CLI's remote forms. A local owner/repo path is not a URL. + const remote = !isAbsolute(value) && (value.startsWith("https://") || (value.startsWith("git@") && value.endsWith(".git"))); + if (remote) { + if (field !== "target") throw new Error("YARA rules must be a local directory."); + prepared[field] = value; + requests.push(`Fetch remote scan target: ${JSON.stringify(value)}`); + } else { + const input = resolve(ctx.cwd, value); + // Ask before resolving external paths, which can probe the host or access + // a Windows network share even when the file is never opened. + if (!isWithin(resolve(ctx.cwd), input)) { + localPaths.push({ field, input }); + requests.push(readRequest(field, input)); + } else { + let resolved: string; + try { + resolved = realpathSync(input); + } catch { + throw new Error(`Could not resolve ${field === "target" ? "scan target" : "YARA rules directory"}. Check that it exists and is accessible.`); + } + localPaths.push({ field, input, resolved }); + if (!isWithin(workspace, resolved)) requests.push(readRequest(field, resolved)); + } + } + } + await approve(requests); + const aliasRequests: string[] = []; + for (const path of localPaths) { + try { + path.resolved ??= realpathSync(path.input); + } catch { + throw new Error(`Could not resolve ${path.field === "target" ? "scan target" : "YARA rules directory"}. Check that it exists and is accessible.`); + } + if (!isWithin(resolve(ctx.cwd), path.input) && !isWithin(workspace, path.resolved) && path.resolved !== path.input) { + aliasRequests.push(readRequest(path.field, path.resolved)); + } + // Keep the original target path so the CLI can enforce its no-symlink + // input policy. YARA directories are canonicalised by the CLI too. + prepared[path.field] = path.field === "target" ? path.input : path.resolved; + } + await approve(aliasRequests); + signal?.throwIfAborted(); + if (realpathSync(ctx.cwd) !== workspace || localPaths.some(({ input, resolved }) => realpathSync(input) !== resolved)) { + throw new Error("Scan input path changed while awaiting confirmation."); + } + return prepared; +} + function reportOutputPath(cwd: string, value?: string): string | undefined { if (!value) return undefined; const output = resolve(cwd, value); @@ -102,8 +171,8 @@ function publishReport(source: string, destination: string): void { } } -function buildScanArgs(params: SkillSpectorScanParams, cwd: string, output?: string): string[] { - const args = ["scan", resolveMaybePath(cwd, params.target) ?? params.target]; +function buildScanArgs(params: SkillSpectorScanParams, output?: string): string[] { + const args = ["scan", params.target]; args.push("--format", params.format ?? "terminal"); const noLlm = params.noLlm ?? true; @@ -111,8 +180,7 @@ function buildScanArgs(params: SkillSpectorScanParams, cwd: string, output?: str if (output) args.push("--output", output); - const yaraRulesDir = resolveMaybePath(cwd, params.yaraRulesDir); - if (yaraRulesDir) args.push("--yara-rules-dir", yaraRulesDir); + if (params.yaraRulesDir) args.push("--yara-rules-dir", params.yaraRulesDir); if (params.verbose) args.push("--verbose"); return args; @@ -132,10 +200,11 @@ export default function (pi: ExtensionAPI) { async execute(_toolCallId, params, signal, onUpdate, ctx) { const bin = findSkillSpectorBin(); const outputPath = reportOutputPath(ctx.cwd, params.output); + const prepared = await approveScanInputs(params, ctx, signal); const reportDir = outputPath ? mkdtempSync(join(tmpdir(), "skillspector-report-")) : undefined; const reportPath = reportDir ? join(reportDir, "report") : undefined; try { - const args = buildScanArgs(params, ctx.cwd, reportPath); + const args = buildScanArgs(prepared, reportPath); const env: Record = {}; if (params.provider) env.SKILLSPECTOR_PROVIDER = params.provider; diff --git a/pyproject.toml b/pyproject.toml index e04a8ccaa..800c0702b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "skillspector" -version = "2.12.0" +version = "2.13.0" description = "SkillSpector: Security scanner for AI agent skills (Claude Code, Cursor, and similar). Scans skills for vulnerabilities, malicious patterns, and security risks before installation. Supports Git repos, URLs, zips, and local directories; runs static pattern checks and optional LLM semantic analysis; outputs terminal, JSON, and Markdown reports with risk scoring." readme = "README.md" license = "Apache-2.0" diff --git a/src/skillspector/artifacts.py b/src/skillspector/artifacts.py index 6a49332c7..2876b5e79 100644 --- a/src/skillspector/artifacts.py +++ b/src/skillspector/artifacts.py @@ -64,6 +64,7 @@ class ArtifactRecord(TypedDict): contains_nul: bool misleading_extension: bool referenced: bool + readable_binary: NotRequired[bool] reason: NotRequired[str] inherited_exclusion_reason: NotRequired[str] @@ -99,6 +100,8 @@ class SecurityTextView: text: str source_offsets: array[int] | None = None reconstructions: tuple[SecurityTextReconstruction, ...] = () + right_boundary_is_fixed: bool = False + right_boundary_recovery_start: int | None = None def source_offset(self, derived_offset: int) -> int: """Map a derived character offset to the corresponding source offset.""" @@ -228,6 +231,7 @@ class _ObfuscatedIgnoreState: ".markdown", ".txt", ".py", + ".pyw", ".sh", ".json", ".yaml", @@ -364,6 +368,13 @@ def classify_artifact(path: str, data: bytes, *, referenced: bool = False) -> Ar decoded = data.decode("utf-8", errors="replace") decodable = False + # These printable prefixes are also valid instruction text. Keep the + # claimed binary format's coverage limits, but inspect the readable bytes. + # Lossy decoding must not let one invalid byte or a binary comment hide text. + readable_binary = data.startswith((b"MZ", b"GIF87a", b"GIF89a", b"%PDF-")) and bool( + binary_instruction_view(decoded)[0].text + ) + if has_binary_magic: kind = ContentKind.BINARY elif decodable: @@ -385,6 +396,8 @@ def classify_artifact(path: str, data: bytes, *, referenced: bool = False) -> Ar disposition = ( ArtifactDisposition.PARTIAL if referenced and kind is not ContentKind.TEXT + else ArtifactDisposition.ANALYZED + if readable_binary else ArtifactDisposition.OUT_OF_SCOPE if kind is ContentKind.BINARY else ArtifactDisposition.ANALYZED @@ -398,9 +411,78 @@ def classify_artifact(path: str, data: bytes, *, referenced: bool = False) -> Ar "contains_nul": contains_nul, "misleading_extension": misleading, "referenced": referenced, + "readable_binary": readable_binary, } +BINARY_TEXT_MAX_CHARS = 32_768 +_BINARY_PRINTABLE_RUN = re.compile(r"[^\x00-\x08\x0b\x0c\x0e-\x1f\x7f-\x9f\ufffd]{8,}") + + +def binary_instruction_view(content: str) -> tuple[SecurityTextView, bool]: + """Bound human-readable binary strings without treating binary noise as text.""" + # Packet formatting is recognized once on the whole artifact, before windows. + excluded: list[tuple[int, int]] = [] + if content.startswith("%PDF-"): + header = '' + start = content.find(header) + while start >= 0: + next_start = content.find(header, start + len(header)) + packet_end = next_start if next_start >= 0 else len(content) + metadata = content.find("", metadata, packet_end) if metadata >= 0 else -1 + if close < 0: + start = next_start + continue + padding_start = close + len("") + padding_end = padding_start + while padding_end < packet_end and content[padding_end] in " \t\r\n": + padding_end += 1 + if content.startswith(('', ''), padding_end): + bom = start + header.index("\ufeff") + excluded.extend([(bom, bom + 1), (padding_start, padding_end)]) + start = next_start + offsets = array("I") + parts: list[str] = [] + truncated = False + cursor = 0 + for excluded_start, excluded_end in [*excluded, (len(content), len(content))]: + for match in _BINARY_PRINTABLE_RUN.finditer(content, cursor, excluded_start): + remaining = BINARY_TEXT_MAX_CHARS - len(offsets) + if remaining <= 2: + truncated = True + break + if parts: + parts.append("\n\n") + offsets.extend([match.start(), match.start()]) + remaining -= 2 + end = min(match.end(), match.start() + remaining) + parts.append(content[match.start() : end]) + offsets.extend(range(match.start(), end)) + if end < match.end(): + truncated = True + break + if truncated: + break + cursor = excluded_end + return SecurityTextView("binary_printable_text", "".join(parts), offsets), truncated + + +def promote_artifact_to_decoded_text(artifact: ArtifactRecord) -> None: + """Apply a successful format-aware text decode without erasing prior limits.""" + generic_binary_scope = artifact["content_kind"] is ContentKind.BINARY and ( + artifact["disposition"] is ArtifactDisposition.OUT_OF_SCOPE + or artifact["disposition"] is ArtifactDisposition.PARTIAL + and "reason" not in artifact + ) + artifact["content_kind"] = ContentKind.TEXT + artifact["readable_binary"] = False + artifact["decodable"] = True + artifact["misleading_extension"] = _suffix(artifact["path"]) in _BINARY_EXTENSIONS + if generic_binary_scope: + artifact["disposition"] = ArtifactDisposition.ANALYZED + + def decode_text(data: bytes) -> str: """Return the loss-tolerant local text projection for static analyzers.""" return data.decode("utf-8", errors="replace") @@ -1679,6 +1761,30 @@ def _contextual_default_ignorable_spans( yield span_start, end +def _normalization_ignored_spans_in_gap( + text: str, + gap_start: int, + gap_end: int, +) -> Iterator[tuple[int, int]]: + """Yield the normalized-view removals inside one contextual gap.""" + for match in _DEFAULT_IGNORABLE_RUN_PATTERN.finditer(text, gap_start, gap_end): + start, end = match.span() + ignored_start = ( + start + if _is_unconditionally_ignored(text[start]) + or _is_contextual_default_ignorable_offset(text, start) + else start + 1 + ) + ignored_end = ( + end + if _is_unconditionally_ignored(text[end - 1]) + or _is_contextual_default_ignorable_offset(text, end - 1) + else end - 1 + ) + if ignored_start < ignored_end: + yield ignored_start, ignored_end + + def _normalization_ignored_spans( text: str, check_runtime: Callable[[], None] | None = None, @@ -1689,22 +1795,7 @@ def _normalization_ignored_spans( require_word_boundaries=False, check_runtime=check_runtime, ): - for match in _DEFAULT_IGNORABLE_RUN_PATTERN.finditer(text, gap_start, gap_end): - start, end = match.span() - ignored_start = ( - start - if _is_unconditionally_ignored(text[start]) - or _is_contextual_default_ignorable_offset(text, start) - else start + 1 - ) - ignored_end = ( - end - if _is_unconditionally_ignored(text[end - 1]) - or _is_contextual_default_ignorable_offset(text, end - 1) - else end - 1 - ) - if ignored_start < ignored_end: - yield ignored_start, ignored_end + yield from _normalization_ignored_spans_in_gap(text, gap_start, gap_end) def _contextual_default_ignorable_offsets( @@ -1813,6 +1904,55 @@ def normalized_security_view( return SecurityTextView("normalized", output.getvalue(), offsets) +def normalized_security_prefix(text: str, max_chars: int) -> str: + """Return an exact bounded prefix of the normalized security projection.""" + if max_chars <= 0: + return "" + + output = StringIO() + output_chars = 0 + + def append(character: str) -> bool: + nonlocal output_chars + normalized = unicodedata.normalize("NFKC", character).translate(ASCII_CONFUSABLE_SKELETON) + remaining = max_chars - output_chars + output.write(normalized[:remaining]) + output_chars += min(len(normalized), remaining) + return output_chars >= max_chars + + source_offset = 0 + while source_offset < len(text) and output_chars < max_chars: + if not _is_token_gap_character(text[source_offset]): + if append(text[source_offset]): + break + source_offset += 1 + continue + + gap_start = source_offset + while source_offset < len(text) and _is_token_gap_character(text[source_offset]): + source_offset += 1 + gap_end = source_offset + before_is_word = gap_start > 0 and _is_word_character(text[gap_start - 1]) + after_is_word = gap_end < len(text) and _is_word_character(text[gap_end]) + ignored_spans = iter( + _normalization_ignored_spans_in_gap(text, gap_start, gap_end) + if before_is_word or after_is_word + else () + ) + next_ignored = next(ignored_spans, None) + gap_offset = gap_start + while gap_offset < gap_end and output_chars < max_chars: + if next_ignored is not None and gap_offset == next_ignored[0]: + gap_offset = next_ignored[1] + next_ignored = next(ignored_spans, None) + continue + if not _is_unconditionally_ignored(text[gap_offset]) and append(text[gap_offset]): + break + gap_offset += 1 + + return output.getvalue() + + def obfuscated_instruction_view( text: str, check_runtime: Callable[[], None] | None = None, @@ -2171,6 +2311,35 @@ def _requires_normalized_security_view_uncached( return not text.translate(_REMOVE_ALLOWED_FORMAT_CHARACTERS).isprintable() +def _has_derived_security_view(text: str) -> bool: + """Return whether security projection produces a distinct text view.""" + if text.isascii(): + return ( + _IGNORED_ASCII_CONTROL.search(text) is not None + or _has_letter_spacing_run(text) + or next(_obfuscated_instruction_matches(text), None) is not None + ) + if any( + unicodedata.normalize("NFKC", character).translate(ASCII_CONFUSABLE_SKELETON) != character + for character in text + ): + return True + if any(_is_unconditionally_ignored(character) for character in text): + return True + if next(_normalization_ignored_spans(text), None) is not None: + return True + if "\ufffd" in text or next(_compact_gap_offsets(text), None) is not None: + return True + return ( + _has_letter_spacing_run(text) + or next( + _obfuscated_instruction_matches(text), + None, + ) + is not None + ) + + def security_text_views( text: str, check_runtime: Callable[[], None] | None = None, diff --git a/src/skillspector/cli.py b/src/skillspector/cli.py index 3a4d1b520..3232f2cab 100644 --- a/src/skillspector/cli.py +++ b/src/skillspector/cli.py @@ -57,7 +57,7 @@ ) from skillspector.logging_config import get_logger, set_level from skillspector.mcp_registry import scan_registry -from skillspector.models import Finding +from skillspector.models import OCCURRENCE_FINDING_ID_KEY, Finding from skillspector.multi_skill import MultiSkillDetectionResult, SkillDirectory, detect_skills from skillspector.nodes.analyzers import ANALYZER_MODULES, ANALYZER_NODE_IDS from skillspector.nodes.report import report @@ -75,7 +75,9 @@ dump_baseline, effective_findings, load_baseline, + source_content_key, ) +from skillspector.terminal_text import visible_terminal_text as _visible_terminal_text logger = get_logger(__name__) @@ -177,6 +179,7 @@ class _CachedTransitiveResult: artifact_references: list[dict[str, object]] has_executable_scripts: bool execution_successful: bool + meta_review_required: bool refs: list[str] @@ -340,6 +343,17 @@ def _result_body(result: dict) -> str: return report_body +def _validate_report_destination(input_path: Path, output: Path | None) -> None: + """Reject reports that would overwrite an input, including path aliases.""" + if ( + output is not None + and input_path.is_file() + and output.exists() + and output.samefile(input_path) + ): + raise ValueError("--output points to the input file. Choose a different output path.") + + def _write_result( result: dict[str, object], output: Path | None, @@ -355,7 +369,7 @@ def _write_result( console.print(f"Report saved to: {output}") else: if format == FormatChoice.terminal: - console.print(report_body) + console.print(report_body, markup=False, emoji=False) else: print(report_body) @@ -560,7 +574,7 @@ def scan( SKILLSPECTOR_PROVIDER Active LLM provider: openai | anthropic | anthropic_proxy | bedrock | nv_build | nv_inference | ollama | azure_openai | - openai_compatible | claude_cli | codex_cli | + openai_compatible | claude_cli | gemini_cli | opencode_cli. Defaults to the NVIDIA path (nv_inference, falling back to nv_build in OSS builds). @@ -583,9 +597,10 @@ def scan( SKILLSPECTOR_COMPAT_API_KEY + SKILLSPECTOR_COMPAT_BASE_URL for openai_compatible - ollama uses the local Ollama service. claude_cli, codex_cli, + ollama uses the local Ollama service. claude_cli, gemini_cli, and opencode_cli use their CLI's existing local - authentication session. + authentication session. codex_cli is registered but disabled because + its read-only sandbox permits host-file reads; use another provider. """ if mcp_registry: if recursive or baseline is not None or show_suppressed or yara_rules_dir is not None: @@ -600,6 +615,7 @@ def scan( ) raise typer.Exit(code=2) try: + _validate_report_destination(Path(input_path), output) result = scan_registry(input_path) report = json.dumps(result, indent=2) if output: @@ -625,7 +641,8 @@ def scan( if not input_path.startswith(("http://", "https://", "git@")): try: resolved_path = validate_local_input_path(resolved_path) - except ValueError as e: + _validate_report_destination(resolved_path, output) + except (OSError, ValueError) as e: err_console.print(f"[red]Error:[/red] {e}") raise typer.Exit(code=2) from e try: @@ -876,6 +893,40 @@ def _ledger_work_identity(entry: dict[str, object]) -> str: return f"{record_value}:{entry.get('phase', '')}" +def _ledger_work_identities(value: object) -> dict[str, str]: + """Map each child ledger row's own work ID to the identity it was built from. + + A status ``planned_work`` target carries only the child's work ID, path and + range, not the identity behind it. Rows whose identity is not the + analyzer's own -- the static_yara rule-set row is ``rule_set:static``, not + ``static_yara`` -- must be re-scoped with that same identity in the status + path, or the status target and its ledger row get different scoped IDs + and the target is dropped as unretained. + """ + identities: dict[str, str] = {} + for event in _coerce_dict_list(value): + work_id = event.get("work_id") + if isinstance(work_id, str) and work_id: + identities[work_id] = _ledger_work_identity(event) + return identities + + +def _source_scoped_work_id(identity: str, item: dict[str, object]) -> str: + """Build the scoped work ID for an already re-pathed ledger row or status target. + + Shared by :func:`_source_aware_ledger` and :func:`_source_aware_status_events` + so the two scoping paths cannot derive different IDs for the same work. + """ + start_line = item.get("start_line") + end_line = item.get("end_line") + return inspection_work_id( + identity, + str(item.get("path", "SKILL.md")), + start_line if isinstance(start_line, int) else None, + end_line if isinstance(end_line, int) else None, + ) + + def _source_aware_ledger( value: object, *, @@ -901,15 +952,7 @@ def _source_aware_ledger( for item in ids if isinstance(item, str) ] - scoped_path = str(entry.get("path", "SKILL.md")) - start_line = entry.get("start_line") - end_line = entry.get("end_line") - entry["work_id"] = inspection_work_id( - _ledger_work_identity(entry), - scoped_path, - start_line if isinstance(start_line, int) else None, - end_line if isinstance(end_line, int) else None, - ) + entry["work_id"] = _source_scoped_work_id(_ledger_work_identity(entry), entry) events.append(entry) return events @@ -922,8 +965,10 @@ def _source_aware_status_events( source_digest: str, retained_work_ids: set[str], max_planned_work: int, + work_identities: dict[str, str] | None = None, ) -> list[dict[str, object]]: statuses: list[dict[str, object]] = [] + identities = work_identities or {} planned_retained = 0 for status in _coerce_dict_list(value): if len(statuses) >= _TRANSITIVE_MAX_STATUS_EVENTS: @@ -945,14 +990,11 @@ def _source_aware_status_events( path = scoped_target.get("path") if isinstance(path, str) and path: scoped_target["path"] = _transitive_component_key(source_identity, path) - start_line = scoped_target.get("start_line") - end_line = scoped_target.get("end_line") - scoped_target["work_id"] = inspection_work_id( - analyzer_id, - str(scoped_target.get("path", "SKILL.md")), - start_line if isinstance(start_line, int) else None, - end_line if isinstance(end_line, int) else None, - ) + # Re-scope with the identity the matching ledger row used, so + # both paths agree on the scoped ID; the analyzer ID is only the + # fallback for targets with no child ledger row. + identity = identities.get(str(target.get("work_id", "")), analyzer_id) + scoped_target["work_id"] = _source_scoped_work_id(identity, scoped_target) if scoped_target["work_id"] not in retained_work_ids: continue scoped_work.append(scoped_target) @@ -1029,7 +1071,7 @@ def _source_aware_file_cache( file_cache: dict[str, str], source_identity: str | None ) -> dict[str, str]: return { - _transitive_component_key(source_identity, path): content + (source_content_key(source_identity, path) if source_identity else path): content for path, content in file_cache.items() } @@ -1164,18 +1206,32 @@ def _cache_transitive_result( child_filtered = _coerce_findings_list(child_result.get("filtered_findings")) child_findings = _coerce_findings_list(child_result.get("findings")) all_ids = {finding.finding_id for finding in [*child_filtered, *child_findings]} + all_ids.update( + occurrence_id + for finding in [*child_filtered, *child_findings] + for occurrence in finding.occurrences + if isinstance((occurrence_id := occurrence.get(OCCURRENCE_FINDING_ID_KEY)), str) + ) all_ids.update(_effective_finding_ids(child_result)) finding_id_map = { finding_id: _scoped_finding_id(source_identity, finding_id) for finding_id in all_ids } def _scope_finding(finding: Finding) -> Finding: + occurrences = [] + for raw in finding.occurrences: + occurrence = dict(raw) + occurrence_id = occurrence.get(OCCURRENCE_FINDING_ID_KEY) + if isinstance(occurrence_id, str): + occurrence[OCCURRENCE_FINDING_ID_KEY] = finding_id_map[occurrence_id] + occurrences.append(occurrence) return replace( finding, finding_id=finding_id_map[finding.finding_id], source_url=target, source_identity=source_identity, source_digest=source_digest, + occurrences=occurrences, ) scoped_filtered = [_scope_finding(item) for item in child_filtered[:_TRANSITIVE_MAX_FINDINGS]] @@ -1250,6 +1306,7 @@ def _scope_finding(finding: Finding) -> Finding: source_digest=source_digest, retained_work_ids=retained_work_ids, max_planned_work=len(retained_work_ids), + work_identities=_ledger_work_identities(child_result.get("inspection_ledger")), ) child_metadata = _decorate_component_metadata( _coerce_component_metadata(child_result.get("component_metadata")), @@ -1301,27 +1358,11 @@ def _scope_finding(finding: Finding) -> Finding: has_executable_scripts=bool(child_result.get("has_executable_scripts", False)) or any(bool(entry.get("executable", False)) for entry in child_metadata), execution_successful=child_result.get("execution_successful") is not False, + meta_review_required=child_result.get("meta_review_required") is True, refs=extraction.references, ) -def _visible_terminal_text(value: str) -> str: - """Render untrusted path text without passing terminal control characters.""" - visible: list[str] = [] - for character in value: - if character.isprintable(): - visible.append(character) - continue - codepoint = ord(character) - if codepoint <= 0xFF: - visible.append(f"\\x{codepoint:02x}") - elif codepoint <= 0xFFFF: - visible.append(f"\\u{codepoint:04x}") - else: - visible.append(f"\\U{codepoint:08x}") - return "".join(visible) - - def _discovered_files_tree(paths: list[str]) -> Tree: """Build a literal, control-safe tree for paths from an untrusted bundle.""" tree = Tree(Text("Discovered Files to Scan", style="bold blue")) @@ -1600,7 +1641,8 @@ def _bounded_cache_update( ) -> None: for path in sorted(values): if path in destination: - destination[path] = values[path] + # Source-scoped keys are disjoint from root paths. A repeated key + # refers to the same cached result within this traversal. continue if len(destination) >= limit: traversal.note_truncation(f"{resource} budget {limit} reached") @@ -1912,6 +1954,7 @@ def _scan_transitive( _coerce_component_metadata(initial_result.get("component_metadata")), None )[: traversal.budget.max_components] has_executable_scripts = bool(initial_result.get("has_executable_scripts", False)) + merged_meta_review_required = initial_result.get("meta_review_required") is True root_extraction = transitive.extract_external_refs_with_metadata( local_file_cache, @@ -2126,6 +2169,8 @@ def _scan_transitive( ) if cached.has_executable_scripts: has_executable_scripts = True + if cached.meta_review_required: + merged_meta_review_required = True _bounded_extend( merged_components, cached.components, @@ -2260,6 +2305,7 @@ def _scan_transitive( "artifact_inventory": merged_artifact_inventory, "artifact_references": merged_artifact_references, "has_executable_scripts": has_executable_scripts, + "meta_review_required": merged_meta_review_required, "use_llm": initial_result.get("use_llm", not no_llm), "llm_requested": initial_result.get("llm_requested", not no_llm), "llm_call_log": merged_llm_call_log, @@ -2310,6 +2356,7 @@ def _scan_transitive( merged_result["effective_finding_ids"] = effective_ids report_result = cast(dict[str, object], report(merged_result)) report_result["analysis_completeness"] = merged_result.get("analysis_completeness", {}) + report_result["meta_review_required"] = merged_meta_review_required report_result["temp_dir_for_cleanup"] = initial_result.get("temp_dir_for_cleanup") active_findings = _coerce_findings_list(report_result.get("filtered_findings")) report_result["transitive_finding_count"] = sum( @@ -2710,6 +2757,10 @@ def _scan_multi_skill( aggregate_limitations = [ f"recursive discovery {limitation.resource} limit reached" for limitation in detection.limitations[:256] + # Symlink omissions get their own clearer aggregate message below; + # listing the generic one too would double-report the same entries. + if limitation.resource != "multi_skill_symlinked_entry" + or not detection.omitted_symlink_entries ] retained_public_records = 0 retained_report_characters = 0 @@ -2843,6 +2894,19 @@ def _scan_multi_skill( len(skills) - scanned_skill_count, ) output_omitted_skill_count = max(0, scanned_skill_count - len(processed_skills)) + omitted_symlink_entry_count = detection.omitted_symlink_entries + if omitted_symlink_entry_count: + analysis_incomplete = True + aggregate_limitations.append( + f"{omitted_symlink_entry_count} symlinked recursive skill(s) omitted " + "(directory symlinks are not followed)" + ) + progress_console.print( + f"[yellow]Warning:[/yellow] {omitted_symlink_entry_count} symlinked skill " + "directories were skipped during recursive discovery and are not " + "included in this scan." + ) + skills_omitted_total = unscanned_skill_count + omitted_symlink_entry_count if output_omitted_skill_count: analysis_incomplete = True aggregate_limitations.append( @@ -2856,11 +2920,11 @@ def _scan_multi_skill( ) aggregate_limitations = list(dict.fromkeys(aggregate_limitations))[:256] aggregate_completeness = _multi_skill_analysis_completeness( - total_skills=len(skills), + total_skills=len(skills) + omitted_symlink_entry_count, complete_skills=complete_skill_count, partial_skills=partial_skill_count, failed_skills=failed_skill_count, - omitted_skills=unscanned_skill_count, + omitted_skills=skills_omitted_total, limitations=aggregate_limitations, ) analysis_incomplete = not bool(aggregate_completeness["is_complete"]) @@ -2898,10 +2962,15 @@ def _scan_multi_skill( progress_console.print( f" {'':<30} {'—':<8} {'—':<12} {unscanned_skill_count:<10} {'partial':<10}" ) - if output_omitted_skill_count or unscanned_skill_count: + if omitted_symlink_entry_count: + progress_console.print( + f" {'':<30} {'—':<8} {'—':<12} " + f"{omitted_symlink_entry_count:<10} {'skipped':<10}" + ) + if output_omitted_skill_count or unscanned_skill_count or omitted_symlink_entry_count: progress_console.print( "[yellow]Recursive scan incomplete:[/yellow] one or more skills were omitted " - "after an aggregate safety limit." + "after an aggregate safety limit or skipped as symlinks." ) if format == FormatChoice.json: @@ -2914,7 +2983,7 @@ def _scan_multi_skill( "risk_recommendation": aggregate_risk_assessment["recommendation"], "analysis_completeness": aggregate_completeness, "skills_scanned": scanned_skill_count, - "skills_omitted": unscanned_skill_count, + "skills_omitted": skills_omitted_total, "skills_output_omitted": output_omitted_skill_count, "public_finding_records": retained_public_records, "report_characters": retained_report_characters, @@ -2965,6 +3034,14 @@ def _scan_multi_skill( "reason": "aggregate_scan_limit", } ) + if omitted_symlink_entry_count: + combined_skills.append( + { + "omitted": True, + "omitted_count": omitted_symlink_entry_count, + "reason": "symlink_not_followed", + } + ) rendered = json.dumps(combined, indent=2) if len(rendered) > _MULTI_SKILL_MAX_REPORT_CHARACTERS: analysis_incomplete = True @@ -3064,7 +3141,7 @@ def _scan_multi_skill( Path(output).write_text(rendered, encoding="utf-8") progress_console.print(f"[green]Combined report saved to:[/green] {output}") elif format is FormatChoice.terminal: - console.print(rendered) + console.print(rendered, markup=False, emoji=False) else: sys.stdout.write(rendered) @@ -3088,13 +3165,15 @@ def mcp( typer.Option( "--transport", "-t", - help="Transport: FastMCP stdio for local CLI agents, http for remote/A2A callers.", + help="Transport: FastMCP stdio for local CLI agents, http for loopback HTTP clients.", case_sensitive=False, ), ] = TransportChoice.stdio, host: Annotated[ str, - typer.Option("--host", help="Host to bind (http transport only)."), + typer.Option( + "--host", help="Loopback IP to bind (http transport only; localhost is accepted)." + ), ] = "127.0.0.1", port: Annotated[ int, @@ -3105,7 +3184,7 @@ def mcp( Run SkillSpector as an MCP server. Exposes a single tool, ``scan_skill``, so any MCP-capable agent (Claude Code, - Codex CLI, Gemini CLI) or remote runtime can scan a skill and gate installs + Codex CLI, Gemini CLI) can scan a skill locally and gate installs on the verdict. Requires the optional mcp extra. Reinstall the GitHub tool package with @@ -3120,7 +3199,7 @@ def mcp( from skillspector.mcp_server import run as run_mcp run_mcp(transport=transport.value, host=host, port=port) - except ModuleNotFoundError as e: + except (ModuleNotFoundError, ValueError) as e: err_console.print(f"[red]Error:[/red] {e}") raise typer.Exit(code=2) from e @@ -3181,7 +3260,26 @@ def baseline( state = _scan_state(input_path, FormatChoice.json, no_llm) state["baseline_path"] = os.path.abspath(output.expanduser()) result = graph.invoke(state) - findings = effective_findings(result) + completeness_value = result.get("analysis_completeness") + completeness = completeness_value if isinstance(completeness_value, dict) else {} + if ( + result.get("execution_successful") is False + or completeness.get("execution_successful") is False + or completeness.get("status") == "failed" + ): + raise ValueError( + "Cannot generate baseline because scan execution failed. " + "Run 'skillspector scan' to inspect analysis completeness, " + "resolve the failures, and retry." + ) + if completeness.get("is_complete") is False or completeness.get("status") == "partial": + err_console.print( + "[yellow]Warning:[/yellow] Scan analysis is incomplete; the baseline " + "accepts only observed findings and coverage gaps remain." + ) + # Fingerprint every occurrence the next scan checks. The reported + # findings are deduplicated and keep only one occurrence's evidence. + findings = result["active_findings"] data = build_baseline_dict( findings, reason=reason, diff --git a/src/skillspector/constants.py b/src/skillspector/constants.py index a01f2e51b..06871548f 100644 --- a/src/skillspector/constants.py +++ b/src/skillspector/constants.py @@ -18,7 +18,12 @@ import logging import os -from skillspector.providers import get_metadata_provider, get_model_config_provider +from skillspector.providers import ( + ModelMetadataProvider, + UnknownProviderError, + get_metadata_provider, + get_model_config_provider, +) logger = logging.getLogger(__name__) @@ -45,11 +50,17 @@ # ``resolve_model`` runs the waterfall: ``SKILLSPECTOR_MODEL`` env > slot # default > general default. OSS users pointing at build.nvidia.com or # stock OpenAI inherit ``NvBuildProvider``'s default model automatically. -_provider = get_metadata_provider() +_provider: ModelMetadataProvider | None +try: + _provider = get_metadata_provider() +except UnknownProviderError: + # Importing static checks and CLI help must not require a valid LLM provider. + # Runtime model configuration still validates the selector in build_model_config. + _provider = None # Exposed for analyzers that need a final fallback symbol (e.g., # ``model = state.model or MODEL_CONFIG[ANALYZER_ID] or _SKILLSPECTOR_DEFAULT_MODEL``). -_SKILLSPECTOR_DEFAULT_MODEL = _provider.DEFAULT_MODEL +_SKILLSPECTOR_DEFAULT_MODEL = _provider.DEFAULT_MODEL if _provider is not None else "" _MODEL_SLOTS: tuple[str, ...] = ( "default", @@ -84,7 +95,11 @@ def build_model_config() -> dict[str, str]: return {slot: _resolve_slot_model(slot, provider) for slot in _MODEL_SLOTS} -MODEL_CONFIG: dict[str, str] = {slot: _resolve_slot_model(slot, _provider) for slot in _MODEL_SLOTS} +MODEL_CONFIG: dict[str, str] = ( + {slot: _resolve_slot_model(slot, _provider) for slot in _MODEL_SLOTS} + if _provider is not None + else {} +) def _validate_model_config() -> None: @@ -93,6 +108,8 @@ def _validate_model_config() -> None: When ``SKILLSPECTOR_STRICT_MODEL_VALIDATION=true``, raises ``ValueError`` instead of logging warnings. """ + if _provider is None: + return unknown: list[str] = [] for slot, model in MODEL_CONFIG.items(): ctx = _provider.get_context_length(model) # type: ignore[attr-defined] diff --git a/src/skillspector/input_handler.py b/src/skillspector/input_handler.py index ab1517a13..6afb078e6 100644 --- a/src/skillspector/input_handler.py +++ b/src/skillspector/input_handler.py @@ -81,6 +81,7 @@ _DIRECT_FILE_URL_SUFFIXES = ( ".md", ".py", + ".pyw", ".sh", ) @@ -766,7 +767,10 @@ class InputHandler: Normalizes all inputs to a local directory path for scanning. """ - def __init__(self, transitive_budget: object | None = None) -> None: + def __init__( + self, transitive_budget: object | None = None, *, allow_git_credentials: bool = True + ) -> None: + self._allow_git_credentials = allow_git_credentials self._temp_dir: Path | None = None self._transitive_budget = transitive_budget self.primary_file_path: str | None = None @@ -1128,23 +1132,99 @@ def _resolve_tree_ref(self, repository_url: str, segments: list[str]) -> tuple[s f"{repository_url} ({'/'.join(segments)})" ) + def _git_invocation(self, url: str) -> tuple[list[str], dict[str, str] | None, str]: + """Build a Git policy for both ref discovery and cloning. + + Remote MCP callers must not borrow the server's SSH keys, helpers, + authorization headers, URL rewrites, client certificates, or netrc. + Trusted local callers retain the operator's configured Git identity. + """ + if self._allow_git_credentials: + return ["git"], None, url + parsed = urlparse(url) + if parsed.scheme != "https" or parsed.username is not None or parsed.password is not None: + raise ValueError("Untrusted Git scans require an unauthenticated HTTPS URL") + if parsed.hostname in {"gitlab.com", "bitbucket.org"}: + # Browser URLs need the Git endpoint without following a redirect + # whose destination has not passed the host/IP checks. + path = parsed.path.rstrip("/") + if path and not path.endswith(".git"): + path += ".git" + url = parsed._replace(path=path).geturl() + isolated_home = self._get_temp_dir() / "git-home" + isolated_home.mkdir(mode=0o700, exist_ok=True) + env = { + key: value + for key, value in os.environ.items() + if key.upper() + in { + "PATH", + "SYSTEMROOT", + "WINDIR", + "TEMP", + "TMP", + "TMPDIR", + "HTTP_PROXY", + "HTTPS_PROXY", + "ALL_PROXY", + "NO_PROXY", + "GIT_SSL_CAINFO", + "GIT_SSL_CAPATH", + "SSL_CERT_FILE", + "SSL_CERT_DIR", + "CURL_CA_BUNDLE", + } + } + env.update( + HOME=str(isolated_home), + USERPROFILE=str(isolated_home), + XDG_CONFIG_HOME=str(isolated_home), + GIT_CONFIG_GLOBAL=os.devnull, + GIT_CONFIG_SYSTEM=os.devnull, + GIT_CONFIG_NOSYSTEM="1", + GIT_TERMINAL_PROMPT="0", + GIT_ALLOW_PROTOCOL="https", + GIT_CEILING_DIRECTORIES=str(isolated_home.parent), + ) + return ( + [ + "git", + "-C", + str(isolated_home), + "-c", + "credential.helper=", + "-c", + "core.askPass=", + "-c", + f"core.hooksPath={os.devnull}", + "-c", + "http.extraHeader=", + "-c", + "http.followRedirects=false", + ], + env, + url, + ) + def _list_remote_refs(self, repository_url: str) -> set[str]: """Return the branch/tag names advertised by the remote repository. Bounded by the ingest deadline; the host allowlist and private-IP checks from URL validation apply. """ + git_argv, git_env, repository_url = self._git_invocation(repository_url) self._validate_url_host(repository_url, ALLOWED_GIT_HOSTS) deadline = self._deadline() self._check_deadline(deadline, "git") timeout = max(1.0, deadline - monotonic()) try: process = subprocess.run( - ["git", "ls-remote", repository_url], + [*git_argv, "ls-remote", repository_url], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, timeout=timeout, check=False, + **({"env": git_env} if git_env is not None else {}), ) except subprocess.TimeoutExpired as exc: raise IngestLimitExceededError("Git ref listing exceeded its time limit") from exc @@ -1205,6 +1285,7 @@ def _clone_git(self, url: str, *, branch: str | None = None) -> Path: self._truncate("byte_budget_exhausted", "git") if remaining_artifacts is not None and remaining_artifacts <= 0: self._truncate("artifact_budget_exhausted", "git") + git_argv, git_env, url = self._git_invocation(url) self._validate_url_host(url, ALLOWED_GIT_HOSTS) deadline = self._deadline() self._check_deadline(deadline, "git") @@ -1224,6 +1305,7 @@ def _clone_git(self, url: str, *, branch: str | None = None) -> Path: clone_command[6:6] = ["--branch", branch] if remaining_bytes is not None: clone_command.insert(6, f"--filter=blob:limit={remaining_bytes}") + clone_command[:1] = git_argv process: subprocess.Popen[bytes] | None = None final_measurement: _TreeMeasurement | None = None try: @@ -1232,6 +1314,7 @@ def _clone_git(self, url: str, *, branch: str | None = None) -> Path: stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, shell=False, + **({"env": git_env} if git_env is not None else {}), ) while True: self._check_deadline(deadline, "git") diff --git a/src/skillspector/inspection_ledger.py b/src/skillspector/inspection_ledger.py index 382973287..ac623db14 100644 --- a/src/skillspector/inspection_ledger.py +++ b/src/skillspector/inspection_ledger.py @@ -36,6 +36,15 @@ class LedgerRecordType(StrEnum): WORK_ITEM = "work_item" SYSTEM = "system" SCOPE_BOUNDARY = "scope_boundary" + # An analyzer's own configuration (e.g. its YARA rule set), not a skill + # artifact. Its ``path`` is only a report-safe label, and every relative + # path is also a legal file name, so path-keyed accounting must exclude + # these records by type -- no choice of label can be collision-free. + RULE_SET = "rule_set" + + +RULE_SET_SCOPE: Final = "rule_set" +"""Public ``scope`` of exception rows describing a rule set rather than a file.""" class LedgerReason(StrEnum): @@ -50,6 +59,8 @@ class LedgerReason(StrEnum): MISSING_FILE_CACHE = "missing_file_cache" SIZE_LIMIT = "size_limit" BINARY_CONTENT = "binary_content" + BINARY_TEXT_PROJECTION = "binary_text_projection" + BINARY_EXECUTABLE_TEXT = "binary_executable_text" EVAL_DATASET = "eval_dataset" SYNTAX_ERROR = "syntax_error" LLM_BATCH_FAILED = "llm_batch_failed" @@ -74,6 +85,7 @@ class LedgerReason(StrEnum): ARCHIVE_TRUNCATED = "archive_truncated" ARCHIVE_UNSAFE_MEMBER_PATH = "archive_unsafe_member_path" ARCHIVE_AMBIGUOUS_MEMBER_PATH = "archive_ambiguous_member_path" + ARTIFACT_PATH_COLLISION = "artifact_path_collision" ARCHIVE_LINK_MEMBER = "archive_link_member" ARCHIVE_DEPTH_LIMIT = "archive_depth_limit" ARCHIVE_MEMBER_LIMIT = "archive_member_limit" @@ -98,6 +110,8 @@ class LedgerReason(StrEnum): OUTPUT_LIMIT = "output_limit" TRANSITIVE_CHILD_SCAN_FAILED = "transitive_child_scan_failed" STATIC_PARSE_LIMIT = "static_parse_limit" + PYTHON_SOURCE_AMBIGUOUS = "python_source_ambiguous" + PYTHON_SOURCE_DECODE_ERROR = "python_source_decode_error" OBFUSCATED_INSTRUCTION_TEXT = "obfuscated_instruction_text" @@ -111,6 +125,12 @@ class LedgerReason(StrEnum): LedgerReason.MISSING_FILE_CACHE: "Applicable analyzer could not obtain file content.", LedgerReason.SIZE_LIMIT: "File exceeds this analyzer's character limit.", LedgerReason.BINARY_CONTENT: "Binary content is unsupported by this analyzer.", + LedgerReason.BINARY_TEXT_PROJECTION: ( + "Only bounded printable text was inspected; binary structure and behavior are outside scope." + ), + LedgerReason.BINARY_EXECUTABLE_TEXT: ( + "Readable text was inspected, but binary executable behavior remains outside analyzer coverage." + ), LedgerReason.EVAL_DATASET: ( "Evaluation dataset prose is excluded from static pattern analysis." ), @@ -141,7 +161,7 @@ class LedgerReason(StrEnum): LedgerReason.MANIFEST_ABSENT: ("No compatible manifest was present for this analyzer."), LedgerReason.NO_APPLICABLE_FILES: ("No files matched this analyzer's applicability contract."), LedgerReason.OMS_SIGNATURE: ( - "Recognized OMS signature metadata is excluded from content analysis." + "DSSE content decoding is incomplete or unsupported; raw fields were retained for inspection. Signature authenticity was not verified." ), LedgerReason.BASELINE_FILE: ( "The explicitly selected suppression baseline is excluded from content analysis." @@ -161,6 +181,10 @@ class LedgerReason(StrEnum): LedgerReason.ARCHIVE_AMBIGUOUS_MEMBER_PATH: ( "Archive member name has an ambiguous or duplicate provenance identity." ), + LedgerReason.ARTIFACT_PATH_COLLISION: ( + "A disk path containing '!/' collides with an archive member. Rename the directory " + "ending in '!'; the colliding archive member was not analyzed." + ), LedgerReason.ARCHIVE_LINK_MEMBER: "Archive link member was not followed or read.", LedgerReason.ARCHIVE_DEPTH_LIMIT: "Nested archive depth limit was reached.", LedgerReason.ARCHIVE_MEMBER_LIMIT: "Cumulative archive member limit was reached.", @@ -210,6 +234,12 @@ class LedgerReason(StrEnum): LedgerReason.STATIC_PARSE_LIMIT: ( "A security-relevant expression exceeded a bounded static parser's span limit." ), + LedgerReason.PYTHON_SOURCE_AMBIGUOUS: ( + "Python execution intent depends on runtime or platform-specific shebang semantics." + ), + LedgerReason.PYTHON_SOURCE_DECODE_ERROR: ( + "Python source bytes could not be decoded under their declared encoding." + ), LedgerReason.OBFUSCATED_INSTRUCTION_TEXT: ( "Obfuscated instruction text could not be fully evaluated by the deterministic layer." ), @@ -293,6 +323,7 @@ class InspectionLedgerException(TypedDict): error_class: NotRequired[str] analyzers: NotRequired[list[str]] fatal: NotRequired[bool] + scope: NotRequired[str] class AnalysisCompleteness(TypedDict): @@ -522,8 +553,10 @@ def analyzer_status_event( def analyzer_status_for_events( analyzer_id: str, events: Iterable[InspectionLedgerEvent] ) -> AnalyzerStatusEvent: - """Summarize an analyzer's terminal work without exposing event payloads.""" - terminal_events = list(events) + """Summarize analyzer work; scope-boundary records remain separate exclusions.""" + terminal_events = [ + event for event in events if event.get("record_type") != LedgerRecordType.SCOPE_BOUNDARY + ] if not terminal_events: return analyzer_status_event( analyzer_id=analyzer_id, @@ -585,6 +618,7 @@ def _exception( error_class: str | None = None, analyzers: Iterable[str] = (), fatal: bool, + scope: str | None = None, ) -> InspectionLedgerException: """Build the public, safe projection of one exceptional ledger fact.""" exception: InspectionLedgerException = { @@ -602,9 +636,16 @@ def _exception( exception["analyzers"] = analyzer_ids if error_class: exception["error_class"] = error_class + if scope: + exception["scope"] = scope return exception +def _is_rule_set_record(event: Mapping[str, object]) -> bool: + """Return whether a ledger row describes a rule set rather than an artifact.""" + return event.get("record_type") == LedgerRecordType.RULE_SET + + def _exception_from_event( event: InspectionLedgerEvent, *, fatal: bool ) -> InspectionLedgerException: @@ -625,6 +666,7 @@ def _exception_from_event( error_class=event.get("error_class"), analyzers=[str(event.get("analyzer_id", ""))], fatal=fatal, + scope=RULE_SET_SCOPE if _is_rule_set_record(event) else None, ) @@ -643,6 +685,9 @@ def _merge_exception_projection( exception["start_line"], exception["end_line"], exception.get("error_class"), + # A rule-set row and a real file's row can share a path label; + # they must never merge into one public row. + exception.get("scope"), ) existing = grouped.get(key) if existing is None: @@ -922,9 +967,17 @@ def accounting_error(path: object = None) -> None: ledger_exceptions = _merge_exception_projection(exceptional_rows) scope_exclusions = _merge_exception_projection(scope_rows) + # Rule-set rows describe an analyzer's configuration, not an artifact. Their + # path is only a label, so folding them into per-component coverage would + # charge the rule set's incompleteness to any real file with that name. + rule_set_work_ids = { + str(event.get("work_id", "")) for event in events if _is_rule_set_record(event) + } per_component: dict[str, list[LedgerOutcome]] = {component: [] for component in components} if primary_targets: for _analyzer_id, target, matches in primary_targets: + if str(target.get("work_id", "")) in rule_set_work_ids: + continue path = _safe_path(target.get("path"), components) outcomes = per_component.setdefault(path, []) if len(matches) == 1: diff --git a/src/skillspector/llm_analyzer_base.py b/src/skillspector/llm_analyzer_base.py index f7ed602fd..47338e40a 100644 --- a/src/skillspector/llm_analyzer_base.py +++ b/src/skillspector/llm_analyzer_base.py @@ -63,6 +63,7 @@ _AgentCLIMessage, _ainvoke_with_usage, _invoke_with_usage, + bind_structured_output, chat_model_provider_name, get_chat_model, new_inference_usage_collector, @@ -607,7 +608,10 @@ def _clamp_start_line(cls, v: int) -> int: @classmethod def _normalize_confidence(cls, v: object) -> float: # Accept 0-100 scale values from some models, then clamp into [0, 1]. - value = float(cast(Any, v)) + try: + value = float(cast(Any, v)) + except (TypeError, ValueError) as exc: + raise ValueError("confidence must be numeric") from exc if value > 2.0: value = value / 100.0 return min(1.0, max(0.0, value)) @@ -933,7 +937,9 @@ def __init__( max_retries=native_retries, ) self._structured_llm = ( - self._llm.with_structured_output(self.response_schema) if self.response_schema else None + self._bind_structured_output(self._llm, self.response_schema) + if self.response_schema + else None ) self._usage_collector = new_inference_usage_collector( node=node, @@ -942,6 +948,18 @@ def __init__( chat_model=self._llm, ) + def _structured_output_preference(self, llm: object) -> str | None: + """Return an analyzer-specific binding preference, if any.""" + return None + + def _bind_structured_output(self, llm: object, schema: type) -> object: + return bind_structured_output( + llm, + schema, + self.model, + preferred_method=self._structured_output_preference(llm), + ) + def _remaining_timeout(self) -> float | None: if callable(self._timeout): remaining = self._timeout() @@ -986,7 +1004,9 @@ def _model_for_call(self) -> tuple[object, object | None]: chat_model_controls(llm), ) structured = ( - llm.with_structured_output(self.response_schema) if self.response_schema else None + self._bind_structured_output(llm, self.response_schema) + if self.response_schema + else None ) return llm, structured diff --git a/src/skillspector/llm_provenance.py b/src/skillspector/llm_provenance.py index b41f15c2e..23157c40c 100644 --- a/src/skillspector/llm_provenance.py +++ b/src/skillspector/llm_provenance.py @@ -210,6 +210,42 @@ def capture_llm_provenance(model_config: Mapping[str, object]) -> dict[str, obje # This provider resolves model defaults before any client is constructed. # Runtime response telemetry supplies the actual effective provider later. resolved_adapter = provider_name(get_model_config_provider()) + return _capture_llm_provenance( + model_config, + configured_adapter=configured_adapter, + resolved_adapter=resolved_adapter, + routing_adapter=resolved_adapter, + ) + + +def capture_static_llm_provenance( + fallback_model_config: Mapping[str, object], +) -> dict[str, object]: + """Capture static-scan configuration without resolving an inactive provider.""" + default_model = os.environ.get("SKILLSPECTOR_MODEL", "").strip() + model_config = { + slot: os.environ.get(f"SKILLSPECTOR_MODEL_{slot.upper()}", "").strip() + or default_model + or fallback_model_config.get(slot) + for slot in LLM_ANALYZER_SLOTS + } + configured_adapter = os.environ.get("SKILLSPECTOR_PROVIDER", "").strip().lower() + return _capture_llm_provenance( + model_config, + configured_adapter=configured_adapter or "unknown", + resolved_adapter="unknown", + routing_adapter=configured_adapter, + ) + + +def _capture_llm_provenance( + model_config: Mapping[str, object], + *, + configured_adapter: str, + resolved_adapter: str, + routing_adapter: str, +) -> dict[str, object]: + """Build a sanitized capture from already selected configuration labels.""" package_version = version("skillspector") source_revision, source_revision_source = _capture_source_revision() @@ -259,7 +295,7 @@ def capture_llm_provenance(model_config: Mapping[str, object]) -> dict[str, obje "provider": { "configured_adapter": _safe_label(configured_adapter), "resolved_adapter": _safe_label(resolved_adapter), - "routing": _capture_provider_routing(resolved_adapter), + "routing": _capture_provider_routing(routing_adapter), "service": "unknown", }, "analyzers": analyzers, @@ -531,6 +567,11 @@ def sanitize_llm_provenance( if effective_adapters else "unknown" ) + routing_adapter = ( + "azure_openai" + if not use_llm and resolved_adapter == "unknown" and configured_adapter == "azure_openai" + else resolved_adapter + ) raw_analyzers = raw.get("analyzers") by_id: dict[str, Mapping[object, object]] = {} @@ -634,7 +675,7 @@ def sanitize_llm_provenance( "effective_adapters": effective_adapters, "service": "unknown", "routing": _sanitize_provider_routing( - provider.get("routing"), resolved_adapter=resolved_adapter + provider.get("routing"), resolved_adapter=routing_adapter ), }, "analyzers": analyzers, diff --git a/src/skillspector/llm_utils.py b/src/skillspector/llm_utils.py index c8f4d6872..9a3174843 100644 --- a/src/skillspector/llm_utils.py +++ b/src/skillspector/llm_utils.py @@ -38,13 +38,16 @@ import asyncio import concurrent.futures import json +import os import threading import weakref from collections.abc import Coroutine from typing import Any, NoReturn +from langchain_core.exceptions import OutputParserException from langchain_core.language_models.chat_models import BaseChatModel -from langchain_core.runnables import Runnable +from langchain_core.runnables import Runnable, RunnableConfig, RunnableLambda +from langchain_core.utils.function_calling import convert_to_openai_tool from skillspector.inference_usage import ( InferenceUsageCollector, @@ -54,6 +57,7 @@ ) from skillspector.model_info import get_max_input_tokens, get_max_output_tokens from skillspector.providers import ( + UnknownProviderError, create_chat_model, create_chat_model_with_provider, get_active_provider, @@ -138,7 +142,10 @@ def is_llm_available() -> tuple[bool, str | None]: :func:`get_chat_model`; unbound HTTP providers keep the credential-resolution and OpenAI fallback path. """ - provider = get_active_provider() + try: + provider = get_active_provider() + except UnknownProviderError as exc: + return False, str(exc) if has_cli_capability(provider): return provider.is_available() # type: ignore[attr-defined] @@ -354,7 +361,10 @@ def invoke(self, prompt: str) -> _AgentCLIMessage: async def ainvoke(self, prompt: str) -> _AgentCLIMessage: return await asyncio.to_thread(self.invoke, prompt) - def with_structured_output(self, schema: type) -> _StructuredAgentCLIModel: + def with_structured_output( + self, schema: type, method: str | None = None + ) -> _StructuredAgentCLIModel: + del method # Agent CLI adapters always validate the response against the schema. return _StructuredAgentCLIModel(self, schema) def set_timeout(self, timeout: float | None) -> None: @@ -362,6 +372,184 @@ def set_timeout(self, timeout: float | None) -> None: self._timeout = timeout +STRUCTURED_OUTPUT_METHODS = ("function_calling", "json_schema") + + +def structured_output_kwargs( + model: str, + provider: object | None = None, + *, + preferred_method: str | None = None, +) -> dict[str, str]: + """Keyword arguments for ``with_structured_output`` when binding a schema for *model*. + + ``SKILLSPECTOR_STRUCTURED_OUTPUT_METHOD`` wins, then the active provider's + ``structured_output_method(model)`` hint, then an analyzer preference, else + LangChain's default (no kwargs). + + Raises: + ValueError: when the environment override or preference is not a known method. + """ + if preferred_method is not None and preferred_method not in STRUCTURED_OUTPUT_METHODS: + raise ValueError( + "preferred structured output method must be one of " + f"{', '.join(STRUCTURED_OUTPUT_METHODS)}; got {preferred_method!r}" + ) + override = os.environ.get("SKILLSPECTOR_STRUCTURED_OUTPUT_METHOD", "").strip().lower() + if override: + if override not in STRUCTURED_OUTPUT_METHODS: + raise ValueError( + "SKILLSPECTOR_STRUCTURED_OUTPUT_METHOD must be one of " + f"{', '.join(STRUCTURED_OUTPUT_METHODS)}; got {override!r}" + ) + return {"method": override} + if provider is None: + provider = get_active_provider() + hint = getattr(provider, "structured_output_method", None) + method = hint(model) if callable(hint) else None + if method: + return {"method": method} + method = preferred_method + return {"method": method} if method else {} + + +def bind_structured_output( + llm: object, + schema: type, + model: str, + provider: object | None = None, + *, + preferred_method: str | None = None, +) -> object: + """``llm.with_structured_output(schema)`` with the method *model* needs. + + A chat model restricted to ``toolChoice`` ``auto`` (Bedrock models that + reject a forced tool call, or a ``ChatOpenAI`` with ``tool_choice`` + disabled) and bound with a tool method gets the tool call asked for in + the prompt, and a prose answer raises :class:`StructuredOutputParseError`, + which the analyzers retry like any other malformed structured response. + """ + kwargs = structured_output_kwargs(model, provider, preferred_method=preferred_method) + unforced = _binds_unforced_tool_call(llm, kwargs.get("method")) + include_raw = isinstance(llm, BaseChatModel) and ( + unforced or kwargs.get("method") == "function_calling" + ) + binding_kwargs: dict[str, Any] = dict(kwargs) + if include_raw: + # Native parsers may silently keep only the first tool call. Retain the + # raw message so validation can reject extra or invalid calls instead. + binding_kwargs["include_raw"] = True + structured = llm.with_structured_output(schema, **binding_kwargs) # type: ignore[attr-defined] + if unforced: + return _require_tool_call(structured, schema, include_raw=include_raw) + if kwargs.get("method") == "function_calling" and not isinstance( + structured, _StructuredAgentCLIModel + ): + return _validate_tool_call_result(structured, schema, include_raw=include_raw) + return structured + + +def _binds_unforced_tool_call(llm: object, method: str | None) -> bool: + """``True`` when *method* binds the schema as a tool *llm* is not forced to call. + + ``ChatBedrockConverse`` binds a tool by default; ``ChatOpenAI`` defaults to + ``json_schema``, so its ``tool_choice``-disabled form needs an explicit + ``function_calling``. + """ + if getattr(llm, "supports_tool_choice_values", None) == ("auto",): + return method in (None, "function_calling") + disabled = getattr(llm, "disabled_params", None) + return ( + method == "function_calling" + and isinstance(disabled, dict) + and "tool_choice" in disabled + and disabled["tool_choice"] is None + ) + + +_TOOL_CALL_INSTRUCTION = ( + "Report your result by calling the {tool} tool exactly once. Do not answer in prose." +) + + +def _require_tool_call( + structured: Runnable, schema: type, *, include_raw: bool = False +) -> Runnable: + """Ask for the tool call in the prompt and fail closed when it does not happen.""" + tool = ( + convert_to_openai_tool(schema)["function"]["name"] + if include_raw + else schema.__name__ + if isinstance(schema, type) + else "response" + ) + + def _ask(prompt: str) -> str: + return f"{prompt}\n\n{_TOOL_CALL_INSTRUCTION.format(tool=tool)}" + + return RunnableLambda(_ask) | _validate_tool_call_result( + structured, schema, include_raw=include_raw + ) + + +def _validate_tool_call_result( + structured: Runnable, schema: type, *, include_raw: bool = False +) -> Runnable: + """Validate forced/auto tool results without changing the bound model prompt.""" + tool = ( + convert_to_openai_tool(schema)["function"]["name"] + if include_raw + else schema.__name__ + if isinstance(schema, type) + else "response" + ) + + def _check(result: object) -> object: + if include_raw: + if not isinstance(result, dict): + raise StructuredOutputParseError(f"model returned invalid {tool} tool output") + raw = result.get("raw") + calls = getattr(raw, "tool_calls", None) + if ( + not isinstance(calls, list) + or len(calls) != 1 + or calls[0].get("name") != tool + or getattr(raw, "invalid_tool_calls", None) + ): + raise StructuredOutputParseError( + f"model must return exactly one valid {tool} tool call" + ) + error = result.get("parsing_error") + if error is not None: + raise StructuredOutputParseError( + f"model returned invalid {tool} tool arguments" + ) from error + result = result.get("parsed") + if result is None: + raise StructuredOutputParseError( + f"model answered in prose instead of calling the {tool} tool" + ) + return result + + def _invoke(prompt: Any, config: RunnableConfig) -> object: + try: + return _check(structured.invoke(prompt, config=config)) + except OutputParserException as exc: + raise StructuredOutputParseError( + f"model returned invalid {tool} tool arguments" + ) from exc + + async def _ainvoke(prompt: Any, config: RunnableConfig) -> object: + try: + return _check(await structured.ainvoke(prompt, config=config)) + except OutputParserException as exc: + raise StructuredOutputParseError( + f"model returned invalid {tool} tool arguments" + ) from exc + + return RunnableLambda(_invoke, _ainvoke) + + def get_chat_model( model: str | None = None, *, timeout: float | None = None ) -> BaseChatModel | AgentCLIChatModel: diff --git a/src/skillspector/mcp_server.py b/src/skillspector/mcp_server.py index 16c0332ea..0bb74bd24 100644 --- a/src/skillspector/mcp_server.py +++ b/src/skillspector/mcp_server.py @@ -15,8 +15,8 @@ """MCP server exposing SkillSpector scanning as an agent-callable tool. -This lets any MCP-capable agent (Claude Code, Codex CLI, Gemini CLI) or remote -runtime call ``scan_skill`` and gate skill/MCP installs on the verdict, turning +This lets local MCP-capable agents (Claude Code, Codex CLI, Gemini CLI) call +``scan_skill`` and gate skill/MCP installs on the verdict, turning SkillSpector from an out-of-band audit tool into a runtime guardrail. The scan core (:func:`run_scan`) is deliberately independent of the ``mcp`` SDK @@ -28,8 +28,10 @@ from __future__ import annotations from collections.abc import Mapping +from ipaddress import ip_address from pathlib import Path from typing import TYPE_CHECKING, Any +from urllib.parse import urlparse from skillspector import __version__ from skillspector.cleanup import cleanup_result @@ -108,7 +110,7 @@ async def run_scan( :data:`VALID_FORMATS`. allow_local_targets: Whether local filesystem targets are allowed. HTTP MCP calls set this to ``False`` so routable servers do not - accept caller-controlled local paths. + accept caller-controlled local paths or use ambient Git credentials. yara_rules_dir: Optional directory of additional YARA rules. Returns: @@ -125,12 +127,20 @@ async def run_scan( local_yara_rules = yara_rules_dir is not None and _is_local_target(yara_rules_dir) if local_target or local_yara_rules: raise ValueError("local targets are disabled for this MCP transport") + parsed_target = urlparse(target.strip()) + if ( + parsed_target.scheme != "https" + or parsed_target.username is not None + or parsed_target.password is not None + ): + raise ValueError("this MCP transport requires an unauthenticated HTTPS target") llm_preflight_available, _ = is_llm_available() llm_enabled = use_llm and llm_preflight_available state: dict[str, Any] = { "input_path": target, + "allow_git_credentials": allow_local_targets, "output_format": output_format, "use_llm": llm_enabled, "llm_requested": use_llm, @@ -273,7 +283,21 @@ async def scan_skill( def run(transport: str = "stdio", host: str = "127.0.0.1", port: int = 8000) -> None: - """Run the MCP server over ``stdio`` (local agents) or ``http`` (remote/A2A).""" + """Run the MCP server over ``stdio`` or loopback-only ``http``.""" + if transport == "http": + # Never resolve caller-selected names or expose this unauthenticated server. + host = "127.0.0.1" if host.lower() == "localhost" else host + try: + host = str(ip_address(host)) + # FastMCP permits these Host headers, not the full IPv4 loopback range. + loopback = host in {"127.0.0.1", "::1"} + except ValueError: + loopback = False + if not loopback: + raise ValueError( + "HTTP MCP has no authentication and must bind to a loopback IP " + "(127.0.0.1 or ::1). Use an authenticating reverse proxy for remote access." + ) server = build_server(allow_local_targets=transport == "stdio") if transport == "stdio": server.run(transport="stdio") diff --git a/src/skillspector/models.py b/src/skillspector/models.py index 061477cd5..be2e1c5cf 100644 --- a/src/skillspector/models.py +++ b/src/skillspector/models.py @@ -120,6 +120,11 @@ def _new_finding_id() -> str: return f"finding-{uuid4().hex}" +OCCURRENCE_FINDING_ID_KEY = "_skillspector_report_finding_id" +OCCURRENCE_CODE_SNIPPET_KEY = "_skillspector_report_code_snippet" +_PRIVATE_OCCURRENCE_KEYS = frozenset({OCCURRENCE_FINDING_ID_KEY, OCCURRENCE_CODE_SNIPPET_KEY}) + + @dataclass class Finding: """Finding model for graph state and report output (shape aligned with to_dict).""" @@ -218,7 +223,9 @@ def _serialized_occurrences(self) -> list[dict[str, object]]: ] serialized: list[dict[str, object]] = [] for raw in occurrences: - occurrence = dict(raw) + occurrence = { + key: value for key, value in raw.items() if key not in _PRIVATE_OCCURRENCE_KEYS + } if self.source_identity: occurrence.setdefault("source_identity", self.source_identity) if self.source_digest: diff --git a/src/skillspector/multi_skill.py b/src/skillspector/multi_skill.py index 0081bc740..4503d3b51 100644 --- a/src/skillspector/multi_skill.py +++ b/src/skillspector/multi_skill.py @@ -132,6 +132,7 @@ class MultiSkillDetectionResult: entries_examined: int = 0 structured_candidates_examined: int = 0 structured_input_bytes_examined: int = 0 + omitted_symlink_entries: int = 0 @property def complete(self) -> bool: @@ -234,10 +235,10 @@ def detect_skills(directory: Path) -> MultiSkillDetectionResult: A directory is considered multi-skill when it has no root ``SKILL.md`` and at least two immediate child directories contain a manifest or supported - structured skill bundle. Any discovery limit or filesystem ambiguity - discards all partial classifications and returns ``complete == False``. - Callers can then fall back to a bounded monolithic scan and propagate the - supplied limitation to their public completeness surfaces. + structured skill bundle. Discovery limits and filesystem ambiguities return + ``complete == False``. A symlinked immediate child is the narrow exception: + it is never traversed, but already classified non-link siblings remain + available for a bounded recursive scan with the supplied limitation. """ absolute_directory = Path(os.path.abspath(directory)) try: @@ -276,18 +277,33 @@ def detect_skills(directory: Path) -> MultiSkillDetectionResult: return MultiSkillDetectionResult(is_multi_skill=False, has_root_skill=True) skills: list[SkillDirectory] = [] + limitations: list[MultiSkillDetectionLimitation] = [] + omitted_symlink_entries = 0 for entry in _bounded_scandir(directory, budget=budget): budget.check_runtime() child = Path(entry.path) + if entry.name in _SKIP_DIRS: + # Intentionally ignored names (e.g. `.git`, `.venv`, + # `node_modules`) are not a discovery gap even when they are + # symlinks; skip them before recording any limitation. Any + # other symlinked name is recorded below, so an eligible + # dot-prefixed skill such as `.review-helper` is never + # silently excluded. + continue try: if entry.is_symlink() or _is_link_or_junction(child): + limitations.append( + MultiSkillDetectionLimitation( + reason_code="read_error", + resource="multi_skill_symlinked_entry", + ) + ) + omitted_symlink_entries += 1 continue if not entry.is_dir(follow_symlinks=False): continue except OSError as exc: raise _read_error("multi_skill_directory_entry") from exc - if entry.name in _SKIP_DIRS: - continue has_manifest = _has_skill_md(child, budget=budget) is_structured = False @@ -314,9 +330,11 @@ def detect_skills(directory: Path) -> MultiSkillDetectionResult: is_multi_skill=len(skills) >= 2, skills=skills, has_root_skill=False, + limitations=tuple(limitations), entries_examined=budget.entries, structured_candidates_examined=budget.structured_candidates, structured_input_bytes_examined=budget.structured_bytes, + omitted_symlink_entries=omitted_symlink_entries, ) @@ -524,6 +542,11 @@ def _extract_skill_name(skill_dir: Path, *, budget: _DetectionBudget) -> str: raise _read_error("multi_skill_manifest_content") from exc prefix = observed[:MAX_MULTI_SKILL_MANIFEST_FRONTMATTER_BYTES] + while prefix.startswith(b"\xef\xbb\xbf"): + # Byte-level counterpart to the BOM strip in build_context._parse_manifest: + # this sniff runs on raw bytes before decode, so a BOM-prefixed SKILL.md + # would otherwise fall through to `fallback` (the directory name) below. + prefix = prefix[3:] if not prefix.startswith(b"---"): return fallback content = prefix.decode("utf-8", errors="replace") diff --git a/src/skillspector/nested_artifacts.py b/src/skillspector/nested_artifacts.py index 811586f46..2fb1f31eb 100644 --- a/src/skillspector/nested_artifacts.py +++ b/src/skillspector/nested_artifacts.py @@ -20,6 +20,8 @@ from dataclasses import dataclass, field from pathlib import Path, PurePosixPath +import regex # type: ignore[import-untyped] + from skillspector.artifacts import ( ArtifactDisposition, ArtifactRecord, @@ -40,6 +42,7 @@ LedgerRecordType, ledger_event, ) +from skillspector.python_ast import PythonSourceClassification, classify_python_source ARCHIVE_MAX_DEPTH = 3 ARCHIVE_MAX_MEMBERS = 1_000 @@ -79,6 +82,7 @@ ".phtml", ".ps1", ".py", + ".pyw", ".pyc", ".pyo", ".rb", @@ -113,6 +117,11 @@ class NestedInspectionResult: components: list[str] = field(default_factory=list) file_cache: dict[str, str] = field(default_factory=dict) raw_file_cache: dict[str, bytes] = field(default_factory=dict) + # Classify with the archive member's execution path, while retaining the + # virtual path as the stable cache/report key used by downstream analyzers. + python_source_classifications: dict[str, PythonSourceClassification] = field( + default_factory=dict + ) artifact_inventory: list[ArtifactRecord] = field(default_factory=list) metadata: list[dict[str, object]] = field(default_factory=list) outer_metadata: dict[str, dict[str, object]] = field(default_factory=dict) @@ -120,6 +129,10 @@ class NestedInspectionResult: # Expected extensions and format mismatches are not content recognition. recognized_zip_paths: set[str] = field(default_factory=set) ledger_events: list[InspectionLedgerEvent] = field(default_factory=list) + # Keep the actual event objects: disk and virtual paths can share both a + # spelling and a work ID, so neither identifies which input produced it. + disk_container_events: list[InspectionLedgerEvent] = field(default_factory=list) + _active_disk_path: str | None = None uncompressed_bytes: int = 0 # Exceptions can target a top-level container before a virtual artifact row # exists. Preserve their canonical disposition so build_context can apply @@ -389,21 +402,134 @@ def _zip_member_is_link(info: zipfile.ZipInfo) -> bool: b"\xbf\xba\xfe\xca", ) +_TYPESCRIPT_DECLARATION_SUFFIXES = (".d.ts", ".d.cts", ".d.mts") +_TYPESCRIPT_DECLARATION_TIMEOUT_SECONDS = 0.05 +_TYPESCRIPT_AMBIENT_MEMBER = ( + r"(?:export\s+)?(?:" + r"interface\s+[A-Za-z_$][\w$]*(?:\s*<[^\r\n{}]*>)?\s*\{[^\r\n{}]*\}|" + r"type\s+[A-Za-z_$][\w$]*(?:\s*<[^\r\n;{}]*>)?\s*=\s*[^\r\n;{}]+;|" + r"(?:const|let|var)\s+[A-Za-z_$][\w$]*(?:\s*:\s*[^\r\n;{}]+)?\s*;|" + r"function\s+[A-Za-z_$][\w$]*\s*\([^\r\n{}]*\)\s*:\s*[^\r\n;{}]+;|" + r"class\s+[A-Za-z_$][\w$]*\s*\{[^\r\n{}]*\}" + r")" +) +_TYPESCRIPT_DECLARATION_FILE = regex.compile( + r"\A\s*(?:(?:" + r"(?:(?:export\s+)?declare\s+(?:const|let|var|function|class)\b[^\r\n;{}]*;)|" + rf"(?:declare\s+(?:export\s+)?namespace\s+[A-Za-z_$][\w$]*\s*\{{(?:\s*{_TYPESCRIPT_AMBIENT_MEMBER})*\s*\}}\s*;?)|" + rf"(?:declare\s+module\s+(?:[\"'][^\"']+[\"']|[A-Za-z_$][\w$]*)\s*\{{(?:\s*{_TYPESCRIPT_AMBIENT_MEMBER})*\s*\}}\s*;?)|" + r"(?:(?:export\s+)?interface\s+[A-Za-z_$][\w$]*(?:\s*<[^\r\n{}]*>)?\s*\{[^\r\n{}]*\}\s*;?)|" + r"(?:(?:export\s+)?type\s+[A-Za-z_$][\w$]*(?:\s*<[^;{}]*>)?\s*=\s*[^;\r\n{}]+;)|" + r"(?:import\s+type\s+(?:[A-Za-z_$][\w$]*(?:\s*,\s*\{[^{}]*\})?|\{[^{}]*\}|\*\s+as\s+[A-Za-z_$][\w$]*)\s+from\s+[\"'][^\"'\r\n]+[\"']\s*;)|" + r"(?:export\s*\{[^{}]*\}\s*;?)" + r")\s*)+\Z", + regex.DOTALL, +) + + +def _strip_typescript_comments(text: str) -> str | None: + """Remove comments without interpreting comment markers inside literals.""" + result: list[str] = [] + index = 0 + while index < len(text): + character = text[index] + if character in "'\"`": + quote = character + start = index + index += 1 + while index < len(text): + if text[index] == "\\": + index += 2 + continue + if quote == "`" and text.startswith("${", index): + # Interpolated templates need a JavaScript parser to prove + # inert; fail closed rather than hide their expression. + return None + if text[index] == quote: + index += 1 + break + index += 1 + else: + return None + result.append(text[start:index]) + continue + if text.startswith("//", index): + while index < len(text) and text[index] not in "\r\n": + result.append(" ") + index += 1 + continue + if text.startswith("/*", index): + end = text.find("*/", index + 2) + if end < 0: + return None + comment = text[index : end + 2] + result.extend( + "\n" if char == "\n" else "\r" if char == "\r" else " " for char in comment + ) + index = end + 2 + continue + result.append(character) + index += 1 + return "".join(result) + + +def _looks_like_typescript_declaration(path: str, data: bytes, *, complete_content: bool) -> bool: + """Recognize clearly inert TypeScript declaration content conservatively. + + Declaration suffixes alone are not trusted: a file named ``evil.d.cts`` + can still contain executable CommonJS. Unknown or non-text content stays + executable so this check cannot create a name-based security bypass. + """ + name = Path(path).name.lower() + if not complete_content or not name.endswith(_TYPESCRIPT_DECLARATION_SUFFIXES) or not data: + return False + try: + text = data.decode("utf-8") + except UnicodeDecodeError: + return False + uncommented = _strip_typescript_comments(text) + if not uncommented or not uncommented.strip(): + return False + try: + return ( + _TYPESCRIPT_DECLARATION_FILE.fullmatch( + uncommented, timeout=_TYPESCRIPT_DECLARATION_TIMEOUT_SECONDS + ) + is not None + ) + except TimeoutError: + # An unproven declaration must remain executable, including on exhaustion. + return False + def has_binary_executable_magic(data: bytes) -> bool: """Return whether canonical bytes begin with supported executable magic.""" return has_dex_magic(data) or data.startswith(_BINARY_EXECUTABLE_MAGICS) -def is_executable_content(path: str, data: bytes, mode: int = 0) -> bool: - """Classify filesystem and archive content with one static-only policy.""" +def is_executable_content( + path: str, data: bytes, mode: int = 0, *, complete_content: bool = False +) -> bool: + """Classify content; only complete declarations may receive the inert exemption.""" suffix = Path(path).suffix.lower() executable_magic = data.startswith(b"#!") or has_binary_executable_magic(data) - return suffix in _EXECUTABLE_SUFFIXES or executable_magic or bool(mode & 0o111) + declaration_only = _looks_like_typescript_declaration( + path, data, complete_content=complete_content + ) + return ( + (suffix in _EXECUTABLE_SUFFIXES and not declaration_only) + or executable_magic + or bool(mode & 0o111) + ) def _member_executable(info: zipfile.ZipInfo, safe_name: str, data: bytes) -> bool: - return is_executable_content(safe_name, data, info.external_attr >> 16) + return is_executable_content( + safe_name, + data, + info.external_attr >> 16, + complete_content=len(data) == info.file_size, + ) def _nested_path(outer_path: str, virtual_path: str) -> str: @@ -435,14 +561,22 @@ def _record_outer_metadata( } -def _virtual_type(path: str, data: bytes, nested_type: str | None) -> str: +def _virtual_type( + path: str, + data: bytes, + nested_type: str | None, + source_classification: PythonSourceClassification, +) -> str: if nested_type is not None: return nested_type + if source_classification is PythonSourceClassification.PYTHON: + return "python" suffix = Path(path).suffix.lower() return { ".md": "markdown", ".markdown": "markdown", ".py": "python", + ".pyw": "python", ".sh": "shell", ".bash": "shell", ".zsh": "shell", @@ -502,27 +636,26 @@ def _exception( limit_seconds: float | None = None, ) -> None: _mark_inventory_exception(result, path=path, reason=reason) - result.ledger_events.append( - ledger_event( - outcome=( - LedgerOutcome.PARTIAL - if reason in _PARTIAL_INVENTORY_REASONS - else LedgerOutcome.SKIPPED - ), - record_type=LedgerRecordType.SYSTEM, - phase="nested_artifact_inspection", - path=path, - reason=reason, - observed_bytes=observed_bytes, - limit_bytes=limit_bytes, - observed_artifacts=observed_artifacts, - limit_artifacts=limit_artifacts, - observed_depth=observed_depth, - limit_depth=limit_depth, - observed_seconds=observed_seconds, - limit_seconds=limit_seconds, - ) + event = ledger_event( + outcome=( + LedgerOutcome.PARTIAL if reason in _PARTIAL_INVENTORY_REASONS else LedgerOutcome.SKIPPED + ), + record_type=LedgerRecordType.SYSTEM, + phase="nested_artifact_inspection", + path=path, + reason=reason, + observed_bytes=observed_bytes, + limit_bytes=limit_bytes, + observed_artifacts=observed_artifacts, + limit_artifacts=limit_artifacts, + observed_depth=observed_depth, + limit_depth=limit_depth, + observed_seconds=observed_seconds, + limit_seconds=limit_seconds, ) + result.ledger_events.append(event) + if path == result._active_disk_path: + result.disk_container_events.append(event) def _time_exception( @@ -737,7 +870,9 @@ def _inspect_zip_bytes( if safe_name in seen_names or virtual_path in result.file_cache: _exception( result, - path=virtual_path, + # A different disk container may already own this path. + # Attribute that skipped copy to its current container. + path=virtual_path if safe_name in seen_names else container_virtual_path, reason=LedgerReason.ARCHIVE_AMBIGUOUS_MEMBER_PATH, ) continue @@ -994,10 +1129,17 @@ def _inspect_zip_bytes( executable = _member_executable(info, safe_name, member_data) member_hidden = _is_hidden_path(safe_name) concealed = executable and bool(concealment_reasons) - virtual_type = _virtual_type(safe_name, member_data, nested_type) + source_classification = classify_python_source(safe_name, member_data) + virtual_type = _virtual_type( + safe_name, + member_data, + nested_type, + source_classification, + ) result.components.append(virtual_path) result.file_cache[virtual_path] = member_data.decode("utf-8", errors="replace") result.raw_file_cache[virtual_path] = member_data + result.python_source_classifications[virtual_path] = source_classification artifact = classify_artifact(virtual_path, member_data) result.artifact_inventory.append(artifact) result.metadata.append( @@ -1105,6 +1247,7 @@ def inspect_nested_artifacts( last_checked_at=started_at, ) for path in dict.fromkeys(components): + result._active_disk_path = path if budget.halted: break if budget.expired(): @@ -1208,6 +1351,7 @@ def inspect_nested_artifacts( result=result, ) + result._active_disk_path = None result.components = list(dict.fromkeys(result.components)) result.uncompressed_bytes = budget.uncompressed_bytes _apply_inventory_overrides(result) diff --git a/src/skillspector/nodes/analyzers/artifact_integrity.py b/src/skillspector/nodes/analyzers/artifact_integrity.py index 6907e27bc..aaf823d09 100644 --- a/src/skillspector/nodes/analyzers/artifact_integrity.py +++ b/src/skillspector/nodes/analyzers/artifact_integrity.py @@ -21,6 +21,7 @@ _concealed_instruction_run_spans, _contextual_default_ignorable_boundary_spans, _obfuscated_instruction_matches, + binary_instruction_view, multiline_prompt_injection_view, prompt_injection_letter_spacing_view, ) @@ -1054,14 +1055,36 @@ def node(state: SkillspectorState) -> AnalyzerNodeResponse: confidence=1.0, ) ) - if artifact.get("content_kind") not in { + if artifact.get("readable_binary") or artifact.get("content_kind") not in { ContentKind.BINARY, ContentKind.OPAQUE, }: - format_density, mixed_script, first_nul_line, first_obfuscation_line = ( - _text_signals(content, budget) - ) - if artifact.get("contains_nul") and first_nul_line is not None: + signal_content = content + binary_view = None + if artifact.get("readable_binary"): + binary_view = binary_instruction_view(content)[0] + signal_content = binary_view.text + ( + format_density, + mixed_script, + first_nul_line, + first_obfuscation_line, + ) = _text_signals(signal_content, budget) + if first_obfuscation_line is not None and binary_view is not None: + projected_start = sum( + len(line) + for line in signal_content.splitlines(keepends=True)[ + : first_obfuscation_line - 1 + ] + ) + first_obfuscation_line = get_line_number( + content, binary_view.source_offset(projected_start) + ) + if ( + artifact.get("contains_nul") + and not artifact.get("readable_binary") + and first_nul_line is not None + ): budget.emit( _finding( "AE3", @@ -1072,7 +1095,10 @@ def node(state: SkillspectorState) -> AnalyzerNodeResponse: line=first_nul_line, ) ) - if format_density >= 0.01 or mixed_script: + # Compressed binary bytes can resemble mixed-script Unicode text. + if not artifact.get("readable_binary") and ( + format_density >= 0.01 or mixed_script + ): budget.emit( _finding( "AE4", diff --git a/src/skillspector/nodes/analyzers/behavioral_ast.py b/src/skillspector/nodes/analyzers/behavioral_ast.py index 72f2b0388..78e895221 100644 --- a/src/skillspector/nodes/analyzers/behavioral_ast.py +++ b/src/skillspector/nodes/analyzers/behavioral_ast.py @@ -32,7 +32,12 @@ ) from skillspector.logging_config import get_logger from skillspector.models import AnalyzerFinding, Finding, Location, Severity -from skillspector.python_ast import ParsedPythonFile, get_python_ast +from skillspector.python_ast import ( + ParsedPythonFile, + PythonSourceClassification, + get_python_ast, + resolve_python_source_classification, +) from skillspector.state import ( AnalyzerNodeResponse, SkillspectorState, @@ -437,6 +442,7 @@ def context_for(lineno: int, column: int) -> str: return context def _emit( + node_index: int, rule_id: str, ast_node: ast.Call, msg_override: str | None = None, @@ -446,8 +452,8 @@ def _emit( complete_match = python_ast.source_segment(ast_node) if complete_match is None: complete_match = get_complete_source_segment(lines, lineno, end_lineno) - start_byte_column = getattr(ast_node, "col_offset", 0) - end_byte_column = getattr(ast_node, "end_col_offset", start_byte_column) + start_byte_column = getattr(ast_node, "col_offset", None) + end_byte_column = getattr(ast_node, "end_col_offset", None) start_column = python_ast.character_column(lineno, start_byte_column) end_column = python_ast.character_column(end_lineno or lineno, end_byte_column) finding = AnalyzerFinding( @@ -466,13 +472,20 @@ def _emit( context=context_for(lineno, start_column if start_column is not None else 0), matched_text=complete_match[:200], complete_match=complete_match, + # Evidence participates in report compaction. Keep distinct syntax + # nodes distinguishable when their source spans are incomplete. + evidence=( + {"python_ast_node_index": node_index} + if start_column is None or end_column is None + else {} + ), ) if budget is None: findings.append(finding) else: budget.emit(finding) - for ast_node in ast.walk(tree): + for node_index, ast_node in enumerate(ast.walk(tree)): if budget is not None: budget.check_runtime() if isinstance(ast_node, ast.Subscript): @@ -482,6 +495,7 @@ def _emit( if isinstance(key, ast.Constant): if isinstance(key.value, str) and key.value in _DANGEROUS_GETATTR_NAMES: _emit( + node_index, "AST9", ast_node, f"Reflective dangerous access via {module}.__dict__ subscript " @@ -489,6 +503,7 @@ def _emit( ) else: _emit( + node_index, "AST7", ast_node, f"Dynamic attribute access via {module}.__dict__ subscript", @@ -513,6 +528,7 @@ def _emit( if isinstance(key, ast.Constant): if isinstance(key.value, str) and key.value in _DANGEROUS_GETATTR_NAMES: _emit( + node_index, "AST9", ast_node, f"Reflective dangerous access via {module}.__dict__.{method}() " @@ -520,6 +536,7 @@ def _emit( ) else: _emit( + node_index, "AST7", ast_node, f"Dynamic attribute access via {module}.__dict__.{method}()", @@ -542,8 +559,10 @@ def _emit( budget.check_runtime if budget is not None else None, ) if source: - _emit("AST8", ast_node, f"Dangerous chain: exec() wrapping {source}") - _emit("AST1", ast_node) + _emit( + node_index, "AST8", ast_node, f"Dangerous chain: exec() wrapping {source}" + ) + _emit(node_index, "AST1", ast_node) elif call_name == "eval": if _is_chain_sink(ast_node, aliases) and ast_node.args: @@ -553,27 +572,29 @@ def _emit( budget.check_runtime if budget is not None else None, ) if source: - _emit("AST8", ast_node, f"Dangerous chain: eval() wrapping {source}") - _emit("AST2", ast_node) + _emit( + node_index, "AST8", ast_node, f"Dangerous chain: eval() wrapping {source}" + ) + _emit(node_index, "AST2", ast_node) elif call_name == "__import__": - _emit("AST3", ast_node) + _emit(node_index, "AST3", ast_node) elif call_name == "compile": - _emit("AST6", ast_node) + _emit(node_index, "AST6", ast_node) elif call_name.startswith("subprocess."): attr = call_name.split(".", 1)[1] if attr in _SUBPROCESS_CALLS: - _emit("AST4", ast_node) + _emit(node_index, "AST4", ast_node) elif call_name.startswith("os."): attr = call_name.split(".", 1)[1] if attr in _OS_EXEC_CALLS: - _emit("AST5", ast_node) + _emit(node_index, "AST5", ast_node) elif (deser_msg := _deserialization_message(call_name, ast_node)) is not None: - _emit("AST10", ast_node, deser_msg) + _emit(node_index, "AST10", ast_node, deser_msg) elif call_name == "getattr" and len(ast_node.args) >= 2: second_arg = ast_node.args[1] @@ -584,9 +605,9 @@ def _emit( if isinstance(second_arg, ast.Constant) and not isinstance(second_arg.value, str): continue if resolved_name in _DANGEROUS_GETATTR_NAMES: - _emit("AST9", ast_node) + _emit(node_index, "AST9", ast_node) elif resolved_name is None or not isinstance(second_arg, ast.Constant): - _emit("AST7", ast_node) + _emit(node_index, "AST7", ast_node) return findings if budget is None else list(budget.current_findings) @@ -632,6 +653,14 @@ def node(state: SkillspectorState) -> AnalyzerNodeResponse: """Parse Python files via AST and detect dangerous execution patterns.""" components: list[str] = state.get("components") or [] file_cache: dict[str, str] = state.get("local_file_cache") or state.get("file_cache") or {} + raw_file_cache = state.get("raw_file_cache") + source_classifications = ( + state.get("python_source_classifications") + if "python_source_classifications" in state + else None + ) + source_classification_limitations = state.get("python_source_classification_limitations") or {} + source_decode_failures = state.get("python_source_decode_failures") or {} python_ast_cache_key = state.get("python_ast_cache_key") all_findings: list[Finding] = [] ledger_events: list[InspectionLedgerEvent] = [] @@ -639,7 +668,27 @@ def node(state: SkillspectorState) -> AnalyzerNodeResponse: terminal_limit: _BehavioralResourceLimitError | None = None for path in components: - if not path.endswith(".py"): + content = file_cache.get(path) + source_classification: PythonSourceClassification | None = None + if source_classifications is not None and path in source_classifications: + source_classification = resolve_python_source_classification( + path, + content, + source_classifications=source_classifications, + raw_file_cache=raw_file_cache, + ) + if source_classification is PythonSourceClassification.NON_PYTHON: + continue + if path in source_classification_limitations: + ledger_events.append( + ledger_event( + outcome=LedgerOutcome.PARTIAL, + phase="behavioral", + analyzer_id=ANALYZER_ID, + path=path, + reason=LedgerReason.RUNTIME_LIMIT, + ) + ) continue if terminal_limit is None and budget.analyzer_exhausted(): terminal_limit = _BehavioralResourceLimitError( @@ -649,11 +698,41 @@ def node(state: SkillspectorState) -> AnalyzerNodeResponse: "limit_findings": MAX_FINDINGS_PER_ANALYZER, }, ) + if terminal_limit is None: + try: + budget.check_runtime() + except _BehavioralResourceLimitError as exc: + terminal_limit = exc if terminal_limit is not None: event = _partial_limit_event(path, terminal_limit) ledger_events.append(event) continue - content = file_cache.get(path) + if path in source_decode_failures: + ledger_events.append( + ledger_event( + outcome=LedgerOutcome.PARTIAL, + phase="behavioral", + analyzer_id=ANALYZER_ID, + path=path, + reason=LedgerReason.PYTHON_SOURCE_DECODE_ERROR, + ) + ) + continue + if source_classification is None: + source_classification = resolve_python_source_classification( + path, + content, + source_classifications=source_classifications, + raw_file_cache=raw_file_cache, + ) + try: + budget.check_runtime() + except _BehavioralResourceLimitError as exc: + terminal_limit = exc + ledger_events.append(_partial_limit_event(path, exc)) + continue + if source_classification is PythonSourceClassification.NON_PYTHON: + continue if content is None: event = ledger_event( outcome=LedgerOutcome.FAILED, @@ -709,10 +788,19 @@ def node(state: SkillspectorState) -> AnalyzerNodeResponse: ) else: event = ledger_event( - outcome=LedgerOutcome.COMPLETED, + outcome=( + LedgerOutcome.PARTIAL + if source_classification is PythonSourceClassification.AMBIGUOUS + else LedgerOutcome.COMPLETED + ), phase="behavioral", analyzer_id=ANALYZER_ID, path=path, + reason=( + LedgerReason.PYTHON_SOURCE_AMBIGUOUS + if source_classification is PythonSourceClassification.AMBIGUOUS + else None + ), emitted_finding_ids=[finding.finding_id for finding in path_findings], ) ledger_events.append(event) diff --git a/src/skillspector/nodes/analyzers/behavioral_taint_tracking.py b/src/skillspector/nodes/analyzers/behavioral_taint_tracking.py index 349768875..4645618b8 100644 --- a/src/skillspector/nodes/analyzers/behavioral_taint_tracking.py +++ b/src/skillspector/nodes/analyzers/behavioral_taint_tracking.py @@ -13,18 +13,21 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Behavioral taint-tracking analyzer (TT1–TT5): sources -> sinks data-flow analysis. +"""Behavioral taint-tracking analyzer (TT1–TT6): sources -> sinks data-flow analysis. Parses Python AST to identify data sources (env vars, file reads, network input) and sinks (network output, exec, file writes), then tracks flows between them -to flag potential credential/data exfiltration chains. +to flag potential credential/data exfiltration chains. Taint follows Python's +lexical scopes and crosses direct calls to functions and methods defined in the +same file (arguments into parameters, return values back to callers). """ from __future__ import annotations import ast import time -from collections.abc import Callable +from collections import deque +from collections.abc import Callable, Iterable, Sequence from dataclasses import dataclass, field from typing import NamedTuple @@ -38,7 +41,12 @@ ) from skillspector.logging_config import get_logger from skillspector.models import AnalyzerFinding, Finding, Location, Severity -from skillspector.python_ast import ParsedPythonFile, get_python_ast +from skillspector.python_ast import ( + ParsedPythonFile, + PythonSourceClassification, + get_python_ast, + resolve_python_source_classification, +) from skillspector.state import ( AnalyzerNodeResponse, SkillspectorState, @@ -122,6 +130,7 @@ "httpx.patch", "httpx.get", "urllib.request.urlopen", + "urllib.request.build_opener.open", "socket.socket.send", "socket.socket.sendall", "socket.socket.sendto", @@ -297,6 +306,14 @@ def _resolve_sink_name( ``importlib.import_module('subprocess').run(...)`` resolves to ``'subprocess.run'`` and re-enters ``_EXEC_SINKS`` like the statically-imported form would. """ + if ( + isinstance(node.func, ast.Attribute) + and node.func.attr == "open" + and isinstance(node.func.value, ast.Call) + and resolve_call_name_typed(node.func.value, type_map, aliases) + == "urllib.request.build_opener" + ): + return "urllib.request.build_opener.open" name = resolve_call_name_typed(node, type_map, aliases) if name is None: name = resolve_dynamic_import_call(node, aliases) @@ -326,6 +343,13 @@ def _pick_rule(source_name: str, sink_name: str, is_direct: bool) -> str: class _TaintedVar(NamedTuple): + """One reported taint fact: data from ``source_call`` reaches ``name``. + + ``lineno`` is the line of the statement that last tainted ``name`` (the + assignment, or the call that bound a parameter), the wording earlier + releases used, so messages and message-based baselines keep matching. + """ + name: str source_call: str lineno: int @@ -343,28 +367,29 @@ def _is_open_for_write(node: ast.Call) -> bool: return False -def _find_source_in_expr( - node: ast.expr, - type_map: dict[str, str] | None = None, - aliases: dict[str, str] | None = None, - check_runtime: Callable[[], None] | None = None, +def _call_source( + node: ast.Call, + type_map: dict[str, str] | None, + aliases: dict[str, str] | None, ) -> str | None: - """Find a source call anywhere in an expression tree (handles chained calls). + """Return the source a call reads (``os.getenv``, ``open`` for reading, ...).""" + name = resolve_call_name_typed(node, type_map, aliases) + if name is None or name not in _ALL_SOURCES: + return None + if name == "open" and _is_open_for_write(node): + return None + return name - Handles patterns like ``open("f").read()``, ``requests.get(url).text``, - and plain ``os.environ.get("K")``. - """ - for child in ast.walk(node): - if check_runtime is not None: - check_runtime() - if not isinstance(child, ast.Call): - continue - name = resolve_call_name_typed(child, type_map, aliases) - if name is None or name not in _ALL_SOURCES: - continue - if name == "open" and _is_open_for_write(child): - continue - return name + +def _credential_subscript(node: ast.AST, aliases: dict[str, str]) -> str | None: + """``os.environ["K"]`` (also with ``os`` aliased) as a whole value is a source.""" + if not isinstance(node, ast.Subscript): + return None + base = resolve_dotted_name(node.value) + if base is not None: + base = apply_import_aliases(base, aliases) + if base and base in _CREDENTIAL_SOURCES: + return base return None @@ -389,66 +414,1707 @@ def _find_nested_sources( return results -def _find_tainted_names_in_args( - node: ast.Call, - tainted: dict[str, _TaintedVar], +# ── Lexical scopes and direct calls ───────────────────────────────────── +# +# Taint is keyed by lexical scope rather than by bare name, so a tainted local +# in one function cannot taint an unrelated same-named variable or parameter +# in another. Keys are small integers; a key is built from a scope id plus a +# reference to the identifier string already in the tree, never from a +# qualified name, so memory does not grow with identifier length. +# +# Interprocedural flow is deliberately narrow. Only calls whose callee is +# known from syntax alone, in O(1), bind arguments and read return values: +# +# * ``f(...)`` where ``f`` resolves lexically to functions (or a lambda +# assigned to the name) defined in this file; every definition bound to the +# name is one callee, so redefinitions cost nothing per call; +# * ``self.m(...)`` / ``cls.m(...)`` in a method, ``Class.m(...)``, +# ``super().m(...)`` and ``Class().m(...)`` / ``cls().m(...)``, through the +# class's own methods or those of its nearest in-file base (a bounded, +# precomputed lookup in C3 order); +# * ``Class(...)`` / ``cls(...)``, which bind to ``__init__`` and ``__new__``. +# +# Receivers of any other shape (``obj.m(...)``) are not resolved and bind +# nothing, exactly as before. ``self.x`` / ``cls.x`` attributes are per class: +# a read sees stores in the class and its in-file bases, never in siblings. + +_MODULE = 0 +_FUNCTION = 1 +_LAMBDA = 2 +_COMPREHENSION = 3 +_CLASS = 4 + +_FunctionNode = ast.FunctionDef | ast.AsyncFunctionDef +_COMPREHENSIONS = (ast.ListComp, ast.SetComp, ast.GeneratorExp, ast.DictComp) + +# Call-site positional arguments at or beyond this parameter index share one +# wildcard slot, and an ``*args`` unpacking fills a wildcard slot keyed by the +# parameter index it starts at (at most this one), so binding work per +# definition is bounded by a constant. +_MAX_POSITIONAL_SLOTS = 16 +# The in-file base lookup visits at most this many classes (the class itself +# included). Each class merges its bases' (already cut) orders once, so the +# work is linear in the bases the file lists. +_MAX_ANCESTORS = 8 + +# How a flow changes the fact it carries (see ``_mark_targets``). +_ASSIGN = 0 # cites the flow's own line +_PASS = 1 # keeps the line of the fact it forwards +_RETURN = 2 # into a function's return summary: never carries bound facts +_BIND = 3 # from a call-site argument into a callee's parameter slot + +# A key keeps one fact per lane; when the lanes agree (almost always) they +# share one fact, which propagates once: +# +# * lane 0, the most severe source for network output: credential (TT3) > +# file read (TT4) > external input (TT2), so a helper's network or file +# source never hides a credential; +# * lane 1, the most severe source for code execution and deserialization: +# external input (TT5/TT6) > file read (TT6) > credential (TT2), so a +# helper's credential never hides user or network input reaching ``exec`` +# or ``pickle.loads``; +# * lane 2, the first source to arrive, unranked, in the order earlier +# releases propagated (the source a ``main`` message cites). +# +# Lanes 0 and 1 decide the rule at a sink (file writes, TT2 for every +# source, read lane 0). The message cites the lane 2 fact whenever it gives +# that same rule, so a more severe source replaces the cited one only when it +# changes the rule (see ``_TaintGraph.cited``). +_SOURCE_NAMES: tuple[str, ...] = tuple(sorted(_ALL_SOURCES)) +_SOURCE_INDEX: dict[str, int] = {name: index for index, name in enumerate(_SOURCE_NAMES)} +_LANE_RANKS: tuple[tuple[int, int, int], ...] = tuple( + (2, 0, 0) + if name in _CREDENTIAL_SOURCES + else (0, 2, 0) + if name in _EXTERNAL_INPUT_SOURCES + else (1, 1, 0) + for name in _SOURCE_NAMES +) +_LANES = (0, 1, 2) +_FIRST_LANE = 2 +_ALL_LANES = 7 + +# A fact is (source index, line). +_Fact = tuple[int, int] + + +def _noop() -> None: + return None + + +class _Scope: + """One Python namespace: module, function, lambda, comprehension or class body.""" + + __slots__ = ( + "kind", + "parent", + "id", + "name", + "in_function", + "bound", + "declared_global", + "declared_nonlocal", + "children", + "names", + "definitions", + "function", + "class_def", + ) + + def __init__(self, kind: int, parent: _Scope | None, scope_id: int, name: str) -> None: + self.kind = kind + self.parent = parent + self.id = scope_id + self.name = name + # Whether facts bound from a call site stay bound in this scope's keys. + self.in_function = kind in (_FUNCTION, _LAMBDA) or ( + kind == _COMPREHENSION and parent is not None and parent.in_function + ) + self.bound: set[str] | None = None + self.declared_global: set[str] | None = None + self.declared_nonlocal: set[str] | None = None + self.children: list[_Scope] | None = None + # Name nodes, and def/class statements, whose names resolve in this scope. + self.names: list[ast.Name] | None = None + self.definitions: list[tuple[str, _Function | _ClassDef]] | None = None + self.function: _Function | None = None + self.class_def: _ClassDef | None = None + + def bind(self, name: str) -> None: + if self.bound is None: + self.bound = set() + self.bound.add(name) + + def request(self, node: ast.Name) -> None: + if self.names is None: + self.names = [] + self.names.append(node) + + def define(self, name: str, definition: _Function | _ClassDef) -> None: + self.bind(name) + if self.definitions is None: + self.definitions = [] + self.definitions.append((name, definition)) + + +class _Function: + """A ``def``/``async def`` or lambda, with the callee group it belongs to.""" + + __slots__ = ("node", "scope", "kind", "key", "group") + + def __init__(self, node: _FunctionNode | ast.Lambda, scope: _Scope) -> None: + self.node = node + self.scope = scope + self.kind = "function" # function | instance | class | static + self.key = -1 # key of the name the definition binds + self.group: _Group | None = None + + +class _ClassDef: + __slots__ = ("node", "scope", "key", "cls") + + def __init__(self, node: ast.ClassDef, scope: _Scope) -> None: + self.node = node + self.scope = scope + self.key = -1 + self.cls: _Class | None = None + + +class _Class: + """Every ``class`` statement bound to one name (redefinitions merge).""" + + __slots__ = ("id", "defs", "bases", "methods", "ancestors", "_body_names") + + def __init__(self, class_id: int) -> None: + self.id = class_id + self.defs: list[_ClassDef] = [] + self.bases: list[_Class] = [] + self.methods: dict[str, dict[str, _Group]] = {} + self.ancestors: list[_Class] | None = None + self._body_names: set[str] | None = None + + def body_names(self) -> set[str]: + """Names any of the class bodies binds (assignment, ``def``, import, ...).""" + if self._body_names is None: + names: set[str] = set() + for class_def in self.defs: + names.update(class_def.scope.bound or ()) + self._body_names = names + return self._body_names + + +class _Group: + """One callee: every function bound to a name, or every same-kind method + of a class with one name. Call sites bind into its argument slots, which + flow into each definition's parameters once. + + Each function belongs to exactly one group. A lambda bound to several + names (``a = b = lambda ...``) is one group that every such name shares, + so its parameters are linked once however many names it has. + """ + + __slots__ = ( + "id", + "kind", + "functions", + "merged", + "slots", + "named", + "kwarg_named", + ) + + def __init__(self, group_id: int, kind: str) -> None: + self.id = group_id + self.kind = kind + self.functions: list[_Function] = [] + # The group this one was merged into, if any (union-find parent). + self.merged: _Group | None = None + self.slots: dict[object, int] = {} + # Computed once, on the first call bound into the group (see + # ``signature``): names a keyword argument can bind to, and the names + # every ``**kwargs`` definition of the group binds by name instead. + self.named: set[str] | None = None + self.kwarg_named: set[str] | None = None + + def add(self, function: _Function) -> None: + self.functions.append(function) + function.group = self + + def root(self) -> _Group: + group = self + while group.merged is not None: + group = group.merged + node = self + while node.merged is not None and node.merged is not group: + following = node.merged + node.merged = group # path compression + node = following + return group + + def signature(self) -> tuple[set[str], set[str] | None]: + """(names a keyword can bind, names every ``**kwargs`` definition takes + by name or ``None`` when no definition has ``**kwargs``). + + A keyword naming a parameter binds only that parameter; it reaches a + ``**kwargs`` only when some definition with ``**kwargs`` lacks it + (positional-only names do reach ``**kwargs``). + """ + if self.named is None: + named: set[str] = set() + kwarg_named: set[str] | None = None + for function in self.functions: + args = function.node.args + names = {arg.arg for arg in args.args} + names.update(arg.arg for arg in args.kwonlyargs) + named |= names + if args.kwarg is not None: + kwarg_named = names if kwarg_named is None else kwarg_named & names + self.named = named + self.kwarg_named = kwarg_named + return self.named, self.kwarg_named + + +class _CallTargets(NamedTuple): + # (callee, receiver offset) pairs the call binds its arguments into. + members: tuple[tuple[_Group, int], ...] + # Callees whose return summary the call's value reads. + results: tuple[_Group, ...] + + +_NO_TARGETS = _CallTargets((), ()) + + +def _parameters(args: ast.arguments) -> list[ast.arg]: + """Every parameter a function binds, in signature order.""" + params = [*args.posonlyargs, *args.args] + if args.vararg is not None: + params.append(args.vararg) + params.extend(args.kwonlyargs) + if args.kwarg is not None: + params.append(args.kwarg) + return params + + +def _decorator_name(decorator: ast.expr) -> str | None: + if isinstance(decorator, ast.Call): + decorator = decorator.func + if isinstance(decorator, ast.Name): + return decorator.id + if isinstance(decorator, ast.Attribute): + return decorator.attr + return None + + +def _method_kind(node: _FunctionNode) -> str: + names = {_decorator_name(decorator) for decorator in node.decorator_list} + if "staticmethod" in names or node.name == "__new__": + return "static" + if "classmethod" in names or node.name in ("__init_subclass__", "__class_getitem__"): + return "class" + return "instance" + + +def _receiver_offset(kind: str, class_object: bool) -> int: + """Leading parameters a call binds implicitly (``self`` / ``cls``). + + ``self.m(x)`` binds ``x`` to an instance method's second parameter; + ``Class.m(obj, x)`` and ``cls.m(obj, x)`` pass ``obj`` explicitly. + """ + if kind == "class": + return 1 + if kind == "instance": + return 0 if class_object else 1 + return 0 + + +def _c3(cls: _Class, bases: list[_Class]) -> list[_Class]: + """C3 linearization of *cls* from its bases' linearizations, cut at ``_MAX_ANCESTORS``. + + Every base takes part in the merge; only the result is cut. Bases are + already linearized, and cut, so a hierarchy deeper than the cap can + differ from Python's order past the classes kept. A base still being + linearized (a cycle) stands for itself. An inconsistent hierarchy, which + Python rejects, continues with the leftmost remaining head. + """ + result = [cls] + if not bases: + return result + sequences = [ + [c for c in (base.ancestors if base.ancestors is not None else [base]) if c is not cls] + for base in bases + ] + if len(sequences) == 1: # single inheritance: the base's order + result.extend(sequences[0][: _MAX_ANCESTORS - 1]) + return result + sequences.append([base for base in bases if base is not cls]) + # How many sequences hold each class after their head. + tails: dict[int, int] = {} + for sequence in sequences: + for c in sequence[1:]: + tails[c.id] = tails.get(c.id, 0) + 1 + positions = [0] * len(sequences) + placed = {cls.id} + while len(result) < _MAX_ANCESTORS: + candidate: _Class | None = None + leftmost: _Class | None = None + for sequence, position in zip(sequences, positions, strict=True): + if position < len(sequence): + head = sequence[position] + if leftmost is None: + leftmost = head + if not tails.get(head.id): + candidate = head + break + candidate = candidate or leftmost + if candidate is None: + break + if candidate.id not in placed: + placed.add(candidate.id) + result.append(candidate) + for i, sequence in enumerate(sequences): + position = positions[i] + if position < len(sequence) and sequence[position] is candidate: + position += 1 + positions[i] = position + if position < len(sequence): + tails[sequence[position].id] -= 1 + return result + + +class _ScopeIndex: + """Lexical scopes of one module plus the in-file callees calls can bind to.""" + + def __init__(self) -> None: + self.scopes: list[_Scope] = [] + # Statements and expressions that create flows, with their scope, in + # ``ast.walk`` (breadth-first) order: Assign, Return and Call. + self.statements: list[tuple[ast.AST, _Scope]] = [] + self.functions: list[_Function] = [] + self.lambdas: dict[ast.Lambda, _Function] = {} + self.lambda_bindings: list[tuple[ast.Name, ast.Lambda]] = [] + self.class_defs: list[_ClassDef] = [] + self.classes: list[_Class] = [] + self.attributes: list[ast.Attribute] = [] + self.super_scopes: dict[ast.Call, _Scope] = {} + # Keys: an id per (scope id, name) or per synthetic descriptor. + self._ids: dict[tuple[object, ...], int] = {} + self.key_scope: list[_Scope | None] = [] + self.key_name: list[str] = [] + # 1 when a fact bound from a call site stays bound in the key (function + # locals); 0 for module, class and attribute state, where it becomes + # an ordinary fact. + self.local = bytearray() + self.name_key: dict[ast.Name, int] = {} + # A class-body read of a name the class also binds sees the class's + # value or, before the class binds it, the module global (Python's + # LOAD_NAME). It reads a view fed by both keys, so the class's own + # value (and ``self.x``) never takes the global's taint: + # (view, class key, module key), and the read nodes' views. + self.class_reads: dict[tuple[int, int, int], None] = {} + self.class_read_views: dict[ast.Name, int] = {} + # Keys of ``**kwargs`` parameters: forwarding one (``f(**kwargs)``) + # binds nothing in the callee, as on ``main``. + self.kwarg_params: set[int] = set() + self.class_of_key: dict[int, _Class] = {} + self.function_groups: dict[int, _Group] = {} + self.groups: list[_Group] = [] + # Key of each method's ``self``/``cls`` parameter -> (class, is ``cls``). + self.receivers: dict[int, tuple[_Class, bool]] = {} + # Attribute read keys (``self.x`` / ``cls.x``) by node and by (class, attr). + self.views: dict[ast.Attribute, int] = {} + self.view_list: list[tuple[_Class, str, int]] = [] + self._calls: dict[ast.Call, _CallTargets] = {} + self._lookups: dict[tuple[int, str, int], tuple[_Group, ...]] = {} + self._lambda_groups: dict[ast.Lambda, _Group] = {} + + @property + def module(self) -> _Scope: + return self.scopes[0] + + def new_scope(self, kind: int, parent: _Scope | None, name: str) -> _Scope: + scope = _Scope(kind, parent, len(self.scopes), name) + self.scopes.append(scope) + if parent is not None: + if parent.children is None: + parent.children = [] + parent.children.append(scope) + return scope + + # ── keys ── + + def _new_key(self, scope: _Scope | None, name: str, local: bool) -> int: + key = len(self.key_scope) + self.key_scope.append(scope) + self.key_name.append(name) + self.local.append(1 if local else 0) + return key + + def key(self, scope: _Scope, name: str) -> int: + """Key of *name* bound in *scope* itself.""" + ident = (scope.id, name) + key = self._ids.get(ident) + if key is None: + key = self._new_key(scope, name, scope.in_function) + self._ids[ident] = key + return key + + def synthetic(self, ident: tuple[object, ...], local: bool, name: str = "") -> int: + key = self._ids.get(ident) + if key is None: + key = self._new_key(None, name, local) + self._ids[ident] = key + return key + + def visible(self, key: int, root: _Scope) -> bool: + """Whether a read evaluated from *root* carries *key*'s data. + + A lambda's parameters and a comprehension's targets live only inside + it: an expression that merely contains the lambda or comprehension + does not hold their values (a function object does not carry its + arguments), so they are skipped unless the walk starts in that scope. + """ + owner = self.key_scope[key] + return owner is None or owner is root or owner.kind not in (_LAMBDA, _COMPREHENSION) + + # ── callees ── + + def function_group(self, key: int) -> _Group: + group = self.function_groups.get(key) + if group is None: + group = self._new_group("function") + self.function_groups[key] = group + return group + + def _new_group(self, kind: str) -> _Group: + group = _Group(len(self.groups), kind) + self.groups.append(group) + return group + + def method_group(self, cls: _Class, name: str, kind: str) -> _Group: + kinds = cls.methods.setdefault(name, {}) + group = kinds.get(kind) + if group is None: + group = self._new_group(kind) + kinds[kind] = group + return group + + def lambda_group(self, node: ast.Lambda, function: _Function) -> _Group: + """The one callee of a lambda, shared by every name it is bound to.""" + group = self._lambda_groups.get(node) + if group is None: + group = self._new_group("function") + group.add(function) + self._lambda_groups[node] = group + return group.root() + + def bind_name(self, key: int, group: _Group) -> None: + """Make *group* (a lambda's) a callee of the name *key*. + + The first binding shares the group; a name with another definition + merges the two groups (redefinitions are one callee), so a lambda + bound to many names is never copied per name. + """ + current = self.function_groups.get(key) + if current is None: + self.function_groups[key] = group + return + current, group = current.root(), group.root() + if current is group: + return + if len(current.functions) < len(group.functions): + current, group = group, current + group.merged = current + for function in group.functions: + function.group = current + current.functions.extend(group.functions) + group.functions = [] + + def finish_groups(self) -> None: + for key, group in self.function_groups.items(): + if group.merged is not None: + self.function_groups[key] = group.root() + + def ancestors(self, cls: _Class, tick: Callable[[], None]) -> list[_Class]: + """*cls* then its in-file bases in C3 (method resolution) order, bounded. + + Each class merges the (cut) linearizations of all its bases and keeps + the first ``_MAX_ANCESTORS`` classes, so the work per class is linear + in the bases it lists. Classes are linearized bases + first, iteratively, so a long inheritance chain cannot exhaust the + stack; a cycle (possible when redefinitions merge) is cut. + """ + if cls.ancestors is not None: + return cls.ancestors + visiting: set[int] = set() + stack = [cls] + while stack: + tick() + current = stack[-1] + if current.ancestors is not None: + stack.pop() + continue + bases = current.bases + if current.id not in visiting: + visiting.add(current.id) + pending = [ + base for base in bases if base.ancestors is None and base.id not in visiting + ] + if pending: + stack.extend(reversed(pending)) + continue + stack.pop() + current.ancestors = _c3(current, bases) + return cls.ancestors + + def lookup( + self, cls: _Class, name: str, skip: int, tick: Callable[[], None] + ) -> tuple[_Group, ...]: + """Method groups named *name* on the nearest class in *cls*'s lookup chain. + + *skip* is 1 for ``super()``, which starts after the class itself. + """ + memo = (cls.id, name, skip) + found = self._lookups.get(memo) + if found is None: + found = () + for candidate in self.ancestors(cls, tick)[skip:]: + kinds = candidate.methods.get(name) + if kinds: + found = tuple(kinds.values()) + break + self._lookups[memo] = found + return found + + def super_class(self, call: ast.Call) -> _Class | None: + """The class whose bases ``super()`` / ``super(C, obj)`` starts from.""" + func = call.func + if not isinstance(func, ast.Name) or func.id != "super": + return None + key = self.name_key.get(func) + if key is None or self.key_scope[key] is not self.module: + return None + if self.module.bound is not None and "super" in self.module.bound: + return None # shadowed builtin + if call.args: + first = call.args[0] + if isinstance(first, ast.Name): + return self.class_of_key.get(self.name_key.get(first, -1)) + return None + scope = self.super_scopes.get(call) + while scope is not None and scope.kind in (_LAMBDA, _COMPREHENSION): + scope = scope.parent + if scope is None or scope.kind != _FUNCTION: + return None + parent = scope.parent + if parent is None or parent.class_def is None: + return None + return parent.class_def.cls + + def constructed_class(self, key: int) -> _Class | None: + """The class ``Name(...)`` constructs: an in-file class name, or ``cls``.""" + receiver = self.receivers.get(key) + if receiver is not None: + return receiver[0] if receiver[1] else None # cls(...) in a classmethod + return self.class_of_key.get(key) + + def call_targets(self, call: ast.Call, tick: Callable[[], None]) -> _CallTargets: + """The in-file callees a call reaches, resolved from syntax alone.""" + cached = self._calls.get(call) + if cached is not None: + return cached + members: list[tuple[_Group, int]] = [] + results: list[_Group] = [] + func = call.func + if isinstance(func, ast.Name): + key = self.name_key.get(func, -1) + group = self.function_groups.get(key) + if group is not None: + members.append((group, 0)) + results.append(group) + constructed = self.constructed_class(key) + if constructed is not None: + for group in self.lookup(constructed, "__init__", 0, tick): + members.append((group, 0 if group.kind == "static" else 1)) + # ``C(x)`` also calls ``C.__new__(C, x)``: ``cls`` is implicit. + for group in self.lookup(constructed, "__new__", 0, tick): + members.append((group, 1)) + elif isinstance(func, ast.Lambda): + function = self.lambdas.get(func) + if function is not None: + members.append((self.lambda_group(func, function), 0)) + elif isinstance(func, ast.Attribute): + value = func.value + cls: _Class | None = None + class_object = False + skip = 0 + if isinstance(value, ast.Name): + key = self.name_key.get(value, -1) + receiver = self.receivers.get(key) + if receiver is not None: + cls, class_object = receiver + else: + cls = self.class_of_key.get(key) + class_object = True + elif isinstance(value, ast.Call): + cls = self.super_class(value) + if cls is not None: + skip = 1 + elif isinstance(value.func, ast.Name): + # ``C().m(...)`` / ``cls().m(...)``: an instance of an in-file class. + cls = self.constructed_class(self.name_key.get(value.func, -1)) + if cls is not None: + for group in self.lookup(cls, func.attr, skip, tick): + members.append((group, _receiver_offset(group.kind, class_object))) + results.append(group) + found = _CallTargets(tuple(members), tuple(results)) if members else _NO_TARGETS + self._calls[call] = found + return found + + # ── attributes ── + + def receiver_class(self, node: ast.Attribute) -> _Class | None: + """The class of ``self`` / ``cls`` in ``self.x`` / ``cls.x``, if it is one.""" + if not isinstance(node.value, ast.Name): + return None + receiver = self.receivers.get(self.name_key.get(node.value, -1)) + return receiver[0] if receiver is not None else None + + def attribute_store(self, node: ast.Attribute) -> int | None: + cls = self.receiver_class(node) + if cls is None: + return None + return self.synthetic(("store", cls.id, node.attr), False) + + def class_body_store(self, cls: _Class, name: str) -> int: + """Key of a name a class body assigns, read as ``self.name`` / ``cls.name``. + + Kept apart from ``self.x`` stores: a subclass that rebinds the name in + its own body shadows a base's class-body value. + """ + return self.synthetic(("class", cls.id, name), False) + + def attribute_view(self, node: ast.Attribute) -> int | None: + """Key read by ``self.x`` / ``cls.x``: stores in the class and its bases.""" + key = self.views.get(node) + if key is not None: + return key + if not isinstance(node.ctx, ast.Load): + return None + cls = self.receiver_class(node) + if cls is None: + return None + ident = ("view", cls.id, node.attr) + key = self._ids.get(ident) + if key is None: + key = self.synthetic(ident, False) + self.view_list.append((cls, node.attr, key)) + self.views[node] = key + return key + + # ── debugging / tests ── + + def qualname(self, scope: _Scope) -> str: + parts: list[str] = [] + current: _Scope | None = scope + while current is not None and current.kind != _MODULE: + parent = current.parent + if parent is not None and parent.kind in (_FUNCTION, _LAMBDA, _COMPREHENSION): + parts.append(f".{current.name}") + else: + parts.append(current.name) + current = parent + return ".".join(reversed(parts)) + + +def _build_scope_index( + tree: ast.AST, check_runtime: Callable[[], None] | None = None, -) -> list[_TaintedVar]: - """Find references to tainted variables in a call's arguments and keywords.""" - seen: set[str] = set() - hits: list[_TaintedVar] = [] - for child in ast.walk(node): - if check_runtime is not None: - check_runtime() - if child is node: +) -> _ScopeIndex: + """Assign every name to its lexical scope and index in-file callees. + + Three linear passes: a breadth-first visit (the same order as + ``ast.walk``) that records scopes, bindings and flow statements; a + depth-first pass over the scope tree that resolves every name with one + stack per identifier, O(1) per name however deep the nesting; and an + indexing pass over class and function definitions. + """ + tick = check_runtime or _noop + index = _ScopeIndex() + module = index.new_scope(_MODULE, None, "") + # Nodes evaluated in a different scope than their parent (bodies of + # functions, classes and lambdas; comprehension elements). + switch: dict[ast.AST, _Scope] = {} + # Walrus targets bind in the nearest enclosing non-comprehension scope. + walrus: dict[ast.Name, _Scope] = {} + annotation_only: set[ast.Name] = set() + + queue: deque[tuple[ast.AST, _Scope]] = deque([(tree, module)]) + while queue: + tick() + node, scope = queue.popleft() + if isinstance(node, ast.Name): + if isinstance(node.ctx, ast.Load): + scope.request(node) + elif node in annotation_only: + pass # ``x: T`` in a class body binds nothing + else: + binding = walrus.pop(node, scope) + binding.bind(node.id) + binding.request(node) continue - var_name: str | None = None - if isinstance(child, ast.Name): - var_name = child.id - elif isinstance(child, ast.Subscript): - var_name = resolve_dotted_name(child.value) - if var_name and var_name not in seen: - tv = tainted.get(var_name) - if tv: - seen.add(var_name) - hits.append(tv) - return hits + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + inner = index.new_scope(_FUNCTION, scope, node.name) + function = _Function(node, inner) + inner.function = function + index.functions.append(function) + scope.define(node.name, function) + for arg in _parameters(node.args): + inner.bind(arg.arg) + # Only the body runs in the new scope; decorators, defaults and + # annotations are evaluated where the def stands. + for statement in node.body: + switch[statement] = inner + elif isinstance(node, ast.ClassDef): + inner = index.new_scope(_CLASS, scope, node.name) + class_def = _ClassDef(node, inner) + inner.class_def = class_def + index.class_defs.append(class_def) + scope.define(node.name, class_def) + for statement in node.body: + switch[statement] = inner + elif isinstance(node, ast.Lambda): + inner = index.new_scope(_LAMBDA, scope, "") + function = _Function(node, inner) + inner.function = function + index.functions.append(function) + index.lambdas[node] = function + for arg in _parameters(node.args): + inner.bind(arg.arg) + switch[node.body] = inner + elif isinstance(node, _COMPREHENSIONS): + # Python 3 comprehensions have their own scope; only the first + # iterable is evaluated in the enclosing one. + inner = index.new_scope(_COMPREHENSION, scope, f"<{type(node).__name__.lower()}>") + if isinstance(node, ast.DictComp): + switch[node.key] = inner + switch[node.value] = inner + else: + switch[node.elt] = inner + for position, generator in enumerate(node.generators): + switch[generator.target] = inner + for condition in generator.ifs: + switch[condition] = inner + if position: + switch[generator.iter] = inner + elif isinstance(node, ast.NamedExpr): + binding = scope + while binding.kind == _COMPREHENSION and binding.parent is not None: + binding = binding.parent + if isinstance(node.target, ast.Name): + walrus[node.target] = binding + elif isinstance(node, ast.Global): + if scope.declared_global is None: + scope.declared_global = set() + scope.declared_global.update(node.names) + elif isinstance(node, ast.Nonlocal): + if scope.declared_nonlocal is None: + scope.declared_nonlocal = set() + scope.declared_nonlocal.update(node.names) + elif isinstance(node, (ast.Import, ast.ImportFrom)): + for alias in node.names: + if alias.name != "*": + scope.bind(alias.asname or alias.name.split(".")[0]) + elif isinstance(node, ast.ExceptHandler): + if node.name: + scope.bind(node.name) + elif isinstance(node, (ast.MatchAs, ast.MatchStar)): + if node.name: + scope.bind(node.name) + elif isinstance(node, ast.MatchMapping): + if node.rest: + scope.bind(node.rest) + elif isinstance(node, ast.AnnAssign): + target = node.target + if node.value is None and scope.kind == _CLASS and isinstance(target, ast.Name): + annotation_only.add(target) + elif isinstance(node, ast.Assign): + index.statements.append((node, scope)) + if isinstance(node.value, ast.Lambda): + for target in node.targets: + if isinstance(target, ast.Name): + index.lambda_bindings.append((target, node.value)) + elif isinstance(node, ast.Return): + index.statements.append((node, scope)) + elif isinstance(node, ast.Call): + index.statements.append((node, scope)) + if isinstance(node.func, ast.Name) and node.func.id == "super": + index.super_scopes[node] = scope + elif isinstance(node, ast.Attribute): + if isinstance(node.value, ast.Name): + index.attributes.append(node) + for child in ast.iter_child_nodes(node): + if not isinstance(child, ast.expr_context): + queue.append((child, switch.pop(child, scope))) + + _resolve_names(index, tick) + _index_definitions(index, tick) + return index + + +def _resolve_names(index: _ScopeIndex, tick: Callable[[], None]) -> None: + """Resolve every recorded name by Python's rules, O(1) per name. + + A depth-first walk of the scope tree keeps, per identifier, a stack of + the function-like scopes that bind it (``global`` pushes the module). + A name read in a function, lambda or comprehension is local if bound + there unless declared ``global``/``nonlocal``, and otherwise resolves to + the nearest enclosing binder, then the module. Class bodies see their own + names but are invisible to the functions and comprehensions inside them. + A class-body read of a name the class binds sees the class's value or, + before the class binds it, the module global (CPython's LOAD_NAME; an + enclosing function's local is not consulted), so it reads both. + """ + module = index.module + stacks: dict[str, list[_Scope]] = {} + + def binding_scope(scope: _Scope, name: str) -> _Scope: + if scope.declared_global is not None and name in scope.declared_global: + return module + if scope.kind == _MODULE: + return module + if scope.kind == _CLASS: + nonlocal_names = scope.declared_nonlocal + if ( + scope.bound is not None + and name in scope.bound + and (nonlocal_names is None or name not in nonlocal_names) + ): + return scope + stack = stacks.get(name) + return stack[-1] if stack else module + + work: list[tuple[_Scope, list[str] | None]] = [(module, None)] + while work: + scope, pushed = work.pop() + if pushed is not None: + for name in pushed: + stacks[name].pop() + continue + tick() + names: list[str] = [] + if scope.kind not in (_MODULE, _CLASS): + declared_global = scope.declared_global or set() + declared_nonlocal = scope.declared_nonlocal or set() + for name in declared_global: + stacks.setdefault(name, []).append(module) + names.append(name) + for name in scope.bound or (): + if name not in declared_global and name not in declared_nonlocal: + stacks.setdefault(name, []).append(scope) + names.append(name) + for node in scope.names or (): + tick() + binder = binding_scope(scope, node.id) + key = index.key(binder, node.id) + index.name_key[node] = key + if binder is scope and scope.kind == _CLASS and isinstance(node.ctx, ast.Load): + view = index.synthetic(("class read", scope.id, node.id), False, node.id) + index.class_read_views[node] = view + index.class_reads[(view, key, index.key(module, node.id))] = None + for name, definition in scope.definitions or (): + tick() + definition.key = index.key(binding_scope(scope, name), name) + work.append((scope, names)) + if scope.children: + work.extend((child, None) for child in reversed(scope.children)) + + +def _index_definitions(index: _ScopeIndex, tick: Callable[[], None]) -> None: + """Merge class redefinitions, link in-file bases, and group callees.""" + for class_def in index.class_defs: + tick() + cls = index.class_of_key.get(class_def.key) + if cls is None: + cls = _Class(len(index.classes)) + index.classes.append(cls) + index.class_of_key[class_def.key] = cls + cls.defs.append(class_def) + class_def.cls = cls + for cls in index.classes: + linked = {cls.id} + for class_def in cls.defs: + for base in class_def.node.bases: + tick() + if isinstance(base, ast.Subscript): # Base[T] + base = base.value + if not isinstance(base, ast.Name): + continue + parent = index.class_of_key.get(index.name_key.get(base, -1)) + if parent is not None and parent.id not in linked: + linked.add(parent.id) + cls.bases.append(parent) + for function in index.functions: + tick() + node = function.node + if node.args.kwarg is not None: + index.kwarg_params.add(index.key(function.scope, node.args.kwarg.arg)) + if isinstance(node, ast.Lambda): + continue + parent = function.scope.parent + if parent is None or parent.class_def is None or parent.class_def.cls is None: + index.function_group(function.key).add(function) + continue + # A method is reached through its class, never by its bare name. + cls = parent.class_def.cls + kind = _method_kind(node) + function.kind = kind + index.method_group(cls, node.name, kind).add(function) + positional = [*node.args.posonlyargs, *node.args.args] + if kind != "static" and positional: + receiver_key = index.key(function.scope, positional[0].arg) + index.receivers[receiver_key] = (cls, kind == "class") + for target, lambda_node in index.lambda_bindings: + tick() + function = index.lambdas.get(lambda_node) + key = index.name_key.get(target) + if function is not None and key is not None: + index.bind_name(key, index.lambda_group(lambda_node, function)) + index.finish_groups() def _mark_targets( - targets: list[ast.expr], - tainted: dict[str, _TaintedVar], - src_name: str, - lineno: int, -) -> None: - for target in targets: - if isinstance(target, ast.Name): - tainted[target.id] = _TaintedVar(target.id, src_name, lineno) - elif isinstance(target, ast.Tuple): - for elt in target.elts: - if isinstance(elt, ast.Name): - tainted[elt.id] = _TaintedVar(elt.id, src_name, lineno) - - -def _find_tainted_in_expr( - node: ast.expr, - tainted: dict[str, _TaintedVar], + graph: _TaintGraph, + targets: Sequence[int], + kind: int, + line: int, + fact: _Fact, + bound: int, + lanes: int = _ALL_LANES, +) -> list[tuple[int, int, int]]: + """Give each target *fact*, in each of *lanes*, where it outranks what it holds. + + Monotone: a target's fact in a lane is only ever replaced by a strictly + more severe source for that lane (never in lane 2, where every source + ranks the same and the first to arrive stays), so each key changes at + most three times per lane and slot, and the worklist terminates. A key + has two slots: ordinary facts, and facts bound from a call site into a + function's parameters (``bound``). A bound fact stays bound in function + locals; module, class and attribute state turns it into an ordinary + fact. Returns the (key, slot, changed lanes) triples to propagate from. + """ + source, fact_line = fact + if kind != _PASS: + fact_line = line + if kind == _BIND: + bound = 1 + new_fact = (source, fact_line) + ranks = _LANE_RANKS[source] + local = graph.index.local + facts = graph.facts + ordinary = facts[0] + tick = graph.tick + changed: list[tuple[int, int, int]] = [] + for position, target in enumerate(targets): + if position and not position & 1023: + tick() + slot = 1 if bound and local[target] else 0 + held = facts[slot] + mask = 0 + for lane in _LANES: + if not lanes >> lane & 1: + continue + rank = ranks[lane] + current = ordinary[lane].get(target) + if current is not None and _LANE_RANKS[current[0]][lane] >= rank: + continue + if slot: + current = held[lane].get(target) + if current is not None and _LANE_RANKS[current[0]][lane] >= rank: + continue + held[lane][target] = new_fact + mask |= 1 << lane + if mask: + changed.append((target, slot, mask)) + return changed + + +class _TaintGraph: + """Scope-aware taint, independent of AST visit order. + + Taint is reachability in a flow graph over keys, computed with a monotone + worklist that fires each flow at most once per (lane, source rank, slot): + at most 14 times whatever the number of sources (three ranks in lanes 0 + and 1, one in lane 2, two slots each). + + * Phase A replays exactly the assignment flows earlier releases used + (``x = `` reads every name in ```` outside source calls), + in the same order, plus a class body's read of the module global under + a name the class also binds (earlier releases had one name for both). + In lane 2, which keeps the first source to arrive, a key that earlier + releases tainted the same way holds the source and line they cite. + Their fact differs only where it came from a same-named variable in + another scope, a name clash this scoping removes. + * Phase B adds what scoping makes precise: arguments of direct in-file + calls bind into parameters (as bound facts), parameter defaults, + return summaries read by callers, and ``self.x`` / ``cls.x`` attributes. + Phase B never replaces a fact by one of equal rank, so it only fills + keys phase A left clean or upgrades lane 0 or 1 to a more severe + source; lane 2 is never replaced. + + At a sink, the sink's ranked lane decides the rule, and the message cites + the lane 2 fact whenever that gives the same rule (``cited``). Variables + phase A tainted are cited before those only phase B taints. A more + severe source therefore changes a finding's message only where it + changes the rule (and, at that sink, a weaker finding may be subsumed or + cite another variable). + + A callee's return summary carries only facts that do not depend on its + parameters: bound facts never enter it. What a call returns from its + own arguments is read at that call site, from those arguments (every + call's value includes its arguments, as before), so one caller's secret + cannot taint another caller's result, also through closures and lambdas. + """ + + def __init__( + self, + index: _ScopeIndex, + type_map: dict[str, str], + aliases: dict[str, str], + check_runtime: Callable[[], None] | None = None, + ) -> None: + self.index = index + self.type_map = type_map + self.aliases = aliases + self.tick = check_runtime or _noop + # facts[slot][lane]: key -> fact; slot 1 holds facts bound from a call site. + self.facts: tuple[tuple[dict[int, _Fact], ...], ...] = ( + ({}, {}, {}), + ({}, {}, {}), + ) + # Each flow's (targets, kind, line), keyed by its index (a stable id). + # ``prop_a`` / ``prop_b`` map a key read by a phase A / B flow to its ids. + self.flows: list[tuple[tuple[int, ...], int, int]] = [] + self.prop_a: dict[int, list[int]] = {} + self.prop_b: dict[int, list[int]] = {} + self.worklist: deque[tuple[int, int, int]] = deque() + self.seeds: list[tuple[tuple[int, ...], int, int, _Fact, int]] = [] + # Per flow, one bit per (lane, source rank, slot) it has already propagated. + self.fired: list[int] = [] + self.returns: dict[int, int] = {} # group id -> return summary key + # Keys phase A tainted: sinks cite these first (``main``'s flows). + self.phase_a_keys: frozenset[int] = frozenset() + self._values: dict[ast.Call, int] = {} + self._class_links: set[tuple[int, str]] = set() + + # ── building ── + + def add_flow( + self, reads: Iterable[int], targets: Sequence[int], kind: int, line: int, phase_a: bool + ) -> None: + if not targets: + return + keys = list(dict.fromkeys(reads)) + if not keys: + return + flow_id = len(self.flows) + self.flows.append((tuple(targets), kind, line)) + table = self.prop_a if phase_a else self.prop_b + for key in keys: + table.setdefault(key, []).append(flow_id) + + def seed( + self, targets: Sequence[int], kind: int, line: int, sources: list[tuple[str, int]] + ) -> None: + """Queue phase B sources for *targets* (applied after phase A).""" + if targets: + for source, lanes in sources: + fact = (_SOURCE_INDEX[source], line) + self.seeds.append((tuple(targets), kind, line, fact, lanes)) + + def ret_key(self, group: _Group) -> int: + key = self.returns.get(group.id) + if key is None: + key = self.index.synthetic(("return", group.id), False) + self.returns[group.id] = key + return key + + def value_key(self, call: ast.Call, root: _Scope) -> int | None: + """Key of an in-file call's value, for an enclosing call's arguments. + + Reading a nested in-file call through its own value key walks each + argument once, however deeply such calls nest. + """ + if self.index.call_targets(call, self.tick) is _NO_TARGETS: + return None + key = self._values.get(call) + if key is None: + key = self.index.synthetic(("value", id(call)), root.in_function) + self._values[call] = key + return key + + def scan( + self, node: ast.AST, root: _Scope, nested_values: bool, opaque_lambdas: bool = False + ) -> tuple[list[int], list[int], list[tuple[str, int]]]: + """Walk an expression once: (names read, other keys read, sources). + + Names are the reads earlier releases used. Other keys are the return + summaries of in-file calls and ``self.x`` attributes. With + *nested_values*, an in-file call is read through its value key and not + walked. With *opaque_lambdas* a lambda is not walked: a default holds + the function object, and each lambda's own defaults are scanned once, + for that lambda; only the body of an immediately called lambda, whose + value the default holds, is walked. A source call is not walked + either: its value is what it reads, not its arguments. Sources, each + with the lanes it is for: in lanes 0 and 1 the most severe source + call (the first in walk order among equals), in lane 2 the first + source call in walk order (the one earlier releases took); or an + ``os.environ[...]`` subscript as the whole expression. + """ + index = self.index + name_key = index.name_key + class_reads = index.class_read_views + tick = self.tick + names: dict[int, None] = {} + others: dict[int, None] = {} + best: list[str | None] = [None, None, None] + best_rank = [-1, -1, -1] + queue: deque[ast.AST] = deque([node]) + while queue: + tick() + child = queue.popleft() + if isinstance(child, ast.Name): + key = class_reads.get(child) + if key is None: + key = name_key.get(child) + if key is not None and index.visible(key, root): + names[key] = None + continue + if isinstance(child, ast.Call): + if nested_values: + value = self.value_key(child, root) + if value is not None: + others[value] = None + continue + source = _call_source(child, self.type_map, self.aliases) + if source is not None: + ranks = _LANE_RANKS[_SOURCE_INDEX[source]] + for lane in _LANES: + if ranks[lane] > best_rank[lane]: + best[lane], best_rank[lane] = source, ranks[lane] + # The call's value is what it reads (a file, a response, + # an environment value), not its own arguments. + continue + for group in index.call_targets(child, tick).results: + others[self.ret_key(group)] = None + if opaque_lambdas and isinstance(child.func, ast.Lambda): + # An immediately called lambda yields its body's value. + queue.append(child.func.body) + elif isinstance(child, ast.Attribute): + view = index.attribute_view(child) + if view is not None: + others[view] = None + elif opaque_lambdas and isinstance(child, ast.Lambda): + continue + for grandchild in ast.iter_child_nodes(child): + if not isinstance(grandchild, ast.expr_context): + queue.append(grandchild) + if best[0] is None: + best = [_credential_subscript(node, self.aliases)] * len(_LANES) + lanes_of: dict[str, int] = {} + for lane, source in enumerate(best): + if source is not None: + lanes_of[source] = lanes_of.get(source, 0) | 1 << lane + return list(names), list(others), list(lanes_of.items()) + + def assign(self, node: ast.Assign, scope: _Scope) -> None: + index = self.index + names, others, sources = self.scan(node.value, scope, False) + # Phase A targets are the ones earlier releases tainted: names and + # tuples of names. ``self.x`` / ``cls.x`` targets are phase B. + plain: list[int] = [] + attributes: list[int] = [] + for target in node.targets: + elements = target.elts if isinstance(target, ast.Tuple) else [target] + for element in elements: + if isinstance(element, ast.Name): + plain.append(index.name_key[element]) + if scope.kind == _CLASS: + self.class_attribute(scope, element.id, index.name_key[element]) + elif isinstance(element, ast.Attribute): + store = index.attribute_store(element) + if store is not None: + attributes.append(store) + line = node.lineno + if sources: + # A direct source seeds its targets in phase A, as before; a more + # severe source among the names it reads may still upgrade them. + for source, lanes in sources: + fact = (_SOURCE_INDEX[source], line) + self.worklist.extend(_mark_targets(self, plain, _ASSIGN, line, fact, 0, lanes)) + self.seed(attributes, _ASSIGN, line, sources) + self.add_flow([*names, *others], [*plain, *attributes], _ASSIGN, line, False) + else: + self.add_flow(names, plain, _ASSIGN, line, True) + self.add_flow(others, plain, _ASSIGN, line, False) + self.add_flow([*names, *others], attributes, _ASSIGN, line, False) + + def class_attribute(self, scope: _Scope, name: str, key: int) -> None: + """A name assigned in a class body is also the class's attribute.""" + class_def = scope.class_def + if class_def is None or class_def.cls is None: + return + link = (scope.id, name) + if link in self._class_links: + return + self._class_links.add(link) + store = self.index.class_body_store(class_def.cls, name) + self.add_flow((key,), (store,), _PASS, 0, False) + + def returned(self, value: ast.expr, scope: _Scope, line: int) -> None: + function = scope.function + if function is None or function.group is None: + return + targets = (self.ret_key(function.group),) + names, others, sources = self.scan(value, scope, False) + self.seed(targets, _RETURN, line, sources) + self.add_flow([*names, *others], targets, _RETURN, line, False) + + def call(self, node: ast.Call, scope: _Scope) -> None: + targets = self.index.call_targets(node, self.tick) + value = self._values.get(node) + if not targets.members and value is None: + return + parts = [arg.value if isinstance(arg, ast.Starred) else arg for arg in node.args] + parts.extend(keyword.value for keyword in node.keywords) + scans = [self.scan(part, scope, True) for part in parts] + line = node.lineno + if value is not None: + reads: list[int] = [] + for names, others, sources in scans: + reads.extend(names) + reads.extend(others) + self.seed((value,), _ASSIGN, line, sources) + reads.extend(self.ret_key(group) for group in targets.results) + self.add_flow(reads, (value,), _ASSIGN, line, False) + if targets.members: + self.bind_arguments(node, targets.members, scans, line) + + def bind_arguments( + self, + node: ast.Call, + members: tuple[tuple[_Group, int], ...], + scans: list[tuple[list[int], list[int], list[tuple[str, int]]]], + line: int, + ) -> None: + """Bind a call's arguments into each callee's slots. + + Positional arguments bind by parameter index (the receiver *offset* + added); from the first ``*args`` on they share a wildcard slot keyed + by the parameter index it starts at, and arguments at index + ``_MAX_POSITIONAL_SLOTS`` or beyond share the last one. ``**mapping`` + binds a wildcard keyed by the offset, so an implicit ``self`` / ``cls`` + is never filled by an unpacking. A forwarded ``**kwargs`` (a function's + own ``**`` parameter) binds nothing, as on ``main``: which names it + holds is not tracked, so no parameter can be chosen for it. Every + other argument costs one slot per callee, whatever its shape. + """ + slots: list[list[int]] = [] + starred_at = -1 + for position, arg in enumerate(node.args): + self.tick() + if starred_at < 0 and isinstance(arg, ast.Starred): + starred_at = position + keys: list[int] = [] + for group, offset in members: + slot: object + if starred_at >= 0: + slot = ("*", min(starred_at + offset, _MAX_POSITIONAL_SLOTS)) + elif position + offset >= _MAX_POSITIONAL_SLOTS: + slot = ("*", _MAX_POSITIONAL_SLOTS) + else: + slot = position + offset + keys.append(self.slot(group, slot)) + slots.append(keys) + for keyword in node.keywords: + self.tick() + keys = [] + if keyword.arg is None and self.forwards_kwargs(keyword.value): + slots.append(keys) + continue + for group, offset in members: + if keyword.arg is None: + keys.append(self.slot(group, ("**", offset))) + continue + named, kwarg_named = group.signature() + if keyword.arg in named: + keys.append(self.slot(group, ("kw", keyword.arg))) + if kwarg_named is not None and keyword.arg not in kwarg_named: + keys.append(self.slot(group, "kw*")) + slots.append(keys) + for keys, (names, others, sources) in zip(slots, scans, strict=True): + self.seed(keys, _BIND, line, sources) + self.add_flow([*names, *others], keys, _BIND, line, False) + + def forwards_kwargs(self, value: ast.expr) -> bool: + """Whether ``**value`` forwards a function's own ``**kwargs`` parameter.""" + return isinstance(value, ast.Name) and self.index.name_key.get(value) in ( + self.index.kwarg_params + ) + + def slot(self, group: _Group, slot: object) -> int: + key = group.slots.get(slot) + if key is None: + key = self.index.synthetic(("slot", group.id, slot), True) + group.slots[slot] = key + return key + + def defaults(self, function: _Function) -> None: + args = function.node.args + defining = function.scope.parent or self.index.module + positional = [*args.posonlyargs, *args.args] + defaulted = positional[len(positional) - len(args.defaults) :] + pairs: list[tuple[ast.arg, ast.expr]] = list(zip(defaulted, args.defaults, strict=True)) + for arg, default in zip(args.kwonlyargs, args.kw_defaults, strict=True): + if default is not None: + pairs.append((arg, default)) + for arg, default in pairs: + self.tick() + target = (self.index.key(function.scope, arg.arg),) + names, others, sources = self.scan(default, defining, False, opaque_lambdas=True) + self.seed(target, _ASSIGN, default.lineno, sources) + self.add_flow([*names, *others], target, _ASSIGN, default.lineno, False) + + def link_callees(self) -> None: + """Flow each callee's argument slots into the parameters they bind. + + Parameter *i* reads positional slot *i*, the ``*args`` wildcards that + start at or before *i*, its keyword slot and the ``**mapping`` + wildcards whose receiver offset is at most *i* (keywords and mappings + never reach positional-only parameters). A ``**kwargs`` parameter + reads the keywords no named parameter takes and the ``**mapping`` + wildcards. + """ + index = self.index + for group in index.groups: + if not group.slots: + continue + slots = group.slots + stars = [ + (start, slots[("*", start)]) + for start in range(_MAX_POSITIONAL_SLOTS + 1) + if ("*", start) in slots + ] + mappings = [ + (offset, slots[("**", offset)]) for offset in (0, 1) if ("**", offset) in slots + ] + rest = slots.get("kw*") + for function in group.functions: + self.tick() + args = function.node.args + scope = function.scope + positional = [*args.posonlyargs, *args.args] + posonly = len(args.posonlyargs) + for position, arg in enumerate(positional): + reads = [slots.get(position)] + reads.extend(key for start, key in stars if start <= position) + if position >= posonly: + reads.append(slots.get(("kw", arg.arg))) + reads.extend(key for offset, key in mappings if offset <= position) + self.bind_parameter(reads, index.key(scope, arg.arg)) + for arg in args.kwonlyargs: + reads = [slots.get(("kw", arg.arg)), *(key for _, key in mappings)] + self.bind_parameter(reads, index.key(scope, arg.arg)) + if args.vararg is not None: + reads = [key for _, key in stars] + reads.extend( + slots.get(i) for i in range(len(positional), _MAX_POSITIONAL_SLOTS) + ) + self.bind_parameter(reads, index.key(scope, args.vararg.arg)) + if args.kwarg is not None: + reads = [rest, *(key for _, key in mappings)] + self.bind_parameter(reads, index.key(scope, args.kwarg.arg)) + + def bind_parameter(self, reads: list[int | None], target: int) -> None: + self.add_flow([key for key in reads if key is not None], (target,), _PASS, 0, False) + + def link_attributes(self) -> None: + """A ``self.x`` read sees ``self.x`` stores in its class and in-file + bases, and the class-body ``x`` of the nearest of them whose body + binds ``x`` in any way (an assignment, annotated assignment, method, + import, ...); only an assignment's value is modelled.""" + index = self.index + ids = index._ids + for node in index.attributes: + self.tick() + index.attribute_view(node) + for cls, attr, view in index.view_list: + self.tick() + reads = [] + shadowed = False + for owner in index.ancestors(cls, self.tick): + store = ids.get(("store", owner.id, attr)) + if store is not None: + reads.append(store) + if not shadowed: + body = ids.get(("class", owner.id, attr)) + if body is not None: + reads.append(body) + shadowed = True + elif attr in owner.body_names(): + shadowed = True # bound without a modelled value + self.add_flow(reads, (view,), _PASS, 0, False) + + def build(self) -> None: + index = self.index + for view, class_key, module_key in index.class_reads: + self.tick() + self.add_flow((module_key,), (view,), _PASS, 0, True) + self.add_flow((class_key,), (view,), _PASS, 0, True) + for node, scope in index.statements: + self.tick() + if isinstance(node, ast.Assign): + self.assign(node, scope) + elif isinstance(node, ast.Return): + if node.value is not None: + self.returned(node.value, scope, node.lineno) + elif isinstance(node, ast.Call): + self.call(node, scope) + for function in index.functions: + self.tick() + self.defaults(function) + self.link_attributes() + self.link_callees() + + # ── fixpoint ── + + def run(self) -> None: + self.build() + self.propagate() + + def propagate(self) -> None: + # Phase A: the assignment flows earlier releases propagated, in order. + self.drain(self.worklist, (self.prop_a,)) + self.phase_a_keys = frozenset(self.facts[0][_FIRST_LANE]) + # Phase B: every flow. Phase A facts are read again through the new + # flows (each phase A flow already fired for them and is skipped). + worklist: deque[tuple[int, int, int]] = deque( + (key, 0, _ALL_LANES) for key in self.facts[0][_FIRST_LANE] + ) + for targets, kind, line, fact, lanes in self.seeds: + self.tick() + worklist.extend(_mark_targets(self, targets, kind, line, fact, 0, lanes)) + self.drain(worklist, (self.prop_a, self.prop_b)) + + def drain( + self, worklist: deque[tuple[int, int, int]], tables: tuple[dict[int, list[int]], ...] + ) -> None: + flows = self.flows + fired = self.fired + if len(fired) < len(flows): + fired.extend([0] * (len(flows) - len(fired))) + facts = self.facts + while worklist: + self.tick() + key, slot, lanes = worklist.popleft() + held = facts[slot] + # Lanes holding the same fact propagate it together. + carried: list[list] = [] + for lane in _LANES: + if lanes >> lane & 1: + fact = held[lane].get(key) + if fact is None: + continue + for entry in carried: + if entry[0] is fact: + entry[1] |= 1 << lane + break + else: + carried.append([fact, 1 << lane]) + for fact, fact_lanes in carried: + ranks = _LANE_RANKS[fact[0]] + # One bit per (lane, rank, slot): 14 bits per flow. + want = 0 + for lane in _LANES: + if fact_lanes >> lane & 1: + want |= 1 << (6 * lane + 2 * ranks[lane] + slot) + for table in tables: + for count, flow_id in enumerate(table.get(key, ())): + if count and not count & 1023: + self.tick() # a key read by very many flows + mark = fired[flow_id] + todo_bits = want & ~mark + if not todo_bits: + # Already propagated a source of this rank: its targets + # hold one at least as severe. Skip to stay linear. + continue + targets, kind, line = flows[flow_id] + if slot and kind == _RETURN: + continue + fired[flow_id] = mark | todo_bits + todo = ( + (1 if todo_bits & 0x3F else 0) + | (2 if todo_bits & 0xFC0 else 0) + | (4 if todo_bits & 0x3000 else 0) + ) + worklist.extend(_mark_targets(self, targets, kind, line, fact, slot, todo)) + + # ── results ── + + def fact(self, key: int, lane: int = 0) -> _Fact | None: + """The most severe fact at *key* for *lane* (an ordinary one on ties).""" + ordinary = self.facts[0][lane].get(key) + bound = self.facts[1][lane].get(key) + if bound is not None and ( + ordinary is None or _LANE_RANKS[bound[0]][lane] > _LANE_RANKS[ordinary[0]][lane] + ): + return bound + return ordinary + + def cited(self, key: int, sink_name: str, lane: int) -> _Fact | None: + """The fact a sink of *lane* reports for *key*. + + The ranked lane gives the most severe rule. The first source to + arrive (lane 2, what earlier releases cite) is reported instead + whenever it gives that same rule, so a more severe source replaces + the cited one only when it changes the rule. + """ + ranked = self.fact(key, lane) + if ranked is None: + return None + first = self.fact(key, _FIRST_LANE) + if first is not None and first is not ranked: + rule = _pick_rule(_SOURCE_NAMES[ranked[0]], sink_name, is_direct=False) + if _pick_rule(_SOURCE_NAMES[first[0]], sink_name, is_direct=False) == rule: + return first + return ranked + + def tainted_names(self) -> dict[str, _TaintedVar]: + """Tainted variables by readable name (module globals keep their bare name).""" + index = self.index + result: dict[str, _TaintedVar] = {} + for key in (*self.facts[0][0], *self.facts[1][0]): + scope = index.key_scope[key] + fact = self.fact(key) + if scope is None or fact is None: + continue + name = index.key_name[key] + if scope.kind == _MODULE: + label = name + elif scope.kind == _CLASS: + label = f"{index.qualname(scope)}.{name}" + else: + label = f"{index.qualname(scope)}..{name}" + result.setdefault(label, _TaintedVar(name, _SOURCE_NAMES[fact[0]], fact[1])) + return result + + +def _collect_tainted( + tree: ast.AST, + type_map: dict[str, str], + aliases: dict[str, str], + check_runtime: Callable[[], None] | None = None, +) -> dict[str, _TaintedVar]: + """Compute taint for *tree*: readable variable name -> its fact. + + Module globals keep their bare name; function locals read + ``qualname..name``. See ``_TaintGraph`` for the propagation rules. + """ + index = _build_scope_index(tree, check_runtime) + graph = _TaintGraph(index, type_map, aliases, check_runtime) + graph.run() + return graph.tainted_names() + + +# Identifiers longer than this are shortened in messages (``prefix...``), so +# message bytes stay bounded per finding instead of growing with identifier +# length times the number of sinks. Real identifiers are far shorter, so their +# messages are unchanged. +_MAX_NAME_CHARS = 120 + + +def _shorten(name: str) -> str: + return name if len(name) <= _MAX_NAME_CHARS else name[: _MAX_NAME_CHARS - 3] + "..." + + +def _spelling(node: ast.AST) -> str: + """How a sink spells what it reads: ``name``, ``self.x``, ``helper()``, ``self.m()``. + + Built from the sink's own nodes, so a message never copies an identifier + spelled elsewhere in the file, and shortened to a bounded length. + """ + if isinstance(node, ast.Name): + return _shorten(node.id) + if isinstance(node, ast.Attribute) and isinstance(node.value, ast.Name): + return _shorten(f"{_shorten(node.value.id)}.{_shorten(node.attr)}") + if isinstance(node, ast.Call): + func = node.func + if isinstance(func, ast.Name): + return _shorten(f"{_shorten(func.id)}()") + if isinstance(func, ast.Attribute): + value = func.value + if isinstance(value, ast.Name): + return _shorten(f"{_shorten(value.id)}.{_shorten(func.attr)}()") + if isinstance(value, ast.Call) and isinstance(value.func, ast.Name): + return _shorten(f"{_shorten(value.func.id)}().{_shorten(func.attr)}()") + return "()" + + +def _find_tainted_names_in_args( + node: ast.Call, + graph: _TaintGraph, + sink_name: str, check_runtime: Callable[[], None] | None = None, -) -> _TaintedVar | None: - """Return the first tainted variable referenced in *node*, or None. +) -> list[_TaintedVar]: + """Find tainted variables, ``self.x`` attributes and helper results a sink reads. - Handles Name references, container literals (dict, list, tuple, set), - and f-strings so that taint propagates through re-assignment and - data packaging (e.g. ``payload = {"key": secret}``). + Plain variables that earlier releases' flows taint (phase A) come first, + in ``ast.walk`` order, so a flow earlier releases reported keeps its + message; plain variables only the new flows taint (bound parameters, + helper results) follow, then attributes and in-file call results. Each + is named as the sink spells it, and reported once. """ + tick = check_runtime or _noop + index = graph.index + class_reads = index.class_read_views + phase_a = graph.phase_a_keys + lane = 1 if sink_name in _EXEC_SINKS or sink_name in _DESERIALIZATION_SINKS else 0 + seen: set[int] = set() + names: list[_TaintedVar] = [] + new_names: list[_TaintedVar] = [] + others: list[_TaintedVar] = [] for child in ast.walk(node): - if check_runtime is not None: - check_runtime() + tick() + if child is node: + continue + read: ast.Name | None = None if isinstance(child, ast.Name): - tv = tainted.get(child.id) - if tv: - return tv - return None + read = child + elif isinstance(child, ast.Subscript) and isinstance(child.value, ast.Name): + read = child.value + reads: list[tuple[int | None, ast.AST, list[_TaintedVar]]] + if read is not None: + key = class_reads.get(read) + if key is None: + key = index.name_key.get(read) + reads = [(key, read, names if key in phase_a else new_names)] + elif isinstance(child, ast.Attribute): + reads = [(index.views.get(child), child, others)] + elif isinstance(child, ast.Call): + reads = [ + (graph.returns.get(group.id), child, others) + for group in index.call_targets(child, tick).results + ] + else: + continue + for key, spelled, out in reads: + if key is None or key in seen: + continue + fact = graph.cited(key, sink_name, lane) + if fact is None: + continue + seen.add(key) + out.append(_TaintedVar(_spelling(spelled), _SOURCE_NAMES[fact[0]], fact[1])) + return names + new_names + others def _analyze_python( @@ -464,8 +2130,11 @@ def _analyze_python( type_map = build_type_map(tree, aliases) lines = python_ast.lines findings: list[AnalyzerFinding] = [] - tainted: dict[str, _TaintedVar] = {} - seen: set[tuple[str, int, int, int | None, int | None]] = set() + check_runtime = budget.check_runtime if budget is not None else None + index = _build_scope_index(tree, check_runtime) + graph = _TaintGraph(index, type_map, aliases, check_runtime) + graph.run() + seen: set[tuple[str, ast.Call]] = set() contexts: dict[int, str] = {} def context_for(lineno: int) -> str: @@ -476,15 +2145,18 @@ def context_for(lineno: int) -> str: return context def _emit( + node_index: int, rule_id: str, ast_node: ast.Call, msg: str, ) -> None: lineno = getattr(ast_node, "lineno", 1) end_lineno = getattr(ast_node, "end_lineno", None) - start_byte_column = getattr(ast_node, "col_offset", 0) + start_byte_column = getattr(ast_node, "col_offset", None) end_byte_column = getattr(ast_node, "end_col_offset", None) - key = (rule_id, lineno, start_byte_column, end_lineno, end_byte_column) + # Deduplicate flows into the same sink without merging distinct nodes + # whose optional source columns are unavailable. + key = (rule_id, ast_node) if key in seen: return seen.add(key) @@ -492,11 +2164,7 @@ def _emit( if complete_match is None: complete_match = get_complete_source_segment(lines, lineno, end_lineno) start_column = python_ast.character_column(lineno, start_byte_column) - end_column = ( - python_ast.character_column(end_lineno or lineno, end_byte_column) - if end_byte_column is not None - else None - ) + end_column = python_ast.character_column(end_lineno or lineno, end_byte_column) finding = AnalyzerFinding( rule_id=rule_id, message=msg, @@ -513,48 +2181,25 @@ def _emit( context=context_for(lineno), matched_text=complete_match[:200], complete_match=complete_match, + # Evidence participates in report compaction. Keep distinct syntax + # nodes distinguishable when their source spans are incomplete. + evidence=( + {"python_ast_node_index": node_index} + if start_column is None or end_column is None + else {} + ), ) if budget is None: findings.append(finding) else: budget.emit(finding) - for ast_node in ast.walk(tree): + # Taint is fully computed above, independent of traversal order, so this + # pass only needs to check sink call sites against it. + for node_index, ast_node in enumerate(ast.walk(tree)): if budget is not None: budget.check_runtime() - # Record tainted assignments. - if isinstance(ast_node, ast.Assign): - src_name = _find_source_in_expr( - ast_node.value, - type_map, - aliases, - budget.check_runtime if budget is not None else None, - ) - # Subscript sources like os.environ["KEY"] (also os aliased as `o`) - if src_name is None and isinstance(ast_node.value, ast.Subscript): - base = resolve_dotted_name(ast_node.value.value) - if base is not None: - base = apply_import_aliases(base, aliases) - if base and base in _CREDENTIAL_SOURCES: - src_name = base - - # Propagate taint through re-assignment and container construction: - # data = secret, payload = {"k": secret}, items = [secret], msg = f"{secret}" - if src_name is None: - tv = _find_tainted_in_expr( - ast_node.value, - tainted, - budget.check_runtime if budget is not None else None, - ) - if tv: - src_name = tv.source_call - - if src_name: - _mark_targets(ast_node.targets, tainted, src_name, ast_node.lineno) - continue - - # Detect flows at sink call sites. if not isinstance(ast_node, ast.Call): continue @@ -569,7 +2214,7 @@ def _emit( ast_node, type_map, aliases, - budget.check_runtime if budget is not None else None, + check_runtime, ): if src_name == "open" and _is_open_for_write(src_node): continue @@ -577,20 +2222,18 @@ def _emit( src_cat = _classify(src_name, _SOURCE_CATEGORIES, "data source") sink_cat = _classify(sink_name, _SINK_CATEGORIES, "data sink") _emit( + node_index, rule, ast_node, f"Direct flow: {src_name} ({src_cat}) \u2192 {sink_name} ({sink_cat})", ) - for tv in _find_tainted_names_in_args( - ast_node, - tainted, - budget.check_runtime if budget is not None else None, - ): + for tv in _find_tainted_names_in_args(ast_node, graph, sink_name, check_runtime): rule = _pick_rule(tv.source_call, sink_name, is_direct=False) src_cat = _classify(tv.source_call, _SOURCE_CATEGORIES, "data source") sink_cat = _classify(sink_name, _SINK_CATEGORIES, "data sink") _emit( + node_index, rule, ast_node, f"Tainted flow: '{tv.name}' from {tv.source_call} (line {tv.lineno}, " @@ -641,6 +2284,14 @@ def node(state: SkillspectorState) -> AnalyzerNodeResponse: """Parse Python files and detect source\u2192sink data flows.""" components: list[str] = state.get("components") or [] file_cache: dict[str, str] = state.get("local_file_cache") or state.get("file_cache") or {} + raw_file_cache = state.get("raw_file_cache") + source_classifications = ( + state.get("python_source_classifications") + if "python_source_classifications" in state + else None + ) + source_classification_limitations = state.get("python_source_classification_limitations") or {} + source_decode_failures = state.get("python_source_decode_failures") or {} python_ast_cache_key = state.get("python_ast_cache_key") all_findings: list[Finding] = [] ledger_events: list[InspectionLedgerEvent] = [] @@ -648,7 +2299,27 @@ def node(state: SkillspectorState) -> AnalyzerNodeResponse: terminal_limit: _BehavioralResourceLimitError | None = None for path in components: - if not path.endswith(".py"): + content = file_cache.get(path) + source_classification: PythonSourceClassification | None = None + if source_classifications is not None and path in source_classifications: + source_classification = resolve_python_source_classification( + path, + content, + source_classifications=source_classifications, + raw_file_cache=raw_file_cache, + ) + if source_classification is PythonSourceClassification.NON_PYTHON: + continue + if path in source_classification_limitations: + ledger_events.append( + ledger_event( + outcome=LedgerOutcome.PARTIAL, + phase="behavioral", + analyzer_id=ANALYZER_ID, + path=path, + reason=LedgerReason.RUNTIME_LIMIT, + ) + ) continue if terminal_limit is None and budget.analyzer_exhausted(): terminal_limit = _BehavioralResourceLimitError( @@ -658,11 +2329,41 @@ def node(state: SkillspectorState) -> AnalyzerNodeResponse: "limit_findings": MAX_FINDINGS_PER_ANALYZER, }, ) + if terminal_limit is None: + try: + budget.check_runtime() + except _BehavioralResourceLimitError as exc: + terminal_limit = exc if terminal_limit is not None: event = _partial_limit_event(path, terminal_limit) ledger_events.append(event) continue - content = file_cache.get(path) + if path in source_decode_failures: + ledger_events.append( + ledger_event( + outcome=LedgerOutcome.PARTIAL, + phase="behavioral", + analyzer_id=ANALYZER_ID, + path=path, + reason=LedgerReason.PYTHON_SOURCE_DECODE_ERROR, + ) + ) + continue + if source_classification is None: + source_classification = resolve_python_source_classification( + path, + content, + source_classifications=source_classifications, + raw_file_cache=raw_file_cache, + ) + try: + budget.check_runtime() + except _BehavioralResourceLimitError as exc: + terminal_limit = exc + ledger_events.append(_partial_limit_event(path, exc)) + continue + if source_classification is PythonSourceClassification.NON_PYTHON: + continue if content is None: event = ledger_event( outcome=LedgerOutcome.FAILED, @@ -718,10 +2419,19 @@ def node(state: SkillspectorState) -> AnalyzerNodeResponse: ) else: event = ledger_event( - outcome=LedgerOutcome.COMPLETED, + outcome=( + LedgerOutcome.PARTIAL + if source_classification is PythonSourceClassification.AMBIGUOUS + else LedgerOutcome.COMPLETED + ), phase="behavioral", analyzer_id=ANALYZER_ID, path=path, + reason=( + LedgerReason.PYTHON_SOURCE_AMBIGUOUS + if source_classification is PythonSourceClassification.AMBIGUOUS + else None + ), emitted_finding_ids=[finding.finding_id for finding in path_findings], ) ledger_events.append(event) diff --git a/src/skillspector/nodes/analyzers/bundled_execution_surface.py b/src/skillspector/nodes/analyzers/bundled_execution_surface.py index 97278fe99..fad1ebbf7 100644 --- a/src/skillspector/nodes/analyzers/bundled_execution_surface.py +++ b/src/skillspector/nodes/analyzers/bundled_execution_surface.py @@ -11,7 +11,7 @@ import posixpath import re import socket -from collections.abc import Callable +from collections.abc import Callable, Mapping from dataclasses import dataclass from typing import Final from urllib.parse import urlsplit @@ -42,6 +42,10 @@ ".claude/settings.local.json", } ) +# Claude Code also loads plugin hooks from the `hooks` field of +# `.claude-plugin/plugin.json` (a path, a list of paths, or an inline object). +_PLUGIN_MANIFEST_PATH: Final = ".claude-plugin/plugin.json" +_MAX_PLUGIN_HOOK_PATHS: Final = 16 _MAX_DECLARATIONS: Final = 2_048 _MAX_DECLARATION_CHARS: Final = 16_384 _VALID_DEFAULT_MODES: Final = frozenset( @@ -193,12 +197,57 @@ "$schema", "disableAllHooks", "env", + "fileSuggestion", "hooks", "includeCoAuthoredBy", "model", "permissions", + "statusLine", } ) +# Settings keys whose values are shell commands Claude Code runs on its own +# once the settings are active (same execution surface as hook handlers). +# apiKeyHelper and the auth/telemetry helpers run regardless of hooks; +# statusLine and fileSuggestion commands are disabled by disableAllHooks. +_SETTINGS_COMMAND_STRING_KEYS: Final = frozenset( + { + "apiKeyHelper", + "awsAuthRefresh", + "awsCredentialExport", + "gcpAuthRefresh", + "otelHeadersHelper", + } +) +_SETTINGS_COMMAND_OBJECT_KEYS: Final = frozenset({"fileSuggestion", "statusLine"}) +_SETTINGS_COMMAND_KEYS: Final = _SETTINGS_COMMAND_STRING_KEYS | _SETTINGS_COMMAND_OBJECT_KEYS +# Command keys that disableAllHooks turns off (per the settings reference). +_DISABLE_SUPPRESSED_COMMAND_KEYS: Final = frozenset({"fileSuggestion", "statusLine"}) +# Settings `env` names that reroute model/API traffic when set in a bundled file. +_ENV_TRAFFIC_REDIRECT_URL_NAMES: Final = frozenset( + { + "ANTHROPIC_BASE_URL", + "ANTHROPIC_BEDROCK_BASE_URL", + "ANTHROPIC_VERTEX_BASE_URL", + } +) +_ENV_PROXY_URL_NAMES: Final = frozenset({"HTTP_PROXY", "HTTPS_PROXY", "http_proxy", "https_proxy"}) +# Settings `env` names whose values make Claude Code or its subprocesses run code. +_ENV_COMMAND_INJECTION_NAMES: Final = frozenset( + { + "CLAUDE_CODE_SHELL_PREFIX", + "DYLD_INSERT_LIBRARIES", + "GIT_SSH_COMMAND", + "LD_PRELOAD", + } +) +# NODE_OPTIONS flags that load attacker-chosen modules into the Node runtime. +# "-r" is Node's own short alias for "--require" and is honored identically +# when set via NODE_OPTIONS (confirmed via `process.allowedNodeEnvironmentFlags` +# on Node 22, which lists both). It has no long-form spelling to fall back on. +_NODE_OPTIONS_CODE_FLAGS: Final = frozenset( + {"-r", "--require", "--import", "--loader", "--experimental-loader"} +) +_ANTHROPIC_DEFAULT_BASE_URL: Final = "https://api.anthropic.com" _HookIdentity = tuple[str, str, str] @@ -234,10 +283,17 @@ class _PermissionDeclaration: activation_state: str = "conditional" +@dataclass(frozen=True) +class _SettingsCommandDeclaration: + key: str + command: str + + @dataclass(frozen=True) class _DeclarationScan: hooks: list[_HookDeclaration] permissions: list[_PermissionDeclaration] + settings_commands: list[_SettingsCommandDeclaration] partial: bool observed: int @@ -1258,13 +1314,154 @@ def _permission_declarations( return declarations, partial, observed +def _settings_command_value(key: str, value: object) -> str | None: + """Return the shell command a settings key causes Claude Code to run. + + Returns "" for a recognized non-command variant (no command runs) and None + when the value shape is unmodeled. + """ + if key in _SETTINGS_COMMAND_STRING_KEYS: + return value if isinstance(value, str) else None + if key in _SETTINGS_COMMAND_OBJECT_KEYS: + if not isinstance(value, dict): + return None + command_type = value.get("type") + if command_type != "command": + return "" if isinstance(command_type, str) else None + command = value.get("command") + if isinstance(command, str): + return command + if isinstance(command, list) and all(isinstance(part, str) for part in command): + return " ".join(command) + return None + return None + + +def _settings_command_object_is_valid(value: object) -> bool: + if not isinstance(value, dict): + return False + command_type = value.get("type") + if command_type is not None and not _is_bounded_string(command_type): + return False + command = value.get("command") + if command is not None and not ( + _is_bounded_string(command) + or (isinstance(command, list) and all(_is_bounded_string(part) for part in command)) + ): + return False + return True + + +def _settings_command_declarations( + document: dict[str, object], *, hooks_disabled: bool, limit: int +) -> tuple[list[_SettingsCommandDeclaration], bool, int]: + """Extract command-running settings keys (BH4 declarations).""" + declarations: list[_SettingsCommandDeclaration] = [] + partial = False + observed = 0 + for key in sorted(set(document) & _SETTINGS_COMMAND_KEYS): + observed += 1 + if observed > limit: + return declarations, True, observed + if key in _DISABLE_SUPPRESSED_COMMAND_KEYS and hooks_disabled: + continue + command = _settings_command_value(key, document[key]) + if command is None: + partial = True + continue + if not command.strip(): + continue + if not _is_bounded_string(command): + partial = True + continue + declarations.append(_SettingsCommandDeclaration(key=key, command=command)) + return declarations, partial, observed + + +def _env_node_options_runs_code(value: str) -> bool: + return any( + token == flag or token.startswith(f"{flag}=") + for token in value.split() + for flag in _NODE_OPTIONS_CODE_FLAGS + ) + + +def _env_value_is_flagged(name: str, value: str) -> bool: + """Decide whether a settings `env` entry deserves a finding. + + Only the documented names are classified, and only their clearly risky + values are flagged: a base URL other than the known default, a proxy URL + that points at a remote host, a loader-injection flag in NODE_OPTIONS, or + a nonempty shell-prefix/loader/SSH-command override. + """ + stripped = value.strip() + if not stripped: + return False + if name in _ENV_TRAFFIC_REDIRECT_URL_NAMES: + return stripped.rstrip("/").lower() != _ANTHROPIC_DEFAULT_BASE_URL + if name in _ENV_PROXY_URL_NAMES: + return _is_external_http_url(stripped) + if name in _ENV_COMMAND_INJECTION_NAMES: + return True + return name == "NODE_OPTIONS" and _env_node_options_runs_code(stripped) + + +def _settings_surface_declarations( + document: dict[str, object], *, limit: int +) -> tuple[list[_PermissionDeclaration], bool, int]: + """Extract risky settings-level surface declarations (BH3 declarations).""" + declarations: list[_PermissionDeclaration] = [] + partial = False + observed = 0 + env = document.get("env") + if env is not None: + observed += 1 + if observed > limit: + return declarations, True, observed + if not _is_bounded_string_map(env): + partial = True + else: + assert isinstance(env, dict) + for name in sorted(env): + if _env_value_is_flagged(name, env[name]): + kind = ( + "env_code_execution" + if name in _ENV_COMMAND_INJECTION_NAMES or name == "NODE_OPTIONS" + else "traffic_redirect" + ) + declarations.append(_PermissionDeclaration(Severity.HIGH, kind)) + if "enableAllProjectMcpServers" in document: + observed += 1 + if observed > limit: + return declarations, True, observed + value = document.get("enableAllProjectMcpServers") + if not _is_bool(value): + partial = True + elif value: + declarations.append(_PermissionDeclaration(Severity.CRITICAL, "mcp_auto_approve")) + servers = document.get("enabledMcpjsonServers") + if servers is not None: + observed += 1 + if observed > limit: + return declarations, True, observed + if not _is_bounded_string_list(servers): + partial = True + elif servers: + declarations.append(_PermissionDeclaration(Severity.HIGH, "mcp_auto_approve")) + return declarations, partial, observed + + def _scan_declarations( path: str, document: object, previous_settings_hook_ids: set[_HookIdentity] | None = None, + *, + hooks_disabled: bool = False, ) -> _DeclarationScan: if not isinstance(document, dict): - return _DeclarationScan(hooks=[], permissions=[], partial=True, observed=0) + return _DeclarationScan( + hooks=[], permissions=[], settings_commands=[], partial=True, observed=0 + ) hooks, hook_partial, hook_observed = _hook_declarations( document, limit=_MAX_DECLARATIONS, @@ -1274,13 +1471,34 @@ def _scan_declarations( permissions, permission_partial, permission_observed = _permission_declarations( path, document, limit=remaining ) + observed = hook_observed + permission_observed is_settings = path in {".claude/settings.json", ".claude/settings.local.json"} + settings_commands: list[_SettingsCommandDeclaration] = [] + settings_partial = False + settings_observed = 0 + surface_partial = False + surface_observed = 0 + if is_settings: + settings_commands, settings_partial, settings_observed = _settings_command_declarations( + document, + hooks_disabled=hooks_disabled, + limit=max(0, _MAX_DECLARATIONS - observed), + ) + observed += settings_observed + surface_declarations, surface_partial, surface_observed = _settings_surface_declarations( + document, limit=max(0, _MAX_DECLARATIONS - observed) + ) + permissions.extend(surface_declarations) + observed += surface_observed return _DeclarationScan( hooks=hooks, permissions=permissions, + settings_commands=settings_commands, partial=( hook_partial or permission_partial + or settings_partial + or surface_partial or (is_settings and not _modeled_settings_fields_are_valid(document)) or ( is_settings @@ -1289,7 +1507,7 @@ def _scan_declarations( ) or (path == "hooks/hooks.json" and "hooks" not in document) ), - observed=hook_observed + permission_observed, + observed=observed, ) @@ -1408,6 +1626,20 @@ def _modeled_settings_fields_are_valid(document: dict[str, object]) -> bool: return False if "env" in document and not _is_bounded_string_map(document.get("env")): return False + for key in _SETTINGS_COMMAND_STRING_KEYS: + if key in document and not _is_bounded_string(document.get(key)): + return False + for key in _SETTINGS_COMMAND_OBJECT_KEYS: + if key in document and not _settings_command_object_is_valid(document.get(key)): + return False + if "enableAllProjectMcpServers" in document and not _is_bool( + document.get("enableAllProjectMcpServers") + ): + return False + if "enabledMcpjsonServers" in document and not _is_bounded_string_list( + document.get("enabledMcpjsonServers") + ): + return False return _permissions_schema_allows_disable(document) @@ -1617,6 +1849,29 @@ def _bh3_finding(path: str, declarations: list[_PermissionDeclaration]) -> Findi return finding +def _bh4_finding(path: str, declarations: list[_SettingsCommandDeclaration]) -> Finding: + analyzer_finding = AnalyzerFinding( + rule_id="BH4", + message="Bundled settings keys run commands when the settings activate.", + severity=Severity.MEDIUM, + location=Location(path, 1), + confidence=0.95, + remediation=( + "Review bundled settings command keys before install; remove or narrow " + "any helper the skill does not need." + ), + tags=["Bundled Execution Surface", "Settings"], + matched_text=f"document:{path}", + evidence={ + "activation_reason": "requires_settings_activation", + "activation_state": "conditional", + "command_keys": sorted({declaration.key for declaration in declarations})[:32], + "declaration_count": len(declarations), + }, + ) + return analyzer_finding_to_finding(analyzer_finding) + + def _analyze_document( path: str, content: str, @@ -1646,7 +1901,9 @@ def _analyze_document( reason=LedgerReason.OPAQUE_CONTENT, ) - scan = _scan_declarations(path, document, previous_settings_hook_ids) + scan = _scan_declarations( + path, document, previous_settings_hook_ids, hooks_disabled=hooks_disabled + ) declarations = [] if hooks_disabled else scan.hooks proofs = [proof for declaration in declarations if (proof := _bh2_proof(declaration))] payload_unmodeled = _payload_analysis_level(declarations) == "unmodeled" @@ -1658,6 +1915,8 @@ def _analyze_document( findings.append(_bh2_finding(path, proofs)) if permission_declarations: findings.append(_bh3_finding(path, permission_declarations)) + if scan.settings_commands: + findings.append(_bh4_finding(path, scan.settings_commands)) if scan.partial or payload_unmodeled: return findings, ledger_event( outcome=LedgerOutcome.PARTIAL, @@ -1678,6 +1937,92 @@ def _analyze_document( ) +def _plugin_hook_documents( + file_cache: Mapping[str, str], +) -> tuple[dict[str, str], dict[str, InspectionLedgerEvent]]: + """Resolve bounded plugin hooks and retain every uninspected declaration.""" + documents: dict[str, str] = {} + partial: dict[str, InspectionLedgerEvent] = {} + + def incomplete(path: str, reason: LedgerReason, **bounds: int) -> None: + partial[path] = ledger_event( + outcome=LedgerOutcome.PARTIAL, + phase="static", + analyzer_id=ANALYZER_ID, + path=path, + reason=reason, + **bounds, + ) + + content = file_cache.get(_PLUGIN_MANIFEST_PATH) + if content is None: + return documents, partial + if len(content) > MAX_FILE_CHARS: + incomplete( + _PLUGIN_MANIFEST_PATH, + LedgerReason.SIZE_LIMIT, + observed_characters=len(content), + limit_characters=MAX_FILE_CHARS, + ) + return documents, partial + try: + manifest = _parse_document(content) + except (RecursionError, ValueError): + incomplete(_PLUGIN_MANIFEST_PATH, LedgerReason.OPAQUE_CONTENT) + return documents, partial + if not isinstance(manifest, dict): + incomplete(_PLUGIN_MANIFEST_PATH, LedgerReason.OPAQUE_CONTENT) + return documents, partial + raw_hooks = manifest.get("hooks") + if raw_hooks is None: + return documents, partial + candidates = raw_hooks if isinstance(raw_hooks, list) else [raw_hooks] + inline: dict[str, list] = {} + for candidate in candidates[:_MAX_PLUGIN_HOOK_PATHS]: + if isinstance(candidate, dict): + for event, groups in candidate.items(): + if isinstance(groups, list): + inline.setdefault(event, []).extend(groups) + else: + incomplete(_PLUGIN_MANIFEST_PATH, LedgerReason.OPAQUE_CONTENT) + continue + if not isinstance(candidate, str): + incomplete(_PLUGIN_MANIFEST_PATH, LedgerReason.OPAQUE_CONTENT) + continue + try: + candidate.encode("utf-8") + except UnicodeEncodeError: + incomplete(_PLUGIN_MANIFEST_PATH, LedgerReason.REFERENCED_UNINSPECTED) + continue + resolved = posixpath.normpath(candidate) + if ( + not resolved + or resolved in {".", ".."} + or resolved.startswith("../") + or posixpath.isabs(candidate) + or "\\" in candidate + or (len(candidate) > 1 and candidate[1] == ":") + or (len(resolved) > 1 and resolved[1] == ":") + ): + incomplete(_PLUGIN_MANIFEST_PATH, LedgerReason.REFERENCED_UNINSPECTED) + continue + hook_content = file_cache.get(resolved) + if hook_content is None: + incomplete(resolved, LedgerReason.REFERENCED_UNINSPECTED) + continue + documents[resolved] = hook_content + if inline: + documents[_PLUGIN_MANIFEST_PATH] = json.dumps({"hooks": inline}, ensure_ascii=False) + if len(candidates) > _MAX_PLUGIN_HOOK_PATHS: + incomplete( + _PLUGIN_MANIFEST_PATH, + LedgerReason.OUTPUT_LIMIT, + observed_records=len(candidates), + limit_records=_MAX_PLUGIN_HOOK_PATHS, + ) + return documents, partial + + def node(state: SkillspectorState) -> AnalyzerNodeResponse: """Inspect exact bundled hook/settings paths using bounded literal classifiers.""" components = state.get("components") or [] @@ -1690,9 +2035,20 @@ def node(state: SkillspectorState) -> AnalyzerNodeResponse: ledger_events: list[InspectionLedgerEvent] = [] previous_settings_hook_ids: set[_HookIdentity] = set() applicable_paths = set(components).intersection(_APPLICABLE_PATHS) + plugin_documents, plugin_partial = _plugin_hook_documents(file_cache) + if decodable.get(_PLUGIN_MANIFEST_PATH) is False: + plugin_partial[_PLUGIN_MANIFEST_PATH] = ledger_event( + outcome=LedgerOutcome.PARTIAL, + phase="static", + analyzer_id=ANALYZER_ID, + path=_PLUGIN_MANIFEST_PATH, + reason=LedgerReason.OPAQUE_CONTENT, + ) + applicable_paths |= set(plugin_documents) hooks_disabled = _bundled_hooks_are_disabled(applicable_paths, file_cache, decodable) for path in sorted(applicable_paths): + partial_event = plugin_partial.pop(path, None) if decodable.get(path) is False: ledger_events.append( ledger_event( @@ -1704,7 +2060,7 @@ def node(state: SkillspectorState) -> AnalyzerNodeResponse: ) ) continue - content = file_cache.get(path) + content = plugin_documents.get(path, file_cache.get(path)) if content is None: ledger_events.append( ledger_event( @@ -1739,8 +2095,12 @@ def node(state: SkillspectorState) -> AnalyzerNodeResponse: ) continue findings.extend(path_findings) + if partial_event is not None: + event.update(partial_event) + event["emitted_finding_ids"] = [finding.finding_id for finding in path_findings] ledger_events.append(event) + ledger_events.extend(plugin_partial.values()) status = analyzer_status_for_events(ANALYZER_ID, ledger_events) return { "findings": findings, diff --git a/src/skillspector/nodes/analyzers/mcp_tool_poisoning.py b/src/skillspector/nodes/analyzers/mcp_tool_poisoning.py index 63e86b6ee..001b51987 100644 --- a/src/skillspector/nodes/analyzers/mcp_tool_poisoning.py +++ b/src/skillspector/nodes/analyzers/mcp_tool_poisoning.py @@ -22,13 +22,14 @@ import re import time import unicodedata -from collections.abc import Callable, Iterator +from collections.abc import Callable, Iterator, Mapping from dataclasses import dataclass, field from typing import cast +from langchain_openai.chat_models.base import BaseChatOpenAI from pydantic import BaseModel, Field, field_validator -from skillspector.inference_usage import InferenceUsageRecord +from skillspector.inference_usage import InferenceUsageRecord, chat_model_controls from skillspector.inspection_ledger import ( InspectionLedgerEvent, LedgerOutcome, @@ -45,6 +46,7 @@ append_output_language_instruction, estimate_tokens, ) +from skillspector.llm_utils import chat_model_provider_name from skillspector.model_info import get_max_input_tokens from skillspector.models import Finding, compute_match_fingerprint from skillspector.nodes.analyzers.static_runner import MAX_FINDINGS_PER_ANALYZER @@ -54,6 +56,7 @@ padding_run_match_fingerprint, ) from skillspector.providers import get_active_provider +from skillspector.providers.chat_models import resolve_reasoning_effort from skillspector.state import ( AnalyzerNodeResponse, LLMCallRecord, @@ -878,6 +881,28 @@ def _check_tp3( {"python", "javascript", "typescript", "shell", "ruby", "go", "rust"} ) +# Track the exact-label gateway workaround in the public PR: +# https://github.com/NVIDIA/SkillSpector/pull/691 +# Dated/suffixed aliases are deliberately excluded. Only the openai route has +# reproduced wrapper evidence; other providers retain their own method hints. +_TP4_TOOL_OUTPUT_MODELS = frozenset({"azure/anthropic/claude-opus-5"}) + + +def _tp4_reasoning_configured(llm: BaseChatOpenAI) -> bool: + """Avoid an automatic forced tool choice when reasoning controls are set.""" + if resolve_reasoning_effort() is not None: + return True + if chat_model_controls(llm).get("reasoning_effort") is not None: + return True + if any(getattr(llm, name, None) is not None for name in ("reasoning_effort", "reasoning")): + return True + for controls in (llm.model_kwargs, llm.extra_body): + if isinstance(controls, Mapping) and any( + controls.get(name) is not None for name in ("reasoning_effort", "reasoning", "thinking") + ): + return True + return False + class _TP4AnalysisResult(BaseModel): """Validated response from the description-behavior mismatch check.""" @@ -902,6 +927,21 @@ class _TP4Analyzer(LLMAnalyzerBase): response_schema = _TP4AnalysisResult + def _structured_output_preference(self, llm: object) -> str | None: + # The OpenAI-compatible Opus 5 gateway sometimes wraps json_schema + # output in a {"json": ...} object instead of the requested schema. + # Tool calling returns the exact TP4 schema on the same gateway, but + # forcing a tool may conflict with configured reasoning/thinking. + # Explicit environment/provider method choices still take precedence. + if ( + isinstance(llm, BaseChatOpenAI) + and chat_model_provider_name(llm) == "openai" + and self.model in _TP4_TOOL_OUTPUT_MODELS + and not _tp4_reasoning_configured(llm) + ): + return "function_calling" + return None + def __init__( self, model: str, diff --git a/src/skillspector/nodes/analyzers/pattern_defaults.py b/src/skillspector/nodes/analyzers/pattern_defaults.py index 6fde2fe86..ee8059593 100644 --- a/src/skillspector/nodes/analyzers/pattern_defaults.py +++ b/src/skillspector/nodes/analyzers/pattern_defaults.py @@ -49,6 +49,7 @@ class PatternCategory(StrEnum): "BH1": "Bundled lifecycle hooks can run automatically when their configured events occur, so their reach and handler capability require review before installation.", "BH2": "The bundled hook declaration directly proves that sensitive event or local file content is sent to a non-loopback remote destination.", "BH3": "Bundled project settings contain permission-related configuration; activation evidence distinguishes conditional grants from modes ignored on this surface.", + "BH4": "Bundled settings keys (apiKeyHelper, statusLine, fileSuggestion, auth and telemetry helpers) run shell commands on their own once the settings are active, so their commands require review before installation.", "P1": "This pattern attempts to override system instructions or ignore safety constraints. Without LLM analysis, manual review is recommended.", "P2": "Hidden instructions were detected in comments or invisible text. These could contain malicious directives. Manual review is recommended.", "P3": "Instructions found that direct the agent to transmit conversation context or user data to external services.", diff --git a/src/skillspector/nodes/analyzers/static_patterns_excessive_agency.py b/src/skillspector/nodes/analyzers/static_patterns_excessive_agency.py index 24a6396c5..13b83511e 100644 --- a/src/skillspector/nodes/analyzers/static_patterns_excessive_agency.py +++ b/src/skillspector/nodes/analyzers/static_patterns_excessive_agency.py @@ -196,7 +196,10 @@ def _frontmatter_bounds(content: str, file_path: str) -> tuple[int, int] | None: """Return the YAML-frontmatter byte offsets for a SKILL.md file.""" if file_path.rsplit("/", 1)[-1].lower() != "skill.md": return None - opening = re.match(r"\A---[ \t]*\r?\n", content) + # A leading BOM is matched here rather than stripped: the offsets this + # function returns index the caller's original `content` (line numbers and + # context are derived from it), so removing characters would shift them. + opening = re.match(r"\A\ufeff*---[ \t]*\r?\n", content) if opening is None: return None closing = re.search(r"^---[ \t]*$", content[opening.end() :], re.MULTILINE) diff --git a/src/skillspector/nodes/analyzers/static_patterns_privilege_escalation.py b/src/skillspector/nodes/analyzers/static_patterns_privilege_escalation.py index 3b457e57e..357b6c5c6 100644 --- a/src/skillspector/nodes/analyzers/static_patterns_privilege_escalation.py +++ b/src/skillspector/nodes/analyzers/static_patterns_privilege_escalation.py @@ -17,12 +17,21 @@ from __future__ import annotations +import ast +import posixpath import re import sys from bisect import bisect_right +from contextvars import ContextVar from skillspector.logging_config import get_logger from skillspector.models import AnalyzerFinding, Location, Severity +from skillspector.python_ast import ( + ParsedPythonFile, + parse_python_source, + peek_python_ast, + peek_python_ast_any_content, +) from skillspector.state import AnalyzerNodeResponse, SkillspectorState from . import static_runner @@ -34,12 +43,23 @@ get_context, get_context_from_lines, get_line_number, + resolve_call_name, ) from .pattern_defaults import PatternCategory logger = get_logger(__name__) ANALYZER_ID = "static_patterns_privilege_escalation" +USES_PYTHON_SOURCE_TYPE = True + +# Scan-scoped handle on the runner's shared Python AST cache, published by +# node() for the analyze() calls the runner makes on its behalf. This module +# stays lexical (no USES_PYTHON_AST opt-in) so its windowed and normalized +# views keep running; the constructed-path analysis below consults the shared +# cache through this key instead of reparsing. +_scan_python_ast_cache_key: ContextVar[str | None] = ContextVar( + "privilege_escalation_python_ast_cache_key", default=None +) PE1_CODE_PATTERNS = [ (r"permissions?\s*:\s*\[?\s*['\"]?\*['\"]?\s*\]?", 0.8), @@ -659,7 +679,179 @@ def _is_qualified_benign_access_requirement( return heading_index >= 0 and lines[heading_index].strip() == "## Access Requirements" -def analyze(content: str, file_path: str, file_type: str) -> list[AnalyzerFinding]: +# Cheap pre-check before walking for constructed join calls: the AST walk +# below is only worthwhile when the text plausibly contains a join() call. +# The gate covers spelling aliases collected from the imports (``from +# os.path import join as j`` binds ``j`` to ``os.path.join``, so the call +# site reads ``j(`` with no literal ``join`` in sight). Keeping the module +# lexical plus this alias-aware gate preserves the runner's parse-once +# invariant for files without any join call. +_JOIN_CALL_BASE_NAMES = {"join"} +_JOIN_CALL_ALIAS_TARGETS = {"os.path", "os.path.join"} + +# Lexical pre-check for a windowed fragment that binds a renamed ``join`` +# without ever spelling ``join(`` (``from os.path import join as j`` calls +# ``j(``). On a whole-file cache miss the fragment is parsed to collect its +# import aliases before the alias-aware join gate below runs, so such +# fragments must also clear the parse gate; import-only fragments are +# filtered out again by the alias-aware gate. +_JOIN_IMPORT_HINT = re.compile(r"^\s*from\s+os\.path\s+import\b", re.MULTILINE) + + +def _join_call_hint(aliases: dict[str, str]) -> re.Pattern[str]: + """Return a pre-check pattern matching ``join(`` and imported join aliases. + + Every local name the file binds to ``os.path`` or ``os.path.join`` is a + possible call spelling (``j(`` for ``from os.path import join as j``), + alongside the plain ``join(`` used by ``os.path.join(``, ``p.join(``, + and direct ``join(`` imports. + """ + names = set(_JOIN_CALL_BASE_NAMES) + for local, qualified in aliases.items(): + if qualified in _JOIN_CALL_ALIAS_TARGETS: + names.add(local) + return re.compile(r"\b(?:" + "|".join(sorted(re.escape(name) for name in names)) + r")\s*\(") + + +def _fragment_whole_file_line_offset(fragment: str, whole_content: str) -> int | None: + """Return how many whole-file lines precede a windowed view fragment. + + Windowed view fragments are contiguous slices of the scanned file, so a + whole-file AST line number maps onto a fragment-relative line by + subtracting this offset. Return ``None`` when the fragment is not a + slice of the whole file (for example a normalized view), so callers keep + the standalone-parse fallback. + """ + start = whole_content.find(fragment) + if start < 0: + return None + return whole_content.count("\n", 0, start) + + +def _constructed_sensitive_paths( + content: str, + file_path: str, + python_ast: ParsedPythonFile | None = None, +) -> list[tuple[int, int, str, float]]: + """Return literal sensitive paths assembled with ``os.path.join`` in Python. + + Each hit is a ``(start_line, end_line, path, confidence)`` tuple anchored + to the call's source span so callers can deduplicate raw findings across + the whole occurrence, including calls wrapped over several lines. + Resolved from the Python AST so calls split across lines and supported + import spellings (``import os.path as p``, ``from os.path import join``, + ``from os.path import join as j``, ``from os import path``) are recognized + without reparsing tricks. The scan's shared parse is reused whenever this + runs inside the runner (the cache key published by node()); standalone + callers get a single on-demand parse. Windowed view fragments under a scan + miss the whole-file cache entry; behind the same textual join-or-import + gate they are evaluated against the whole file's cached tree instead of + being parsed standalone, so a fragment starting mid-block (for example + inside a function body, which is not a valid module on its own) keeps its + findings instead of silently dropping them. The whole file's import-alias + map is carried in from the scan cache for the join gate and call + resolution, so a renamed ``os.path.join`` spelling (``from os.path import + join as j``) is recognized even when the import lives in an earlier window + than the call. Call spans map onto fragment-relative lines and only + calls starting inside the fragment are owned by it, preserving the + runner's source-coordinate and dedupe behavior. When the whole-file tree + is unavailable, or the fragment is not a slice of the whole file + (normalized views), the fragment is parsed directly as before. Only + fully-literal positional argument lists are resolved; anything dynamic is + left to the existing pattern loop. Unparseable content simply yields no + findings here. + """ + whole_file_aliases: dict[str, str] | None = None + whole_file_tree: ast.Module | None = None + whole_file_source: str | None = None + whole_file_line_offset: int | None = None + if python_ast is None: + cache_key = _scan_python_ast_cache_key.get() + if cache_key is not None: + python_ast = peek_python_ast(cache_key, content, file_path) + if python_ast is None: + # A windowed view fragment, not the scan's whole file: the + # shared tree does not cover this slice. Carry the whole + # file's import-alias map and tree from the scan cache first + # (a fragment is a slice of the same scanned file), so the + # textual gate below also fires on renamed call spellings + # like ``j(`` when the ``from os.path import join as j`` + # import lives in an earlier window. Fragments without a + # plausible join call or join import never parse. + whole_file = peek_python_ast_any_content(cache_key, file_path) + if whole_file is not None and whole_file.tree is not None: + whole_file_aliases = whole_file.import_aliases + whole_file_tree = whole_file.tree + whole_file_source = whole_file.content + if _join_call_hint(whole_file_aliases or {}).search( + content + ) or _JOIN_IMPORT_HINT.search(content): + if whole_file_tree is not None and whole_file_source is not None: + # Evaluate the whole-file tree rather than parsing the + # fragment standalone: a slice starting mid-block is + # not a valid module, so its standalone parse fails + # and the finding would be silently dropped. + whole_file_line_offset = _fragment_whole_file_line_offset( + content, whole_file_source + ) + if whole_file_line_offset is None: + python_ast = parse_python_source(content, file_path) + else: + python_ast = parse_python_source(content, file_path) + if whole_file_tree is not None and whole_file_line_offset is not None: + tree = whole_file_tree + aliases = whole_file_aliases or {} + owned_span: tuple[int, int] | None = ( + whole_file_line_offset, + whole_file_line_offset + len(content.splitlines()), + ) + else: + if python_ast is None: + return [] + tree = python_ast.tree + if tree is None: + return [] + aliases = ( + whole_file_aliases if whole_file_aliases is not None else python_ast.import_aliases + ) + owned_span = None + if not _join_call_hint(aliases).search(content): + return [] + resolved: list[tuple[int, int, str, float]] = [] + for node in ast.walk(tree): + if not isinstance(node, ast.Call): + continue + if resolve_call_name(node, aliases) != "os.path.join": + continue + if node.keywords or len(node.args) < 2: + continue + parts = [arg.value for arg in node.args if isinstance(arg, ast.Constant)] + if len(parts) != len(node.args) or not all(isinstance(part, str) for part in parts): + continue + value = posixpath.join(*parts) + start_line = node.lineno + end_line = node.end_lineno or node.lineno + if owned_span is not None: + # Only calls starting inside this fragment are owned by it; the + # runner's own owned-range filter and cross-window dedupe keep + # exactly one finding per call at whole-file coordinates. + if not owned_span[0] < start_line <= owned_span[1]: + continue + start_line -= owned_span[0] + end_line -= owned_span[0] + for pattern, confidence in PE3_PATTERNS: + if re.search(pattern, value, re.IGNORECASE): + resolved.append((start_line, end_line, value, confidence)) + break + return resolved + + +def analyze( + content: str, + file_path: str, + file_type: str, + python_ast: ParsedPythonFile | None = None, +) -> list[AnalyzerFinding]: """Analyze content for privilege escalation patterns (PE1–PE5).""" findings: list[AnalyzerFinding] = [] line_starts, line_ends = _source_line_metadata(content) @@ -808,6 +1000,40 @@ def context_at(offset: int) -> str: complete_match=match.group(0), ) ) + if file_type == "python": + for start_line, end_line, path, confidence in _constructed_sensitive_paths( + content, file_path, python_ast + ): + constructed = AnalyzerFinding( + rule_id="PE3", + message="Credential Access", + severity=Severity.HIGH, + location=loc(start_line), + confidence=confidence, + tags=list(tag), + context=get_context(content, line_starts[start_line - 1]), + matched_text=path, + ) + # One PE3 per source occurrence: the pattern loop above may already + # have fired inside the join call's line span (for example the + # literal '.ssh/id_rsa' on a wrapped argument line, while this + # finding anchors to the call's first line). Keep the + # best-confidence finding per call span, mirroring the PE4/PE5 + # per-line aggregation below; unrelated occurrences on other + # lines are preserved. + duplicate = next( + ( + existing + for existing in findings + if existing.rule_id == "PE3" + and start_line <= existing.location.start_line <= end_line + ), + None, + ) + if duplicate is None: + findings.append(constructed) + elif confidence > duplicate.confidence: + findings[findings.index(duplicate)] = constructed # Collect best-confidence PE4 finding per line to avoid double-counting lines # that match multiple patterns (e.g. DockerClient(base_url=".../docker.sock")). pe4_best: dict[int, AnalyzerFinding] = {} @@ -955,6 +1181,10 @@ def _is_negated_safety_constraint( def node(state: SkillspectorState) -> AnalyzerNodeResponse: """Run privilege_escalation patterns and return findings.""" - response = static_runner.run_static_patterns_with_ledger(state, [sys.modules[__name__]]) + token = _scan_python_ast_cache_key.set(state.get("python_ast_cache_key")) + try: + response = static_runner.run_static_patterns_with_ledger(state, [sys.modules[__name__]]) + finally: + _scan_python_ast_cache_key.reset(token) logger.info("%s: %d findings", ANALYZER_ID, len(response["findings"])) return response diff --git a/src/skillspector/nodes/analyzers/static_patterns_prompt_injection.py b/src/skillspector/nodes/analyzers/static_patterns_prompt_injection.py index 3a4ac4255..e5f538ca8 100644 --- a/src/skillspector/nodes/analyzers/static_patterns_prompt_injection.py +++ b/src/skillspector/nodes/analyzers/static_patterns_prompt_injection.py @@ -78,17 +78,24 @@ def _is_p9_skipped_path(file_path: str) -> bool: # P2: Hidden Instructions. Build the character class from the shared P9 # constant so hidden-instruction and padding detection cannot drift apart. _ZERO_WIDTH_PATTERN = "[" + "".join(sorted(ZERO_WIDTH_CHARS)) + "]" +# Bidirectional control characters (Trojan Source, CVE-2021-42574). Evaluated +# unconditionally below, NOT in the markdown-gated P2_PATTERNS list -- see the +# comment at that check for why. +# The class is the 9 paired controls only. The direction marks U+200E/U+200F/ +# U+061C are deliberately excluded: they cannot reorder a span the way the +# paired controls can, which is the same line GCC's -Wbidi-chars draws by +# default. Do not add them here without revisiting that tradeoff. +_BIDI_CONTROL_PATTERN = r"[\u202a-\u202e\u2066-\u2069]" P2_PATTERNS = [ (r"", 0.7), (r"\[//\]:\s*#\s*\(.*?(?:system|instructions?|ignore|POST|GET|send|transmit).*?\)", 0.8), (_ZERO_WIDTH_PATTERN, 0.6), - (r"[\u202a-\u202e\u2066-\u2069]", 0.85), (r"data:text/plain;base64,[A-Za-z0-9+/=]{50,}", 0.7), ] _SINGLE_CHARACTER_P2_PATTERNS = frozenset( { _ZERO_WIDTH_PATTERN, - r"[\u202a-\u202e\u2066-\u2069]", + _BIDI_CONTROL_PATTERN, } ) # P3: Exfiltration Commands @@ -457,6 +464,28 @@ def ctx(start: int) -> str: ) ) + # P2 (extended): Bidirectional control characters (Trojan Source, + # CVE-2021-42574). Runs regardless of file_type, like the Tag-block check + # above — bidi overrides are exploitable in scripts and config files too + # (issue #39). Zero-width detection stays markdown-gated because + # ZERO_WIDTH_CHARS includes U+FEFF (BOM), which would false-positive on + # every BOM-prefixed source file; the bidi range never overlaps it. + for match in _p2_pattern_matches(content, _BIDI_CONTROL_PATTERN, check_runtime): + runtime_check() + findings.append( + AnalyzerFinding( + rule_id="P2", + message="Hidden Instructions", + severity=Severity.HIGH, + location=locations.location(match.start(), match.end()), + confidence=0.85, + tags=tag, + context=ctx(match.start()), + matched_text=match.group(0)[:200], + complete_match=match.group(0), + ) + ) + # P9: Whitespace Padding (skipped for generated/vendored files). if not _is_p9_skipped_path(file_path): runtime_check() diff --git a/src/skillspector/nodes/analyzers/static_patterns_supply_chain.py b/src/skillspector/nodes/analyzers/static_patterns_supply_chain.py index 04db8cac5..43e3a818f 100644 --- a/src/skillspector/nodes/analyzers/static_patterns_supply_chain.py +++ b/src/skillspector/nodes/analyzers/static_patterns_supply_chain.py @@ -30,11 +30,14 @@ from __future__ import annotations +import ast +import base64 import codecs import io import json import os import re +import shlex import sys import time import tomllib @@ -62,6 +65,7 @@ from skillspector.logging_config import get_logger from skillspector.models import AnalyzerFinding, Finding, Location, Severity from skillspector.state import ( + MAX_WORKFLOW_SECONDS, AnalyzerNodeResponse, SkillspectorState, transitive_note_truncation, @@ -91,6 +95,9 @@ logger = get_logger(__name__) ANALYZER_ID = "static_patterns_supply_chain" +# SC2 literal-XOR decoding and SC3 key on file_type, so Python executed via a +# shebang (extensionless, .md, ...) must receive "python" rather than its suffix. +USES_PYTHON_SOURCE_TYPE = True # Dependency work is supplemental to the canonical text scan and therefore # needs its own aggregate ceilings. These apply across every manifest in a @@ -173,14 +180,30 @@ def _decoded_literal_xor_calls(content: str) -> list[tuple[int, str]]: behavioral analyzers. A second parse in static pattern analysis breaks that graph-level cache. """ - function_pattern = re.compile( - r"^def\s+(?P[A-Za-z_]\w*)\([^)]*\):(?P(?:\n[ \t]+.*)+)", - re.MULTILINE, - ) - key_pattern = re.compile(r"\b\w+\s*=\s*b(['\"])(?P(?:\\.|[^'\"])*)\1") + function_pattern = re.compile(r"^def\s+(?P[A-Za-z_]\w*)\(", re.MULTILINE) + body_pattern = re.compile(r"(?:\n[ \t]+.*)+") + # A backslash belongs only to an escape, so a missing closing quote cannot + # explore exponentially many partitions of a run of backslashes. + key_pattern = re.compile(r"\b\w+\s*=\s*b(['\"])(?P(?:\\[\s\S]|[^'\"\\])*)\1") decoded: list[tuple[int, str]] = [] + closing_paren = -1 + consumed_until = 0 for function in function_pattern.finditer(content): - body = function.group("body") + if function.start() < consumed_until: + continue + # Reuse the next closing parenthesis across malformed headers instead + # of searching the same suffix once for every unclosed function. + if closing_paren < function.end(): + closing_paren = content.find(")", function.end()) + if closing_paren < 0: + break + if not content.startswith(":", closing_paren + 1): + continue + body_match = body_pattern.match(content, closing_paren + 2) + if body_match is None: + continue + consumed_until = body_match.end() + body = body_match.group() key_match = key_pattern.search(body) if key_match is None or "bytes(" not in body or "^" not in body or ".decode(" not in body: continue @@ -215,6 +238,121 @@ def _decoded_literal_xor_calls(content: str) -> list[tuple[int, str]]: return decoded +_MAX_SHELL_LITERAL_CHARS = 16_384 +# Shell decoders that print a base64, base32 or hex payload. Flag runs stay +# short and bounded, and the group is atomic, so a flag such as -dddd that +# parses several ways cannot multiply backtracking. +_SC3_SHELL_DECODER = ( + r"(?>\bbase(?:64|32)(?:[ \t]{1,16}--?[A-Za-z][\w=-]{0,20}){0,3}?" + r"[ \t]{1,16}(?:--decode|-[A-Za-z]{0,3}d[A-Za-z]{0,3})\b" + r"|\bxxd(?:[ \t]{1,16}-[A-Za-z]{1,10}){0,3}?[ \t]{1,16}-(?:r|rp|pr|rps|psr|revert)\b" + r"|\bopenssl[ \t]{1,16}(?:base64|enc)(?:[ \t]{1,16}-[A-Za-z0-9-]{1,16}){0,4}?[ \t]{1,16}-d\b)" +) +# One pipeline stage. It ends at a pipe, a list operator, a newline, an +# inline-code backtick or a backslash (so a JSON-escaped \n ends it too), but +# keeps up to two redirections such as 2>&1 or JSON-escaped quotes. Runs are +# possessive, so a failed match never backtracks into them. +_SC3_PIPE_SEGMENT = r"[^|;&\n`\\]{0,256}+(?:(?:&[0-9>-]|\\['\"])[^|;&\n`\\]{0,64}+){0,2}+" +# A shell reads its program from stdin unless -c or a script operand supplies +# one. An interpreter counts only without a program argument: +# `python3 -m json.tool` and `python3 "$dir/render.py"` read data. A sudo +# option argument cannot start with "-", so each sudo word parses one way. +_SC3_PIPE_EXECUTOR = ( + r"\|(?!\|)&?[ \t]{0,16}" + r"(?:sudo(?:[ \t]{1,16}+-[A-Za-z]{1,8}+(?:[ \t]{1,16}+[\w.][\w.-]{0,31}+)?){0,4}[ \t]{1,16}+)?" + r"(?:(?:/[\w.-]{1,32}){0,4}/)?" + r"(?:(?:ba|da|z|k|a)?sh(?![\w.-])" + r"(?![ \t]{1,16}(?:-[A-Za-z]{1,8}[ \t]{1,16}){0,3}-[A-Za-z]{0,3}c)" + r"(?![ \t]{1,16}(?!-|/dev/stdin\b|[0-9]{0,2}[<>])[^\s|;&)#])" + r"|(?:python(?:[23](?:\.[0-9]{1,2})?)?|perl|ruby|node)(?:[ \t]{1,16}-[A-Za-z]{0,4}){0,2}" + r"(?=['\"]|[ \t]{0,16}(?:$|[\r;&|)`#]|[0-9]{0,2}>)))" +) +# eval, -c and -e run their argument whichever substitution supplies it. A bare +# shell, interpreter or source runs only content it reads from <(...), +# < <(...) or a here-string; given "$(...)" it runs a file path. Executor words +# must start a word, so `deploy.sh "$(...)"` and `--source "$(...)"` do not +# count. `.` counts only in command position, at a line start or after an +# operator, a bracket, a backtick or a quote, so `jq . <<< ...` and +# `tar -C . < <(...)` do not count. A backslash admits JSON-escaped quotes in +# hook configs. +_SC3_SUBSTITUTION_EXECUTOR = ( + r"(?:(?(?:\\?['\"])?)(?P[A-Za-z0-9+/=]{{8,{_MAX_SHELL_LITERAL_CHARS}}})" +) +_SC3_PIPED_LITERAL = re.compile( + r"\b(?:echo(?:[ \t]{1,16}-[neE]{1,3})?" + r"|printf(?:[ \t]{1,16}(?P['\"]?)%s(?:\\n)?(?P=format_quote))?)" + rf"[ \t]{{1,16}}{_SC3_SHELL_LITERAL}(?P=quote)" + rf"[ \t]{{0,16}}\|(?!\|)[ \t]{{0,16}}(?P{_SC3_SHELL_DECODER})", + re.IGNORECASE, +) +_SC3_HERE_STRING_LITERAL = re.compile( + rf"(?P{_SC3_SHELL_DECODER}){_SC3_HERE_STRING}{_SC3_SHELL_LITERAL}" + rf"(?![A-Za-z0-9+/=])(?P=quote)", + re.IGNORECASE, +) +_SC3_SHELL_DECODE_COMMAND = re.compile(_SC3_SHELL_DECODER, re.IGNORECASE) +_SC3_DECODED_DATA_SINK = re.compile( + r"(?:[ \t]{1,16}-[A-Za-z0-9-]{1,16}){0,4}[ \t]{0,16}" + r"\|(?!\|)&?[ \t]{0,16}sha256sum(?=$|[\s;&|)`])" +) + + +def _decoded_shell_literal_payloads(content: str) -> list[tuple[int, str]]: + """Decode literal payloads fed to a shell decoder. + + Only an inline echo/printf argument or a here-string is decoded, as base64, + base32 or (for xxd) hex; an ``openssl enc`` cipher is not applied. + Direct checksum consumers are exempt only in a simple shell context without + alias, function or dynamic-command syntax. Other flows remain conservative + because a saved payload can be executed later. + Variables, files and downloads stay opaque. Invalid UTF-8 becomes U+FFFD, + so a stray byte cannot hide the command. Returns ``(offset, text)`` pairs + in source order. + """ + checksum_exemption_allowed: bool | None = None + decoded: list[tuple[int, str]] = [] + for pattern in (_SC3_PIPED_LITERAL, _SC3_HERE_STRING_LITERAL): + for match in pattern.finditer(content): + if _SC3_DECODED_DATA_SINK.match(content, match.end()) is not None: + if checksum_exemption_allowed is None: + # Normalize once: escaped or quoted words can still configure aliases. + tokens = _shell_tokens(content) + checksum_exemption_allowed = tokens is not None and not any( + token in {"alias", "function", "eval", "source", "."} + or token.startswith("sha256sum=") + or any(character.isspace() or character in "()" for character in token) + for token in tokens + ) + if checksum_exemption_allowed: + continue + literal = match.group("literal") + decoder = match.group("decoder").lower() + try: + if decoder.startswith("xxd"): + # Like xxd -r -p, ignore a trailing odd nibble. + payload = bytes.fromhex(literal[: len(literal) // 2 * 2]) + elif decoder.startswith("base32"): + payload = base64.b32decode(literal + "=" * (-len(literal) % 8)) + else: + payload = base64.b64decode(literal + "=" * (-len(literal) % 4), validate=True) + except ValueError: + continue + decoded.append((match.start("literal"), payload.decode("utf-8", errors="replace"))) + decoded.sort() + return decoded + + SC3_CODE_PATTERNS = [ (r"exec\s*\(\s*(?:base64\.)?b64decode\s*\(", 0.95), (r"eval\s*\(\s*(?:base64\.)?b64decode\s*\(", 0.95), @@ -237,6 +375,23 @@ def _decoded_literal_xor_calls(content: str) -> list[tuple[int, str]]: (r"decode\s+(?:this|the)\s+(?:base64|hex)\s+(?:and\s+)?(?:run|execute)", 0.8), ] SC3_PATTERNS = SC3_CODE_PATTERNS + SC3_PROSE_PATTERNS +# Shell decode-to-execute chains run on every file type, like SC2, because +# SKILL.md instructions and hook configs carry shell commands too. +SC3_SHELL_PATTERNS = [ + ( + # A here-string literal can outgrow a pipeline stage, so it is consumed + # whole, and possessively, before the first stage. + rf"{_SC3_SHELL_DECODER}(?:{_SC3_HERE_STRING}\\?['\"]?" + rf"[A-Za-z0-9+/=]{{0,{_MAX_SHELL_LITERAL_CHARS}}}+)?+{_SC3_PIPE_SEGMENT}" + rf"(?:\|(?!\|)&?{_SC3_PIPE_SEGMENT}){{0,3}}?{_SC3_PIPE_EXECUTOR}", + 0.9, + ), + ( + rf"{_SC3_SUBSTITUTION_EXECUTOR}[^()`\n]{{0,{_MAX_SHELL_LITERAL_CHARS}}}?" + rf"{_SC3_SHELL_DECODER}", + 0.9, + ), +] # SC7: Untrusted Container Image — pulling images with signature/registry # verification turned off. These flags disable image trust regardless of the @@ -1523,7 +1678,12 @@ def line_number(start: int) -> int: file_type, line_starts, ) - if _is_safe_supply_chain_pattern(mt): + data_consumer = _interpreter_reads_stdin_as_data( + content, + match.start(), + line_starts, + ) + if _is_safe_supply_chain_pattern(mt) or data_consumer: adj = min(confidence, 0.15) sev = Severity.LOW else: @@ -1532,31 +1692,43 @@ def line_number(start: int) -> int: finding_tags = list(tag) if warned_internal_installer: finding_tags.extend(["contextual-triage", "explicit-risk-warning"]) + if data_consumer: + finding_tags.append("data-only-stdin-consumer") + if warned_internal_installer: + message = "Warned Pipe-to-Shell Installer" + remediation = ( + "Keep the warning adjacent to this command. Prefer a checksum, signature, " + "or inspect-before-execute flow instead of piping fetched content directly " + "to a shell." + ) + explanation = ( + "The matched documentation explicitly warns that an internal installer " + "is fetched and piped directly to a shell. The warning provides context, " + "but the command still executes remote code without an inspection step." + ) + elif data_consumer: + message = "Inline Program Reads Download as Data" + remediation = ( + "Prefer structured data formats and validate the fetched content before " + "using it in a program." + ) + explanation = ( + "The fetched content is parsed as data by the interpreter program supplied " + "on the command line; the download is not executed as a script." + ) + else: + message = "External Script Fetching" + remediation = None + explanation = None findings.append( AnalyzerFinding( rule_id="SC2", - message=( - "Warned Pipe-to-Shell Installer" - if warned_internal_installer - else "External Script Fetching" - ), + message=message, severity=sev, location=loc(line_num), confidence=adj, - remediation=( - "Keep the warning adjacent to this command. Prefer a checksum, signature, " - "or inspect-before-execute flow instead of piping fetched content directly " - "to a shell." - if warned_internal_installer - else None - ), - explanation=( - "The matched documentation explicitly warns that an internal installer " - "is fetched and piped directly to a shell. The warning provides context, " - "but the command still executes remote code without an inspection step." - if warned_internal_installer - else None - ), + remediation=remediation, + explanation=explanation, tags=finding_tags, context=ctx(match.start()), matched_text=mt[:200], @@ -1581,28 +1753,52 @@ def line_number(start: int) -> int: ) ) break - if file_type in ("python", "javascript", "shell", "perl", "other"): - for pattern, confidence in SC3_PATTERNS: - matches = ( - static_runner.iter_paragraph_matches - if (pattern, confidence) in SC3_PROSE_PATTERNS - else re.finditer - ) - for match in matches(pattern, content, re.IGNORECASE | re.MULTILINE): - line_num = line_number(match.start()) + # Every shell decode check needs a decoder command, which most files lack. + has_shell_decoder = _SC3_SHELL_DECODE_COMMAND.search(content) is not None + if has_shell_decoder: + for start, command in _decoded_shell_literal_payloads(content): + for pattern, confidence in SC2_PATTERNS: + if not re.search(pattern, command, re.IGNORECASE | re.MULTILINE): + continue findings.append( AnalyzerFinding( - rule_id="SC3", - message="Obfuscated Code", + rule_id="SC2", + message="External Script Fetching", severity=Severity.HIGH, - location=loc(line_num), + location=loc(line_number(start)), confidence=confidence, - tags=tag, - context=ctx(match.start()), - matched_text=match.group(0)[:200], - complete_match=match.group(0), + tags=list(tag), + context=ctx(start), + matched_text=command[:200], ) ) + break + sc3_patterns = ( + SC3_PATTERNS if file_type in ("python", "javascript", "shell", "perl", "other") else [] + ) + if has_shell_decoder: + sc3_patterns = sc3_patterns + SC3_SHELL_PATTERNS + for pattern, confidence in sc3_patterns: + matches = ( + static_runner.iter_paragraph_matches + if (pattern, confidence) in SC3_PROSE_PATTERNS + else re.finditer + ) + for match in matches(pattern, content, re.IGNORECASE | re.MULTILINE): + line_num = line_number(match.start()) + findings.append( + AnalyzerFinding( + rule_id="SC3", + message="Obfuscated Code", + severity=Severity.HIGH, + location=loc(line_num), + confidence=confidence, + tags=tag, + context=ctx(match.start()), + matched_text=match.group(0)[:200], + complete_match=match.group(0), + ) + ) # SC7: untrusted container image. Example filtering is delegated to the runner. for pattern, confidence in SC7_PATTERNS: for match in re.finditer(pattern, content, re.IGNORECASE | re.MULTILINE): @@ -1704,6 +1900,238 @@ def _is_safe_supply_chain_pattern(text: str) -> bool: return _is_trusted_source(text) or bool(_SAFE_INSTALL_PATTERN.search(text)) +# A fetched program supplied on the interpreter command line leaves the +# download as data on stdin. Every other shape stays HIGH. +_MAX_SC2_LOGICAL_LINE_CHARS = 4_096 +_PYTHON_SAFE_CALLS = frozenset( + { + "bool", + "dict", + "float", + "int", + "json.dump", + "json.dumps", + "json.load", + "json.loads", + "len", + "list", + "print", + "str", + "sys.stderr.write", + "sys.stdin.buffer.read", + "sys.stdin.buffer.readline", + "sys.stdin.read", + "sys.stdin.readline", + "sys.stdout.write", + } +) +_PYTHON_SAFE_CALL_ROOTS = frozenset(name.split(".", 1)[0] for name in _PYTHON_SAFE_CALLS) +_PYTHON_SAFE_ATTRIBUTES = frozenset( + { + "buffer", + "dump", + "dumps", + "load", + "loads", + "read", + "readline", + "stderr", + "stdin", + "stdout", + "write", + } +) +_PYTHON_DANGEROUS = re.compile( + r"\b(?:exec|eval|compile|execfile|__import__|getattr|setattr|delattr|" + r"globals|locals|vars|open|input|system|popen|spawn\w*|runpy|pickle|" + r"marshal|dill|shelve|importlib|subprocess|os)\b|__" +) +_PYTHON_BINDING_NODES = ( + ast.AsyncFunctionDef, + ast.ClassDef, + ast.FunctionDef, + ast.Global, + ast.Lambda, + ast.Nonlocal, +) + + +def _sc2_logical_command( + content: str, + match_start: int, + line_starts: tuple[int, ...], +) -> str | None: + """Return the shell logical line containing *match_start*, if it starts it.""" + line_index = bisect_right(line_starts, match_start) - 1 + if line_index < 0: + return None + line_start = line_starts[line_index] + + # Walk back only across escaped shell newlines after locating the current + # logical line in O(log n). Any other logical break is not a shell + # command boundary and makes the exemption unsafe. + while line_index > 0: + separator_start = ( + line_start - 2 + if line_start >= 2 and content[line_start - 2 : line_start] == "\r\n" + else line_start - 1 + ) + separator = LOGICAL_LINE_BREAK.match(content, separator_start) + if separator is None: + return None + if separator.group(0) not in {"\n", "\r\n"}: + return None + if separator_start == 0 or content[separator_start - 1] != "\\": + break + line_index -= 1 + line_start = line_starts[line_index] + if match_start - line_start > _MAX_SC2_LOGICAL_LINE_CHARS: + return None + + if match_start - line_start > _MAX_SC2_LOGICAL_LINE_CHARS: + return None + + line_end = len(content) + search_pos = match_start + search_end = min(len(content), line_start + _MAX_SC2_LOGICAL_LINE_CHARS + 1) + while search_pos <= len(content): + separator = LOGICAL_LINE_BREAK.search(content, search_pos, search_end) + if separator is None: + if len(content) - line_start > _MAX_SC2_LOGICAL_LINE_CHARS: + return None + break + if separator.group(0) not in {"\n", "\r\n"}: + return None + if separator.start() > 0 and content[separator.start() - 1] == "\\": + search_pos = separator.end() + if search_pos - line_start > _MAX_SC2_LOGICAL_LINE_CHARS: + return None + continue + line_end = separator.start() + break + + if line_end - line_start > _MAX_SC2_LOGICAL_LINE_CHARS: + return None + + # Only lower a pipeline that begins the logical line. This rejects + # command substitutions, wrappers such as ``bash -c``, and ``eval``. + if content[line_start:match_start].strip(): + return None + command = content[line_start:line_end] + return re.sub(r"\\\r?\n", " ", command) + + +def _shell_tokens(command: str) -> list[str] | None: + """Tokenize a conservative subset of shell syntax, or return None.""" + if re.search(r"[`$]", command): + return None + lexer = shlex.shlex(command, posix=True, punctuation_chars="|&;<>()") + lexer.whitespace_split = True + lexer.commenters = "" + try: + return list(lexer) + except ValueError: + return None + + +def _dotted_name(node: ast.expr) -> str | None: + """Return a dotted name for ``a.b.c`` expressions, otherwise None.""" + parts: list[str] = [] + current = node + while isinstance(current, ast.Attribute): + parts.append(current.attr) + current = current.value + if not isinstance(current, ast.Name): + return None + parts.append(current.id) + return ".".join(reversed(parts)) + + +def _python_stdin_script_is_data_only(script: str) -> bool: + """Return whether a Python ``-c`` script only treats stdin as data. + + This is deliberately a structural allowlist rather than a name-based + denylist. Rebinding an allowlisted call root, any attribute or subscript + store, and any attribute outside the small data-processing allowlist makes + the script ambiguous and therefore ineligible for the SC2 downgrade. + """ + if not script.strip() or len(script) > _MAX_SC2_LOGICAL_LINE_CHARS: + return False + try: + tree = ast.parse(script, mode="exec") + except (SyntaxError, ValueError): + return False + + for node in ast.walk(tree): + if isinstance(node, _PYTHON_BINDING_NODES): + return False + if isinstance(node, ast.Import): + if any(alias.name not in {"json", "sys"} or alias.asname for alias in node.names): + return False + elif isinstance(node, ast.ImportFrom): + return False + elif isinstance(node, ast.Name): + if isinstance(node.ctx, (ast.Store, ast.Del)): + if node.id in _PYTHON_SAFE_CALL_ROOTS or node.id.startswith("__"): + return False + elif _PYTHON_DANGEROUS.search(node.id): + return False + elif isinstance(node, ast.Attribute): + if node.attr not in _PYTHON_SAFE_ATTRIBUTES: + return False + if isinstance(node.ctx, (ast.Store, ast.Del)): + return False + elif isinstance(node, ast.Subscript): + if isinstance(node.ctx, (ast.Store, ast.Del)): + return False + elif isinstance(node, ast.Call): + name = _dotted_name(node.func) + if name is None or name not in _PYTHON_SAFE_CALLS: + return False + return True + + +def _interpreter_reads_stdin_as_data( + content: str, + match_start: int, + line_starts: tuple[int, ...], +) -> bool: + """Return whether the piped-to interpreter receives a command-line program.""" + command = _sc2_logical_command(content, match_start, line_starts) + if command is None: + return False + tokens = _shell_tokens(command) + if tokens is None: + return False + if any( + token in {"|&", "||", "&&", ";", "&", "<", ">", ">>", "<<", "(", ")"} for token in tokens + ): + return False + pipes = [index for index, token in enumerate(tokens) if token == "|"] + if len(pipes) != 1: + return False + + pipe_index = pipes[0] + fetch = tokens[:pipe_index] + tail = tokens[pipe_index + 1 :] + if not fetch or fetch[0].lower() not in {"curl", "wget"}: + return False + if tail[:1] == ["sudo"]: + tail = tail[1:] + if not tail: + return False + + interpreter = tail[0].lower() + args = tail[1:] + if interpreter in {"python", "python3"}: + return args == ["-m", "json.tool"] or ( + len(args) == 2 and args[0] == "-c" and _python_stdin_script_is_data_only(args[1]) + ) + # Node remains HIGH: arbitrary JavaScript cannot be proven data-only by + # this lightweight allowlist (computed properties can recover Function). + return False + + # --------------------------------------------------------------------------- # SC4–SC6: Dependency-level analysis (runs per dependency file) # --------------------------------------------------------------------------- @@ -2380,6 +2808,8 @@ def _analyze_triggers( MAX_SC8_DISCOVERED_ENTRIES = 10_000 MAX_SC8_DIRECTORY_ENTRIES = 10_000 MAX_SC8_TRAVERSAL_DEPTH = 64 +# Processing budget, measured with ``time.thread_time()`` so that time spent +# waiting for the GIL behind concurrently running analyzers does not count. MAX_SC8_ANALYSIS_SECONDS = 5.0 MAX_SC8_FINDINGS = 10_000 MAX_SC8_LIMITATIONS = 256 @@ -2414,7 +2844,14 @@ def _scan_shipped_bytecode( timeout_seconds: float | None = None, max_findings: int | None = None, ) -> _ShippedBytecodeScanResult: - """Discover shipped bytecode with deterministic aggregate resource bounds.""" + """Discover shipped bytecode with deterministic aggregate resource bounds. + + Analyzers run concurrently in one process, so the walk can spend most of + its wall-clock time waiting for the GIL. The processing budget therefore + counts only this thread's CPU time. Wall-clock time is bounded by the + remaining workflow deadline (``timeout_seconds``), or by the full workflow + limit for callers outside a workflow. + """ findings: list[Finding] = [] limitations: list[_SupplementalLimitation] = [] if not skill_path or not isinstance(skill_path, str): @@ -2424,10 +2861,11 @@ def _scan_shipped_bytecode( return _ShippedBytecodeScanResult(findings, limitations) started_at = time.monotonic() - runtime_limit = max(0.0, MAX_SC8_ANALYSIS_SECONDS) - if timeout_seconds is not None: - runtime_limit = min(runtime_limit, max(0.0, timeout_seconds)) - deadline = started_at + runtime_limit + active_started_at = time.thread_time() + active_limit = max(0.0, MAX_SC8_ANALYSIS_SECONDS) + wall_clock_limit = max( + 0.0, MAX_WORKFLOW_SECONDS if timeout_seconds is None else timeout_seconds + ) requested_finding_limit = MAX_SC8_FINDINGS if max_findings is None else max_findings finding_limit = max(0, min(requested_finding_limit, MAX_SC8_FINDINGS)) discovered_entries = 0 @@ -2451,15 +2889,20 @@ def add_limitation(limitation: _SupplementalLimitation) -> None: ) def runtime_exhausted(relative_directory: str) -> bool: - now = time.monotonic() - if now < deadline: + wall_clock_elapsed = max(0.0, time.monotonic() - started_at) + active_elapsed = max(0.0, time.thread_time() - active_started_at) + if wall_clock_elapsed >= wall_clock_limit: + observed_seconds, limit_seconds = wall_clock_elapsed, wall_clock_limit + elif active_elapsed >= active_limit: + observed_seconds, limit_seconds = active_elapsed, active_limit + else: return False add_limitation( _SupplementalLimitation( path=scope_path(relative_directory), reason=LedgerReason.RUNTIME_LIMIT, - observed_seconds=max(0.0, now - started_at), - limit_seconds=runtime_limit, + observed_seconds=observed_seconds, + limit_seconds=limit_seconds, ) ) return True @@ -2658,6 +3101,7 @@ def _analyze_concealed_executables( concealment_reasons.append("disguised_container") concealment = concealment_reasons[0] excluded_from_analysis = metadata.get("excluded_from_analysis") is True + partially_analyzed_executable = metadata.get("partially_analyzed_executable") is True referenced_uninspected = ( metadata.get("inspection_limitation_reason") == LedgerReason.REFERENCED_UNINSPECTED.value @@ -2670,6 +3114,8 @@ def _analyze_concealed_executables( if referenced_uninspected else "An excluded artifact could not be completely inspected." if inspection_incomplete + else "Readable text was inspected, but binary executable behavior was not analyzed." + if partially_analyzed_executable else "Executable content is excluded from analysis." if excluded_from_analysis else "Executable content is concealed inside a document, hidden, " @@ -2695,6 +3141,9 @@ def _analyze_concealed_executables( else "A resource, read, or archive-safety limit left excluded content " "outside deterministic inspection coverage." if inspection_incomplete + else "Text-based checks covered the readable projection, but they cannot " + "establish the behavior of the binary executable." + if partially_analyzed_executable else "An executable artifact remains available under the skill install path " "but its content is outside analyzer coverage." if excluded_from_analysis @@ -2727,6 +3176,7 @@ def _analyze_concealed_executables( "local_only": True, "referenced": metadata.get("referenced") is True, "excluded_from_analysis": excluded_from_analysis, + "partially_analyzed_executable": partially_analyzed_executable, "excluded_inspection_incomplete": inspection_incomplete, "inherited_exclusion_reason": metadata.get("inherited_exclusion_reason"), "inspection_limitation_reason": metadata.get("inspection_limitation_reason"), diff --git a/src/skillspector/nodes/analyzers/static_patterns_tool_misuse.py b/src/skillspector/nodes/analyzers/static_patterns_tool_misuse.py index eb7fc60d5..dd12f323d 100644 --- a/src/skillspector/nodes/analyzers/static_patterns_tool_misuse.py +++ b/src/skillspector/nodes/analyzers/static_patterns_tool_misuse.py @@ -25,31 +25,54 @@ from __future__ import annotations import ast +import heapq import re import sys -from bisect import bisect_right +import time +from bisect import bisect_left, bisect_right from collections.abc import Callable, Iterator from dataclasses import dataclass - +from hashlib import sha256 + +from skillspector.artifacts import ( + SecurityTextView, + _has_derived_security_view, + normalized_security_prefix, + normalized_security_view, + security_text_views, +) from skillspector.logging_config import get_logger -from skillspector.models import AnalyzerFinding, Location, Severity -from skillspector.python_ast import parse_python_source +from skillspector.models import AnalyzerFinding, Finding, Location, Severity +from skillspector.python_ast import ParsedPythonFile, parse_python_source from skillspector.security_reconstruction import validated_json_string_spans from skillspector.state import AnalyzerNodeResponse, SkillspectorState from . import static_runner from .common import ( LINE_BREAK_CHARS, + LOGICAL_LINE_BREAK, MARKDOWN_FENCE_CLOSE, MARKDOWN_FENCE_OPEN, get_context, - get_line_number, ) from .pattern_defaults import PatternCategory logger = get_logger(__name__) ANALYZER_ID = "static_patterns_tool_misuse" +ANALYZE_USES_POSTPROCESS = True +POSTPROCESS_USES_PYTHON_AST = True +POSTPROCESS_USES_RUNTIME_BUDGET = True +_VARIABLE_SHELL_FLAG_EVIDENCE = "_tm1_variable_shell_flag" +LEXICAL_DIRECT_SHELL_EVIDENCE = "_tm1_lexical_direct_shell" +LEXICAL_RAW_OWNER_EVIDENCE = "_tm1_lexical_raw_owner" +LEXICAL_NORMALIZED_MATCH_EVIDENCE = "_tm1_lexical_normalized_match" +LEXICAL_BOUND_IDENTITY_EVIDENCE = "_tm1_lexical_bound_identity" +LEXICAL_BOUND_IDENTITY_SOURCE_EVIDENCE = "_tm1_lexical_bound_identity_source" +LEXICAL_BOUND_CLASSIFICATION_EVIDENCE = "_tm1_lexical_bound_classification" +LEXICAL_REACH_TERMINATED_EVIDENCE = "_tm1_lexical_reach_terminated" +LEXICAL_BOUND_REACHABLE_EVIDENCE = "_tm1_lexical_bound_reachable" +_MAX_DIRECT_SHELL_CONTEXT_CHARS = 1024 _SHELL_COMMAND_WORD_START_RE = re.compile(r"[rRdDeE$'\"`\\]") _SHELL_COMMAND_WORD_CHARS = 4096 @@ -60,12 +83,40 @@ _PRINTF_STATIC_CHARS = 256 _PRINTF_STATIC_ARGUMENTS = 32 _PRINTF_STATIC_WORD_RE = re.compile(r"[-A-Za-z0-9_./*?%]{0,64}") +# A bounded ``printf`` format operand may also carry POSIX backslash escapes. +_PRINTF_FORMAT_WORD_RE = re.compile(r"[-A-Za-z0-9_./*?%\\]{0,64}") +# The old deterministic alphabet plus the control/high-byte values needed by +# byte-oriented probes. Command substitutions are expanded after shell syntax +# has been recognized, so decoded angle brackets remain literal word data +# rather than redirections. Anything else outside this set is either a shell +# separator, a metacharacter the downstream glob model does not handle, or +# otherwise not provably inert, so the substitution stays undecidable. +_PRINTF_SAFE_WORD_RE = re.compile(r"[\x01-\x08\x0e-\x1f\x7f-\xffA-Za-z0-9_./*?%<>-]*") _DESTRUCTIVE_COMMAND_BASENAMES = frozenset({"rm", "del", "erase"}) _QUOTED_GLOB_SENTINEL = "\ue000" _DYNAMIC_SHELL_WORD_SENTINEL = "\ue001" _RUNTIME_SHELL_PARAMETER_SENTINEL = "\ue002" -_SIMPLE_BRACED_PARAMETER_RE = re.compile(r"\$\{(?:[A-Za-z_][A-Za-z0-9_]*|[0-9]+|[@*#?$!-])\}") +# A braced expansion head whose result depends only on parameter values: an +# optional length prefix, a name with an optional literal subscript or a +# numeric/special parameter, then the closing brace or a value operator. +# Indirection, arithmetic subscripts, substring offsets, ``@`` transformations, +# zsh flags and ``${ cmd; }`` can evaluate code, so they do not match. The +# documentation placeholder ``${...}`` is a bad substitution in every shell. +_VALUE_PARAMETER_EXPANSION_HEAD_RE = re.compile( + r"\$\{(?:#?(?:[A-Za-z_][A-Za-z0-9_]*(?:\[(?:[@*]|[0-9]+)\])?|[0-9]+|[@*#?$!-])" + r"(?:\}|:?[-=?+]|##?|%%?|/[/#%]?|\^\^?|,,?)|\.\.\.\})" +) +_VALUE_PARAMETER_EXPANSION_START_RE = re.compile(r"\$\{") +_EXECUTABLE_EXPANSION_MARKERS = ("`", "$(", "$[", "<(", ">(") +_SHELL_ASSIGNMENT_PREFIX_RE = re.compile(r"[A-Za-z_][A-Za-z0-9_]*\+?=") +_SHELL_ASSIGNMENT_NAME_CHARACTERS = frozenset( + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_" +) +_SHELL_CLAUSE_BOUNDARY_CHARACTERS = "\r\n;|&(){}`" _SHELL_DELIMITER_WORD_RE = re.compile(r"[^$'\"`\\(){}<>#;|&!\s]++") +# A ``#`` that starts a word begins a shell comment. Quoting is not tracked, so +# ``echo "a # b"`` also matches; callers only use this to restrict ownership. +_SHELL_COMMENT_START_RE = re.compile(r"(?])#") _PERL_LITERAL_PRINT_RE = re.compile( r"^[ \t]*+print\b(?:[ \t]++(?:STDOUT|STDERR)\b)?[ \t]*+(?P\()?[ \t]*+" r"(?P\"(?:\\[\\\"'nrt]|[^\\\"$@`\r\n])*+\"" @@ -90,9 +141,18 @@ _SHELL_COMMAND_STRING_SHELLS = frozenset({"sh", "ash", "bash", "dash", "ksh", "yash", "zsh"}) _SHELL_COMMAND_STRING_ARGUMENTS = 32 _SHELL_CLAUSE_PREFIX_WORDS = frozenset({"do", "else", "elif", "then", "time", "!"}) +_SHELL_ASSIGNMENT_PREFIX_WORDS = _SHELL_CLAUSE_PREFIX_WORDS | frozenset( + {"if", "while", "until", "export", "local", "declare", "readonly", "typeset"} +) _SHELL_REDIRECTION_PREFIX_RE = re.compile(r"(?:[0-9]+)?(?:&>>?|<>|>>?|<<-?|>&|<&|[<>])") _RECURSIVE_OPTION_SOURCE_RE = re.compile(r"-(?:[A-Za-z]*[rR]|-recursive)") _FORCE_OPTION_SOURCE_RE = re.compile(r"-(?:[A-Za-z]*f|-force)") +_MARKDOWN_CONTRACTION_PROSE_RE = re.compile( + r"(?:The|This|That|These|Those|It|They|We|You|I|A|An)[ \t]+[A-Za-z \t,]*\b" + r"(?:doesn|isn|aren|wasn|weren|don|didn|hasn|haven|hadn|can|couldn|shouldn|wouldn|won|mustn)" + r"(?P')t\b[A-Za-z \t,]*[.!?]", + re.IGNORECASE, +) _ROOT_GLOB_DOCUMENTATION_LINE_RE = re.compile( r"[ \t]*(?:(?:[-*+]|#{1,6})[ \t]+)?" r"(?:(?:(?:documentation|note|example)[ \t]*:[ \t]*)" @@ -143,23 +203,29 @@ # Match a literal True assigned to a local name shortly before it is passed as # shell= to a subprocess invocation. The bounded newline gap and the -# intervening-write guard keep this a local data-flow fact; Python scope +# AST binding-evidence guard keep this a local data-flow fact; Python scope # visibility is enforced separately in analyze() via _VARIABLE_SHELL_FLAG_RE. +_VARIABLE_SHELL_ASSIGNMENT_PATTERN = r"(?m)^\s*([A-Za-z_]\w*)\s*=\s*True\s*$\n" _VARIABLE_SHELL_FLAG_PATTERN = ( - r"(?m)^\s*([A-Za-z_]\w*)\s*=\s*True\s*$\n" - r"(?:(?![^\n]*\b\1\s*=)[^\n]{0,240}\n){0,4}?[^\n]{0,240}" + _VARIABLE_SHELL_ASSIGNMENT_PATTERN + r"(?:[^\n]{0,240}\n){0,4}?[^\n]{0,240}" r"(?:subprocess\.\w+|Popen)\s*\([^)]*\bshell\s*=\s*\1\b" ) _VARIABLE_SHELL_FLAG_RE = re.compile(_VARIABLE_SHELL_FLAG_PATTERN, re.IGNORECASE | re.MULTILINE) +_VARIABLE_SHELL_ASSIGNMENT_RE = re.compile( + _VARIABLE_SHELL_ASSIGNMENT_PATTERN, re.IGNORECASE | re.MULTILINE +) +_SHELL_NAME_ARGUMENT_RE = re.compile(r"\bshell\s*=\s*\w+", re.IGNORECASE) # TM1: Tool Parameter Abuse — dangerous parameter values -TM1_CODE_PATTERNS = [ +DIRECT_SHELL_TRUE_PATTERNS: tuple[tuple[str, float], ...] = ( # shell=True is a classic command injection vector - (r"subprocess\.\w+\s*\([^)]*shell\s*=\s*True", 0.8), - (r"Popen\s*\([^)]*shell\s*=\s*True", 0.8), - # Preserve the direct-call signal when a Python boolean is assigned to a - # local name immediately before the invocation. The bounded newline gap - # avoids treating an arbitrary distant assignment as a data-flow fact. + (r"subprocess\.\w+\s*\([^)]*shell\s*=\s*True\b", 0.8), + (r"Popen\s*\([^)]*shell\s*=\s*True\b", 0.8), +) +_DIRECT_SHELL_TRUE_VALUE = re.compile(r"shell\s*=\s*True\b", re.IGNORECASE) +TM1_CODE_PATTERNS = [ + *DIRECT_SHELL_TRUE_PATTERNS, + # Preserve the current-main fallback when the Python AST is unavailable. (_VARIABLE_SHELL_FLAG_PATTERN, 0.8), # Bound command names on both sides so prefixes such as rmm/ (RAPIDS # Memory Manager headers) are not interpreted as destructive commands. @@ -932,14 +998,27 @@ def next_word() -> str | None: return True, arguments +def _is_value_parameter_expansion(content: str, start: int, end: int) -> bool: + """Return whether a braced expansion and its nested expansions substitute only values.""" + expansion = content[start:end] + if any(marker in expansion for marker in _EXECUTABLE_EXPANSION_MARKERS): + return False + return all( + _VALUE_PARAMETER_EXPANSION_HEAD_RE.match(expansion, nested.start()) is not None + for nested in _VALUE_PARAMETER_EXPANSION_START_RE.finditer(expansion) + ) + + def _invocation_expansion_marker(content: str, start: int, end: int) -> str: """Distinguish runtime-only parameters from possible command reconstruction.""" if content.startswith("$(", start) or ( - content.startswith("${", start) - and _SIMPLE_BRACED_PARAMETER_RE.fullmatch(content, start, end) is None + content.startswith("${", start) and not _is_value_parameter_expansion(content, start, end) ): - # Complex parameter expansions may contain nested command substitutions. + # Other parameter expansions may contain nested command substitutions + # or evaluate a value as code. return _DYNAMIC_SHELL_WORD_SENTINEL + # ``${NAME:-default}``, ``${ARR[@]}`` and similar forms select the same + # runtime value as ``$NAME`` and need the same printf invocation evidence. return _RUNTIME_SHELL_PARAMETER_SENTINEL @@ -1167,6 +1246,77 @@ def _next_shell_invocation_word( return "".join(output) if word_started else None, cursor, False +_PRINTF_HEX_ESCAPE_RE = re.compile(r"[0-9A-Fa-f]{1,2}") +_PRINTF_OCTAL_ESCAPE_RE = re.compile(r"[0-7]{1,3}") +_PRINTF_ESCAPE_SEQUENCES = { + "a": "\a", + "b": "\b", + "e": "\x1b", + "E": "\x1b", + "f": "\f", + "n": "\n", + "r": "\r", + "t": "\t", + "v": "\v", + "\\": "\\", + "'": "'", + '"': '"', +} + + +def _decode_printf_escapes(text: str) -> str | None: + """Decode bounded escapes as modeled by Bash builtin/coreutils ``printf``. + + This intentionally models the Bash/GNU-coreutils escape subset used by the + static parser. Extensions such as ``\\xHH``, ``\\e``/``\\E``, ``\\'`` and + ``\\"`` are not portable to a dash ``printf`` and are not a general POSIX + guarantee. Unknown or trailing escapes make the substitution undecidable + and return ``None`` so the caller keeps its fail-closed behaviour. + """ + output: list[str] = [] + cursor = 0 + while cursor < len(text): + character = text[cursor] + if character != "\\": + output.append(character) + cursor += 1 + continue + if cursor + 1 >= len(text): + return None + escaped = text[cursor + 1] + simple = _PRINTF_ESCAPE_SEQUENCES.get(escaped) + if simple is not None: + output.append(simple) + cursor += 2 + continue + if escaped == "x": + digits = _PRINTF_HEX_ESCAPE_RE.match(text, cursor + 2) + if digits is None: + return None + output.append(chr(int(digits.group(0), 16))) + cursor = digits.end() + continue + if escaped in "01234567": + digits = _PRINTF_OCTAL_ESCAPE_RE.match(text, cursor + 1) + if digits is None: + return None + value = int(digits.group(0), 8) + if value > 0o377: + return None + output.append(chr(value)) + cursor = digits.end() + continue + return None + return "".join(output) + + +def _bounded_printf_word(result: str) -> str | None: + """Return a decoded ``printf`` result only when it stays one inert word.""" + if _PRINTF_SAFE_WORD_RE.fullmatch(result) is None: + return None + return result + + def _static_printf_substitution( content: str, start: int, @@ -1190,13 +1340,29 @@ def _static_printf_substitution( if not arguments or len(arguments) > _PRINTF_STATIC_ARGUMENTS + 1: return None format_word, *values = arguments - if _PRINTF_STATIC_WORD_RE.fullmatch(format_word) is None or any( + if _PRINTF_FORMAT_WORD_RE.fullmatch(format_word) is None or any( _PRINTF_STATIC_WORD_RE.fullmatch(value) is None for value in values ): return None segments: list[str | None] = [] literal: list[str] = [] + + def flush_literal() -> bool: + # POSIX ``printf`` decodes escapes in the format operand before any + # conversion runs, so decode each literal segment in place. Decoding + # after the conversion split keeps an escaped percent sign from + # re-entering conversion parsing. + text = "".join(literal) + literal.clear() + if "\\" in text: + decoded = _decode_printf_escapes(text) + if decoded is None: + return False + text = decoded + segments.append(text) + return True + cursor = 0 while cursor < len(format_word): character = format_word[cursor] @@ -1206,18 +1372,17 @@ def _static_printf_substitution( continue if cursor + 1 >= len(format_word) or format_word[cursor + 1] not in "s%": return None - if literal: - segments.append("".join(literal)) - literal.clear() + if literal and not flush_literal(): + return None conversion = format_word[cursor + 1] segments.append(None if conversion == "s" else "%") cursor += 2 - if literal: - segments.append("".join(literal)) + if literal and not flush_literal(): + return None conversions = sum(segment is None for segment in segments) if conversions == 0: - return format_word + return _bounded_printf_word("".join(segments)) remaining = iter(values) output: list[str] = [] cycles = max(1, (len(values) + conversions - 1) // conversions) @@ -1226,8 +1391,7 @@ def _static_printf_substitution( output.append(next(remaining, "") if segment is None else segment) if sum(len(piece) for piece in output) > 64: return None - result = "".join(output) - return result if _PRINTF_STATIC_WORD_RE.fullmatch(result) is not None else None + return _bounded_printf_word("".join(output)) def _may_have_destructive_outer_operands(content: str) -> bool: @@ -1413,6 +1577,50 @@ def _skip_backtick_substitution( ) +def _previous_word_end(content: str, start: int) -> int | None: + """Return where the previous word in this clause ends, or ``None`` at a clause start.""" + cursor = start + while cursor > 0 and content[cursor - 1] in " \t": + cursor -= 1 + if cursor == 0 or content[cursor - 1] in _SHELL_CLAUSE_BOUNDARY_CHARACTERS: + return None + return cursor + + +def _is_assignment_word(content: str, start: int) -> bool: + """Return whether ``start`` begins a prefix assignment word or its quoted value.""" + word_start = start + # The exhaustion sweep already skips the checks for a quote after ``=``. + # Recognizing it here keeps the parser consistent for any caller and avoids + # evaluating printf reconstruction whose result would be discarded. + if 0 < start < len(content) and content[start] in "'\"" and content[start - 1] == "=": + word_start = start - 1 + if word_start > 0 and content[word_start - 1] == "+": + word_start -= 1 + while word_start > 0 and content[word_start - 1] in _SHELL_ASSIGNMENT_NAME_CHARACTERS: + word_start -= 1 + prefix = _SHELL_ASSIGNMENT_PREFIX_RE.match(content, word_start) + if prefix is None or (word_start != start and prefix.end() != start): + return False + # The shell recognizes assignments only before the command name. After + # ``alias``, ``echo`` or a path prefix, ``NAME=value`` is an ordinary word. + previous_end = _previous_word_end(content, word_start) + if previous_end is None: + return True + if previous_end == word_start: + return False + previous_start = previous_end + while ( + previous_start > 0 + and content[previous_start - 1] not in " \t" + _SHELL_CLAUSE_BOUNDARY_CHARACTERS + ): + previous_start -= 1 + return ( + content[previous_start:previous_end] in _SHELL_ASSIGNMENT_PREFIX_WORDS + and _previous_word_end(content, previous_start) is None + ) + + def _parse_shell_command_word( content: str, start: int, @@ -1421,9 +1629,31 @@ def _parse_shell_command_word( backtick_end_cache: dict[int, int | None] | None = None, *, check_runtime: Callable[[], None] | None = None, + owned_word_positions: set[int] | None = None, + enclosing_delimiter: str | None = None, ) -> _ShellCommandWord | None: runtime_check = check_runtime or (lambda: None) + # The shell recognizes ``NAME=value`` before expansion and never runs the + # value as the command name. Its substitutions keep their own candidate + # positions, and a later ``$NAME`` command word is still checked, so only + # this word's printf reconstruction check is skipped. + assignment = _is_assignment_word(content, start) + + def reconstructs_command( + substitution_start: int, substitution_end: int, *, backtick: bool = False + ) -> bool: + return not assignment and _is_printf_substitution( + content, + substitution_start, + substitution_end, + backtick=backtick, + check_runtime=runtime_check, + ) + output: list[str] = [] + wrapper_quote = enclosing_delimiter or ( + _command_wrapper_quote(content, start) if content[start] in "$`" else None + ) quote: str | None = None ansi_c_quote = False dynamic = False @@ -1437,6 +1667,8 @@ def _parse_shell_command_word( character = content[cursor] if quote is not None: if character == quote: + if owned_word_positions is not None: + owned_word_positions.add(cursor) quote = None ansi_c_quote = False elif character == "\\" and ansi_c_quote: @@ -1471,12 +1703,7 @@ def _parse_shell_command_word( if static_value is None: output.append("$()") dynamic = True - limited = limited or _is_printf_substitution( - content, - cursor, - substitution_end, - check_runtime=runtime_check, - ) + limited = limited or reconstructs_command(cursor, substitution_end) else: output.append(static_value) cursor = substitution_end @@ -1521,12 +1748,8 @@ def _parse_shell_command_word( if static_value is None: output.append("$()") dynamic = True - limited = limited or _is_printf_substitution( - content, - cursor, - substitution_end, - backtick=True, - check_runtime=runtime_check, + limited = limited or reconstructs_command( + cursor, substitution_end, backtick=True ) else: output.append(static_value) @@ -1569,12 +1792,7 @@ def _parse_shell_command_word( if static_value is None: output.append("$()") dynamic = True - limited = limited or _is_printf_substitution( - content, - cursor, - substitution_end, - check_runtime=runtime_check, - ) + limited = limited or reconstructs_command(cursor, substitution_end) else: output.append(static_value) cursor = substitution_end @@ -1598,6 +1816,11 @@ def _parse_shell_command_word( cursor = parameter_end continue elif character == "`": + if enclosing_delimiter == "`": + # The enclosing substitution ends here; this is not an opener. + if owned_word_positions is not None: + owned_word_positions.add(cursor) + break substitution_end = _skip_backtick_substitution( content, cursor, @@ -1618,18 +1841,14 @@ def _parse_shell_command_word( if static_value is None: output.append("$()") dynamic = True - limited = limited or _is_printf_substitution( - content, - cursor, - substitution_end, - backtick=True, - check_runtime=runtime_check, - ) + limited = limited or reconstructs_command(cursor, substitution_end, backtick=True) else: output.append(static_value) cursor = substitution_end continue elif character in "'\"": + if character == wrapper_quote: + break quote = character elif character == "\\" and cursor + 1 < limit: if content[cursor + 1] == "\n": @@ -1726,6 +1945,9 @@ def _has_shell_command_word_exhaustion( ) -> bool: """Find candidate command words whose deterministic parse hit a safety bound.""" parsed_through = 0 + # Completed words own closing quotes, never executable expansion starts. + # Inner commands remain independent candidates; never suppress their bodies. + owned_word_positions: set[int] = set() parameter_end_cache: dict[int, _ParameterExpansionEnd] = {} substitution_end_cache: dict[int, int | None] = {} backtick_end_cache: dict[int, int | None] = {} @@ -1734,17 +1956,23 @@ def _has_shell_command_word_exhaustion( for candidate in _SHELL_COMMAND_WORD_START_RE.finditer(content): check_runtime() start = candidate.start() + if start in owned_word_positions: + continue if structural_quote_closers is not None and start in structural_quote_closers: continue json_string_start = structural_quote_openers is not None and ( start in structural_quote_openers or start - 1 in structural_quote_openers ) + assignment_quote = start > 0 and content[start - 1] == "=" and content[start] in "'\"" if start < parsed_through or ( - not json_string_start and not _is_shell_command_word_start(content, start) + not json_string_start + and not assignment_quote + and not _is_shell_command_word_start(content, start) ): continue if _has_quoted_assignment_prefix(content, start): continue + candidate_word_positions: set[int] = set() parsed = _parse_shell_command_word( content, start, @@ -1752,6 +1980,10 @@ def _has_shell_command_word_exhaustion( substitution_end_cache, backtick_end_cache, check_runtime=check_runtime, + owned_word_positions=candidate_word_positions, + enclosing_delimiter=( + _assignment_value_wrapper(content, start) if assignment_quote else None + ), ) if parsed is None: substitution_start = ( @@ -1789,6 +2021,21 @@ def _has_shell_command_word_exhaustion( if unresolved_end - start > _SHELL_COMMAND_WORD_CHARS: return True continue + # Comments end at a newline, and a quote after ``=`` in prose or host + # source may be mis-paired. Such a word cannot own a later line. + confined_word = "\n" in content[start : parsed.end] and ( + assignment_quote + or _SHELL_COMMENT_START_RE.search(content, content.rfind("\n", 0, start) + 1, start) + is not None + ) + if not confined_word: + # A quote that opens a runtime-selected word stays an independent + # candidate, so a mis-paired claim cannot hide its operands. + owned_word_positions.update( + position + for position in candidate_word_positions + if content[position + 1 : position + 2] not in ("$", "`") + ) # Only executable nested substitutions retain independent command # positions. A plain dynamic data argument still owns its inner bytes; # revisiting those as commands would turn quoted printf data into code. @@ -1805,17 +2052,30 @@ def _has_shell_command_word_exhaustion( or "$" not in raw_word or not any(marker in raw_word for marker in ("$(", "`")) or simple_backtick_parameter - ): + ) and not (assignment_quote and confined_word): parsed_through = max(parsed_through, parsed.end) + if assignment_quote: + # An assignment value never names the command, and main never + # parsed this position, so it only claims ownership. + continue if parsed.limited: return True if parsed.dynamic and may_have_destructive_outer_operands: - tokens, _, exhausted = _bounded_shell_tokens( + tokens, command_end, exhausted = _bounded_shell_tokens( content, start, parsed.end, check_runtime=check_runtime, ) + if exhausted and command_end - start < _ROOT_GLOB_COMMAND_CHARS: + # Earlier prose (for example "row-first") cannot supply this + # command's operands. Refine only this candidate's exhausted + # span, including nested commands inside the executable word. + # Hitting the span bound always remains partial, including when + # real operands occur beyond that bound. This rescan is bounded + # by the same constant as the tokenizer. + check_runtime() + exhausted = _may_have_destructive_outer_operands(content[start:command_end]) if ( exhausted or _has_unsupported_brace_expansion(tokens) @@ -2089,6 +2349,19 @@ def _command_wrapper_quote(content: str, command_start: int) -> str | None: return content[command_start - 1] if backslashes % 2 == 0 else None +def _assignment_value_wrapper(content: str, value_start: int) -> str | None: + """Return an unescaped quote or backtick directly enclosing an assignment.""" + name_start = value_start - 1 + if name_start > 0 and content[name_start - 1] == "+": + name_start -= 1 + while name_start > 0 and ( + content[name_start - 1] == "_" + or (content[name_start - 1].isascii() and content[name_start - 1].isalnum()) + ): + name_start -= 1 + return _command_wrapper_quote(content, name_start) + + def _perl_literal_print_shell_text( content: str, check_runtime: Callable[[], None], @@ -2818,262 +3091,1275 @@ def _has_unsupported_brace_expansion(tokens: tuple[_ShellToken, ...]) -> bool: ) -_SCOPE_NODE_TYPES = (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef, ast.Lambda) +def _projected_subprocess_qualifier( + content: str, + popen_start: int, + maximum_lookbehind: int, +) -> tuple[int, int, str, bool] | None: + """Return a security-view ``subprocess.`` suffix and its raw source span.""" + qualifier = "subprocess." + if popen_start <= 0: + return None + maximum = min(popen_start, max(0, maximum_lookbehind)) + lookbehind = min(maximum, 64) + while lookbehind: + prefix_start = popen_start - lookbehind + # Include the first method character so contextual default-ignorable + # handling sees the same right boundary as the full security view. + needs_more_source = False + source = content[prefix_start : popen_start + 1] + for view in security_text_views(source): + derived_popen_start = ( + popen_start - prefix_start + if view.source_offsets is None + else bisect_left(view.source_offsets, popen_start - prefix_start) + ) + prefix = view.text[:derived_popen_start] + if len(prefix) < len(qualifier): + needs_more_source = True + continue + suffix_start = len(prefix) - len(qualifier) + suffix = prefix[suffix_start:] + if suffix.casefold() == qualifier: + return ( + prefix_start + view.source_offset(suffix_start), + prefix_start + view.source_offset(derived_popen_start - 1) + 1, + suffix, + view.name != "raw", + ) + if not needs_more_source: + return None + if lookbehind == maximum: + return None + lookbehind = min(maximum, lookbehind * 2) + return None -def _scope_chain(tree: ast.Module, target: ast.AST) -> tuple[ast.AST, ...] | None: - """Return the chain of enclosing scopes for *target*, outermost first. +def _subprocess_qualifier_start(content: str, popen_start: int) -> int | None: + """Find a bounded security-view ``subprocess.`` suffix before bare Popen.""" + qualifier = _projected_subprocess_qualifier( + content, + popen_start, + static_runner._WINDOW_OVERLAP_CHARS + len("subprocess."), + ) + return qualifier[0] if qualifier is not None else None - The module itself is the outermost scope. Returns None when *target* is - not part of *tree*. - """ - parents: dict[ast.AST, ast.AST] = {} - stack: list[ast.AST] = [tree] - found = False - while stack: - node = stack.pop() - if node is target: - found = True - break - for child in ast.iter_child_nodes(node): - parents[child] = node - stack.append(child) - if not found: - return None - chain: list[ast.AST] = [] - current: ast.AST | None = target - while current is not None: - if isinstance(current, _SCOPE_NODE_TYPES) or current is tree: - chain.append(current) - current = parents.get(current) - chain.reverse() - return tuple(chain) +def _outer_shell_true_anchor(matched_text: str) -> int | None: + """Locate the first outer-call shell literal, ignoring quoted lookalikes.""" + stack: list[str] = [] + quote: str | None = None + triple = False + in_comment = False + cursor = 0 + pairs = {")": "(", "]": "[", "}": "{"} + while cursor < len(matched_text): + character = matched_text[cursor] + if in_comment: + if character in LINE_BREAK_CHARS: + in_comment = False + cursor += 1 + continue + if quote is not None: + if character == "\\": + cursor = min(len(matched_text), cursor + 2) + continue + marker = quote * (3 if triple else 1) + if matched_text.startswith(marker, cursor): + cursor += len(marker) + quote = None + triple = False + else: + cursor += 1 + continue + if character in "'\"": + quote = character + triple = matched_text.startswith(character * 3, cursor) + cursor += 3 if triple else 1 + continue + if character == "#": + in_comment = True + cursor += 1 + continue + if character in "([{": + stack.append(character) + cursor += 1 + continue + if character in pairs: + if stack and stack[-1] == pairs[character]: + stack.pop() + cursor += 1 + continue + if ( + stack == ["("] + and ( + cursor == 0 + or not (matched_text[cursor - 1].isalnum() or matched_text[cursor - 1] == "_") + ) + and _DIRECT_SHELL_TRUE_VALUE.match(matched_text, cursor) is not None + ): + return cursor + cursor += 1 + return None -def _scope_binds_name(scope_node: ast.AST, name: str) -> bool: - """Return whether *name* is bound directly in *scope_node*. - Nested function/class/lambda bodies are not descended into: their bindings - belong to those scopes, not this one. - """ - if isinstance(scope_node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.Lambda)): - args = scope_node.args - named = [arg.arg for arg in (*args.posonlyargs, *args.args, *args.kwonlyargs)] - if args.vararg is not None: - named.append(args.vararg.arg) - if args.kwarg is not None: - named.append(args.kwarg.arg) - if name in named: - return True - bodies: list[ast.AST] = ( - [scope_node.body] if isinstance(scope_node, ast.Lambda) else list(scope_node.body) - ) - elif isinstance(scope_node, (ast.ClassDef, ast.Module)): - bodies = list(scope_node.body) - else: - return False - stack = list(bodies) - while stack: - node = stack.pop() - if isinstance(node, _SCOPE_NODE_TYPES): - continue - if isinstance(node, ast.Name) and isinstance(node.ctx, ast.Store) and node.id == name: - return True - stack.extend(ast.iter_child_nodes(node)) - return False +_SCOPE_NODE_TYPES = (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef, ast.Lambda) +_COMPREHENSION_SCOPE_TYPES = (ast.ListComp, ast.SetComp, ast.DictComp, ast.GeneratorExp) +_NATIVE_SHELL_RECEIVERS = frozenset({"subprocess", "Popen"}) -def _direct_global_nonlocal(scope_node: ast.AST, name: str) -> str | None: - """Return 'global'/'nonlocal' when *scope_node* declares *name* as such. +def _paired_store_values(target: ast.AST, value: ast.AST) -> Iterator[tuple[ast.AST, ast.AST]]: + """Yield each store target with the RHS element it receives, when the shapes match.""" + pairs = [(target, value)] + while pairs: + destination, source = pairs.pop() + if ( + isinstance(destination, (ast.Tuple, ast.List)) + and isinstance(source, (ast.Tuple, ast.List)) + and len(destination.elts) == len(source.elts) + and not any(isinstance(item, ast.Starred) for item in (*destination.elts, *source.elts)) + ): + pairs.extend(zip(destination.elts, source.elts, strict=True)) + else: + yield destination, source - Only declarations directly in the scope are considered; nested scopes are - not descended into. - """ - if isinstance(scope_node, ast.Lambda): - return None - bodies: list[ast.stmt] | None = None - if isinstance(scope_node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef, ast.Module)): - bodies = scope_node.body - if bodies is None: - return None - stack: list[ast.AST] = list(bodies) - while stack: - node = stack.pop() - if isinstance(node, _SCOPE_NODE_TYPES): - continue - if isinstance(node, ast.Global) and name in node.names: - return "global" - if isinstance(node, ast.Nonlocal) and name in node.names: - return "nonlocal" - stack.extend(ast.iter_child_nodes(node)) - return None +def _captures_native_receiver(value: ast.AST, receiver: str) -> bool: + """Return whether *value* reads the native *receiver* callable or module.""" + if isinstance(value, ast.Name): + return value.id == receiver + return ( + receiver == "Popen" + and isinstance(value, ast.Attribute) + and value.attr == "Popen" + and isinstance(value.value, ast.Name) + and value.value.id == "subprocess" + ) -def _variable_shell_flag_same_scope(content: str, file_path: str, match: re.Match[str]) -> bool: - """Return whether a variable-shell-flag match is a same-scope data flow. - The regex cannot see Python scopes, so ``use_shell = True`` in one - function followed by ``shell=use_shell`` in another still matches. Resolve - the matched assignment and the ``shell=`` use through the Python AST and - require the assignment to be visible from the use: identical scope chains, - a closure read from an enclosing scope, or a matching global/nonlocal - declaration. Unparseable content keeps the candidate so a syntax error - cannot silence the signal. - """ - var_name = match.group(1) - assign_line = content.count("\n", 0, match.start()) + 1 - use_line = content.count("\n", 0, match.end()) + 1 - tree = parse_python_source(content, file_path).tree - if tree is None: - return True - assign_node: ast.Assign | None = None - call_node: ast.Call | None = None +def _native_receiver_aliases(tree: ast.Module) -> dict[str, str]: + """Map each name whose every binding captures one native receiver to it.""" + paired: dict[ast.AST, ast.AST] = {} for node in ast.walk(tree): - if ( - assign_node is None - and isinstance(node, ast.Assign) - and node.lineno == assign_line - and isinstance(node.value, ast.Constant) - and node.value.value is True - and any( - isinstance(target, ast.Name) and target.id == var_name for target in node.targets - ) - ): - assign_node = node - if ( - call_node is None - and isinstance(node, ast.Call) - and any( - keyword.arg == "shell" - and isinstance(keyword.value, ast.Name) - and keyword.value.id == var_name - and assign_line <= keyword.value.lineno <= use_line - for keyword in node.keywords - ) - ): - call_node = node - if assign_node is None or call_node is None: - return True - assign_chain = _scope_chain(tree, assign_node) - use_chain = _scope_chain(tree, call_node) - if assign_chain is None or use_chain is None: - return True - if assign_chain == use_chain: - return True - if len(assign_chain) < len(use_chain) and use_chain[: len(assign_chain)] == assign_chain: - # Closure read: the use sits in a scope nested inside the assignment's - # scope, so the name resolves to the assigned value. - return True - use_scope = use_chain[-1] - if use_scope is not tree: - declaration = _direct_global_nonlocal(use_scope, var_name) - if declaration == "global": - return assign_chain == (tree,) - if declaration == "nonlocal": - binding = next( + if isinstance(node, ast.Assign): + for target in node.targets: + paired.update(_paired_store_values(target, node.value)) + elif isinstance(node, (ast.AnnAssign, ast.NamedExpr)) and node.value is not None: + paired[node.target] = node.value + captures: dict[str, set[str | None]] = {} + for node in ast.walk(tree): + if isinstance(node, (ast.Import, ast.ImportFrom)): + for alias in node.names: + if alias.name == "*": + continue + native = ( + "subprocess" + if isinstance(node, ast.Import) and alias.name == "subprocess" + else "Popen" + if isinstance(node, ast.ImportFrom) + and node.level == 0 + and node.module == "subprocess" + and alias.name == "Popen" + else None + ) + bound = alias.asname or alias.name.split(".", 1)[0] + captures.setdefault(bound, set()).add(native) + continue + name: str | None = None + native = None + if isinstance(node, ast.Name) and isinstance(node.ctx, (ast.Store, ast.Del)): + name = node.id + value = paired.get(node) if isinstance(node.ctx, ast.Store) else None + native = next( ( - scope - for scope in use_chain[-2::-1] - if isinstance(scope, (ast.FunctionDef, ast.AsyncFunctionDef, ast.Lambda)) - and _scope_binds_name(scope, var_name) + receiver + for receiver in sorted(_NATIVE_SHELL_RECEIVERS) + if value is not None and _captures_native_receiver(value, receiver) ), None, ) - return binding is not None and assign_chain == _scope_chain(tree, binding) - return False + elif isinstance(node, ast.arg): + name = node.arg + elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): + name = node.name + elif isinstance(node, (ast.ExceptHandler, ast.MatchAs, ast.MatchStar)): + name = node.name + elif isinstance(node, ast.MatchMapping): + name = node.rest + if name is not None: + captures.setdefault(name, set()).add(native) + return { + name: receiver + for name, values in captures.items() + if len(values) == 1 and (receiver := next(iter(values))) is not None + } -def _tm1_candidates( - content: str, - *, - shell_content: str | None = None, -) -> Iterator[tuple[int, int, str, float]]: - for pattern, confidence in TM1_PATTERNS: - matches = ( - static_runner.iter_paragraph_matches - if (pattern, confidence) in TM1_PROSE_PATTERNS - else re.finditer +def _store_keeps_shell_signal(name: str, value: ast.AST, aliases: dict[str, str]) -> bool | None: + """Classify one simple-name store for lexical TM1 reconciliation. + + ``None`` means an identity store that leaves the binding unchanged. ``True`` + means the store keeps the native receiver or a truthy shell flag. ``False`` + means the store may replace the binding. + """ + if isinstance(value, ast.Name) and value.id == name: + return None + if name in _NATIVE_SHELL_RECEIVERS: + return _captures_native_receiver(value, name) or ( + isinstance(value, ast.Name) and aliases.get(value.id) == name ) - for match in matches(pattern, content, re.IGNORECASE | re.MULTILINE): - yield match.start(), match.end(), match.group(0), confidence + if name.casefold() in {"subprocess", "popen"}: + return False + return isinstance(value, ast.Constant) and bool(value.value) - yield from _tm1_shell_candidates(content) - if shell_content is not None and shell_content != content: - yield from _tm1_shell_candidates(shell_content) +@dataclass(frozen=True) +class _ScopedShellNode: + """One relevant AST node plus its source and lexical-scope identity.""" -def _tm1_shell_candidates(content: str) -> Iterator[tuple[int, int, str, float]]: - seen_commands: set[tuple[int, int]] = set() - covered_until = 0 - for command_start, body_start in _destructive_command_words(content): - if command_start < covered_until: + node: ast.Assign | ast.Call + scope_chain: tuple[ast.AST, ...] + start: int + + +@dataclass(frozen=True) +class _VariableShellAstIndex: + """Linear-time index used to reconcile every bounded regex candidate.""" + + tree: ast.Module + line_character_starts: tuple[int, ...] + assignments: dict[tuple[int, str], tuple[_ScopedShellNode, ...]] + calls: dict[str, tuple[_ScopedShellNode, ...]] + call_starts: dict[str, tuple[int, ...]] + bindings: dict[ast.AST, frozenset[str]] + class_binding_starts: dict[ast.ClassDef, dict[str, tuple[int, ...]]] + declarations: dict[ast.AST, dict[str, str]] + binding_events: dict[tuple[ast.AST, ast.AST], dict[str, tuple[tuple[int, bool], ...]]] + eager_effect_starts: dict[ast.AST, tuple[int, ...]] + reference_starts: dict[str, tuple[int, ...]] + + +@dataclass(frozen=True) +class _VariableShellCandidate: + """One raw variable-shell regex candidate resolved against the shared AST.""" + + name: str + call: ast.Call + same_scope: bool + visible: bool + assignment_start: int + assignment_scope: ast.AST + call_scope_chain: tuple[ast.AST, ...] + + +def _resolved_name_scope( + index: _VariableShellAstIndex, + use_chain: tuple[ast.AST, ...], + name: str, + use_start: int, +) -> ast.AST: + """Return the Python scope that resolves *name* at one use site.""" + crossed_function = False + nonlocal_lookup = False + seen_class_scope = False + for scope in reversed(use_chain): + if scope is index.tree: + return index.tree + declaration = index.declarations.get(scope, {}).get(name) + if declaration == "global": + return index.tree + if declaration == "nonlocal": + nonlocal_lookup = True + crossed_function = True continue - command_key = (command_start, body_start) - if command_key in seen_commands: + if isinstance(scope, (ast.FunctionDef, ast.AsyncFunctionDef, ast.Lambda)): + if name in index.bindings.get(scope, frozenset()): + return scope + crossed_function = True continue - seen_commands.add(command_key) - # Documentation is excluded regardless of the shell parse result. Apply - # that existing semantic gate first so large manuals do not pay for a - # character-by-character shell parse for every explanatory ``rm`` noun. - if _is_root_glob_documentation(content, command_start, body_start): + if isinstance(scope, _COMPREHENSION_SCOPE_TYPES): + if name in index.bindings.get(scope, frozenset()): + return scope + # Comprehensions use an implicit function scope. Free names skip a + # surrounding class namespace just like names in a method body. + crossed_function = True continue - tokens, command_end, _ = _bounded_shell_tokens( - content, - command_start, - body_start, - ) - covered_until = max(covered_until, command_end) - command = content[command_start:command_end] - if _has_destructive_root_glob(tokens) or _has_destructive_root_path(tokens): - yield command_start, command_end, command, 0.9 + if isinstance(scope, ast.ClassDef): + # A method does not close over its class namespace. A call evaluated + # directly in the class body sees assignments already executed, but + # a later class assignment does not create a compile-time local. + starts = index.class_binding_starts.get(scope, {}).get(name, ()) + if ( + not crossed_function + and not nonlocal_lookup + and not seen_class_scope + and bisect_left(starts, use_start) + ): + return scope + seen_class_scope = True + return index.tree + + +def _node_character_span(parsed: ParsedPythonFile, node: ast.AST) -> tuple[int, int] | None: + """Return one AST node's absolute character span.""" + line = getattr(node, "lineno", None) + end_line = getattr(node, "end_lineno", None) + byte_column = getattr(node, "col_offset", None) + end_byte_column = getattr(node, "end_col_offset", None) + if not all(isinstance(value, int) for value in (line, end_line, byte_column, end_byte_column)): + return None + assert isinstance(line, int) + assert isinstance(end_line, int) + assert isinstance(byte_column, int) + assert isinstance(end_byte_column, int) + start_column = parsed.character_column(line, byte_column) + end_column = parsed.character_column(end_line, end_byte_column) + if start_column is None or end_column is None: + return None + start = parsed.line_character_starts[line - 1] + start_column + end = parsed.line_character_starts[end_line - 1] + end_column + return start, end -def _markdown_block_separator(line: str, check_runtime: Callable[[], None]) -> bool: - """Recognize Setext underlines/thematic breaks with bounded, linear work.""" - line = line.strip(" \t") - marker = line[:1] - if marker not in {"=", "-", "*", "_"}: - return False - count = 0 - internal_gap = False - for index, character in enumerate(line): - if index % 256 == 0: - check_runtime() - if character == marker: - count += 1 - elif character in " \t" and marker != "=": - internal_gap = True - else: - return False - return count >= (3 if internal_gap or marker in {"*", "_"} else 1) +def _parameter_names(arguments: ast.arguments) -> tuple[str, ...]: + names = [ + argument.arg + for argument in (*arguments.posonlyargs, *arguments.args, *arguments.kwonlyargs) + ] + if arguments.vararg is not None: + names.append(arguments.vararg.arg) + if arguments.kwarg is not None: + names.append(arguments.kwarg.arg) + return tuple(names) + + +def _direct_shell_name(call: ast.Call) -> str | None: + function = call.func + direct = isinstance(function, ast.Name) and function.id == "Popen" + direct = direct or ( + isinstance(function, ast.Attribute) + and isinstance(function.value, ast.Name) + and function.value.id == "subprocess" + ) + if not direct: + return None + for keyword in call.keywords: + if keyword.arg == "shell" and isinstance(keyword.value, ast.Name): + return keyword.value.id + return None -def _markdown_table_cells( - line: str, start: int, check_runtime: Callable[[], None] -) -> list[tuple[int, int]]: - """Locate GFM cells without copying content or changing source offsets.""" - end = len(line) - while start < end and line[start] in " \t": - if start % 256 == 0: - check_runtime() - start += 1 - while end > start and line[end - 1] in " \t": - if end % 256 == 0: - check_runtime() - end -= 1 - if start == end: - return [] - # Edge pipes are optional. In cmark-gfm a backslash immediately before a - # pipe escapes it even when that backslash follows another backslash. - if line[start] == "|": - start += 1 - if line[end - 1] == "|" and (end == 1 or line[end - 2] != "\\"): - end -= 1 - if start > end: - return [] - cells: list[tuple[int, int]] = [] - cell_start = start - for cursor in range(start, end): +def _build_variable_shell_ast_index(parsed: ParsedPythonFile) -> _VariableShellAstIndex | None: + """Index relevant nodes, bindings, and declarations in one AST traversal.""" + tree = parsed.tree + if tree is None: + return None + assignments: dict[tuple[int, str], list[_ScopedShellNode]] = {} + calls: dict[str, list[_ScopedShellNode]] = {} + bindings: dict[ast.AST, set[str]] = {tree: set()} + class_binding_starts: dict[ast.ClassDef, dict[str, list[int]]] = {} + declarations: dict[ast.AST, dict[str, str]] = {} + binding_events: dict[ast.AST, dict[str, list[tuple[int, bool]]]] = {} + eager_effect_starts: dict[ast.AST, list[int]] = {} + reference_starts: dict[str, list[int]] = {} + parents = {child: parent for parent in ast.walk(tree) for child in ast.iter_child_nodes(parent)} + attribute_identity_stores: set[ast.Attribute] = set() + # Simple-name stores that keep the current binding, including unpacked + # self-stores. They neither replace nor restore a binding. + identity_name_stores: set[ast.Name] = set() + native_aliases = _native_receiver_aliases(tree) + + def mark_attribute_identity_stores(target: ast.AST, value: ast.AST) -> None: + for destination, source in _paired_store_values(target, value): + if ( + isinstance(destination, ast.Attribute) + and isinstance(destination.value, ast.Name) + and isinstance(source, ast.Attribute) + and isinstance(source.value, ast.Name) + and destination.attr == source.attr + and destination.value.id == source.value.id + ): + attribute_identity_stores.add(destination) + + def record_name_store_values(scope: ast.AST, target: ast.AST, value: ast.AST, end: int) -> None: + # Pair unpacked targets with their RHS elements so a native alias, a + # truthy flag, or a self-store is not mistaken for a replacement. + for destination, source in _paired_store_values(target, value): + if not isinstance(destination, ast.Name): + continue + keeps_signal = _store_keeps_shell_signal(destination.id, source, native_aliases) + if keeps_signal is None: + identity_name_stores.add(destination) + elif keeps_signal: + record_binding(scope, destination.id, end, True) + + def record_binding(scope: ast.AST, name: str, start: int, restores: bool = False) -> None: + # A conditional store is not affirmative replacement evidence. Only a + # literal branch whose execution is proven can revoke the lexical owner. + child = node + if child in identity_name_stores: + return + parent = parents.get(child) + if isinstance(parent, ast.AnnAssign) and parent.value is None and child is parent.target: + return + while parent is not None and parent is not scope: + if isinstance(parent, ast.If): + if not isinstance(parent.test, ast.Constant): + return + taken = parent.body if bool(parent.test.value) else parent.orelse + if child not in taken: + return + elif isinstance(parent, ast.BoolOp): + # Later operands run only after every earlier operand lets the + # short circuit continue. Only constant operands prove that. + continues = isinstance(parent.op, ast.And) + for operand in parent.values: + if operand is child: + break + if ( + not isinstance(operand, ast.Constant) + or bool(operand.value) is not continues + ): + return + elif isinstance(parent, ast.IfExp) and child is not parent.test: + if not isinstance(parent.test, ast.Constant): + return + if child is not (parent.body if bool(parent.test.value) else parent.orelse): + return + elif isinstance( + parent, + ( + ast.For, + ast.AsyncFor, + ast.While, + ast.With, + ast.AsyncWith, + ast.Try, + ast.TryStar, + ast.Match, + ast.comprehension, + *_COMPREHENSION_SCOPE_TYPES, + ), + ): + return + child, parent = parent, parents.get(parent) + binding_events.setdefault(scope, {}).setdefault(name, []).append((start, restores)) + + def bind(scope: ast.AST, name: str, start: int) -> None: + bindings.setdefault(scope, set()).add(name) + record_binding(scope, name, start) + if isinstance(scope, ast.ClassDef): + class_binding_starts.setdefault(scope, {}).setdefault(name, []).append(start) + + stack: list[tuple[ast.AST, tuple[ast.AST, ...], int | None]] = [(tree, (tree,), None)] + while stack: + node, scope_chain, binding_start = stack.pop() + scope = scope_chain[-1] + span = _node_character_span(parsed, node) + start = span[0] if span is not None else 0 + end = span[1] if span is not None else start + # A direct observed slot write is useful only before another eager + # effect. Cached-module lifetime across imports belongs to the companion. + if isinstance( + node, + ( + ast.Call, + ast.Attribute, + ast.Subscript, + ast.BinOp, + ast.BoolOp, + ast.Compare, + ast.UnaryOp, + ast.NamedExpr, + ast.JoinedStr, + ast.Import, + ast.ImportFrom, + ast.Delete, + ast.AugAssign, + ast.If, + ast.For, + ast.AsyncFor, + ast.While, + ast.With, + ast.AsyncWith, + ast.Try, + ast.TryStar, + ast.Match, + ast.FunctionDef, + ast.AsyncFunctionDef, + ast.ClassDef, + ast.Await, + ast.Yield, + ast.YieldFrom, + *_COMPREHENSION_SCOPE_TYPES, + ), + ): + execution = next( + ( + item + for item in reversed(scope_chain) + if isinstance(item, (ast.FunctionDef, ast.AsyncFunctionDef, ast.Lambda)) + ), + tree, + ) + eager_effect_starts.setdefault(execution, []).append(start) + # Any later reference may invoke a deferred body, so keep every load. + if isinstance(node, ast.Name) and isinstance(node.ctx, ast.Load): + reference_starts.setdefault(node.id, []).append(start) + elif isinstance(node, ast.Attribute) and isinstance(node.ctx, ast.Load): + reference_starts.setdefault(node.attr, []).append(start) + + if isinstance(node, ast.Assign): + for target in node.targets: + mark_attribute_identity_stores(target, node.value) + elif isinstance(node, ast.AnnAssign) and node.value is not None: + mark_attribute_identity_stores(node.target, node.value) + if isinstance(node, ast.Assign): + # The RHS is captured before any target is assigned. A final + # same-slot store restores that snapshot after earlier writes in + # this statement, including chained and repeated unpacking targets. + stores: dict[tuple[str, str], list[ast.Attribute]] = {} + targets = list(reversed(node.targets)) + while targets: + target = targets.pop() + if isinstance(target, ast.Attribute) and isinstance(target.value, ast.Name): + stores.setdefault((target.value.id, target.attr), []).append(target) + elif isinstance(target, (ast.Tuple, ast.List)): + targets.extend(reversed(target.elts)) + for slot_stores in stores.values(): + if slot_stores[-1] in attribute_identity_stores: + attribute_identity_stores.update(slot_stores) + + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + bind(scope, node.name, end) + nested_chain = (*scope_chain, node) + bindings.setdefault(node, set()).update(_parameter_names(node.args)) + header_nodes: list[ast.AST] = [*node.decorator_list, node.args] + if node.returns is not None: + header_nodes.append(node.returns) + header_nodes.extend(getattr(node, "type_params", [])) + stack.extend((child, scope_chain, None) for child in header_nodes) + stack.extend((child, nested_chain, None) for child in node.body) + continue + if isinstance(node, ast.Lambda): + nested_chain = (*scope_chain, node) + bindings.setdefault(node, set()).update(_parameter_names(node.args)) + stack.append((node.args, scope_chain, None)) + stack.append((node.body, nested_chain, None)) + continue + if isinstance(node, ast.ClassDef): + bind(scope, node.name, end) + nested_chain = (*scope_chain, node) + bindings.setdefault(node, set()) + header_nodes = [*node.decorator_list, *node.bases] + header_nodes.extend(keyword.value for keyword in node.keywords) + header_nodes.extend(getattr(node, "type_params", [])) + stack.extend((child, scope_chain, None) for child in header_nodes) + stack.extend((child, nested_chain, None) for child in node.body) + continue + if isinstance(node, _COMPREHENSION_SCOPE_TYPES): + nested_chain = (*scope_chain, node) + bindings.setdefault(node, set()) + generators = node.generators + if generators: + first, *remaining = generators + # The leftmost iterable is the only comprehension expression + # evaluated in the enclosing scope. + stack.append((first.iter, scope_chain, None)) + stack.append((first.target, nested_chain, None)) + stack.extend((condition, nested_chain, None) for condition in first.ifs) + for generator in remaining: + stack.append((generator.iter, nested_chain, None)) + stack.append((generator.target, nested_chain, None)) + stack.extend((condition, nested_chain, None) for condition in generator.ifs) + if isinstance(node, ast.DictComp): + stack.append((node.key, nested_chain, None)) + stack.append((node.value, nested_chain, None)) + else: + stack.append((node.elt, nested_chain, None)) + continue + + if ( + isinstance(node, ast.Attribute) + and isinstance(node.ctx, ast.Store) + and isinstance(node.value, ast.Name) + ): + assignment = parents.get(node) + if ( + node not in attribute_identity_stores + and isinstance(assignment, ast.Assign) + and len(assignment.targets) == 1 + and assignment.targets[0] is node + and tree.body + and assignment is tree.body[0] + and not any( + isinstance(value, ast.Name) + and isinstance(value.ctx, ast.Load) + and value.id in {node.value.id, "Popen"} + for value in ast.walk(assignment.value) + ) + ): + # Retain the legacy first-statement direct shadow control only. + # An RHS using the receiver or Popen may preserve a native callable. + # Prior stores/protocol effects need the companion's explicit proof. + record_binding( + scope, + node.value.id + "." + node.attr, + binding_start if binding_start is not None else end, + ) + + if isinstance(node, ast.Global): + for name in node.names: + declarations.setdefault(scope, {})[name] = "global" + elif isinstance(node, ast.Nonlocal): + for name in node.names: + declarations.setdefault(scope, {})[name] = "nonlocal" + elif isinstance(node, ast.Name) and isinstance(node.ctx, (ast.Store, ast.Del)): + bind(scope, node.id, binding_start if binding_start is not None else start) + elif isinstance(node, (ast.Import, ast.ImportFrom)): + for alias in node.names: + if alias.name != "*": + bound_name = alias.asname or alias.name.split(".", 1)[0] + bind(scope, bound_name, end) + restores = ( + isinstance(node, ast.Import) + and alias.name == "subprocess" + and bound_name == "subprocess" + ) or ( + isinstance(node, ast.ImportFrom) + and node.level == 0 + and node.module == "subprocess" + and alias.name == "Popen" + and bound_name == "Popen" + ) + if restores: + record_binding(scope, bound_name, end, True) + elif isinstance(node, ast.ExceptHandler) and node.name: + type_span = _node_character_span(parsed, node.type) if node.type is not None else None + bind(scope, node.name, type_span[1] if type_span is not None else start) + elif isinstance(node, (ast.MatchAs, ast.MatchStar)) and node.name: + bind(scope, node.name, start) + elif isinstance(node, ast.MatchMapping) and node.rest: + bind(scope, node.rest, start) + + if ( + isinstance(node, ast.Assign) + and span is not None + and isinstance(node.value, ast.Constant) + and node.value.value is True + ): + for target in node.targets: + if isinstance(target, ast.Name): + scoped = _ScopedShellNode(node=node, scope_chain=scope_chain, start=start) + assignments.setdefault((node.lineno, target.id), []).append(scoped) + elif isinstance(node, ast.Call) and span is not None: + shell_name = _direct_shell_name(node) + if shell_name is not None: + scoped = _ScopedShellNode(node=node, scope_chain=scope_chain, start=start) + calls.setdefault(shell_name, []).append(scoped) + + if isinstance(node, ast.Assign): + stack.append((node.value, scope_chain, None)) + stack.extend((target, scope_chain, end) for target in node.targets) + for target in node.targets: + record_name_store_values(scope, target, node.value, end) + elif isinstance(node, (ast.AnnAssign, ast.AugAssign)): + if node.value is not None: + stack.append((node.value, scope_chain, None)) + stack.append((node.target, scope_chain, end)) + if isinstance(node, ast.AnnAssign): + stack.append((node.annotation, scope_chain, None)) + if node.value is not None: + record_name_store_values(scope, node.target, node.value, end) + elif isinstance(node, ast.NamedExpr): + stack.append((node.value, scope_chain, None)) + # PEP 572 makes a walrus target inside one or more + # comprehensions local to the nearest containing real scope. + # Comprehension ``for`` targets remain local to their implicit + # scope, but the named-expression target skips those scopes. + target_chain = scope_chain + while len(target_chain) > 1 and isinstance( + target_chain[-1], _COMPREHENSION_SCOPE_TYPES + ): + target_chain = target_chain[:-1] + stack.append((node.target, target_chain, end)) + record_name_store_values(target_chain[-1], node.target, node.value, end) + elif isinstance(node, (ast.For, ast.AsyncFor)): + iteration_span = _node_character_span(parsed, node.iter) + iteration_end = iteration_span[1] if iteration_span is not None else end + stack.append((node.iter, scope_chain, None)) + stack.append((node.target, scope_chain, iteration_end)) + stack.extend((child, scope_chain, None) for child in (*node.body, *node.orelse)) + elif isinstance(node, ast.withitem): + context_span = _node_character_span(parsed, node.context_expr) + context_end = context_span[1] if context_span is not None else end + stack.append((node.context_expr, scope_chain, None)) + if node.optional_vars is not None: + stack.append((node.optional_vars, scope_chain, context_end)) + else: + stack.extend( + (child, scope_chain, binding_start) for child in ast.iter_child_nodes(node) + ) + + def execution_scope(scope: ast.AST) -> ast.AST: + current: ast.AST | None = scope + while current is not None: + if isinstance(current, (ast.FunctionDef, ast.AsyncFunctionDef, ast.Lambda)): + return current + current = parents.get(current) + return tree + + # Keep the execution scope of every store. A global/nonlocal declaration + # changes the binding namespace; it never proves a deferred body was run. + scoped_events: dict[tuple[ast.AST, ast.AST], dict[str, list[tuple[int, bool]]]] = {} + sorted_class_starts = { + scope: {name: tuple(sorted(starts)) for name, starts in names.items()} + for scope, names in class_binding_starts.items() + } + for scope, names in binding_events.items(): + for name, events in names.items(): + receiver_name = name.partition(".")[0] + declaration = declarations.get(scope, {}).get(receiver_name) + for event in events: + target_scope = scope + if "." in name and declaration is None: + # A class body can change where its receiver resolves + # between stores. Resolve each slot store at its own offset. + current: ast.AST | None = scope + crossed_function = False + while current is not None: + if current is tree: + target_scope = tree + break + if isinstance(current, (ast.FunctionDef, ast.AsyncFunctionDef, ast.Lambda)): + if receiver_name in bindings.get(current, set()): + target_scope = current + break + crossed_function = True + elif isinstance(current, ast.ClassDef) and not crossed_function: + starts = sorted_class_starts.get(current, {}).get(receiver_name, ()) + if bisect_right(starts, event[0]): + target_scope = current + break + current = parents.get(current) + if declaration == "global": + target_scope = tree + elif declaration == "nonlocal": + outer = parents.get(scope) + while outer is not None: + if isinstance( + outer, (ast.FunctionDef, ast.AsyncFunctionDef, ast.Lambda) + ) and receiver_name in bindings.get(outer, set()): + target_scope = outer + break + outer = parents.get(outer) + key = (target_scope, execution_scope(scope)) + scoped_events.setdefault(key, {}).setdefault(name, []).append(event) + + frozen_calls = { + name: tuple(sorted(nodes, key=lambda item: item.start)) for name, nodes in calls.items() + } + return _VariableShellAstIndex( + tree=tree, + line_character_starts=parsed.line_character_starts, + assignments={key: tuple(nodes) for key, nodes in assignments.items()}, + calls=frozen_calls, + call_starts={ + name: tuple(item.start for item in nodes) for name, nodes in frozen_calls.items() + }, + bindings={scope: frozenset(names) for scope, names in bindings.items()}, + class_binding_starts=sorted_class_starts, + declarations=declarations, + eager_effect_starts={ + scope: tuple(sorted(set(starts))) for scope, starts in eager_effect_starts.items() + }, + reference_starts={name: tuple(sorted(starts)) for name, starts in reference_starts.items()}, + binding_events={ + scope: {name: tuple(sorted(set(events))) for name, events in names.items()} + for scope, names in scoped_events.items() + }, + ) + + +def _assignment_name_scope( + index: _VariableShellAstIndex, + scope_chain: tuple[ast.AST, ...], + name: str, +) -> ast.AST: + """Return the scope mutated by a simple-name assignment.""" + scope = scope_chain[-1] + declaration = index.declarations.get(scope, {}).get(name) + if declaration == "global": + return index.tree + if declaration == "nonlocal": + for outer in reversed(scope_chain[:-1]): + if isinstance(outer, (ast.FunctionDef, ast.AsyncFunctionDef, ast.Lambda)) and name in ( + index.bindings.get(outer, frozenset()) + ): + return outer + return scope + + +def _variable_shell_call_start(match: re.Match[str]) -> int | None: + """Return the final call selected by one bounded variable-shell match.""" + callees = tuple(re.finditer(r"(?:subprocess\.\w+|Popen)\s*\(", match.group(0), re.IGNORECASE)) + return match.start() + callees[-1].start() if callees else None + + +def _is_true_prefixed_name(name: str) -> bool: + """Return whether a variable window keeps a call-anchored fallback for a name.""" + return name.casefold().startswith("true") + + +def _variable_shell_matches(content: str) -> list[re.Match[str]]: + """Return bounded variable-shell windows, one per call for true-prefixed names. + + The non-overlapping scan pairs an assignment with one call and skips an + assignment inside an earlier window. The ``\\b``-bounded direct pattern + does not report ``shell=true_value`` and folds identical same-line + ``shell=true`` calls into one candidate, so every assignment of a + true-prefixed name is also paired with each call its bounded window + reaches. The nearest assignment owns each call. + """ + windows: dict[int, re.Match[str]] = {} + + def keep(window: re.Match[str]) -> None: + previous = windows.get(window.end()) + if previous is None or window.start() > previous.start(): + windows[window.end()] = window + + for match in _VARIABLE_SHELL_FLAG_RE.finditer(content): + keep(match) + for assignment in _VARIABLE_SHELL_ASSIGNMENT_RE.finditer(content): + if not _is_true_prefixed_name(assignment.group(1)): + continue + search_end = assignment.end() - 1 + for _ in range(5): + search_end = content.find("\n", search_end + 1) + if search_end == -1: + search_end = len(content) + break + for argument in _SHELL_NAME_ARGUMENT_RE.finditer(content, assignment.end(), search_end): + window = _VARIABLE_SHELL_FLAG_RE.fullmatch(content, assignment.start(), argument.end()) + if window is not None: + keep(window) + return sorted(windows.values(), key=lambda window: (window.start(), window.end())) + + +def _resolve_variable_shell_candidate( + index: _VariableShellAstIndex, + match: re.Match[str], + assignment_line: int, +) -> _VariableShellCandidate | None: + """Resolve a bounded regex match to its exact assignment, call, and binding.""" + name = match.group(1) + assignment_nodes = [ + item + for item in index.assignments.get((assignment_line, name), ()) + if match.start() <= item.start < match.end() + ] + call_nodes = index.calls.get(name, ()) + starts = index.call_starts.get(name, ()) + expected_call_start = _variable_shell_call_start(match) + if expected_call_start is None: + return None + call_index = bisect_left(starts, expected_call_start) + if ( + len(assignment_nodes) != 1 + or call_index >= len(call_nodes) + or call_nodes[call_index].start != expected_call_start + ): + return None + assignment = assignment_nodes[0] + call = call_nodes[call_index] + if call.start >= match.end(): + return None + assignment_scope = _assignment_name_scope(index, assignment.scope_chain, name) + resolved_scope = _resolved_name_scope(index, call.scope_chain, name, call.start) + return _VariableShellCandidate( + name=name, + call=call.node, + same_scope=assignment.scope_chain == call.scope_chain, + visible=resolved_scope is assignment_scope, + assignment_start=assignment.start, + assignment_scope=assignment_scope, + call_scope_chain=call.scope_chain, + ) + + +def _earliest_deferred_invocation( + index: _VariableShellAstIndex, + parsed: ParsedPythonFile, + call_scope_chain: tuple[ast.AST, ...], + owner_execution: ast.AST, +) -> float: + """Return the earliest owner-scope offset at which a deferred call may run. + + A function, method, or lambda can run once it exists and something refers to + it: a call in any statement (including ``if``/``try``/main guards), a callback + registration, a decorator, or another body. Without a visible reference it + runs only after the owner scope finishes. + """ + owner_position = next( + (position for position, scope in enumerate(call_scope_chain) if scope is owner_execution), + None, + ) + if owner_position is None or owner_position + 1 >= len(call_scope_chain): + return 0 + entries = call_scope_chain[owner_position + 1 :] + span = _node_character_span(parsed, entries[0]) + if span is None: + return 0 + defined = span[1] + if any( + isinstance(entry, (ast.Lambda, ast.GeneratorExp)) or getattr(entry, "decorator_list", None) + for entry in entries + ): + return defined + names = { + entry.name + for entry in entries + if isinstance(entry, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)) + } + references = [ + start + for name in names + for start in index.reference_starts.get(name, ()) + if not span[0] <= start < span[1] + ] + if not references: + return float("inf") + return max(defined, min(references)) + + +def _deferred_receiver_is_replaced( + index: _VariableShellAstIndex, + parsed: ParsedPythonFile, + candidate: _VariableShellCandidate, + receiver: str, + receiver_scope: ast.AST, + receiver_execution: ast.AST, + current_execution: ast.AST, + call_start: int, +) -> bool: + """Return whether every possible invocation of a deferred call sees a replacement.""" + local_events = index.binding_events.get((receiver_scope, current_execution), {}).get( + receiver, () + ) + local_index = bisect_right(local_events, (call_start, True)) + if local_index: + # A store earlier in the same invocation decides the receiver. + return not local_events[local_index - 1][1] + outer_events = index.binding_events.get((receiver_scope, receiver_execution), {}).get( + receiver, () + ) + first_invocation = _earliest_deferred_invocation( + index, parsed, candidate.call_scope_chain, receiver_execution + ) + keeps_receiver: dict[int, bool] = {} + for start, restores in outer_events: + keeps_receiver[start] = keeps_receiver.get(start, False) or restores + starts = sorted(keeps_receiver) + earlier = bisect_left(starts, first_invocation) + if not earlier: + return False + # The binding seen by the first possible invocation, then every later one. + return not any(keeps_receiver[start] for start in starts[earlier - 1 :]) + + +def _lexical_shell_has_counterevidence( + index: _VariableShellAstIndex, + parsed: ParsedPythonFile, + candidate: _VariableShellCandidate, + *, + receiver_trusted: bool | None, +) -> bool: + """Reject a lexical candidate only for an observed binding replacement. + + The companion's unknown receiver trust is an abstention. Generic calls, + protocol effects, and unsupported statements do not prove a replacement. + Stores/imports do; use their source order and Python binding scopes. + """ + call_span = _node_character_span(parsed, candidate.call) + if call_span is None: + return False + call_start = call_span[0] + shell = next((item.value for item in candidate.call.keywords if item.arg == "shell"), None) + shell_span = _node_character_span(parsed, shell) if shell is not None else None + shell_start = shell_span[0] if shell_span is not None else call_start + current_execution = next( + ( + scope + for scope in reversed(candidate.call_scope_chain) + if isinstance(scope, (ast.FunctionDef, ast.AsyncFunctionDef, ast.Lambda)) + ), + index.tree, + ) + + def binding_events(scope: ast.AST, name: str) -> tuple[tuple[int, bool], ...]: + owner_execution = ( + scope + if isinstance(scope, (ast.FunctionDef, ast.AsyncFunctionDef, ast.Lambda)) + else index.tree + ) + events = index.binding_events.get((scope, owner_execution), {}).get(name, ()) + if current_execution is not owner_execution: + local_events = index.binding_events.get((scope, current_execution), {}).get(name, ()) + return tuple(sorted((*events, *local_events))) + return events + + flag_events = binding_events(candidate.assignment_scope, candidate.name) + flag_index = bisect_right(flag_events, (shell_start, True)) + if flag_index: + last_flag = flag_events[flag_index - 1] + if last_flag[0] > candidate.assignment_start and not last_flag[1]: + return True + if receiver_trusted is True: + return False + function = candidate.call.func + receiver = function.id if isinstance(function, ast.Name) else function.value.id + receiver_scope = _resolved_name_scope(index, candidate.call_scope_chain, receiver, call_start) + receiver_execution = ( + receiver_scope + if isinstance(receiver_scope, (ast.FunctionDef, ast.AsyncFunctionDef, ast.Lambda)) + else index.tree + ) + # Called-module slot replacement is proved only by the companion cache + # state, which discards stale proof after an unknown eager effect. + receiver_events = binding_events(receiver_scope, receiver) + receiver_index = bisect_right(receiver_events, (call_start, True)) + if receiver_execution is not current_execution: + # Source order does not order a deferred body against outer stores. + # Require a replacement at every invocation the outer scope can reach. + if _deferred_receiver_is_replaced( + index, + parsed, + candidate, + receiver, + receiver_scope, + receiver_execution, + current_execution, + call_start, + ): + return True + elif receiver_index: + if not receiver_events[receiver_index - 1][1]: + return True + elif isinstance(receiver_scope, (ast.FunctionDef, ast.AsyncFunctionDef, ast.Lambda)): + # Compile-time local bindings shadow a global receiver even before store. + if receiver in index.bindings.get(receiver_scope, frozenset()): + return True + if isinstance(function, ast.Attribute): + # Preserve a direct same-name replacement only in this execution scope, + # after the completed store and before another effect or receiver binding. + method_events = index.binding_events.get((receiver_scope, current_execution), {}).get( + receiver + "." + function.attr, () + ) + method_index = bisect_right(method_events, (call_start, True)) + if method_index: + method_start, restores = method_events[method_index - 1] + effects = index.eager_effect_starts.get(current_execution, ()) + effect_index = bisect_right(effects, method_start) + later_binding = receiver_index and receiver_events[receiver_index - 1][0] > method_start + if ( + not restores + and not later_binding + and (effect_index == len(effects) or effects[effect_index] >= call_start) + ): + return True + return False + + +def _tm1_candidates( + content: str, + *, + direct_shell_only: bool = False, + shell_content: str | None = None, +) -> Iterator[tuple[int, int, str, float, bool, int | None, int | None, str | None]]: + def direct_matches( + pattern_index: int, + pattern: str, + confidence: float, + ) -> Iterator[tuple[int, int, re.Match[str], float]]: + for match in re.finditer(pattern, content, re.IGNORECASE | re.MULTILINE): + yield match.start(), pattern_index, match, confidence + + def direct_candidates() -> Iterator[ + tuple[int, int, str, float, bool, int | None, int | None, str | None] + ]: + direct_iterators = ( + direct_matches(pattern_index, pattern, confidence) + for pattern_index, (pattern, confidence) in enumerate(DIRECT_SHELL_TRUE_PATTERNS) + ) + qualified_popen_starts: set[int] = set() + for _, pattern_index, match, confidence in heapq.merge( + *direct_iterators, + key=lambda candidate: (candidate[0], candidate[1]), + ): + alternate_start: int | None = None + alternate_matched_text: str | None = None + if pattern_index == 0: + method = re.match(r"subprocess\.(?P\w+)", match.group(0), re.IGNORECASE) + if method is not None and method.group("method").casefold() == "popen": + alternate_start = match.start() + method.start("method") + alternate_matched_text = content[alternate_start : match.end()] + qualified_popen_starts.add(alternate_start) + elif match.start() in qualified_popen_starts: + # Emit one candidate for a qualified ``subprocess.Popen`` call, + # but retain the bare-method coordinate as an ownership fallback + # when the qualifier belongs to an adjacent scan window. + qualified_popen_starts.remove(match.start()) + continue + elif ( + qualifier_start := _subprocess_qualifier_start(content, match.start()) + ) is not None: + alternate_start = qualifier_start + alternate_matched_text = content[qualifier_start : match.end()] + relative_anchor = _outer_shell_true_anchor(match.group(0)) + if relative_anchor is None: + for shell_value in _DIRECT_SHELL_TRUE_VALUE.finditer(match.group(0)): + relative_anchor = shell_value.start() + anchor = match.start() + relative_anchor if relative_anchor is not None else None + yield ( + match.start(), + match.end(), + match.group(0), + confidence, + True, + anchor, + alternate_start, + alternate_matched_text, + ) + + if direct_shell_only: + yield from direct_candidates() + return + + def ordinary_matches( + pattern_index: int, + pattern: str, + confidence: float, + ) -> Iterator[tuple[int, int, re.Match[str], float]]: + if pattern == _VARIABLE_SHELL_FLAG_PATTERN: + for match in _variable_shell_matches(content): + yield match.start(), pattern_index, match, confidence + return + matches = ( + static_runner.iter_paragraph_matches + if (pattern, confidence) in TM1_PROSE_PATTERNS + else re.finditer + ) + for match in matches(pattern, content, re.IGNORECASE | re.MULTILINE): + yield match.start(), pattern_index, match, confidence + + def ordinary_candidates() -> Iterator[ + tuple[int, int, str, float, bool, int | None, int | None, str | None] + ]: + ordinary_iterators = ( + ordinary_matches(pattern_index, pattern, confidence) + for pattern_index, (pattern, confidence) in enumerate( + TM1_PATTERNS[len(DIRECT_SHELL_TRUE_PATTERNS) :] + ) + ) + for _, _, match, confidence in heapq.merge( + *ordinary_iterators, + key=lambda candidate: (candidate[0], candidate[1]), + ): + yield ( + match.start(), + match.end(), + match.group(0), + confidence, + False, + None, + None, + None, + ) + + def destructive_candidates( + candidate_content: str, + ) -> Iterator[tuple[int, int, str, float, bool, int | None, int | None, str | None]]: + seen_commands: set[tuple[int, int]] = set() + covered_until = 0 + for command_start, body_start in _destructive_command_words(candidate_content): + if command_start < covered_until: + continue + command_key = (command_start, body_start) + if command_key in seen_commands: + continue + seen_commands.add(command_key) + # Documentation is excluded regardless of the shell parse result. Apply + # that existing semantic gate first so large manuals do not pay for a + # character-by-character shell parse for every explanatory ``rm`` noun. + if _is_root_glob_documentation(candidate_content, command_start, body_start): + continue + tokens, command_end, _ = _bounded_shell_tokens( + candidate_content, + command_start, + body_start, + ) + covered_until = max(covered_until, command_end) + command = candidate_content[command_start:command_end] + if _has_destructive_root_glob(tokens) or _has_destructive_root_path(tokens): + yield command_start, command_end, command, 0.9, False, None, None, None + + destructive_iterators = [destructive_candidates(content)] + if shell_content is not None and shell_content != content: + destructive_iterators.append(destructive_candidates(shell_content)) + yield from heapq.merge( + direct_candidates(), + ordinary_candidates(), + *destructive_iterators, + key=lambda candidate: candidate[0], + ) + + +def _markdown_block_separator(line: str, check_runtime: Callable[[], None]) -> bool: + """Recognize Setext underlines/thematic breaks with bounded, linear work.""" + line = line.strip(" \t") + marker = line[:1] + if marker not in {"=", "-", "*", "_"}: + return False + count = 0 + internal_gap = False + for index, character in enumerate(line): + if index % 256 == 0: + check_runtime() + if character == marker: + count += 1 + elif character in " \t" and marker != "=": + internal_gap = True + else: + return False + return count >= (3 if internal_gap or marker in {"*", "_"} else 1) + + +def _markdown_table_cells( + line: str, start: int, check_runtime: Callable[[], None] +) -> list[tuple[int, int]]: + """Locate GFM cells without copying content or changing source offsets.""" + end = len(line) + while start < end and line[start] in " \t": + if start % 256 == 0: + check_runtime() + start += 1 + while end > start and line[end - 1] in " \t": + if end % 256 == 0: + check_runtime() + end -= 1 + if start == end: + return [] + # Edge pipes are optional. In cmark-gfm a backslash immediately before a + # pipe escapes it even when that backslash follows another backslash. + if line[start] == "|": + start += 1 + if line[end - 1] == "|" and (end == 1 or line[end - 2] != "\\"): + end -= 1 + if start > end: + return [] + cells: list[tuple[int, int]] = [] + cell_start = start + for cursor in range(start, end): if (cursor - start) % 256 == 0: check_runtime() if line[cursor] == "|" and (cursor == start or line[cursor - 1] != "\\"): @@ -3144,6 +4430,7 @@ def _markdown_shell_text( inside their bodies. Pair equal-length runs in linear time. """ output = list(content) + prose_apostrophes: set[int] = set() runs: list[tuple[int, int]] = [] list_marker = re.compile(r"(?:[-+*]|[0-9]{1,9}[.)])(?=[ \t])") backtick_runs = re.compile(r"`+") @@ -3354,6 +4641,20 @@ def mask_inline_delimiters() -> None: if not cells: table_columns = None elif not separator and not empty_list_item: + if ( + complete_context + and not runs + and not heading + and len(leading) <= 512 + and (prose := _MARKDOWN_CONTRACTION_PROSE_RE.fullmatch(leading)) + ): + # A whole plain-language sentence, including a parsed + # list-item body, owns its contraction; + # it is not an unclosed shell quote spanning later prose. + # The restrictive grammar excludes code delimiters and + # shell syntax. Fences, indented code, HTML, tables, and + # pending multiline inline spans never reach this rule. + prose_apostrophes.add(offset + prefix + prose.start("apostrophe")) for match in backtick_runs.finditer(line): check_runtime() runs.append((offset + match.start(), offset + match.end())) @@ -3370,7 +4671,40 @@ def mask_inline_delimiters() -> None: paragraph_list_indent = 0 offset += len(line) mask_inline_delimiters() - return "".join(output) + projected = "".join(output) + if prose_apostrophes: + # A prose-looking line can still be inside an earlier shell word, such + # as a multiline bash -c string. Reuse the shell parser to preserve that + # ownership before masking any apostrophe. Unresolved words retain all + # remaining bytes; this pass never grants ownership past a parse limit. + cursor = 0 + next_runtime_check = 0 + last_apostrophe = max(prose_apostrophes) + parameter_ends: dict[int, _ParameterExpansionEnd] = {} + substitution_ends: dict[int, int | None] = {} + backtick_ends: dict[int, int | None] = {} + while cursor <= last_apostrophe: + if cursor >= next_runtime_check: + check_runtime() + next_runtime_check = cursor + 4096 + if cursor in prose_apostrophes: + output[cursor] = " " + elif projected[cursor] in "'\"`$\\": + parsed = _parse_shell_command_word( + projected, + cursor, + parameter_ends, + substitution_ends, + backtick_ends, + check_runtime=check_runtime, + ) + if parsed is None or parsed.limited: + break + cursor = max(cursor + 1, parsed.end) + continue + cursor += 1 + return "".join(output) + return projected def has_bounded_parse_exhaustion( @@ -3452,62 +4786,246 @@ def _line_containing(content: str, start: int, end: int) -> str: return content[line_start:line_end] -def analyze(content: str, file_path: str, file_type: str) -> list[AnalyzerFinding]: +def _bounded_context(content: str, match_start: int) -> str: + """Return a fixed-size context centered on one direct shell call.""" + left = max(0, match_start - _MAX_DIRECT_SHELL_CONTEXT_CHARS // 2) + right = min(len(content), left + _MAX_DIRECT_SHELL_CONTEXT_CHARS) + left = max(0, right - _MAX_DIRECT_SHELL_CONTEXT_CHARS) + return content[left:right].rstrip(LINE_BREAK_CHARS) + + +def _raw_classification_bounds(content: str, source_start: int) -> tuple[int, int]: + """Return the raw scanner-window bounds that own *source_start*.""" + if len(content) <= static_runner.SECURITY_VIEW_WINDOW_CHARS: + return 0, len(content) + owned_start = ( + source_start // static_runner._RAW_WINDOW_OWNED_CHARS + ) * static_runner._RAW_WINDOW_OWNED_CHARS + owned_end = min(len(content), owned_start + static_runner._RAW_WINDOW_OWNED_CHARS) + raw_start = max(0, owned_start - static_runner._WINDOW_OVERLAP_CHARS) + raw_end = min(len(content), owned_end + static_runner._WINDOW_OVERLAP_CHARS) + return raw_start, raw_end + + +def _classify_tm1( + context: str, + matched_text: str, + matched_line: str, + confidence: float, + file_type: str, + *, + safe_context: bool | None = None, + safe_matched_line: bool | None = None, +) -> tuple[Severity, float]: + """Apply the existing TM1 contextual classification to one candidate.""" + if safe_context is None: + safe_context = _is_safe_container_command(context) or _is_safe_dockerfile_idiom( + context, matched_text + ) + if safe_matched_line is None: + safe_matched_line = _is_safe_cache_cleanup(matched_line) + if safe_context or safe_matched_line: + return Severity.LOW, min(confidence, 0.15) + adjusted = ( + min(1.0, confidence + 0.1) if file_type in ("python", "shell", "javascript") else confidence + ) + return Severity.HIGH, adjusted + + +def analyze( + content: str, + file_path: str, + file_type: str, + *, + _direct_shell_only: bool = False, + defer_variable_reconciliation: bool = False, +) -> list[AnalyzerFinding]: """Analyze content for tool misuse patterns (TM1–TM3).""" findings: list[AnalyzerFinding] = [] def loc(ln: int) -> Location: return Location(file=file_path, start_line=ln) + line_starts = (0, *(match.end() for match in LOGICAL_LINE_BREAK.finditer(content))) + + def line_number(start: int) -> int: + return bisect_right(line_starts, start) + + context_by_line: dict[int, str] = {} + def ctx(start: int) -> str: - return get_context(content, start) + line = line_number(start) + context = context_by_line.get(line) + if context is None: + context = get_context(content, start) + context_by_line[line] = context + return context tag = [PatternCategory.TOOL_MISUSE.value] tm1_findings_by_key: dict[tuple[int, str], AnalyzerFinding] = {} - - # The variable-shell-flag regex cannot see Python scopes, so an assignment - # in one function and a shell= use in another still match. Drop those - # cross-scope candidates for Python files. - cross_scope_starts: set[int] = set() - if file_type == "python": - for variable_match in _VARIABLE_SHELL_FLAG_RE.finditer(content): - if not _variable_shell_flag_same_scope(content, file_path, variable_match): - cross_scope_starts.add(variable_match.start()) + direct_safety_by_line: dict[int, tuple[bool, bool]] = {} + + variable_matches = ( + {} + if _direct_shell_only + else { + (match.start(), match.end()): (match.group(1), match) + for match in _variable_shell_matches(content) + } + ) + invisible_variable_matches: set[tuple[int, int]] = set() + if file_type == "python" and variable_matches and not defer_variable_reconciliation: + parsed = parse_python_source(content, file_path) + ast_index = _build_variable_shell_ast_index(parsed) + if ast_index is not None: + for span, (_, match) in variable_matches.items(): + assignment_line = bisect_right(ast_index.line_character_starts, match.start(1)) + candidate = _resolve_variable_shell_candidate(ast_index, match, assignment_line) + if candidate is not None and ( + not candidate.visible + or _lexical_shell_has_counterevidence( + ast_index, parsed, candidate, receiver_trusted=None + ) + ): + invisible_variable_matches.add(span) shell_content = ( _perl_literal_print_shell_text(content, lambda: None) if file_type == "perl" else None ) - for match_start, match_end, matched_text, confidence in _tm1_candidates( - content, shell_content=shell_content + for ( + match_start, + match_end, + matched_text, + confidence, + is_direct_shell_true, + direct_shell_anchor, + alternate_start, + alternate_matched_text, + ) in _tm1_candidates( + content, + direct_shell_only=_direct_shell_only, + shell_content=shell_content, ): - if match_start in cross_scope_starts: + variable_match = variable_matches.get((match_start, match_end)) + if variable_match is not None and (match_start, match_end) in invisible_variable_matches: continue - line_num = get_line_number(content, match_start) - context_text = ctx(match_start) + line_num = line_number(match_start) + classification_context = ctx(match_start) + context_text = ( + _bounded_context(content, match_start) + if is_direct_shell_true + else classification_context + ) matched = matched_text[:200] matched_line = _line_containing(content, match_start, match_end) - if ( - _is_safe_container_command(context_text) - or _is_safe_dockerfile_idiom(context_text, matched) - or _is_safe_cache_cleanup(matched_line) - ): - adj = min(confidence, 0.15) - sev = Severity.LOW + if is_direct_shell_true: + safety = direct_safety_by_line.get(line_num) + if safety is None: + safety = ( + _is_safe_container_command(classification_context) + or _is_safe_dockerfile_idiom(classification_context, matched), + _is_safe_cache_cleanup(matched_line), + ) + direct_safety_by_line[line_num] = safety + sev, adj = _classify_tm1( + classification_context, + matched, + matched_line, + confidence, + file_type, + safe_context=safety[0], + safe_matched_line=safety[1], + ) else: - adj = ( - min(1.0, confidence + 0.1) - if file_type in ("python", "shell", "javascript") - else confidence + sev, adj = _classify_tm1( + classification_context, + matched, + matched_line, + confidence, + file_type, ) - sev = Severity.HIGH - candidate_key = (line_num, " ".join(matched.strip().split())) + candidate_identity = ( + matched_text if is_direct_shell_true or variable_match is not None else matched + ) + complete_identity_match = matched_text + candidate_key = ( + line_num, + sha256(" ".join(candidate_identity.strip().split()).encode()).hexdigest(), + ) existing = tm1_findings_by_key.get(candidate_key) if existing is not None: if adj > existing.confidence: existing.confidence = adj existing.severity = sev continue + evidence: dict[str, object] = {static_runner._VIEW_START_EVIDENCE: match_start} + if is_direct_shell_true: + canonical_start = ( + alternate_start + if alternate_start is not None and alternate_start < match_start + else match_start + ) + shell_match = ( + _DIRECT_SHELL_TRUE_VALUE.match(content, direct_shell_anchor) + if direct_shell_anchor is not None + else None + ) + canonical_match = ( + content[canonical_start : shell_match.end()] + if shell_match is not None + else ( + alternate_matched_text + if alternate_start is not None + and alternate_start < match_start + and alternate_matched_text is not None + else matched_text + ) + ) + complete_identity_match = normalized_security_view(canonical_match).text + reach_end = content.find(")", match_start) + reach_terminated = reach_end != -1 + if not reach_terminated: + reach_end = len(content) + evidence.update( + { + static_runner._PRESERVE_SOURCE_START_EVIDENCE: True, + static_runner._VIEW_REACH_END_EVIDENCE: reach_end, + static_runner._VIEW_REPLACEMENT_START_LIMIT_EVIDENCE: max( + 0, + len(content) - len("True"), + ), + LEXICAL_REACH_TERMINATED_EVIDENCE: reach_terminated, + LEXICAL_DIRECT_SHELL_EVIDENCE: True, + LEXICAL_NORMALIZED_MATCH_EVIDENCE: normalized_security_prefix( + canonical_match, + 200, + ), + } + ) + if direct_shell_anchor is not None: + evidence[static_runner._VIEW_ANCHOR_EVIDENCE] = direct_shell_anchor + if alternate_start is not None: + evidence[static_runner._VIEW_ALTERNATE_START_EVIDENCE] = alternate_start + if alternate_matched_text is not None: + evidence[static_runner._ALTERNATE_MATCHED_TEXT_EVIDENCE] = alternate_matched_text[ + :200 + ] + if variable_match is not None and defer_variable_reconciliation: + # Without Python bindings, keep only call-anchored windows for + # true-prefixed names; coalescing drops one a direct owner reports. + if file_type != "python" and not _is_true_prefixed_name(variable_match[0]): + continue + evidence[_VARIABLE_SHELL_FLAG_EVIDENCE] = variable_match[0] + evidence[static_runner._PRESERVE_SOURCE_START_EVIDENCE] = True + call_start = _variable_shell_call_start(variable_match[1]) + if call_start is not None: + if variable_match[0].casefold().startswith("true"): + evidence[static_runner._VIEW_START_EVIDENCE] = call_start + # Like a direct owner, a call-anchored window keeps one + # identity across raw and normalized security views. + complete_identity_match = normalized_security_view(matched_text).text + evidence[static_runner._VIEW_ANCHOR_EVIDENCE] = call_start finding = AnalyzerFinding( rule_id="TM1", message="Tool Parameter Abuse", @@ -3517,11 +5035,15 @@ def ctx(start: int) -> str: tags=tag, context=context_text, matched_text=matched, - complete_match=matched_text, - evidence={static_runner._VIEW_START_EVIDENCE: match_start}, + complete_match=complete_identity_match, + evidence=evidence, ) tm1_findings_by_key[candidate_key] = finding findings.append(finding) + + if _direct_shell_only: + return findings + for pattern, confidence in TM2_PATTERNS: matches = ( static_runner.iter_paragraph_matches @@ -3529,7 +5051,7 @@ def ctx(start: int) -> str: else re.finditer ) for match in matches(pattern, content, re.IGNORECASE | re.MULTILINE): - line_num = get_line_number(content, match.start()) + line_num = line_number(match.start()) context_text = ctx(match.start()) matched = match.group(0)[:200] @@ -3559,7 +5081,7 @@ def ctx(start: int) -> str: else re.finditer ) for match in matches(pattern, content, re.IGNORECASE | re.MULTILINE): - line_num = get_line_number(content, match.start()) + line_num = line_number(match.start()) findings.append( AnalyzerFinding( rule_id="TM3", @@ -3576,7 +5098,7 @@ def ctx(start: int) -> str: # TM4: privileged K8s workload. Example filtering is delegated to the runner. for pattern, confidence in TM4_PATTERNS: for match in re.finditer(pattern, content, re.IGNORECASE | re.MULTILINE): - line_num = get_line_number(content, match.start()) + line_num = line_number(match.start()) findings.append( AnalyzerFinding( rule_id="TM4", @@ -3593,8 +5115,1623 @@ def ctx(start: int) -> str: return findings +def coalesce_path_findings(content: str, findings: list[Finding]) -> list[Finding]: + """Select one owner per direct call without discarding private coordinates.""" + from . import static_python_shell_truthiness + + full_match_identities = { + finding.finding_id: finding.match_fingerprint + for finding in findings + if finding.match_fingerprint is not None + } + bound_call_starts = { + call_start + for finding in findings + if finding.evidence.get(static_python_shell_truthiness.BOUND_SHELL_EVIDENCE) is True + and type( + call_start := finding.evidence.get( + static_python_shell_truthiness.BOUND_CALL_START_EVIDENCE + ) + ) + is int + } + # The case-insensitive direct ``shell=True`` pattern also matches a name + # spelled ``true``, so its lexical owner already reports that exact call + # even when no Python AST can reconcile the variable window. + true_name_call_starts = bound_call_starts | { + coordinate + for finding in findings + if finding.evidence.get(LEXICAL_DIRECT_SHELL_EVIDENCE) is True + for key in ( + static_runner._ABSOLUTE_START_EVIDENCE, + static_runner._ABSOLUTE_ALTERNATE_START_EVIDENCE, + ) + if type(coordinate := finding.evidence.get(key)) is int + } + findings = [ + finding + for finding in findings + if not ( + isinstance(variable_name := finding.evidence.get(_VARIABLE_SHELL_FLAG_EVIDENCE), str) + and finding.evidence.get(static_runner._ABSOLUTE_ANCHOR_EVIDENCE) + in (true_name_call_starts if variable_name.casefold() == "true" else bound_call_starts) + ) + ] + canonical_bound_calls: dict[tuple[str, int, str], set[int]] = {} + for finding in findings: + canonical = finding.evidence.get( + static_python_shell_truthiness.BOUND_CANONICAL_FINGERPRINT_EVIDENCE + ) + call_start = finding.evidence.get(static_python_shell_truthiness.BOUND_CALL_START_EVIDENCE) + if isinstance(canonical, str) and type(call_start) is int: + canonical_bound_calls.setdefault( + (finding.file, finding.start_line, canonical), + set(), + ).add(call_start) + colliding_bound_identities = { + key for key, call_starts in canonical_bound_calls.items() if len(call_starts) > 1 + } + + lexical_records: list[tuple[Finding, int]] = [] + parents: dict[int, int] = {} + + def find(coordinate: int) -> int: + parent = parents.setdefault(coordinate, coordinate) + while parent != coordinate: + grandparent = parents[parent] + parents[coordinate] = grandparent + coordinate = parent + parent = grandparent + return coordinate + + def union(first: int, second: int) -> None: + first_root = find(first) + second_root = find(second) + if first_root == second_root: + return + canonical = min(first_root, second_root) + parents[max(first_root, second_root)] = canonical + + for finding in findings: + if finding.evidence.get(LEXICAL_DIRECT_SHELL_EVIDENCE) is not True: + continue + start = finding.evidence.get(static_runner._ABSOLUTE_START_EVIDENCE) + if type(start) is int: + find(start) + alternate_start = finding.evidence.get(static_runner._ABSOLUTE_ALTERNATE_START_EVIDENCE) + if type(alternate_start) is int: + union(start, alternate_start) + lexical_records.append((finding, start)) + + lexical_groups: dict[int, list[Finding]] = {} + for finding, start in lexical_records: + lexical_groups.setdefault(find(start), []).append(finding) + + severity_rank = {"LOW": 0, "MEDIUM": 1, "HIGH": 2, "CRITICAL": 3} + + def lexical_owner_rank( + finding: Finding, + call_start: int, + ) -> tuple[bool, bool, bool, int, float]: + """Rank public classification donors independently of scan order.""" + return ( + finding.evidence.get(LEXICAL_RAW_OWNER_EVIDENCE) is True, + "normalized-view" not in finding.tags, + finding.evidence.get(static_runner._ABSOLUTE_START_EVIDENCE) == call_start, + severity_rank.get(finding.severity, -1), + finding.confidence, + ) + + def select_lexical_owner(group: list[Finding], call_start: int) -> Finding: + return max( + enumerate(group), + key=lambda item: (*lexical_owner_rank(item[1], call_start), -item[0]), + )[1] + + def fingerprint_source_rank( + finding: Finding, + call_start: int, + ) -> tuple[bool, bool, int, bool, int]: + """Rank immutable identity donors independently of scan order.""" + anchor = finding.evidence.get(static_runner._ABSOLUTE_ANCHOR_EVIDENCE) + reach_end = finding.evidence.get(static_runner._ABSOLUTE_REACH_END_EVIDENCE) + return ( + finding.evidence.get(static_runner._ABSOLUTE_START_EVIDENCE) == call_start, + "normalized-view" in finding.tags, + anchor if type(anchor) is int else -1, + finding.evidence.get(LEXICAL_REACH_TERMINATED_EVIDENCE) is True, + reach_end if type(reach_end) is int else -1, + ) + + def fingerprint_record_rank( + record: dict[str, object], + call_start: int, + ) -> tuple[bool, bool, int, bool, int]: + """Re-rank a retained identity donor at the current group coordinate.""" + anchor = record.get("anchor") + reach_end = record.get("reach_end") + return ( + record.get("start") == call_start, + record.get("normalized") is True, + anchor if type(anchor) is int else -1, + record.get("reach_terminated") is True, + reach_end if type(reach_end) is int else -1, + ) + + def classification_record( + finding: Finding, + ) -> dict[str, object]: + """Snapshot the bounded public fields needed across cap finalizers.""" + return { + "severity": finding.severity, + "confidence": finding.confidence, + "base_tags": tuple(finding.tags), + "tags": tuple(finding.tags), + "raw_owner": finding.evidence.get(LEXICAL_RAW_OWNER_EVIDENCE) is True, + "raw_view": "normalized-view" not in finding.tags, + "start": finding.evidence.get(static_runner._ABSOLUTE_START_EVIDENCE), + } + + def classification_record_rank( + record: dict[str, object], + call_start: int, + ) -> tuple[bool, bool, bool, int, float]: + """Re-rank a retained public donor at the current group coordinate.""" + severity = record.get("severity") + confidence = record.get("confidence") + return ( + record.get("raw_owner") is True, + record.get("raw_view") is True, + record.get("start") == call_start, + severity_rank.get(severity, -1) if isinstance(severity, str) else -1, + ( + float(confidence) + if isinstance(confidence, (int, float)) and not isinstance(confidence, bool) + else -1.0 + ), + ) + + def select_fingerprint_source(group: list[Finding], call_start: int) -> Finding: + """Prefer the canonical-coordinate normalized match, independent of scan order.""" + return max( + enumerate(group), + key=lambda item: (*fingerprint_source_rank(item[1], call_start), -item[0]), + )[1] + + def lexical_fingerprint(finding: Finding) -> str | None: + """Hash immutable lexical text instead of a prior coalescing override.""" + canonical_match = finding.evidence.get(LEXICAL_NORMALIZED_MATCH_EVIDENCE) + if not isinstance(canonical_match, str) and finding.matched_text is None: + return finding.fingerprint() + if not isinstance(canonical_match, str): + canonical_match = normalized_security_prefix(finding.matched_text or "", 200) + normalized = " ".join(canonical_match.strip().split()) + return sha256(f"{finding.rule_id}\x1f{normalized}".encode()).hexdigest() + + def actual_shell_is_reachable( + candidate: Finding, + shell_value_start: int, + direct_match_end: int, + ) -> bool | None: + """Compare an AST shell value with this candidate's exact regex reach.""" + reach_end = candidate.evidence.get(static_runner._ABSOLUTE_REACH_END_EVIDENCE) + reach_terminated = candidate.evidence.get(LEXICAL_REACH_TERMINATED_EVIDENCE) + if type(reach_end) is not int or type(reach_terminated) is not bool: + return None + replacement_start_limit = candidate.evidence.get( + static_runner._ABSOLUTE_REPLACEMENT_START_LIMIT_EVIDENCE + ) + if type(replacement_start_limit) is int and shell_value_start > replacement_start_limit: + replacement_recovery_start = candidate.evidence.get( + static_runner._ABSOLUTE_REPLACEMENT_RECOVERY_START_EVIDENCE + ) + if type(replacement_recovery_start) is not int or not any( + type(candidate_start) is int and candidate_start >= replacement_recovery_start + for candidate_start in ( + candidate.evidence.get(static_runner._ABSOLUTE_START_EVIDENCE), + candidate.evidence.get(static_runner._ABSOLUTE_ALTERNATE_START_EVIDENCE), + ) + ): + return False + if reach_terminated: + return reach_end > shell_value_start + return reach_end >= direct_match_end + + lexical_owners: dict[int, Finding] = {} + lexical_member_starts: dict[str, int] = {} + for call_start, group in lexical_groups.items(): + owner = select_lexical_owner(group, call_start) + canonical = select_fingerprint_source(group, call_start) + if "normalized-view" not in owner.tags: + owner.evidence[LEXICAL_RAW_OWNER_EVIDENCE] = True + owner_start = owner.evidence.get(static_runner._ABSOLUTE_START_EVIDENCE) + if owner is not canonical: + # Coalescing can run repeatedly while a finding cap is being + # finalized. Persist the canonical identity coordinates on the + # retained public owner even when both candidates start together, + # otherwise a later pass re-hashes the earlier lexical lookalike. + canonical_match = canonical.evidence.get(LEXICAL_NORMALIZED_MATCH_EVIDENCE) + if isinstance(canonical_match, str): + owner.evidence[LEXICAL_NORMALIZED_MATCH_EVIDENCE] = canonical_match + for evidence_key in ( + static_runner._ABSOLUTE_ANCHOR_EVIDENCE, + static_runner._ABSOLUTE_REACH_END_EVIDENCE, + static_runner._ABSOLUTE_REPLACEMENT_START_LIMIT_EVIDENCE, + static_runner._ABSOLUTE_REPLACEMENT_RECOVERY_START_EVIDENCE, + LEXICAL_REACH_TERMINATED_EVIDENCE, + ): + if evidence_key in canonical.evidence: + owner.evidence[evidence_key] = canonical.evidence[evidence_key] + else: + owner.evidence.pop(evidence_key, None) + if ( + owner is not canonical + and type(owner_start) is int + and owner_start != call_start + and canonical.matched_text is not None + ): + # A raw bare-Popen owner can be the only classification context + # that existed before a cross-window normalized qualifier was + # reconstructed. Preserve that public owner while borrowing only + # the canonical call identity from the derived candidate. + owner.matched_text = canonical.matched_text + owner.finding = canonical.finding + owner.evidence[static_runner._ABSOLUTE_ALTERNATE_START_EVIDENCE] = call_start + if "normalized-view" in canonical.tags and "normalized-view" not in owner.tags: + owner.tags.append("normalized-view") + lexical_canonical_fingerprint = lexical_fingerprint(canonical) + complete_identity = full_match_identities.get(canonical.finding_id) + owner.match_fingerprint = ( + complete_identity + if isinstance(complete_identity, str) + else lexical_canonical_fingerprint + ) + lexical_owners[call_start] = owner + lexical_member_starts.update((candidate.finding_id, call_start) for candidate in group) + + ast_call_starts: set[int] = set() + ast_owners: dict[str, Finding] = {} + for finding in findings: + if finding.evidence.get(static_python_shell_truthiness.BOUND_SHELL_EVIDENCE) is not True: + continue + canonical_fingerprint = finding.evidence.get( + static_python_shell_truthiness.BOUND_CANONICAL_FINGERPRINT_EVIDENCE + ) + lexical_identity = finding.evidence.get(LEXICAL_BOUND_IDENTITY_EVIDENCE) + previously_reachable = finding.evidence.get(LEXICAL_BOUND_REACHABLE_EVIDENCE) is True + complete_identity = full_match_identities.get(finding.finding_id) + canonical_collision = ( + isinstance(canonical_fingerprint, str) + and ( + finding.file, + finding.start_line, + canonical_fingerprint, + ) + in colliding_bound_identities + ) + fingerprint = ( + lexical_identity + if isinstance(lexical_identity, str) + else complete_identity + if canonical_collision and isinstance(complete_identity, str) + else canonical_fingerprint + ) + if canonical_collision and isinstance(fingerprint, str): + finding.evidence[LEXICAL_BOUND_IDENTITY_EVIDENCE] = fingerprint + if isinstance(fingerprint, str): + finding.match_fingerprint = fingerprint + ast_call_start = finding.evidence.get( + static_python_shell_truthiness.BOUND_CALL_START_EVIDENCE + ) + ast_call_end = finding.evidence.get(static_python_shell_truthiness.BOUND_CALL_END_EVIDENCE) + direct_match_end = finding.evidence.get( + static_python_shell_truthiness.BOUND_DIRECT_MATCH_END_EVIDENCE + ) + popen_start = finding.evidence.get( + static_python_shell_truthiness.BOUND_POPEN_START_EVIDENCE + ) + shell_anchor = finding.evidence.get( + static_python_shell_truthiness.BOUND_SHELL_ANCHOR_EVIDENCE + ) + ast_owner: Finding | None = None + if type(ast_call_start) is int: + ast_call_starts.add(ast_call_start) + lexical_coordinates = [ast_call_start] + if type(popen_start) is int: + # Parenthesized, spaced, and explicitly continued receivers + # can only be recognized lexically from the method token. That + # token still belongs to this parsed call even when a quoted + # trailing ``shell=True`` produces a different shell anchor. + ast_call_starts.add(popen_start) + lexical_coordinates.append(popen_start) + coordinate_groups = [ + (coordinate, lexical_groups[coordinate]) + for coordinate in lexical_coordinates + if coordinate in lexical_groups + ] + exact_group = [ + candidate + for _, group in coordinate_groups + for candidate in group + if type(shell_anchor) is int + and candidate.evidence.get(static_runner._ABSOLUTE_ANCHOR_EVIDENCE) == shell_anchor + ] + if exact_group: + ast_owner = select_lexical_owner(exact_group, ast_call_start) + ast_owners[finding.finding_id] = ast_owner + finding.evidence.pop(LEXICAL_BOUND_IDENTITY_EVIDENCE, None) + finding.evidence.pop(LEXICAL_BOUND_IDENTITY_SOURCE_EVIDENCE, None) + finding.evidence.pop(LEXICAL_BOUND_CLASSIFICATION_EVIDENCE, None) + elif type(shell_anchor) is int and type(direct_match_end) is int: + # The legacy regex stops at the first nested ``)``. If an + # earlier parenthesized argument contains ``shell=True``, a + # direct literal is therefore fingerprinted from that retained + # same-call prefix in every file type. Keep a bound-name call + # on that established identity without borrowing its public + # text, tags, or classification. Exact coordinates exclude a + # separate nested call that happens to appear in the arguments. + same_call_candidates = [ + candidate for _, group in coordinate_groups for candidate in group + ] + shell_value_start = direct_match_end - len("True") + reachability = [ + actual_shell_is_reachable( + candidate, + shell_value_start, + direct_match_end, + ) + for candidate in same_call_candidates + ] + actual_shell_is_unreachable = bool(same_call_candidates) and all( + type( + candidate_anchor := candidate.evidence.get( + static_runner._ABSOLUTE_ANCHOR_EVIDENCE + ) + ) + is int + and ast_call_start <= candidate_anchor < shell_anchor + and (type(ast_call_end) is not int or candidate_anchor < ast_call_end) + and reachable is False + for candidate, reachable in zip( + same_call_candidates, + reachability, + strict=True, + ) + ) + if previously_reachable or any(reachable is True for reachable in reachability): + finding.evidence[LEXICAL_BOUND_REACHABLE_EVIDENCE] = True + finding.evidence.pop(LEXICAL_BOUND_IDENTITY_EVIDENCE, None) + finding.evidence.pop(LEXICAL_BOUND_IDENTITY_SOURCE_EVIDENCE, None) + finding.evidence.pop(LEXICAL_BOUND_CLASSIFICATION_EVIDENCE, None) + if isinstance(canonical_fingerprint, str): + finding.match_fingerprint = canonical_fingerprint + elif actual_shell_is_unreachable: + identity_coordinate = min(coordinate for coordinate, _ in coordinate_groups) + identity_source = select_fingerprint_source( + same_call_candidates, + identity_coordinate, + ) + public_owner = select_lexical_owner( + same_call_candidates, + identity_coordinate, + ) + legacy_fingerprint = lexical_fingerprint(identity_source) + if isinstance(legacy_fingerprint, str): + identity_record: dict[str, object] = { + "start": identity_source.evidence.get( + static_runner._ABSOLUTE_START_EVIDENCE + ), + "normalized": "normalized-view" in identity_source.tags, + "anchor": identity_source.evidence.get( + static_runner._ABSOLUTE_ANCHOR_EVIDENCE + ), + "reach_terminated": identity_source.evidence.get( + LEXICAL_REACH_TERMINATED_EVIDENCE + ), + "reach_end": identity_source.evidence.get( + static_runner._ABSOLUTE_REACH_END_EVIDENCE + ), + } + prior_identity_record = finding.evidence.get( + LEXICAL_BOUND_IDENTITY_SOURCE_EVIDENCE + ) + if not isinstance(prior_identity_record, dict) or ( + fingerprint_record_rank(identity_record, identity_coordinate) + > fingerprint_record_rank( + prior_identity_record, + identity_coordinate, + ) + ): + finding.evidence[LEXICAL_BOUND_IDENTITY_EVIDENCE] = legacy_fingerprint + finding.evidence[LEXICAL_BOUND_IDENTITY_SOURCE_EVIDENCE] = ( + identity_record + ) + persisted_identity = finding.evidence.get(LEXICAL_BOUND_IDENTITY_EVIDENCE) + if isinstance(persisted_identity, str): + finding.match_fingerprint = persisted_identity + + prior_classification = finding.evidence.get( + LEXICAL_BOUND_CLASSIFICATION_EVIDENCE + ) + current_classification = classification_record(public_owner) + if not isinstance(prior_classification, dict) or ( + classification_record_rank( + current_classification, + identity_coordinate, + ) + > classification_record_rank( + prior_classification, + identity_coordinate, + ) + ): + prior_classification = current_classification + + base_tags = prior_classification.get("base_tags") + projected_tags = ( + list(base_tags) + if isinstance(base_tags, (list, tuple)) + and all(isinstance(tag, str) for tag in base_tags) + else [] + ) + persisted_identity_record = finding.evidence.get( + LEXICAL_BOUND_IDENTITY_SOURCE_EVIDENCE + ) + if ( + isinstance(persisted_identity_record, dict) + and persisted_identity_record.get("normalized") is True + and persisted_identity_record.get("start") == identity_coordinate + and prior_classification.get("start") != identity_coordinate + and "normalized-view" not in projected_tags + ): + # This is the same projection performed above when + # a raw bare-Popen owner and its normalized qualified + # identity coexist in one pass. Re-derive it when + # those bounded donors arrive in separate cap passes. + projected_tags.append("normalized-view") + prior_classification["tags"] = tuple(projected_tags) + finding.evidence[LEXICAL_BOUND_CLASSIFICATION_EVIDENCE] = ( + prior_classification + ) + if ast_owner is not None and isinstance(canonical_fingerprint, str): + ast_owner.match_fingerprint = canonical_fingerprint + + reconciled: list[Finding] = [] + emitted_call_starts: set[int] = set() + for finding in findings: + is_ast = finding.evidence.get(static_python_shell_truthiness.BOUND_SHELL_EVIDENCE) is True + ast_call_start = finding.evidence.get( + static_python_shell_truthiness.BOUND_CALL_START_EVIDENCE + ) + ast_owner = ast_owners.get(finding.finding_id) + if is_ast: + if type(ast_call_start) is int and ast_call_start in emitted_call_starts: + continue + if ast_owner is not None: + reconciled.append(ast_owner) + else: + reconciled.append(finding) + if type(ast_call_start) is int: + emitted_call_starts.add(ast_call_start) + continue + lexical_start = lexical_member_starts.get(finding.finding_id) + if lexical_start is not None: + if lexical_start not in ast_call_starts and lexical_start not in emitted_call_starts: + reconciled.append(lexical_owners[lexical_start]) + emitted_call_starts.add(lexical_start) + continue + reconciled.append(finding) + + source_line_starts = (0, *(match.end() for match in LOGICAL_LINE_BREAK.finditer(content))) + tm1_slots: list[int] = [] + tm1_by_source: list[tuple[int, int, Finding]] = [] + for index, finding in enumerate(reconciled): + if finding.rule_id != "TM1": + continue + bound_start = finding.evidence.get(static_python_shell_truthiness.BOUND_CALL_START_EVIDENCE) + lexical_coordinate = finding.evidence.get(static_runner._ABSOLUTE_START_EVIDENCE) + source_start = ( + bound_start + if type(bound_start) is int + else lexical_coordinate + if type(lexical_coordinate) is int + else None + ) + if source_start is None: + line_index = min(max(finding.start_line - 1, 0), len(source_line_starts) - 1) + source_start = source_line_starts[line_index] + if type(finding.start_column) is int and finding.start_column > 0: + source_start += finding.start_column + tm1_slots.append(index) + tm1_by_source.append((source_start, index, finding)) + for slot, (_, _, finding) in zip( + tm1_slots, + sorted(tm1_by_source, key=lambda item: (item[0], item[1])), + strict=True, + ): + reconciled[slot] = finding + public: list[Finding] = [] + seen_direct_calls: set[tuple[str, int, str | None]] = set() + for finding in reconciled: + is_direct_shell = ( + finding.evidence.get(LEXICAL_DIRECT_SHELL_EVIDENCE) is True + or finding.evidence.get(static_python_shell_truthiness.BOUND_SHELL_EVIDENCE) is True + ) + if finding.rule_id != "TM1" or not is_direct_shell: + public.append(finding) + continue + key = (finding.file, finding.start_line, finding.fingerprint()) + if key in seen_direct_calls: + continue + seen_direct_calls.add(key) + public.append(finding) + return public + + +def _owning_raw_window( + content: str, + source_start: int, +) -> tuple[int, int, int, int]: + """Return the raw and owned bounds used by the ordinary window scanner.""" + if len(content) <= static_runner.SECURITY_VIEW_WINDOW_CHARS: + return 0, len(content), 0, len(content) + owned_start = ( + source_start // static_runner._RAW_WINDOW_OWNED_CHARS + ) * static_runner._RAW_WINDOW_OWNED_CHARS + owned_end = min(len(content), owned_start + static_runner._RAW_WINDOW_OWNED_CHARS) + raw_start = max(0, owned_start - static_runner._WINDOW_OVERLAP_CHARS) + raw_end = min(len(content), owned_end + static_runner._WINDOW_OVERLAP_CHARS) + return raw_start, raw_end, owned_start, owned_end + + +def _window_popen_qualifiers( + content: str, + popen_starts: set[int], + check_runtime: Callable[[], None], + source_context: static_runner._WindowSourceContext, +) -> dict[int, tuple[int, int, str, str, bool, bool]]: + """Replay each owning bounded window once for retained Popen anchors.""" + targets_by_window: dict[tuple[int, int, int, int], set[int]] = {} + for popen_start in popen_starts: + bounds = _owning_raw_window(content, popen_start) + targets_by_window.setdefault(bounds, set()).add(popen_start) + + qualifiers: dict[int, tuple[int, int, str, str, bool, bool]] = {} + qualified_pattern = DIRECT_SHELL_TRUE_PATTERNS[0][0] + for (raw_start, raw_end, owned_start, owned_end), targets in targets_by_window.items(): + check_runtime() + raw_window = content[raw_start:raw_end] + context_prefix = static_runner._markdown_context_prefix( + content, + raw_start, + raw_end, + source_context.fence_states, + source_context.fence_transitions, + ) + for raw_view in security_text_views(context_prefix + raw_window): + full_view = static_runner._window_view_with_markdown_context( + raw_view, + len(context_prefix), + ) + check_runtime() + for view in static_runner._bounded_view_slices(full_view): + check_runtime() + for match in re.finditer( + qualified_pattern, view.text, re.IGNORECASE | re.MULTILINE + ): + method = re.match( + r"subprocess\.(?P\w+)", + match.group(0), + re.IGNORECASE, + ) + if method is None or method.group("method").casefold() != "popen": + continue + derived_popen_start = match.start() + method.start("method") + popen_start = raw_start + view.source_offset(derived_popen_start) + if popen_start not in targets: + continue + qualifier_start = raw_start + view.source_offset(match.start()) + qualifier_end = raw_start + view.source_offset(derived_popen_start - 1) + 1 + qualifier_text = view.text[match.start() : derived_popen_start] + raw_match_end = raw_start + view.source_offset(match.end() - 1) + 1 + requires_derived_match = ( + view.name != "raw" + and match.group(0) != content[qualifier_start:raw_match_end] + ) + candidate = ( + qualifier_start, + qualifier_end, + qualifier_text, + match.group(0), + requires_derived_match, + owned_start <= qualifier_start < owned_end, + ) + previous = qualifiers.get(popen_start) + if previous is None or (candidate[4], candidate[5]) > ( + previous[4], + previous[5], + ): + qualifiers[popen_start] = candidate + return qualifiers + + +def _cross_window_subprocess_qualifiers( + content: str, + popen_anchors: dict[int, int], + check_runtime: Callable[[], None], +) -> dict[int, tuple[int, int, str, str, bool]]: + """Recover retained qualifiers while replaying each shared projection once.""" + qualifiers: dict[int, tuple[int, int, str, str, bool]] = {} + qualified_pattern = DIRECT_SHELL_TRUE_PATTERNS[0][0] + targets_by_run: dict[tuple[int, int], dict[int, int]] = {} + runs_by_anchor = static_runner._continuity_runs_for_anchors( + content, + set(popen_anchors.values()), + check_runtime, + ) + for popen_start, anchor in popen_anchors.items(): + run = runs_by_anchor.get(anchor) + if run is not None: + targets_by_run.setdefault(run, {})[popen_start] = anchor + + for _, projection_target_anchors in sorted(targets_by_run.items()): + projection_targets = set(projection_target_anchors) + projection = static_runner._anchored_continuity_view( + content, + min(projection_target_anchors.values()), + check_runtime, + ) + if projection is None or not projection.text or projection.source_offsets is None: + continue + + for full_view in security_text_views(projection.text): + for view in static_runner._bounded_view_slices(full_view): + check_runtime() + for match in re.finditer( + qualified_pattern, + view.text, + re.IGNORECASE | re.MULTILINE, + ): + method = re.match( + r"subprocess\.(?P\w+)", + match.group(0), + re.IGNORECASE, + ) + if method is None or method.group("method").casefold() != "popen": + continue + derived_popen_start = match.start() + method.start("method") + projected_popen_start = view.source_offset(derived_popen_start) + raw_popen_start = projection.source_offset(projected_popen_start) + if raw_popen_start not in projection_targets or raw_popen_start in qualifiers: + continue + projected_start = view.source_offset(match.start()) + projected_end = view.source_offset(derived_popen_start - 1) + qualifiers[raw_popen_start] = ( + projection.source_offset(projected_start), + projection.source_offset(projected_end) + 1, + view.text[match.start() : derived_popen_start], + match.group(0), + True, + ) + return qualifiers + + +def reconcile_retained_findings( + content: str, + findings: list[Finding], + check_runtime: Callable[[], None], + source_context: static_runner._WindowSourceContext, +) -> None: + """Finalize retained bare-Popen identity without discovering new findings.""" + retained: list[tuple[Finding, int, int]] = [] + for finding in findings: + if ( + finding.evidence.get(LEXICAL_DIRECT_SHELL_EVIDENCE) is not True + or re.match(r"Popen\b", finding.matched_text or "", re.IGNORECASE) is None + ): + continue + coordinates = [ + coordinate + for key in ( + static_runner._ABSOLUTE_START_EVIDENCE, + static_runner._ABSOLUTE_ALTERNATE_START_EVIDENCE, + ) + if type(coordinate := finding.evidence.get(key)) is int + ] + if coordinates: + popen_start = max(coordinates) + shell_anchor = finding.evidence.get(static_runner._ABSOLUTE_ANCHOR_EVIDENCE) + retained.append( + ( + finding, + popen_start, + shell_anchor if type(shell_anchor) is int else popen_start, + ) + ) + + window_qualifiers = _window_popen_qualifiers( + content, + {popen_start for _, popen_start, _ in retained}, + check_runtime, + source_context, + ) + cross_window_qualifiers = _cross_window_subprocess_qualifiers( + content, + {popen_start: continuity_anchor for _, popen_start, continuity_anchor in retained}, + check_runtime, + ) + for finding, popen_start, _ in retained: + check_runtime() + window_qualifier = window_qualifiers.get(popen_start) + cross_window_qualifier = cross_window_qualifiers.get(popen_start) + if cross_window_qualifier is not None: + qualifier = cross_window_qualifier + update_public_match = True + elif window_qualifier is not None: + ( + qualifier_start, + qualifier_end, + qualifier_text, + canonical_match, + normalized_view, + qualifier_owned, + ) = window_qualifier + update_public_match = normalized_view or qualifier_owned + qualifier = ( + qualifier_start, + qualifier_end, + qualifier_text, + canonical_match, + normalized_view, + ) + else: + qualifier = None + update_public_match = False + if qualifier is None: + continue + qualifier_start, _, _, canonical_match, normalized_view = qualifier + canonical_projection = normalized_security_view(canonical_match).text + canonical_preview = canonical_projection[:200] + finding.evidence[LEXICAL_NORMALIZED_MATCH_EVIDENCE] = canonical_preview + if update_public_match: + finding.matched_text = canonical_preview + finding.finding = canonical_preview + finding.evidence[static_runner._ABSOLUTE_ALTERNATE_START_EVIDENCE] = qualifier_start + finding.match_fingerprint = sha256( + f"{finding.rule_id}\x1f{' '.join(canonical_projection.strip().split())}".encode() + ).hexdigest() + if normalized_view and "normalized-view" not in finding.tags: + finding.tags.append("normalized-view") + + +def _lexical_call_start(finding: Finding) -> int | None: + """Return a lexical owner's canonical call coordinate.""" + coordinates = ( + coordinate + for key in ( + static_runner._ABSOLUTE_START_EVIDENCE, + static_runner._ABSOLUTE_ALTERNATE_START_EVIDENCE, + ) + if type(coordinate := finding.evidence.get(key)) is int + ) + return min(coordinates, default=None) + + +def _internal_evidence_keys() -> tuple[str, ...]: + """Return private reconciliation keys that must not escape public findings.""" + from . import static_python_shell_truthiness + + return ( + _VARIABLE_SHELL_FLAG_EVIDENCE, + LEXICAL_DIRECT_SHELL_EVIDENCE, + LEXICAL_RAW_OWNER_EVIDENCE, + LEXICAL_NORMALIZED_MATCH_EVIDENCE, + LEXICAL_BOUND_IDENTITY_EVIDENCE, + LEXICAL_BOUND_IDENTITY_SOURCE_EVIDENCE, + LEXICAL_BOUND_CLASSIFICATION_EVIDENCE, + LEXICAL_REACH_TERMINATED_EVIDENCE, + LEXICAL_BOUND_REACHABLE_EVIDENCE, + static_python_shell_truthiness.BOUND_SHELL_EVIDENCE, + static_python_shell_truthiness.BOUND_CALL_START_EVIDENCE, + static_python_shell_truthiness.BOUND_CALL_END_EVIDENCE, + static_python_shell_truthiness.BOUND_SHELL_ANCHOR_EVIDENCE, + static_python_shell_truthiness.BOUND_CANONICAL_FINGERPRINT_EVIDENCE, + static_python_shell_truthiness.BOUND_NORMALIZED_VIEW_EVIDENCE, + static_python_shell_truthiness.BOUND_CLASSIFICATION_MATCH_EVIDENCE, + static_python_shell_truthiness.BOUND_DIRECT_MATCH_END_EVIDENCE, + static_python_shell_truthiness.BOUND_POPEN_START_EVIDENCE, + static_python_shell_truthiness.BOUND_DIRECT_OWNER_START_EVIDENCE, + static_python_shell_truthiness.BOUND_SHELL_VALUE_START_EVIDENCE, + static_python_shell_truthiness.BOUND_SHELL_VALUE_END_EVIDENCE, + static_python_shell_truthiness.DIRECT_LITERAL_METADATA_EVIDENCE, + static_runner._VIEW_START_EVIDENCE, + static_runner._VIEW_ANCHOR_EVIDENCE, + static_runner._VIEW_ALTERNATE_START_EVIDENCE, + static_runner._VIEW_REACH_END_EVIDENCE, + static_runner._VIEW_REPLACEMENT_START_LIMIT_EVIDENCE, + static_runner._SOURCE_START_EVIDENCE, + static_runner._SOURCE_ANCHOR_EVIDENCE, + static_runner._SOURCE_ALTERNATE_START_EVIDENCE, + static_runner._SOURCE_REACH_END_EVIDENCE, + static_runner._SOURCE_REPLACEMENT_START_LIMIT_EVIDENCE, + static_runner._SOURCE_REPLACEMENT_RECOVERY_START_EVIDENCE, + static_runner._PRESERVE_SOURCE_START_EVIDENCE, + static_runner._ABSOLUTE_START_EVIDENCE, + static_runner._ABSOLUTE_ANCHOR_EVIDENCE, + static_runner._ABSOLUTE_ALTERNATE_START_EVIDENCE, + static_runner._ABSOLUTE_REACH_END_EVIDENCE, + static_runner._ABSOLUTE_REPLACEMENT_START_LIMIT_EVIDENCE, + static_runner._ABSOLUTE_REPLACEMENT_RECOVERY_START_EVIDENCE, + static_runner._ALTERNATE_MATCHED_TEXT_EVIDENCE, + ) + + +def cleanup_path_findings(findings: list[Finding]) -> list[Finding]: + """Strip private reconciliation state when the analysis deadline has expired.""" + internal_keys = _internal_evidence_keys() + for finding in findings: + for key in internal_keys: + finding.evidence.pop(key, None) + return findings + + +class _DirectShellReplayLexical: + """Lexical-only facade used for direct-equivalent bounded windows.""" + + USES_PYTHON_SOURCE_TYPE = True + + @staticmethod + def analyze(content: str, file_path: str, file_type: str) -> list[AnalyzerFinding]: + return analyze( + content, + file_path, + file_type, + _direct_shell_only=True, + ) + + +def _bound_direct_replays( + content: str, + findings: list[Finding], + python_ast: ParsedPythonFile | None, + *, + started_at: float | None = None, + timeout_seconds: float | None = None, +) -> dict[int, Finding]: + """Replay only retained owners in one cap-independent virtual artifact.""" + from . import static_python_shell_truthiness + + if python_ast is None or python_ast.tree is None: + return {} + retained_call_starts: set[int] = set() + file_path = "" + for finding in findings: + if ( + finding.evidence.get(static_python_shell_truthiness.BOUND_SHELL_EVIDENCE) is not True + or finding.evidence.get(static_python_shell_truthiness.DIRECT_LITERAL_METADATA_EVIDENCE) + is True + ): + continue + call_start = finding.evidence.get(static_python_shell_truthiness.BOUND_CALL_START_EVIDENCE) + if type(call_start) is int: + retained_call_starts.add(call_start) + file_path = finding.file + if not retained_call_starts: + return {} + + budget_started_at = time.monotonic() if started_at is None else started_at + runtime_limit = static_runner.MAX_STATIC_ANALYSIS_SECONDS_PER_ARTIFACT + if timeout_seconds is not None: + runtime_limit = min(runtime_limit, max(0.0, timeout_seconds)) + replay_budget = static_runner._FindingBudget( + max_findings=static_runner.MAX_FILE_CHARS, + started_at=budget_started_at, + deadline=budget_started_at + runtime_limit, + clock=time.monotonic, + ) + try: + specs = static_python_shell_truthiness.bound_shell_metadata( + python_ast, + file_path, + check_runtime=replay_budget.check_runtime, + ) + except static_runner._StaticResourceLimitError: + return {} + if not specs: + return {} + + cursor = 0 + shift = 0 + virtual_parts: list[str] = [] + virtual_coordinates_by_call: dict[int, tuple[int, int | None, int]] = {} + try: + for spec in specs: + if spec.value_start < cursor: + # The supported straight-line grammar cannot overlap shell-name + # spans. Stay conservative if malformed coordinates ever do. + return {} + replay_budget.check_runtime() + virtual_parts.append(content[cursor : spec.value_start]) + virtual_call_start = spec.call_start + shift + virtual_coordinates_by_call[spec.call_start] = ( + virtual_call_start, + spec.popen_start + shift if spec.popen_start is not None else None, + spec.shell_anchor + shift, + ) + virtual_parts.append("True") + cursor = spec.value_end + shift += len("True") - (spec.value_end - spec.value_start) + virtual_parts.append(content[cursor:]) + virtual_content = "".join(virtual_parts) + replay_budget.check_runtime() + except static_runner._StaticResourceLimitError: + return {} + replacement_starts = [spec.value_start for spec in specs] + replacement_ends: list[int] = [] + cumulative_deltas: list[int] = [0] + for spec in specs: + replacement_ends.append(spec.value_end) + cumulative_deltas.append( + cumulative_deltas[-1] + len("True") - (spec.value_end - spec.value_start) + ) + + def source_to_virtual_boundary(source_offset: int) -> int: + replacement_count = bisect_right(replacement_ends, source_offset) + return source_offset + cumulative_deltas[replacement_count] + + def boundary_is_preserved(source_offset: int, virtual_offset: int) -> bool: + """Return whether a raw-window edge has one unambiguous virtual edge.""" + replacement_index = bisect_right(replacement_starts, source_offset) - 1 + if ( + replacement_index >= 0 + and replacement_starts[replacement_index] + < source_offset + < replacement_ends[replacement_index] + ): + return False + return source_to_virtual_boundary(source_offset) == virtual_offset + + metadata_by_call = {spec.call_start: spec for spec in specs} + fixed_coordinates = { + coordinate + for finding in findings + if finding.evidence.get(LEXICAL_DIRECT_SHELL_EVIDENCE) is True + and type(finding.evidence.get(static_runner._ABSOLUTE_REPLACEMENT_START_LIMIT_EVIDENCE)) + is int + for key in ( + static_runner._ABSOLUTE_START_EVIDENCE, + static_runner._ABSOLUTE_ALTERNATE_START_EVIDENCE, + ) + if type(coordinate := finding.evidence.get(key)) is int + } + retained_shell_anchors = { + spec.shell_anchor for spec in specs if spec.call_start in retained_call_starts + } + try: + continuity_runs = static_runner._continuity_runs_for_anchors( + content, + retained_shell_anchors, + replay_budget.check_runtime, + ) + except static_runner._StaticResourceLimitError: + return {} + continuity_calls = { + spec.call_start + for spec in specs + if spec.call_start in retained_call_starts and spec.shell_anchor in continuity_runs + } + replay_targets: dict[int, int] = {} + target_calls: set[int] = set() + for call_start in retained_call_starts: + target_spec = metadata_by_call.get(call_start) + target_coordinates = virtual_coordinates_by_call.get(call_start) + if target_spec is None or target_coordinates is None: + continue + virtual_call_start, virtual_popen_start, _ = target_coordinates + related_fixed_boundary = bool( + {call_start, target_spec.popen_start}.difference({None}).intersection(fixed_coordinates) + ) + source_coordinates = (call_start, target_spec.popen_start) + virtual_coordinates = (virtual_call_start, virtual_popen_start) + changes_outer_window = any( + not all( + boundary_is_preserved(source_boundary, virtual_boundary) + for source_boundary, virtual_boundary in zip( + _owning_raw_window(content, source_coordinate), + _owning_raw_window(virtual_content, virtual_coordinate), + strict=True, + ) + ) + for source_coordinate, virtual_coordinate in zip( + source_coordinates, + virtual_coordinates, + strict=True, + ) + if source_coordinate is not None and virtual_coordinate is not None + ) or (len(content) <= static_runner.SECURITY_VIEW_WINDOW_CHARS) != ( + len(virtual_content) <= static_runner.SECURITY_VIEW_WINDOW_CHARS + ) + call_end = ( + target_spec.call_end if target_spec.call_end is not None else target_spec.value_end + ) + try: + replay_budget.check_runtime() + call_text = content[target_spec.call_start : call_end] + call_has_derived_view = _has_derived_security_view(call_text) + call_has_direct_lookalike = _DIRECT_SHELL_TRUE_VALUE.search(call_text) is not None + replay_budget.check_runtime() + except static_runner._StaticResourceLimitError: + return {} + if not ( + target_spec.normalized_view + or call_has_derived_view + or call_has_direct_lookalike + or related_fixed_boundary + or changes_outer_window + or call_start in continuity_calls + ): + continue + target_calls.add(call_start) + replay_targets[virtual_call_start] = call_start + if virtual_popen_start is not None: + replay_targets[virtual_popen_start] = call_start + if not target_calls: + return {} + + source_line_starts = ( + 0, + *(match.end() for match in LOGICAL_LINE_BREAK.finditer(virtual_content)), + ) + window_targets: dict[tuple[int, int, int, int], set[int]] = {} + for coordinate in replay_targets: + bounds = _owning_raw_window(virtual_content, coordinate) + window_targets.setdefault(bounds, set()).add(coordinate) + + replay_candidates: list[Finding] = [] + seen_candidates: set[static_runner._ViewFindingKey] = set() + whole_artifact_window = len(virtual_content) <= static_runner.SECURITY_VIEW_WINDOW_CHARS + try: + for (raw_start, raw_end, owned_start, owned_end), _ in window_targets.items(): + replay_budget.check_runtime() + raw_window = virtual_content[raw_start:raw_end] + owned_source_start = owned_start - raw_start + owned_source_end = owned_end - raw_start + outer_right_boundary_is_fixed = raw_end < len(virtual_content) or ( + whole_artifact_window + and len(virtual_content) == static_runner.SECURITY_VIEW_WINDOW_CHARS + ) + right_boundary_recovery_start = ( + owned_start + static_runner._RAW_WINDOW_OWNED_CHARS - raw_start + if raw_end < len(virtual_content) + else static_runner._RAW_WINDOW_OWNED_CHARS + if whole_artifact_window + and len(virtual_content) > static_runner.SECURITY_VIEW_WINDOW_CHARS - len("True") + else None + ) + for full_view in security_text_views(raw_window): + full_view = static_runner._with_fixed_right_boundary( + full_view, + outer_right_boundary_is_fixed, + right_boundary_recovery_start, + ) + for view in static_runner._bounded_view_slices(full_view): + replay_budget.check_runtime() + view_findings, _ = static_runner._scan_view_windows( + file_path, + view, + [_DirectShellReplayLexical], + replay_budget, + None, + python_source=True, + source_text=raw_window, + ) + owned_findings: list[Finding] = [] + for replay in view_findings: + source_start = replay.evidence.get(static_runner._SOURCE_START_EVIDENCE) + if isinstance(source_start, int) and not ( + owned_source_start <= source_start < owned_source_end + ): + alternate_start = replay.evidence.get( + static_runner._SOURCE_ALTERNATE_START_EVIDENCE + ) + if not ( + isinstance(alternate_start, int) + and owned_source_start <= alternate_start < owned_source_end + ): + continue + replay.evidence[static_runner._SOURCE_START_EVIDENCE] = alternate_start + replay.evidence[static_runner._SOURCE_ALTERNATE_START_EVIDENCE] = ( + source_start + ) + owned_findings.append(replay) + static_runner._restore_source_lines( + owned_findings, + raw_window=raw_window, + window_line=1, + view=view, + window_start=raw_start, + source_line_starts=source_line_starts, + ) + for replay in owned_findings: + candidate_coordinates = { + coordinate + for key in ( + static_runner._ABSOLUTE_START_EVIDENCE, + static_runner._ABSOLUTE_ALTERNATE_START_EVIDENCE, + ) + if type(coordinate := replay.evidence.get(key)) is int + } + if not candidate_coordinates.intersection(replay_targets): + continue + key = static_runner._view_finding_key(replay) + if key not in seen_candidates: + seen_candidates.add(key) + replay_candidates.append(replay) + + for call_start in continuity_calls.intersection(target_calls): + replay_budget.check_runtime() + _, _, virtual_shell_anchor = virtual_coordinates_by_call[call_start] + projection = static_runner._anchored_continuity_view( + virtual_content, + virtual_shell_anchor, + replay_budget.check_runtime, + ) + if projection is None or not projection.text or projection.source_offsets is None: + continue + projection_is_fixed = projection.source_offsets[-1] + 1 < len(virtual_content) + for full_view in security_text_views(projection.text): + full_view = SecurityTextView( + name=f"continuity-{full_view.name}", + text=full_view.text, + source_offsets=full_view.source_offsets, + right_boundary_is_fixed=projection_is_fixed, + ) + for view in static_runner._bounded_view_slices(full_view): + replay_budget.check_runtime() + view_findings, _ = static_runner._scan_view_windows( + file_path, + view, + [_DirectShellReplayLexical], + replay_budget, + None, + python_source=True, + source_text=projection.text, + ) + static_runner._restore_source_lines( + view_findings, + raw_window=projection.text, + window_line=1, + view=view, + start_source_offsets=projection.source_offsets, + ) + for replay in view_findings: + candidate_coordinates = { + coordinate + for key in ( + static_runner._ABSOLUTE_START_EVIDENCE, + static_runner._ABSOLUTE_ALTERNATE_START_EVIDENCE, + ) + if type(coordinate := replay.evidence.get(key)) is int + } + if not candidate_coordinates.intersection(replay_targets): + continue + key = static_runner._view_finding_key(replay) + if key not in seen_candidates: + seen_candidates.add(key) + replay_candidates.append(replay) + except static_runner._StaticResourceLimitError: + return {} + + replay_findings = static_runner._deduplicate_view_findings( + coalesce_path_findings(virtual_content, replay_candidates) + ) + replay_by_call: dict[int, Finding] = {} + for replay in replay_findings: + replay_coordinates = { + coordinate + for key in ( + static_runner._ABSOLUTE_START_EVIDENCE, + static_runner._ABSOLUTE_ALTERNATE_START_EVIDENCE, + ) + if type(coordinate := replay.evidence.get(key)) is int + } + for coordinate in sorted(replay_coordinates): + call_start = replay_targets.get(coordinate) + if call_start is not None: + spec = metadata_by_call[call_start] + _, _, virtual_shell_anchor = virtual_coordinates_by_call[call_start] + lexical_anchor = replay.evidence.get(static_runner._ABSOLUTE_ANCHOR_EVIDENCE) + if lexical_anchor == virtual_shell_anchor: + replay.match_fingerprint = spec.canonical_fingerprint + if spec.normalized_view and "normalized-view" not in replay.tags: + replay.tags.append("normalized-view") + replay_by_call.setdefault(call_start, replay) + break + return replay_by_call + + +def _bound_shell_call_key(call: ast.Call) -> tuple[int, int, int, int]: + """Return the call key shared with the flow-sensitive companion.""" + return ( + getattr(call, "lineno", 1), + getattr(call, "col_offset", 0), + getattr(call, "end_lineno", getattr(call, "lineno", 1)), + getattr(call, "end_col_offset", getattr(call, "col_offset", 0)), + ) + + +def _reconcile_variable_shell_findings( + content: str, + findings: list[Finding], + python_ast: ParsedPythonFile | None, +) -> list[Finding]: + """Reconcile the bounded variable regex with the shared Python AST.""" + marked = [ + finding + for finding in findings + if isinstance(finding.evidence.get(_VARIABLE_SHELL_FLAG_EVIDENCE), str) + ] + if not marked: + return findings + + file_path = marked[0].file + if python_ast is None or python_ast.tree is None: + return findings + + from . import static_python_shell_truthiness + + ast_index = _build_variable_shell_ast_index(python_ast) + if ast_index is None: + return findings + resolved: dict[tuple[int, str], list[_VariableShellCandidate]] = {} + for match in _variable_shell_matches(content): + assignment_line = bisect_right(ast_index.line_character_starts, match.start(1)) + candidate = _resolve_variable_shell_candidate(ast_index, match, assignment_line) + if candidate is not None: + call_span = _node_character_span(python_ast, candidate.call) + if call_span is not None: + resolved.setdefault((call_span[0], candidate.name), []).append(candidate) + ownership, emitted, cached_replacements = ( + static_python_shell_truthiness.bound_shell_call_analysis(file_path, python_ast) + ) + supported_calls = set(emitted) + retained_companion_starts: set[int] = set() + retained_companion_sources: set[tuple[int, str]] = set() + for finding in findings: + if finding.evidence.get(static_python_shell_truthiness.BOUND_SHELL_EVIDENCE) is not True: + continue + start = finding.evidence.get(static_python_shell_truthiness.BOUND_CALL_START_EVIDENCE) + end = finding.evidence.get(static_python_shell_truthiness.BOUND_CALL_END_EVIDENCE) + if type(start) is int and type(end) is int and 0 <= start < end <= len(content): + retained_companion_starts.add(start) + retained_companion_sources.add( + (bisect_right(python_ast.line_character_starts, start), content[start:end]) + ) + retained_emitted: set[tuple[int, int, int, int]] = set() + for key in emitted: + start = python_ast.line_character_starts[key[0] - 1] + ( + python_ast.character_column(key[0], key[1]) or 0 + ) + end = python_ast.line_character_starts[key[2] - 1] + ( + python_ast.character_column(key[2], key[3]) or 0 + ) + # The companion shares one report owner for identical same-line calls. + # Compare the complete raw source, since displayed previews can collide. + if ( + start in retained_companion_starts + or (key[0], content[start:end]) in retained_companion_sources + ): + retained_emitted.add(key) + emitted = retained_emitted + + reconciled: list[Finding] = [] + for finding in findings: + variable_name = finding.evidence.get(_VARIABLE_SHELL_FLAG_EVIDENCE) + if not isinstance(variable_name, str): + reconciled.append(finding) + continue + call_start = finding.evidence.get(static_runner._ABSOLUTE_ANCHOR_EVIDENCE) + candidates = ( + resolved.get((call_start, variable_name), []) if type(call_start) is int else [] + ) + if len(candidates) != 1: + reconciled.append(finding) + continue + candidate = candidates[0] + if not candidate.visible: + continue + call_key = _bound_shell_call_key(candidate.call) + if ( + call_key in cached_replacements + or call_key in emitted + or _lexical_shell_has_counterevidence( + ast_index, python_ast, candidate, receiver_trusted=ownership.get(call_key) + ) + ): + continue + if call_key in supported_calls: + retained_owner = static_python_shell_truthiness.bound_shell_finding_for_call( + file_path, python_ast, candidate.call + ) + if retained_owner is not None: + replacement = static_runner.analyzer_finding_to_finding(retained_owner) + replacement.finding_id = finding.finding_id + reconciled.append(replacement) + continue + if variable_name.casefold().startswith("true"): + call = candidate.call + finding.start_line = getattr(call, "lineno", finding.start_line) + finding.end_line = getattr(call, "end_lineno", finding.end_line) + finding.start_column = python_ast.character_column( + finding.start_line, getattr(call, "col_offset", 0) + ) + if finding.end_line is not None: + finding.end_column = python_ast.character_column( + finding.end_line, getattr(call, "end_col_offset", 0) + ) + call_source = python_ast.source_segment(call) + if call_source is not None: + finding.matched_text = call_source[:200] + finding.finding = call_source[:200] + if type(call_start) is int: + call_context = get_context(content, call_start) + finding.context = call_context + finding.code_snippet = call_context + reconciled.append(finding) + return reconciled + + +def postprocess_path_findings( + content: str, + findings: list[Finding], + *, + python_ast: ParsedPythonFile | None = None, + started_at: float | None = None, + timeout_seconds: float | None = None, +) -> list[Finding]: + """Finalize ownership, classification, and private TM1 coordinates.""" + from . import static_python_shell_truthiness + + findings = _reconcile_variable_shell_findings(content, findings, python_ast) + direct_replays = _bound_direct_replays( + content, + findings, + python_ast, + started_at=started_at, + timeout_seconds=timeout_seconds, + ) + reconciled = coalesce_path_findings(content, findings) + if python_ast is not None: + direct_starts = { + start + for finding in reconciled + if finding.evidence.get(LEXICAL_DIRECT_SHELL_EVIDENCE) is True + and type(start := _lexical_call_start(finding)) is int + } + direct_metadata = static_python_shell_truthiness.direct_literal_metadata( + python_ast, + next((finding.file for finding in reconciled), ""), + direct_starts, + ) + canonical_direct_calls: dict[tuple[str, int, str], set[int]] = {} + for finding in reconciled: + direct_call_start = _lexical_call_start(finding) + metadata = ( + direct_metadata.get(direct_call_start) if type(direct_call_start) is int else None + ) + if metadata is not None: + canonical_direct_calls.setdefault( + (finding.file, finding.start_line, metadata.match_fingerprint), + set(), + ).add(metadata.call_start) + colliding_direct_identities = { + key for key, call_starts in canonical_direct_calls.items() if len(call_starts) > 1 + } + for finding in reconciled: + direct_call_start = _lexical_call_start(finding) + metadata = ( + direct_metadata.get(direct_call_start) if type(direct_call_start) is int else None + ) + lexical_anchor = finding.evidence.get(static_runner._ABSOLUTE_ANCHOR_EVIDENCE) + if metadata is None or lexical_anchor != metadata.shell_anchor: + continue + if ( + finding.file, + finding.start_line, + metadata.match_fingerprint, + ) not in colliding_direct_identities: + finding.match_fingerprint = metadata.match_fingerprint + finding.evidence.update( + { + static_python_shell_truthiness.BOUND_SHELL_EVIDENCE: True, + static_python_shell_truthiness.BOUND_CALL_START_EVIDENCE: metadata.call_start, + static_python_shell_truthiness.DIRECT_LITERAL_METADATA_EVIDENCE: True, + } + ) + if metadata.call_end is not None: + finding.evidence[static_python_shell_truthiness.BOUND_CALL_END_EVIDENCE] = ( + metadata.call_end + ) + if metadata.normalized_view and "normalized-view" not in finding.tags: + finding.tags.append("normalized-view") + if metadata.normalized_view: + finding.evidence[static_python_shell_truthiness.BOUND_NORMALIZED_VIEW_EVIDENCE] = ( + True + ) + safety_by_scope: dict[tuple[int, int, bool], tuple[bool, bool]] = {} + raw_lines_by_start: dict[int, tuple[str, ...]] = {} + source_line_starts = (0, *(match.end() for match in LOGICAL_LINE_BREAK.finditer(content))) + + for finding in reconciled: + is_bound = finding.evidence.get(static_python_shell_truthiness.BOUND_SHELL_EVIDENCE) is True + if ( + finding.evidence.get(LEXICAL_DIRECT_SHELL_EVIDENCE) is True + and "normalized-view" in finding.tags + and isinstance( + canonical_match := finding.evidence.get(LEXICAL_NORMALIZED_MATCH_EVIDENCE), + str, + ) + ): + finding.matched_text = canonical_match + finding.finding = canonical_match + is_direct_literal = ( + finding.evidence.get(static_python_shell_truthiness.DIRECT_LITERAL_METADATA_EVIDENCE) + is True + ) + if is_bound and not is_direct_literal: + if ( + finding.evidence.get(static_python_shell_truthiness.BOUND_NORMALIZED_VIEW_EVIDENCE) + is True + and "normalized-view" not in finding.tags + ): + finding.tags.append("normalized-view") + matched = (finding.matched_text or "")[:200] + classification_match = finding.evidence.get( + static_python_shell_truthiness.BOUND_CLASSIFICATION_MATCH_EVIDENCE, + matched, + ) + if not isinstance(classification_match, str): + classification_match = matched + call_start = finding.evidence.get( + static_python_shell_truthiness.BOUND_CALL_START_EVIDENCE + ) + call_end = finding.evidence.get(static_python_shell_truthiness.BOUND_CALL_END_EVIDENCE) + classification_context = finding.context or "" + matched_line = matched + safe_context: bool | None = None + safe_matched_line: bool | None = None + if type(call_start) is int: + preserves_lexical_location = ( + finding.evidence.get( + static_python_shell_truthiness.DIRECT_LITERAL_METADATA_EVIDENCE + ) + is True + ) + if not preserves_lexical_location: + finding.start_line = bisect_right(source_line_starts, call_start) + if type(call_end) is int and call_end > call_start: + finding.end_line = bisect_right(source_line_starts, call_end - 1) + retained_start = finding.evidence.get(static_runner._ABSOLUTE_START_EVIDENCE) + direct_owner_start = finding.evidence.get( + static_python_shell_truthiness.BOUND_DIRECT_OWNER_START_EVIDENCE + ) + classification_start = ( + retained_start + if preserves_lexical_location and type(retained_start) is int + else direct_owner_start + if type(direct_owner_start) is int + else call_start + ) + direct_match_end = finding.evidence.get( + static_python_shell_truthiness.BOUND_DIRECT_MATCH_END_EVIDENCE + ) + popen_start = finding.evidence.get( + static_python_shell_truthiness.BOUND_POPEN_START_EVIDENCE + ) + _, qualifier_window_end = _raw_classification_bounds(content, call_start) + if ( + not preserves_lexical_location + and classification_start == call_start + and type(direct_match_end) is int + and direct_match_end > qualifier_window_end + and type(popen_start) is int + ): + classification_start = popen_start + raw_start, raw_end = _raw_classification_bounds(content, classification_start) + classification_line = bisect_right( + source_line_starts, + classification_start, + ) + normalized_view = ( + finding.evidence.get( + static_python_shell_truthiness.BOUND_NORMALIZED_VIEW_EVIDENCE + ) + is True + ) + scope_key = (raw_start, classification_line, normalized_view) + safety = safety_by_scope.get(scope_key) + if safety is None: + raw_lines = raw_lines_by_start.get(raw_start) + if raw_lines is None: + raw_lines = tuple(content[raw_start:raw_end].splitlines()) + raw_lines_by_start[raw_start] = raw_lines + raw_start_line = bisect_right(source_line_starts, raw_start) + local_line = classification_line - raw_start_line + if 0 <= local_line < len(raw_lines): + context_start = max(0, local_line - 3) + context_end = min(len(raw_lines), local_line + 4) + classification_context = "\n".join(raw_lines[context_start:context_end]) + matched_line = raw_lines[local_line] + if normalized_view: + classification_context = normalized_security_view( + classification_context + ).text + matched_line = normalized_security_view(matched_line).text + classification_match = normalized_security_view(classification_match).text + safety = ( + _is_safe_container_command(classification_context) + or _is_safe_dockerfile_idiom( + classification_context, + classification_match, + ), + _is_safe_cache_cleanup(matched_line), + ) + safety_by_scope[scope_key] = safety + safe_context, safe_matched_line = safety + severity, finding.confidence = _classify_tm1( + classification_context, + classification_match, + matched_line, + ( + 0.8 + if finding.evidence.get( + static_python_shell_truthiness.DIRECT_LITERAL_METADATA_EVIDENCE + ) + is True + else finding.confidence + ), + "python", + safe_context=safe_context, + safe_matched_line=safe_matched_line, + ) + finding.severity = severity.value + borrowed_classification = finding.evidence.get(LEXICAL_BOUND_CLASSIFICATION_EVIDENCE) + if isinstance(borrowed_classification, dict): + borrowed_severity = borrowed_classification.get("severity") + borrowed_confidence = borrowed_classification.get("confidence") + borrowed_tags = borrowed_classification.get("tags") + if isinstance(borrowed_severity, str): + finding.severity = borrowed_severity + if isinstance(borrowed_confidence, (int, float)) and not isinstance( + borrowed_confidence, bool + ): + finding.confidence = float(borrowed_confidence) + if isinstance(borrowed_tags, (list, tuple)) and all( + isinstance(tag, str) for tag in borrowed_tags + ): + finding.tags = list(borrowed_tags) + replay_call_start = finding.evidence.get( + static_python_shell_truthiness.BOUND_CALL_START_EVIDENCE + ) + replay = ( + direct_replays.get(replay_call_start) + if type(replay_call_start) is int and not is_direct_literal + else None + ) + if replay is not None: + finding.match_fingerprint = replay.fingerprint() + finding.severity = replay.severity + finding.confidence = replay.confidence + finding.tags = list(replay.tags) + return cleanup_path_findings(reconciled) + + def node(state: SkillspectorState) -> AnalyzerNodeResponse: """Run tool_misuse patterns and return findings.""" - response = static_runner.run_static_patterns_with_ledger(state, [sys.modules[__name__]]) + from . import static_python_shell_truthiness + + response = static_runner.run_static_patterns_with_ledger( + state, + [sys.modules[__name__], static_python_shell_truthiness], + ) logger.info("%s: %d findings", ANALYZER_ID, len(response["findings"])) return response diff --git a/src/skillspector/nodes/analyzers/static_python_shell_truthiness.py b/src/skillspector/nodes/analyzers/static_python_shell_truthiness.py new file mode 100644 index 000000000..e6ccdba05 --- /dev/null +++ b/src/skillspector/nodes/analyzers/static_python_shell_truthiness.py @@ -0,0 +1,2282 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Find direct subprocess calls using a definitely truthy local name. + +This companion recognizes the straight-line ordinary-Python form reported in +issue #475. Arguments evaluated through ``shell=`` must be passive, and +unsupported expressions or compound statements discard facts rather than +guessing about Python execution. +""" + +from __future__ import annotations + +import ast +import re +from collections.abc import Callable +from dataclasses import dataclass + +from skillspector.artifacts import ( + normalized_security_prefix, + normalized_security_view, + security_text_views, +) +from skillspector.models import AnalyzerFinding, Location, Severity, compute_match_fingerprint +from skillspector.python_ast import ParsedPythonFile, parse_python_source + +from .common import LINE_BREAK_CHARS, get_complete_source_segment, get_context_from_lines +from .pattern_defaults import PatternCategory + +ANALYZER_ID = "static_patterns_tool_misuse" +USES_PYTHON_AST = True +BOUND_SHELL_EVIDENCE = "_tm1_bound_shell_value" +BOUND_CALL_START_EVIDENCE = "_tm1_bound_call_start" +BOUND_CALL_END_EVIDENCE = "_tm1_bound_call_end" +BOUND_SHELL_ANCHOR_EVIDENCE = "_tm1_bound_shell_anchor" +BOUND_CANONICAL_FINGERPRINT_EVIDENCE = "_tm1_bound_canonical_fingerprint" +BOUND_NORMALIZED_VIEW_EVIDENCE = "_tm1_bound_normalized_view" +BOUND_CLASSIFICATION_MATCH_EVIDENCE = "_tm1_bound_classification_match" +BOUND_DIRECT_MATCH_END_EVIDENCE = "_tm1_bound_direct_match_end" +BOUND_POPEN_START_EVIDENCE = "_tm1_bound_popen_start" +BOUND_DIRECT_OWNER_START_EVIDENCE = "_tm1_bound_direct_owner_start" +BOUND_SHELL_VALUE_START_EVIDENCE = "_tm1_bound_shell_value_start" +BOUND_SHELL_VALUE_END_EVIDENCE = "_tm1_bound_shell_value_end" +DIRECT_LITERAL_METADATA_EVIDENCE = "_tm1_direct_literal_metadata" +_DIRECT_CALL_NAMES = frozenset({"subprocess", "Popen"}) +_CACHED_SUBPROCESS_API_SLOTS = frozenset( + {"run", "Popen", "call", "check_call", "check_output", "getoutput", "getstatusoutput"} +) +_DIRECT_CALLEE = re.compile(r"(?:subprocess\.\w+|Popen)", re.IGNORECASE) +_SHELL_KEYWORD_PREFIX = re.compile(r"shell\s*=\s*", re.IGNORECASE) +_MAX_CONTEXT_CHARS = 1024 +_MAX_DIRECT_NAME_CHARS = len("subprocess") + 1 +BoundShellCallKey = tuple[int, int, int, int] + + +def _bound_shell_call_key(call: ast.Call) -> BoundShellCallKey: + """Return a stable source key for one call node.""" + return ( + getattr(call, "lineno", 1), + getattr(call, "col_offset", 0), + getattr(call, "end_lineno", getattr(call, "lineno", 1)), + getattr(call, "end_col_offset", getattr(call, "col_offset", 0)), + ) + + +@dataclass(frozen=True, slots=True) +class DirectLiteralMetadata: + """AST metadata attached only to an already-confirmed lexical finding.""" + + call_start: int + call_end: int | None + shell_anchor: int + match_fingerprint: str + normalized_view: bool + + +@dataclass(frozen=True, slots=True) +class BoundShellMetadata: + """Cap-independent source coordinates for one supported bound shell call.""" + + call_start: int + call_end: int | None + shell_anchor: int + value_start: int + value_end: int + popen_start: int | None + canonical_fingerprint: str + normalized_view: bool + + +def _truth_value( + expression: ast.expr | None, + facts: dict[str, bool], +) -> bool | None: + """Return truth for a small, immutable, side-effect-free expression subset.""" + if expression is None: + return None + + resolved: dict[ast.expr, bool] = {} + pending: list[tuple[ast.expr, bool]] = [(expression, False)] + while pending: + current, expanded = pending.pop() + if isinstance(current, ast.Constant): + resolved[current] = bool(current.value) + elif isinstance(current, ast.Name): + if current.id not in facts: + return None + resolved[current] = facts[current.id] + elif isinstance(current, ast.Tuple): + if not current.elts: + resolved[current] = False + elif any(isinstance(item, ast.Starred) for item in current.elts): + return None + elif all(_is_passive_argument(item) for item in current.elts): + resolved[current] = True + else: + return None + elif isinstance(current, ast.UnaryOp): + if not isinstance(current.op, ast.Not) and not ( + isinstance(current.op, (ast.UAdd, ast.USub)) + and isinstance(current.operand, ast.Constant) + and type(current.operand.value) in (bool, int, float, complex) + ): + return None + if expanded: + operand = resolved[current.operand] + resolved[current] = not operand if isinstance(current.op, ast.Not) else operand + else: + pending.append((current, True)) + pending.append((current.operand, False)) + else: + return None + return resolved[expression] + + +def _update_trusted_names_from_import( + statement: ast.Import | ast.ImportFrom, + trusted_names: set[str], +) -> None: + """Update only direct receiver names that the import actually binds.""" + if isinstance(statement, ast.Import): + for imported in statement.names: + bound = imported.asname or imported.name.partition(".")[0] + if imported.name == "subprocess" and bound == "subprocess": + trusted_names.add(bound) + elif bound in trusted_names: + trusted_names.discard(bound) + return + + if any(imported.name == "*" for imported in statement.names): + trusted_names.clear() + return + for imported in statement.names: + bound = imported.asname or imported.name + if ( + statement.level == 0 + and statement.module == "subprocess" + and imported.name == "Popen" + and bound == "Popen" + ): + trusted_names.add(bound) + elif bound in trusted_names: + trusted_names.discard(bound) + + +class _DirectBindingCollector: + """Collect direct receiver bindings without entering nested scopes.""" + + def __init__(self, tracked_names: set[str] | frozenset[str]) -> None: + self.tracked_names = tracked_names + self.bound: set[str] = set() + self.mutated: set[str] = set() + self.nonlocal_names: set[str] = set() + + @staticmethod + def _function_header_nodes( + node: ast.FunctionDef | ast.AsyncFunctionDef, + ) -> list[ast.AST]: + nodes: list[ast.AST] = [*node.decorator_list, *node.args.defaults] + nodes.extend(item for item in node.args.kw_defaults if item is not None) + arguments = (*node.args.posonlyargs, *node.args.args, *node.args.kwonlyargs) + nodes.extend( + argument.annotation for argument in arguments if argument.annotation is not None + ) + if node.args.vararg is not None and node.args.vararg.annotation is not None: + nodes.append(node.args.vararg.annotation) + if node.args.kwarg is not None and node.args.kwarg.annotation is not None: + nodes.append(node.args.kwarg.annotation) + if node.returns is not None: + nodes.append(node.returns) + nodes.extend(getattr(node, "type_params", ())) + return nodes + + def visit(self, node: ast.AST) -> None: + pending = [node] + while pending: + current = pending.pop() + if isinstance(current, ast.Name): + if ( + isinstance(current.ctx, (ast.Store, ast.Del)) + and current.id in self.tracked_names + ): + self.bound.add(current.id) + continue + if isinstance(current, (ast.Attribute, ast.Subscript)): + if isinstance(current.ctx, (ast.Store, ast.Del)): + root: ast.expr = current.value + while isinstance(root, (ast.Attribute, ast.Subscript)): + root = root.value + if isinstance(root, ast.Name) and root.id in self.tracked_names: + self.mutated.add(root.id) + pending.extend(ast.iter_child_nodes(current)) + continue + if isinstance(current, (ast.FunctionDef, ast.AsyncFunctionDef)): + if current.name in self.tracked_names: + self.bound.add(current.name) + pending.extend(self._function_header_nodes(current)) + continue + if isinstance(current, ast.ClassDef): + if current.name in self.tracked_names: + self.bound.add(current.name) + pending.extend(current.decorator_list) + pending.extend(current.bases) + pending.extend(keyword.value for keyword in current.keywords) + continue + if isinstance(current, ast.Lambda): + pending.extend(current.args.defaults) + pending.extend(item for item in current.args.kw_defaults if item is not None) + continue + if isinstance(current, (ast.ListComp, ast.SetComp, ast.GeneratorExp)): + pending.append(current.elt) + for generator in current.generators: + pending.append(generator.iter) + pending.extend(generator.ifs) + continue + if isinstance(current, ast.DictComp): + pending.extend((current.key, current.value)) + for generator in current.generators: + pending.append(generator.iter) + pending.extend(generator.ifs) + continue + if isinstance(current, ast.Import): + for imported in current.names: + bound = imported.asname or imported.name.partition(".")[0] + if bound in self.tracked_names: + self.bound.add(bound) + continue + if isinstance(current, ast.ImportFrom): + if any(imported.name == "*" for imported in current.names): + self.bound.update(self.tracked_names) + continue + for imported in current.names: + bound = imported.asname or imported.name + if bound in self.tracked_names: + self.bound.add(bound) + continue + if isinstance(current, ast.ExceptHandler): + if isinstance(current.name, str) and current.name in self.tracked_names: + self.bound.add(current.name) + pending.extend(ast.iter_child_nodes(current)) + continue + if isinstance(current, (ast.Global, ast.Nonlocal)): + self.nonlocal_names.update(current.names) + continue + if isinstance(current, ast.MatchAs): + if isinstance(current.name, str) and current.name in self.tracked_names: + self.bound.add(current.name) + if current.pattern is not None: + pending.append(current.pattern) + continue + if isinstance(current, ast.MatchStar): + if isinstance(current.name, str) and current.name in self.tracked_names: + self.bound.add(current.name) + continue + if isinstance(current, ast.MatchMapping): + if isinstance(current.rest, str) and current.rest in self.tracked_names: + self.bound.add(current.rest) + pending.extend(current.patterns) + continue + pending.extend(ast.iter_child_nodes(current)) + + +def _direct_bound_names(node: ast.AST) -> set[str]: + """Return names bound by *node* without entering deferred nested scopes.""" + candidates: set[str] = set() + for current in ast.walk(node): + if isinstance(current, ast.Name) and isinstance(current.ctx, (ast.Store, ast.Del)): + candidates.add(current.id) + elif isinstance(current, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): + candidates.add(current.name) + elif isinstance(current, ast.Import): + candidates.update( + imported.asname or imported.name.partition(".")[0] for imported in current.names + ) + elif isinstance(current, ast.ImportFrom): + candidates.update( + imported.asname or imported.name + for imported in current.names + if imported.name != "*" + ) + elif isinstance(current, ast.ExceptHandler) and isinstance(current.name, str): + candidates.add(current.name) + elif isinstance(current, (ast.MatchAs, ast.MatchStar)) and isinstance( + current.name, + str, + ): + candidates.add(current.name) + elif isinstance(current, ast.MatchMapping) and isinstance(current.rest, str): + candidates.add(current.rest) + collector = _DirectBindingCollector(candidates) + collector.visit(node) + return collector.bound + + +def _function_parameter_names(statement: ast.FunctionDef | ast.AsyncFunctionDef) -> set[str]: + """Return names that may already hold unsafe values on body entry.""" + arguments = statement.args + names = { + argument.arg + for argument in (*arguments.posonlyargs, *arguments.args, *arguments.kwonlyargs) + } + if arguments.vararg is not None: + names.add(arguments.vararg.arg) + if arguments.kwarg is not None: + names.add(arguments.kwarg.arg) + declarations = _DirectBindingCollector(set()) + for child in statement.body: + declarations.visit(child) + names.update(declarations.nonlocal_names) + return names + + +def _function_bound_direct_names( + statement: ast.FunctionDef | ast.AsyncFunctionDef, + tracked_names: set[str], +) -> set[str]: + """Return compile-time local receiver names for one function scope.""" + arguments = statement.args + named = (*arguments.posonlyargs, *arguments.args, *arguments.kwonlyargs) + names = {argument.arg for argument in named} + if arguments.vararg is not None: + names.add(arguments.vararg.arg) + if arguments.kwarg is not None: + names.add(arguments.kwarg.arg) + collector = _DirectBindingCollector(tracked_names) + for child in statement.body: + collector.visit(child) + return names.intersection(tracked_names).union( + collector.bound.difference(collector.nonlocal_names) + ) + + +def _changed_direct_names(nodes: list[ast.AST], tracked_names: set[str]) -> set[str]: + """Return receiver names explicitly rebound or mutated by current-scope nodes.""" + collector = _DirectBindingCollector(tracked_names) + for node in nodes: + collector.visit(node) + return collector.bound.union(collector.mutated) + + +def _class_body_changed_direct_names( + statement: ast.ClassDef, + tracked_names: set[str], +) -> set[str]: + """Return explicit class-execution effects on outer receiver objects.""" + + def nested_classes(node: ast.AST) -> list[ast.ClassDef]: + classes: list[ast.ClassDef] = [] + pending = [node] + while pending: + current = pending.pop() + if isinstance(current, ast.ClassDef): + classes.append(current) + continue + if isinstance(current, (ast.FunctionDef, ast.AsyncFunctionDef, ast.Lambda)): + continue + pending.extend(ast.iter_child_nodes(current)) + return classes + + affected: set[str] = set() + pending_classes = [statement] + while pending_classes: + current_class = pending_classes.pop() + declaration_collector = _DirectBindingCollector(tracked_names) + for child in current_class.body: + declaration_collector.visit(child) + global_names = declaration_collector.nonlocal_names.intersection(tracked_names) + + local_direct: dict[str, bool] = {} + affected.update(global_names.intersection(declaration_collector.bound)) + for child in current_class.body: + collector = _DirectBindingCollector(tracked_names) + collector.visit(child) + affected.update( + name + for name in collector.mutated + if name in global_names or local_direct.get(name, True) + ) + affected.update(collector.bound.intersection(global_names)) + pending_classes.extend(nested_classes(child)) + + local_bound = collector.bound.difference(global_names) + if isinstance(child, ast.Import): + for imported in child.names: + bound = imported.asname or imported.name.partition(".")[0] + if bound in local_bound: + local_direct[bound] = ( + imported.name == "subprocess" and bound == "subprocess" + ) + elif isinstance(child, ast.ImportFrom): + for imported in child.names: + bound = imported.asname or imported.name + if bound in local_bound: + local_direct[bound] = ( + child.level == 0 + and child.module == "subprocess" + and imported.name == "Popen" + and bound == "Popen" + ) + elif isinstance(child, ast.Assign): + prior_local_direct = dict(local_direct) + for name in local_bound: + local_direct[name] = False + for target in child.targets: + if isinstance(target, ast.Name) and target.id in local_bound: + local_direct[target.id] = ( + isinstance(child.value, ast.Name) + and child.value.id == target.id + and prior_local_direct.get( + child.value.id, + child.value.id in tracked_names, + ) + ) + elif isinstance(child, ast.AnnAssign) and child.value is not None: + prior_local_direct = dict(local_direct) + for name in local_bound: + local_direct[name] = False + if isinstance(child.target, ast.Name) and child.target.id in local_bound: + local_direct[child.target.id] = ( + isinstance(child.value, ast.Name) + and child.value.id == child.target.id + and prior_local_direct.get( + child.value.id, + child.value.id in tracked_names, + ) + ) + elif isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): + if child.name in tracked_names and child.name not in global_names: + local_direct[child.name] = False + elif isinstance(child, ast.Delete): + for name in collector.bound: + local_direct.pop(name, None) + elif local_bound: + for name in local_bound: + local_direct.pop(name, None) + return affected + + +def _nodes_contain_eager_call(nodes: list[ast.AST]) -> bool: + """Return whether eager evaluation reaches a call outside deferred bodies.""" + pending = list(nodes) + while pending: + current = pending.pop() + if isinstance(current, ast.Call): + return True + if isinstance(current, (ast.FunctionDef, ast.AsyncFunctionDef)): + if not _function_header_is_passive(current): + return True + continue + if isinstance(current, ast.ClassDef): + if ( + not _class_header_is_passive(current) + or _class_body_may_release_outer_value(current) + or _nodes_contain_eager_call(list(current.body)) + ): + return True + continue + if isinstance(current, ast.Lambda): + defaults = ( + *current.args.defaults, + *(item for item in current.args.kw_defaults if item is not None), + ) + if any(not _is_passive_argument(default) for default in defaults): + return True + continue + pending.extend(ast.iter_child_nodes(current)) + return False + + +def _class_body_may_release_outer_value(statement: ast.ClassDef) -> bool: + """Return whether class execution stores to a declared outer name.""" + declarations = _DirectBindingCollector(set()) + for child in statement.body: + declarations.visit(child) + outer_names = declarations.nonlocal_names + if outer_names: + bindings = _DirectBindingCollector(outer_names) + for child in statement.body: + bindings.visit(child) + if bindings.bound.intersection(outer_names): + return True + + pending: list[ast.AST] = list(statement.body) + while pending: + current = pending.pop() + if isinstance(current, ast.ClassDef): + if _class_body_may_release_outer_value(current): + return True + continue + if isinstance(current, (ast.FunctionDef, ast.AsyncFunctionDef, ast.Lambda)): + continue + pending.extend(ast.iter_child_nodes(current)) + return False + + +def _class_body_has_eager_effects(statement: ast.ClassDef) -> bool: + """Return whether class execution can run user code or unsafe finalizers.""" + declarations = _DirectBindingCollector(set()) + for child in statement.body: + declarations.visit(child) + outer_names = declarations.nonlocal_names + bound_names: set[str] = set() + finalizer_safe_names: set[str] = set() + + for child in statement.body: + if isinstance(child, (ast.Global, ast.Nonlocal, ast.Pass)) or ( + isinstance(child, ast.Expr) and isinstance(child.value, ast.Constant) + ): + continue + if isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef)): + releases_unsafe_value = ( + child.name in bound_names and child.name not in finalizer_safe_names + ) + if ( + child.name in outer_names + or not _function_header_is_passive(child) + or releases_unsafe_value + ): + return True + bound_names.add(child.name) + finalizer_safe_names.add(child.name) + continue + if isinstance(child, ast.ClassDef): + releases_unsafe_value = ( + child.name in bound_names and child.name not in finalizer_safe_names + ) + if ( + child.name in outer_names + or not _class_header_is_passive(child) + or releases_unsafe_value + or _class_body_has_eager_effects(child) + ): + return True + bound_names.add(child.name) + finalizer_safe_names.add(child.name) + continue + if isinstance(child, ast.Assign): + if not all(isinstance(target, ast.Name) for target in child.targets): + return True + target_names = {target.id for target in child.targets if isinstance(target, ast.Name)} + if target_names.intersection(outer_names): + return True + releases_unsafe_value = any( + name in bound_names + and name not in finalizer_safe_names + and not (isinstance(child.value, ast.Name) and child.value.id == name) + for name in target_names + ) + if releases_unsafe_value or not _is_finalizer_safe_value( + child.value, + finalizer_safe_names, + ): + return True + bound_names.update(target_names) + finalizer_safe_names.update(target_names) + continue + if isinstance(child, ast.AnnAssign): + if not isinstance(child.target, ast.Name) or not _annotation_is_passive( + child.annotation + ): + return True + if child.value is None: + continue + target_name = child.target.id + if target_name in outer_names: + return True + releases_unsafe_value = ( + target_name in bound_names + and target_name not in finalizer_safe_names + and not (isinstance(child.value, ast.Name) and child.value.id == target_name) + ) + if releases_unsafe_value or not _is_finalizer_safe_value( + child.value, + finalizer_safe_names, + ): + return True + bound_names.add(target_name) + finalizer_safe_names.add(target_name) + continue + if isinstance(child, ast.Assert) and _is_finalizer_safe_value( + child.test, + finalizer_safe_names, + ): + continue + return True + return False + + +def _class_deferred_receiver_trust( + statement: ast.ClassDef, + trusted_names: set[str], +) -> tuple[set[str], dict[int, set[str]]]: + """Return final and observed-call outer trust for deferred class methods.""" + deferred = set(trusted_names) + trusted_at_call_by_definition: dict[int, set[str]] = {} + active_functions: dict[str, int] = {} + declarations = _DirectBindingCollector(_DIRECT_CALL_NAMES) + for child in statement.body: + declarations.visit(child) + global_names = declarations.nonlocal_names.intersection(_DIRECT_CALL_NAMES) + + for child_index, child in enumerate(statement.body): + call = _passive_direct_call(child) + if call is not None: + assert isinstance(call.func, ast.Name) + owner = active_functions.get(call.func.id) + if owner is not None: + trusted_at_call_by_definition.setdefault(owner, set()).update(deferred) + + if _nodes_contain_eager_call([child]): + # Class-body expressions run before any method can be called and may + # mutate the surrounding module/function receiver binding. + deferred.clear() + + collector = _DirectBindingCollector(_DIRECT_CALL_NAMES) + collector.visit(child) + deferred.difference_update(collector.mutated) + globally_bound = collector.bound.intersection(global_names) + if isinstance(child, ast.Import): + for imported in child.names: + bound = imported.asname or imported.name.partition(".")[0] + if bound not in globally_bound: + continue + if imported.name == "subprocess" and bound == "subprocess": + deferred.add(bound) + else: + deferred.discard(bound) + elif isinstance(child, ast.ImportFrom): + if any(imported.name == "*" for imported in child.names) and global_names: + deferred.clear() + for imported in child.names: + bound = imported.asname or imported.name + if bound not in globally_bound: + continue + if ( + child.level == 0 + and child.module == "subprocess" + and imported.name == "Popen" + and bound == "Popen" + ): + deferred.add(bound) + else: + deferred.discard(bound) + else: + deferred.difference_update(globally_bound) + + changed_names = _direct_bound_names(child) + for name in changed_names: + active_functions.pop(name, None) + if ( + isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef)) + and _function_header_is_passive(child) + and _is_immediate_function(child) + ): + active_functions[child.name] = child_index + return deferred, trusted_at_call_by_definition + + +def _normalized_direct_name(name: str) -> str | None: + """Return the bounded canonical spelling of a direct subprocess receiver.""" + normalized = normalized_security_prefix(name, _MAX_DIRECT_NAME_CHARS).casefold() + return normalized if normalized in {"subprocess", "popen"} else None + + +def _is_direct_subprocess_syntax(call: ast.Call) -> bool: + """Return whether a call uses one of the normalized direct subprocess spellings.""" + function = call.func + if isinstance(function, ast.Name): + return _normalized_direct_name(function.id) == "popen" + return ( + isinstance(function, ast.Attribute) + and isinstance(function.value, ast.Name) + and _normalized_direct_name(function.value.id) == "subprocess" + ) + + +def _is_direct_subprocess_call(call: ast.Call, trusted_names: set[str]) -> bool: + if not _is_direct_subprocess_syntax(call): + return False + function = call.func + receiver = function.id if isinstance(function, ast.Name) else function.value.id + return receiver in trusted_names + + +def _is_passive_argument(expression: ast.expr) -> bool: + """Return whether evaluation cannot invoke user-controlled Python code.""" + normal, hash_required, truth_required, numeric_required, integral_required = range(5) + pending: list[tuple[ast.expr, int]] = [(expression, normal)] + while pending: + current, requirement = pending.pop() + if isinstance(current, ast.Constant): + if requirement == numeric_required and type(current.value) not in ( + bool, + int, + float, + complex, + ): + return False + if requirement == integral_required and type(current.value) not in (bool, int): + return False + continue + if isinstance(current, ast.Name): + if requirement != normal: + return False + continue + if isinstance(current, ast.List): + if requirement in (hash_required, numeric_required, integral_required) or any( + isinstance(item, ast.Starred) for item in current.elts + ): + return False + pending.extend((item, normal) for item in current.elts) + continue + if isinstance(current, ast.Tuple): + if requirement in (numeric_required, integral_required): + return False + if any(isinstance(item, ast.Starred) for item in current.elts): + return False + nested_requirement = hash_required if requirement == hash_required else normal + pending.extend((item, nested_requirement) for item in current.elts) + continue + if isinstance(current, ast.Dict): + if requirement in (hash_required, numeric_required, integral_required) or any( + key is None for key in current.keys + ): + return False + pending.extend((key, hash_required) for key in current.keys if key is not None) + pending.extend((value, normal) for value in current.values) + continue + if isinstance(current, ast.Set): + if requirement in (hash_required, numeric_required, integral_required): + return False + pending.extend((item, hash_required) for item in current.elts) + continue + if isinstance(current, ast.UnaryOp): + if isinstance(current.op, ast.Not): + pending.append((current.operand, truth_required)) + elif isinstance(current.op, (ast.UAdd, ast.USub)): + operand_requirement = ( + integral_required if requirement == integral_required else numeric_required + ) + pending.append((current.operand, operand_requirement)) + elif isinstance(current.op, ast.Invert): + pending.append((current.operand, integral_required)) + else: + return False + continue + if isinstance(current, ast.JoinedStr) and all( + isinstance(item, ast.Constant) for item in current.values + ): + if requirement in (numeric_required, integral_required): + return False + continue + return False + return True + + +def _call_arguments_are_passive(call: ast.Call) -> bool: + return all(_is_passive_argument(argument) for argument in call.args) and all( + keyword.arg is not None and _is_passive_argument(keyword.value) for keyword in call.keywords + ) + + +def _expression_preserves_receiver_trust( + expression: ast.expr, + trusted_names: set[str], + protocol_safe_names: set[str] | None = None, +) -> bool: + """Return whether eager evaluation cannot replace a direct receiver.""" + if _is_passive_argument(expression): + return True + if isinstance(expression, ast.Call): + return ( + _is_direct_subprocess_call(expression, trusted_names) + and _call_arguments_are_passive(expression) + and ( + protocol_safe_names is None + or _call_arguments_are_protocol_safe(expression, protocol_safe_names) + ) + ) + if isinstance(expression, (ast.Tuple, ast.List)): + return all( + not isinstance(item, ast.Starred) + and _expression_preserves_receiver_trust( + item, + trusted_names, + protocol_safe_names, + ) + for item in expression.elts + ) + if isinstance(expression, (ast.ListComp, ast.SetComp, ast.GeneratorExp)): + values = [expression.elt] + elif isinstance(expression, ast.DictComp): + values = [expression.key, expression.value] + else: + return False + for generator in expression.generators: + if generator.is_async or not _is_simple_comprehension_target(generator.target): + return False + if protocol_safe_names is not None and ( + not _is_finalizer_safe_value(generator.iter, protocol_safe_names) + or any( + not _is_finalizer_safe_value(condition, protocol_safe_names) + for condition in generator.ifs + ) + ): + return False + values.extend((generator.iter, *generator.ifs)) + # Comprehensions have historically retained ownership for their direct + # subprocess calls. Preserve that behavior while still rejecting generic + # nested calls and receiver stores. + return all( + _expression_preserves_receiver_trust( + value, + trusted_names, + protocol_safe_names, + ) + for value in values + ) + + +def _is_simple_comprehension_target(target: ast.expr) -> bool: + if isinstance(target, ast.Name): + return True + return isinstance(target, (ast.Tuple, ast.List)) and all( + _is_simple_comprehension_target(item) for item in target.elts + ) + + +def _shell_argument_is_captured_before_effects(call: ast.Call) -> bool: + """Return whether evaluation reaches ``shell=`` without user-code effects. + + Python evaluates every positional argument, including starred expansions, + before keyword arguments. Keyword values are then evaluated in their stored + order. Effects after ``shell=`` cannot change the already captured value. + """ + if any(not _is_passive_argument(argument) for argument in call.args): + return False + for keyword in call.keywords: + if keyword.arg == "shell": + return _is_passive_argument(keyword.value) + if keyword.arg is None or not _is_passive_argument(keyword.value): + return False + return False + + +def _is_finalizer_safe_value(expression: ast.expr, safe_names: set[str]) -> bool: + """Return whether releasing the resulting value cannot run user code.""" + if not _is_passive_argument(expression): + return False + return all( + not isinstance(node, ast.Name) or node.id in safe_names for node in ast.walk(expression) + ) + + +def _call_arguments_are_protocol_safe(call: ast.Call, safe_names: set[str]) -> bool: + """Return whether subprocess argument consumption cannot dispatch user code.""" + return all(_is_finalizer_safe_value(argument, safe_names) for argument in call.args) and all( + keyword.arg is not None and _is_finalizer_safe_value(keyword.value, safe_names) + for keyword in call.keywords + ) + + +def _annotation_is_passive(annotation: ast.expr) -> bool: + """Accept only annotation spellings whose evaluation cannot rebind a name.""" + return all( + isinstance(node, (ast.Name, ast.Constant, ast.Load)) for node in ast.walk(annotation) + ) + + +def _function_header_is_passive( + statement: ast.FunctionDef | ast.AsyncFunctionDef, +) -> bool: + """Reject definition-time expressions that could mutate tracked bindings.""" + if statement.decorator_list or getattr(statement, "type_params", []): + return False + defaults = (*statement.args.defaults, *(item for item in statement.args.kw_defaults if item)) + if any(not _is_passive_argument(default) for default in defaults): + return False + arguments = ( + *statement.args.posonlyargs, + *statement.args.args, + *statement.args.kwonlyargs, + ) + annotations = [argument.annotation for argument in arguments if argument.annotation is not None] + if statement.args.vararg is not None and statement.args.vararg.annotation is not None: + annotations.append(statement.args.vararg.annotation) + if statement.args.kwarg is not None and statement.args.kwarg.annotation is not None: + annotations.append(statement.args.kwarg.annotation) + if statement.returns is not None: + annotations.append(statement.returns) + return all(_annotation_is_passive(annotation) for annotation in annotations) + + +def _class_header_is_passive(statement: ast.ClassDef) -> bool: + """Return whether evaluating a class header cannot rebind a receiver.""" + return not ( + statement.decorator_list + or statement.bases + or statement.keywords + or getattr(statement, "type_params", []) + ) + + +def _is_immediate_function(statement: ast.FunctionDef | ast.AsyncFunctionDef) -> bool: + """Return whether a direct call begins executing this function body.""" + if isinstance(statement, ast.AsyncFunctionDef): + return False + pending: list[ast.AST] = list(statement.body) + while pending: + current = pending.pop() + if isinstance(current, (ast.Yield, ast.YieldFrom)): + return False + if isinstance(current, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef, ast.Lambda)): + continue + pending.extend(ast.iter_child_nodes(current)) + return True + + +def _passive_direct_call(statement: ast.stmt) -> ast.Call | None: + """Return a directly evaluated simple-name call with passive arguments.""" + value: ast.expr | None = None + if isinstance(statement, (ast.Expr, ast.Assign)): + value = statement.value + elif isinstance(statement, ast.AnnAssign): + value = statement.value + if ( + isinstance(value, ast.Call) + and isinstance(value.func, ast.Name) + and _call_arguments_are_passive(value) + ): + return value + return None + + +def _advance_trusted_names( + statement: ast.stmt, + trusted_names: set[str], + bound_names: set[str], + finalizer_safe_names: set[str], + unknown_unsafe_bindings: list[bool], +) -> None: + """Apply one eager statement's receiver-trust and value-release effects.""" + if isinstance(statement, (ast.Import, ast.ImportFrom)): + imported_names = _direct_bound_names(statement) + releases_unsafe_value = any( + name in bound_names and name not in finalizer_safe_names for name in imported_names + ) + bound_names.update(imported_names) + finalizer_safe_names.difference_update(imported_names) + _update_trusted_names_from_import(statement, trusted_names) + imports_unknown_names = isinstance(statement, ast.ImportFrom) and any( + imported.name == "*" for imported in statement.names + ) + if releases_unsafe_value or imports_unknown_names: + unknown_unsafe_bindings[0] = True + if releases_unsafe_value or unknown_unsafe_bindings[0]: + trusted_names.clear() + return + if isinstance(statement, (ast.FunctionDef, ast.AsyncFunctionDef)): + releases_unsafe_value = ( + statement.name in bound_names and statement.name not in finalizer_safe_names + ) + if not _function_header_is_passive(statement) or releases_unsafe_value: + trusted_names.clear() + unknown_unsafe_bindings[0] = True + bound_names.add(statement.name) + finalizer_safe_names.discard(statement.name) + trusted_names.discard(statement.name) + return + if isinstance(statement, ast.Assign): + targets = list(statement.targets) + simple_targets = all(isinstance(target, ast.Name) for target in targets) + result_is_finalizer_safe = _is_finalizer_safe_value( + statement.value, + finalizer_safe_names, + ) + preserves_receiver_trust = simple_targets and _expression_preserves_receiver_trust( + statement.value, + trusted_names, + finalizer_safe_names, + ) + releases_unsafe_value = simple_targets and any( + target.id in bound_names + and target.id not in finalizer_safe_names + and not (isinstance(statement.value, ast.Name) and statement.value.id == target.id) + for target in targets + if isinstance(target, ast.Name) + ) + if not preserves_receiver_trust or releases_unsafe_value: + trusted_names.clear() + finalizer_safe_names.clear() + unknown_unsafe_bindings[0] = True + if not simple_targets: + bound_names.update(_direct_bound_names(statement)) + return + changed = _changed_direct_names( + [statement.value, *statement.targets], + trusted_names, + ) + preserved = { + target.id + for target in statement.targets + if isinstance(target, ast.Name) + and isinstance(statement.value, ast.Name) + and statement.value.id == target.id + and target.id in trusted_names + } + trusted_names.difference_update(changed.difference(preserved)) + for target in targets: + assert isinstance(target, ast.Name) + bound_names.add(target.id) + if preserves_receiver_trust and not releases_unsafe_value and result_is_finalizer_safe: + finalizer_safe_names.add(target.id) + else: + finalizer_safe_names.discard(target.id) + return + if isinstance(statement, ast.AnnAssign): + value = statement.value + target = statement.target + simple_target = isinstance(target, ast.Name) + preserves_receiver_trust = ( + simple_target + and _annotation_is_passive(statement.annotation) + and ( + value is None + or _expression_preserves_receiver_trust( + value, + trusted_names, + finalizer_safe_names, + ) + ) + ) + releases_unsafe_value = ( + value is not None + and simple_target + and target.id in bound_names + and target.id not in finalizer_safe_names + and not (isinstance(value, ast.Name) and value.id == target.id) + ) + result_is_finalizer_safe = value is not None and _is_finalizer_safe_value( + value, + finalizer_safe_names, + ) + if not preserves_receiver_trust or releases_unsafe_value: + trusted_names.clear() + finalizer_safe_names.clear() + unknown_unsafe_bindings[0] = True + if value is not None: + bound_names.update(_direct_bound_names(statement)) + if simple_target: + if ( + preserves_receiver_trust + and not releases_unsafe_value + and result_is_finalizer_safe + ): + finalizer_safe_names.add(target.id) + else: + finalizer_safe_names.discard(target.id) + trusted_names.difference_update(_changed_direct_names([statement], trusted_names)) + return + if isinstance(statement, ast.ClassDef): + releases_unsafe_value = ( + statement.name in bound_names and statement.name not in finalizer_safe_names + ) + unsafe_class_execution = ( + not _class_header_is_passive(statement) + or _class_body_has_eager_effects(statement) + or releases_unsafe_value + ) + if unsafe_class_execution: + trusted_names.clear() + unknown_unsafe_bindings[0] = True + finalizer_safe_names.clear() + else: + finalizer_safe_names.discard(statement.name) + trusted_names.difference_update(_changed_direct_names([statement], trusted_names)) + trusted_names.difference_update(_class_body_changed_direct_names(statement, trusted_names)) + bound_names.update(_direct_bound_names(statement)) + return + if isinstance(statement, ast.Expr): + value = statement.value + preserves_receiver_trust = _expression_preserves_receiver_trust( + value, + trusted_names, + finalizer_safe_names, + ) + if not preserves_receiver_trust: + trusted_names.clear() + unknown_unsafe_bindings[0] = True + finalizer_safe_names.clear() + else: + trusted_names.difference_update(_changed_direct_names([statement], trusted_names)) + bound_names.update(_direct_bound_names(statement)) + return + if isinstance(statement, ast.Assert): + expressions = [statement.test] + if statement.msg is not None: + expressions.append(statement.msg) + if not ( + _is_finalizer_safe_value(statement.test, finalizer_safe_names) + and all( + _expression_preserves_receiver_trust( + item, + trusted_names, + finalizer_safe_names, + ) + for item in expressions + ) + ): + trusted_names.clear() + unknown_unsafe_bindings[0] = True + finalizer_safe_names.clear() + return + if isinstance(statement, (ast.Global, ast.Nonlocal, ast.Pass)): + return + bound_names.update(_direct_bound_names(statement)) + finalizer_safe_names.clear() + trusted_names.clear() + unknown_unsafe_bindings[0] = True + + +class _CachedSubprocessState: + """Track explicit writes to cached module slots independently of imports.""" + + def __init__(self) -> None: + self.module_names = {"subprocess"} + self.changed_methods: set[str] = set() + self.changed_popen = False + self.effect_generation = 0 + self.protocol_unsafe = False + self.bound_names: set[str] = set() + self.safe_names: set[str] = set() + + def copy(self) -> _CachedSubprocessState: + copied = _CachedSubprocessState() + copied.module_names = set(self.module_names) + copied.changed_methods = set(self.changed_methods) + copied.changed_popen = self.changed_popen + copied.effect_generation = self.effect_generation + copied.protocol_unsafe = self.protocol_unsafe + copied.bound_names = set(self.bound_names) + copied.safe_names = set(self.safe_names) + return copied + + def _invalidate(self) -> None: + """An unmodeled eager effect can change receivers and restore old slots.""" + self.module_names.clear() + self.changed_methods.clear() + self.changed_popen = False + self.safe_names.clear() + self.effect_generation += 1 + + def _passive_value(self, value: ast.expr) -> bool: + pending = [value] + while pending: + current = pending.pop() + if _is_passive_argument(current): + continue + if ( + isinstance(current, ast.Attribute) + and isinstance(current.value, ast.Name) + and current.value.id in self.module_names + ): + continue + if isinstance(current, (ast.Tuple, ast.List)) and not any( + isinstance(item, ast.Starred) for item in current.elts + ): + pending.extend(current.elts) + continue + return False + return True + + def blocks(self, call: ast.Call) -> bool: + # A later import or slot write cannot establish that an unknown earlier + # effect left the cached module's protocol and finalizer bindings intact. + if self.protocol_unsafe or self.effect_generation > 0: + return False + function = call.func + if isinstance(function, ast.Name): + return function.id == "Popen" and self.changed_popen + return isinstance(function, ast.Attribute) and function.attr in self.changed_methods + + def advance(self, statement: ast.stmt) -> None: + """Apply only explicit eager bindings; deferred bodies do not execute here.""" + if isinstance(statement, (ast.Import, ast.ImportFrom)): + imported_names = _direct_bound_names(statement) + if any( + name in self.bound_names + and name not in self.safe_names + and name not in self.module_names + for name in imported_names + ): + self._invalidate() + ordinary_subprocess_import = ( + isinstance(statement, ast.Import) + and all(imported.name == "subprocess" for imported in statement.names) + ) or ( + isinstance(statement, ast.ImportFrom) + and statement.level == 0 + and statement.module == "subprocess" + and all( + imported.name == "Popen" and (imported.asname or imported.name) == "Popen" + for imported in statement.names + ) + ) + if not ordinary_subprocess_import: + self._invalidate() + self.module_names.difference_update(imported_names) + self.bound_names.update(imported_names) + self.safe_names.difference_update(imported_names) + if isinstance(statement, ast.Import): + self.module_names.update( + imported.asname or "subprocess" + for imported in statement.names + if imported.name == "subprocess" + ) + elif statement.level == 0 and statement.module == "subprocess": + for imported in statement.names: + if imported.name == "Popen" and (imported.asname or imported.name) == "Popen": + self.changed_popen = "Popen" in self.changed_methods + return + if isinstance(statement, ast.ClassDef): + if not _class_header_is_passive(statement): + self._invalidate() + class_state = self.copy() + declarations = _DirectBindingCollector(set()) + pending_classes = [statement] + while pending_classes: + for child in pending_classes.pop().body: + declarations.visit(child) + if isinstance(child, ast.ClassDef): + pending_classes.append(child) + local_bound = _direct_bound_names(statement).difference(declarations.nonlocal_names) + for child in statement.body: + local_bound.update( + _direct_bound_names(child).difference(declarations.nonlocal_names) + ) + class_state.bound_names.difference_update(local_bound) + class_state.safe_names.difference_update(local_bound) + for child in statement.body: + class_state.advance(child) + self.protocol_unsafe |= class_state.protocol_unsafe + if class_state.effect_generation != self.effect_generation: + self.module_names.clear() + self.changed_popen = False + self.safe_names.clear() + self.effect_generation = class_state.effect_generation + self.changed_methods = class_state.changed_methods + self.module_names.difference_update(declarations.nonlocal_names) + self.module_names.update( + class_state.module_names.intersection(declarations.nonlocal_names) + ) + if "Popen" in declarations.nonlocal_names: + self.changed_popen = class_state.changed_popen + self.bound_names.update( + class_state.bound_names.intersection(declarations.nonlocal_names) + ) + self.safe_names.difference_update(declarations.nonlocal_names) + self.safe_names.update(class_state.safe_names.intersection(declarations.nonlocal_names)) + if ( + statement.name in self.bound_names + and statement.name not in self.safe_names + and statement.name not in self.module_names + ): + self._invalidate() + self.bound_names.add(statement.name) + self.safe_names.discard(statement.name) + self.module_names.discard(statement.name) + return + if isinstance(statement, (ast.FunctionDef, ast.AsyncFunctionDef)): + if not _function_header_is_passive(statement) or ( + statement.name in self.bound_names + and statement.name not in self.safe_names + and statement.name not in self.module_names + ): + self._invalidate() + self.bound_names.add(statement.name) + self.safe_names.discard(statement.name) + self.module_names.discard(statement.name) + return + targets: list[ast.expr] = [] + value: ast.expr | None = None + if isinstance(statement, ast.Assign): + targets, value = list(statement.targets), statement.value + elif isinstance(statement, ast.AnnAssign): + if statement.value is None: + if not _annotation_is_passive(statement.annotation): + self._invalidate() + return + targets, value = [statement.target], statement.value + elif isinstance(statement, ast.AugAssign): + targets = [statement.target] + elif isinstance(statement, ast.Delete): + self._invalidate() + self.bound_names.difference_update(_direct_bound_names(statement)) + return + else: + if not ( + isinstance(statement, (ast.Global, ast.Nonlocal, ast.Pass)) + or isinstance(statement, (ast.Expr, ast.Return)) + and (statement.value is None or self._passive_value(statement.value)) + ): + self._invalidate() + return + + native_api_value = value is not None and any( + isinstance(node, ast.Attribute) + and isinstance(node.value, ast.Name) + and node.value.id in self.module_names + and node.attr in _CACHED_SUBPROCESS_API_SLOTS + and node.attr not in self.changed_methods + or isinstance(node, ast.Name) + and node.id == "Popen" + and not self.changed_popen + for node in ast.walk(value) + ) + same_slot_identity = ( + len(targets) == 1 + and isinstance(targets[0], ast.Attribute) + and isinstance(targets[0].value, ast.Name) + and targets[0].value.id in self.module_names + and isinstance(value, ast.Attribute) + and isinstance(value.value, ast.Name) + and value.value.id in self.module_names + and targets[0].attr == value.attr + ) + # A native callable capture or cross-slot store cannot establish a + # custom replacement. Abstain without tracking callable value aliases. + if native_api_value and not same_slot_identity: + self._invalidate() + if ( + isinstance(statement, ast.AugAssign) + or value is not None + and not self._passive_value(value) + ): + self._invalidate() + rhs_names = ( + {node.id for node in ast.walk(value) if isinstance(node, ast.Name)} + if value is not None + else set() + ) + before_names = rhs_names.intersection(self.module_names) + before_methods = set(self.changed_methods) + before_safe = rhs_names.intersection(self.safe_names) + pending = [(target, value) for target in reversed(targets)] + while pending: + target, source = pending.pop() + if isinstance(target, (ast.Tuple, ast.List)): + if not isinstance(statement, ast.Delete) and not ( + isinstance(source, (ast.Tuple, ast.List)) + and len(target.elts) == len(source.elts) + and not any(isinstance(item, ast.Starred) for item in source.elts) + and not any(isinstance(item, ast.Starred) for item in target.elts) + ): + self._invalidate() + sources = ( + list(source.elts) + if isinstance(source, (ast.Tuple, ast.List)) + and len(target.elts) == len(source.elts) + and not any(isinstance(item, ast.Starred) for item in source.elts) + else [None] * len(target.elts) + ) + pending.extend(reversed(list(zip(target.elts, sources, strict=True)))) + elif isinstance(target, ast.Starred): + pending.append((target.value, None)) + elif isinstance(target, ast.Name): + self_store = isinstance(source, ast.Name) and source.id == target.id + releases_unsafe = ( + target.id in self.bound_names + and target.id not in self.safe_names + and target.id not in self.module_names + and not self_store + ) + safe_value = source is not None and _is_finalizer_safe_value(source, before_safe) + self.module_names.discard(target.id) + if isinstance(source, ast.Name) and source.id in before_names: + self.module_names.add(target.id) + self.bound_names.add(target.id) + self.safe_names.discard(target.id) + if safe_value: + self.safe_names.add(target.id) + if releases_unsafe: + self._invalidate() + elif ( + isinstance(target, ast.Attribute) + and isinstance(target.value, ast.Name) + and target.value.id in self.module_names + ): + if target.attr not in _CACHED_SUBPROCESS_API_SLOTS or self.protocol_unsafe: + self.protocol_unsafe |= target.attr.startswith("__") + self._invalidate() + continue + preserves_slot = ( + isinstance(source, ast.Attribute) + and isinstance(source.value, ast.Name) + and source.value.id in before_names + and source.attr == target.attr + ) + if preserves_slot and target.attr not in before_methods: + self.changed_methods.discard(target.attr) + elif not preserves_slot and target.attr in before_methods: + # Releasing a previously replaced slot can run an arbitrary + # finalizer after the new store and restore its old value. + self._invalidate() + else: + self.changed_methods.add(target.attr) + else: + self._invalidate() + if isinstance(statement, ast.AnnAssign) and not _annotation_is_passive( + statement.annotation + ): + self._invalidate() + + +class _Analyzer: + def __init__( + self, + file_path: str, + parsed: ParsedPythonFile, + *, + emit_findings: bool = True, + check_runtime: Callable[[], None] | None = None, + ) -> None: + self.file_path = file_path + self.parsed = parsed + self.python_ast = parsed + self.content = parsed.content + self.lines = parsed.lines + self._emit_findings = emit_findings + self._check_runtime = check_runtime + self.findings: list[AnalyzerFinding] = [] + self._finding_keys: set[tuple[int, str]] = set() + self.bound_shell_metadata: list[BoundShellMetadata] = [] + + self.bound_shell_call_ownership: dict[BoundShellCallKey, bool] = {} + self.emitted_shell_calls: set[BoundShellCallKey] = set() + self.cached_replacement_by_call: dict[BoundShellCallKey, bool] = {} + self.cached_subprocess = _CachedSubprocessState() + + def _record_cached_replacement(self, call: ast.Call) -> None: + """Keep explicit cached-slot evidence distinct from receiver uncertainty.""" + shell = next((item.value for item in call.keywords if item.arg == "shell"), None) + if not _is_direct_subprocess_syntax(call) or not isinstance(shell, ast.Name): + return + key = _bound_shell_call_key(call) + blocked = self.cached_subprocess.blocks(call) + self.cached_replacement_by_call[key] = ( + self.cached_replacement_by_call.get(key, True) and blocked + ) + + def _record_eager_cached_replacements(self, expression: ast.expr) -> None: + """Visit eager expression order until an unmodeled effect or scope boundary.""" + pending: list[ast.expr | None] = [expression] + while pending: + current = pending.pop() + if current is None: + return + if isinstance(current, ast.Call): + function = current.func + passive_lookup = isinstance(function, ast.Name) or ( + isinstance(function, ast.Attribute) + and isinstance(function.value, ast.Name) + and function.value.id in self.cached_subprocess.module_names + ) + if passive_lookup: + self._record_cached_replacement(current) + children = [function, *current.args, *(item.value for item in current.keywords)] + pending.append(None) + pending.extend(reversed(children)) + elif isinstance(current, (ast.Name, ast.Constant)): + continue + elif isinstance(current, (ast.Tuple, ast.List)): + pending.extend(reversed(current.elts)) + elif isinstance(current, ast.Attribute): + if not ( + isinstance(current.value, ast.Name) + and current.value.id in self.cached_subprocess.module_names + ): + pending.append(None) + pending.append(current.value) + else: + return + + def _record_bound_shell_call(self, call: ast.Call, trusted_names: set[str]) -> None: + """Record whether the companion owns one supported bound-shell call.""" + shell = next((item.value for item in call.keywords if item.arg == "shell"), None) + if not _is_direct_subprocess_syntax(call) or not isinstance(shell, ast.Name): + return + self._record_cached_replacement(call) + key = _bound_shell_call_key(call) + blocked = self.cached_subprocess.blocks(call) + self.bound_shell_call_ownership[key] = bool( + _is_direct_subprocess_call(call, trusted_names) + and not blocked + and _shell_argument_is_captured_before_effects(call) + ) + + def _source_position(self, node: ast.AST, *, end: bool = False) -> int | None: + """Map one AST UTF-8 byte coordinate through the shared parsed source.""" + line_name = "end_lineno" if end else "lineno" + column_name = "end_col_offset" if end else "col_offset" + line = getattr(node, line_name, None) + byte_column = getattr(node, column_name, None) + if not isinstance(line, int) or not isinstance(byte_column, int): + return None + character_column = self.parsed.character_column(line, byte_column) + line_index = line - 1 + if character_column is None or not 0 <= line_index < len(self.parsed.line_character_starts): + return None + return self.parsed.line_character_starts[line_index] + character_column + + def _source_start(self, node: ast.AST) -> int | None: + return self._source_position(node) + + def _source_end(self, node: ast.AST) -> int | None: + return self._source_position(node, end=True) + + def _source_segment(self, node: ast.AST) -> str: + source = self.parsed.source_segment(node) + if source is not None: + return source + line = getattr(node, "lineno", 1) + end_line = getattr(node, "end_lineno", None) + return get_complete_source_segment(self.lines, line, end_line) + + def _canonical_fingerprint( + self, + call: ast.Call, + shell_keyword: ast.keyword, + shell: ast.expr, + ) -> tuple[str, bool, int] | None: + """Mirror the direct lexical match through ``shell=True``.""" + call_start = self._source_start(call) + shell_start = self._source_start(shell) + if call_start is None or shell_start is None or shell_start < call_start: + return None + raw_callee = self._source_segment(call.func) + canonical_start = call_start + if ( + isinstance(call.func, ast.Attribute) + and _normalized_direct_name(call.func.attr) == "popen" + and not any( + view.text.casefold() == "subprocess.popen" + for view in security_text_views(raw_callee) + ) + ): + function_end = self._source_end(call.func) + raw_method = re.search(r"(?P\w+)\s*$", raw_callee) + if function_end is not None and raw_method is not None: + canonical_start = function_end - len(raw_method.group("method")) + raw_canonical = self.content[canonical_start:shell_start] + "True" + canonical = normalized_security_view(raw_canonical).text + normalized_callee = normalized_security_view(raw_callee).text + keyword_start = self._source_start(shell_keyword) + raw_keyword = self.content[keyword_start:shell_start] if keyword_start is not None else "" + normalized_keyword = normalized_security_view(raw_keyword).text + normalization_exposed_direct_spelling = ( + _DIRECT_CALLEE.fullmatch(raw_callee) is None + and _DIRECT_CALLEE.fullmatch(normalized_callee) is not None + or _SHELL_KEYWORD_PREFIX.fullmatch(raw_keyword) is None + and _SHELL_KEYWORD_PREFIX.fullmatch(normalized_keyword) is not None + ) + return ( + compute_match_fingerprint("TM1", canonical), + normalization_exposed_direct_spelling, + canonical_start, + ) + + def _bounded_context(self, call: ast.Call) -> str: + call_start = self._source_start(call) + if call_start is None: + line = getattr(call, "lineno", 1) + return get_context_from_lines(self.lines, line)[:_MAX_CONTEXT_CHARS] + left = max(0, call_start - _MAX_CONTEXT_CHARS // 2) + right = min(len(self.content), left + _MAX_CONTEXT_CHARS) + left = max(0, right - _MAX_CONTEXT_CHARS) + return self.content[left:right].rstrip(LINE_BREAK_CHARS) + + def _append_finding( + self, + call: ast.Call, + shell_keyword: ast.keyword, + shell: ast.expr, + ) -> None: + line = getattr(call, "lineno", 1) + end_line = getattr(call, "end_lineno", None) + start_column = self.parsed.character_column(line, getattr(call, "col_offset", 0)) + end_column = ( + self.parsed.character_column(end_line, getattr(call, "end_col_offset", 0)) + if isinstance(end_line, int) + else None + ) + source_start = self._source_start(call) + source_end = self._source_end(call) + shell_anchor = self._source_start(shell_keyword) + shell_start = self._source_start(shell) + shell_end = self._source_end(shell) + popen_start: int | None = None + if ( + isinstance(call.func, ast.Attribute) + and _normalized_direct_name(call.func.attr) == "popen" + ): + function_end = self._source_end(call.func) + raw_function = self._source_segment(call.func) + raw_method = re.search(r"(?P\w+)\s*$", raw_function) + if function_end is not None and raw_method is not None: + popen_start = function_end - len(raw_method.group("method")) + canonical = self._canonical_fingerprint(call, shell_keyword, shell) + if ( + source_start is not None + and shell_anchor is not None + and shell_start is not None + and shell_end is not None + and canonical is not None + ): + self.bound_shell_metadata.append( + BoundShellMetadata( + call_start=source_start, + call_end=source_end, + shell_anchor=shell_anchor, + value_start=shell_start, + value_end=shell_end, + popen_start=popen_start, + canonical_fingerprint=canonical[0], + normalized_view=canonical[1], + ) + ) + if not self._emit_findings: + return + + source = self._source_segment(call) + evidence: dict[str, object] = {BOUND_SHELL_EVIDENCE: True} + if source_start is not None: + evidence[BOUND_CALL_START_EVIDENCE] = source_start + if source_end is not None: + evidence[BOUND_CALL_END_EVIDENCE] = source_end + if shell_anchor is not None: + evidence[BOUND_SHELL_ANCHOR_EVIDENCE] = shell_anchor + if shell_start is not None: + evidence[BOUND_SHELL_VALUE_START_EVIDENCE] = shell_start + if shell_end is not None: + evidence[BOUND_SHELL_VALUE_END_EVIDENCE] = shell_end + if source_start is not None and shell_start is not None: + evidence[BOUND_CLASSIFICATION_MATCH_EVIDENCE] = ( + self.content[source_start:shell_start] + "True" + )[:200] + evidence[BOUND_DIRECT_MATCH_END_EVIDENCE] = shell_start + len("True") + if popen_start is not None: + evidence[BOUND_POPEN_START_EVIDENCE] = popen_start + finding_key = (line, compute_match_fingerprint("TM1", source)) + if finding_key in self._finding_keys: + return + self._finding_keys.add(finding_key) + if canonical is not None: + fingerprint, is_normalized, direct_owner_start = canonical + evidence[BOUND_CANONICAL_FINGERPRINT_EVIDENCE] = fingerprint + evidence[BOUND_DIRECT_OWNER_START_EVIDENCE] = direct_owner_start + if is_normalized: + evidence[BOUND_NORMALIZED_VIEW_EVIDENCE] = True + self.findings.append( + AnalyzerFinding( + rule_id="TM1", + message="Tool Parameter Abuse", + severity=Severity.HIGH, + location=Location( + file=self.file_path, + start_line=line, + end_line=end_line, + start_column=start_column, + end_column=end_column, + ), + confidence=0.8, + tags=[PatternCategory.TOOL_MISUSE.value], + context=self._bounded_context(call), + matched_text=source[:200], + complete_match=source, + evidence=evidence, + ) + ) + + def _inspect_call(self, call: ast.Call, facts: dict[str, bool]) -> None: + shell_keyword = next((item for item in call.keywords if item.arg == "shell"), None) + if shell_keyword is None: + return + shell = shell_keyword.value + if ( + not isinstance(shell, ast.Name) + or self.cached_subprocess.blocks(call) + or facts.get(shell.id) is not True + ): + return + self.emitted_shell_calls.add(_bound_shell_call_key(call)) + self._append_finding(call, shell_keyword, shell) + + def _scan_assignment( + self, + targets: list[ast.expr], + value: ast.expr, + facts: dict[str, bool], + trusted_names: set[str], + bound_names: set[str], + finalizer_safe_names: set[str], + unknown_unsafe_bindings: list[bool], + ) -> None: + simple_targets = all(isinstance(target, ast.Name) for target in targets) + releases_unsafe_value = simple_targets and any( + target.id in bound_names + and target.id not in finalizer_safe_names + and not (isinstance(value, ast.Name) and value.id == target.id) + for target in targets + if isinstance(target, ast.Name) + ) + result_is_finalizer_safe = _is_finalizer_safe_value(value, finalizer_safe_names) + call_has_protocol_effects = False + preserves_receiver_trust = simple_targets and _expression_preserves_receiver_trust( + value, + trusted_names, + finalizer_safe_names, + ) + if isinstance(value, ast.Call): + self._record_bound_shell_call(value, trusted_names) + if isinstance(value, ast.Call) and _is_direct_subprocess_call(value, trusted_names): + resolved = None + safe_value = _call_arguments_are_passive(value) + if _shell_argument_is_captured_before_effects(value): + self._inspect_call(value, facts) + if safe_value: + call_has_protocol_effects = not _call_arguments_are_protocol_safe( + value, + finalizer_safe_names, + ) + else: + resolved = _truth_value(value, facts) + safe_value = resolved is not None or _is_passive_argument(value) + + if not safe_value or not simple_targets: + facts.clear() + finalizer_safe_names.clear() + for target in targets: + if isinstance(target, ast.Name): + bound_names.add(target.id) + if not preserves_receiver_trust: + trusted_names.clear() + unknown_unsafe_bindings[0] = True + else: + trusted_names.difference_update( + _changed_direct_names([value, *targets], trusted_names) + ) + return + if releases_unsafe_value: + facts.clear() + finalizer_safe_names.clear() + trusted_names.clear() + unknown_unsafe_bindings[0] = True + if call_has_protocol_effects: + facts.clear() + finalizer_safe_names.clear() + trusted_names.clear() + unknown_unsafe_bindings[0] = True + for target in targets: + assert isinstance(target, ast.Name) + bound_names.add(target.id) + if releases_unsafe_value or call_has_protocol_effects or resolved is None: + facts.pop(target.id, None) + else: + facts[target.id] = resolved + if releases_unsafe_value or call_has_protocol_effects or not result_is_finalizer_safe: + finalizer_safe_names.discard(target.id) + else: + finalizer_safe_names.add(target.id) + preserves_binding = ( + isinstance(value, ast.Name) and value.id == target.id and value.id in trusted_names + ) + if not preserves_binding: + trusted_names.discard(target.id) + + def _scan_block( + self, + statements: list[ast.stmt], + *, + trusted_names: set[str] | None = None, + initial_bound_names: set[str] | None = None, + nested_function_trusted_names: set[str] | None = None, + nested_function_trusted_at_call: dict[int, set[str]] | None = None, + cached_subprocess: _CachedSubprocessState | None = None, + ) -> None: + previous_cached = self.cached_subprocess + self.cached_subprocess = (cached_subprocess or previous_cached).copy() + initial_cached = self.cached_subprocess.copy() + trusted_names = set(_DIRECT_CALL_NAMES if trusted_names is None else trusted_names) + facts: dict[str, bool] = {} + bound_names = set(initial_bound_names or ()) + finalizer_safe_names: set[str] = set() + unknown_unsafe_bindings = [False] + + last_invalidation_by_name: dict[str, int] = {} + receiver_trust = set(trusted_names) + receiver_bound_names = set(initial_bound_names or ()) + receiver_finalizer_safe_names: set[str] = set() + receiver_unknown_unsafe_bindings = [False] + for candidate_index, candidate in enumerate(statements): + before = set(receiver_trust) + _advance_trusted_names( + candidate, + receiver_trust, + receiver_bound_names, + receiver_finalizer_safe_names, + receiver_unknown_unsafe_bindings, + ) + for name in before.difference(receiver_trust): + last_invalidation_by_name[name] = candidate_index + + trusted_at_call_by_definition: dict[int, set[str]] = {} + receiver_trust = set(trusted_names) + receiver_bound_names = set(initial_bound_names or ()) + receiver_finalizer_safe_names = set() + receiver_unknown_unsafe_bindings = [False] + active_functions: dict[str, int] = {} + cached_at_call_by_definition: dict[int, _CachedSubprocessState] = {} + deferred_cached = initial_cached.copy() + for candidate_index, candidate in enumerate(statements): + call = _passive_direct_call(candidate) + if call is not None: + assert isinstance(call.func, ast.Name) + owner = active_functions.get(call.func.id) + if owner is not None: + trusted_at_call_by_definition.setdefault(owner, set()).update(receiver_trust) + prior_cached = cached_at_call_by_definition.get(owner) + if prior_cached is None: + cached_at_call_by_definition[owner] = deferred_cached.copy() + else: + prior_cached.changed_methods.intersection_update( + deferred_cached.changed_methods + ) + prior_cached.changed_popen &= deferred_cached.changed_popen + prior_cached.effect_generation = max( + prior_cached.effect_generation, deferred_cached.effect_generation + ) + + changed_names = _direct_bound_names(candidate) + for name in changed_names: + active_functions.pop(name, None) + if ( + isinstance(candidate, (ast.FunctionDef, ast.AsyncFunctionDef)) + and _function_header_is_passive(candidate) + and _is_immediate_function(candidate) + ): + active_functions[candidate.name] = candidate_index + _advance_trusted_names( + candidate, + receiver_trust, + receiver_bound_names, + receiver_finalizer_safe_names, + receiver_unknown_unsafe_bindings, + ) + deferred_cached.advance(candidate) + + for index, statement in enumerate(statements): + if self._check_runtime is not None: + self._check_runtime() + if isinstance(statement, (ast.Expr, ast.Return, ast.Assign, ast.AnnAssign)): + if statement.value is not None: + self._record_eager_cached_replacements(statement.value) + if isinstance(statement, (ast.FunctionDef, ast.AsyncFunctionDef)): + passive_header = _function_header_is_passive(statement) + if nested_function_trusted_names is None: + trusted_at_call = trusted_at_call_by_definition.get(index, set()) + nested_trusted_names = set(trusted_names).union(trusted_at_call) + nested_trusted_names = { + name + for name in nested_trusted_names + if last_invalidation_by_name.get(name, -1) <= index + or name in trusted_at_call + } + else: + nested_trusted_names = set(nested_function_trusted_names).union( + (nested_function_trusted_at_call or {}).get(index, set()) + ) + nested_trusted_names.difference_update( + _function_bound_direct_names(statement, nested_trusted_names) + ) + nested_trusted_names.discard(statement.name) + if not passive_header: + nested_trusted_names.clear() + nested_cached = cached_at_call_by_definition.get(index, deferred_cached).copy() + nested_cached.module_names.difference_update( + _function_bound_direct_names(statement, nested_cached.module_names) + ) + local_bound = _function_bound_direct_names(statement, nested_cached.bound_names) + nested_cached.bound_names.difference_update(local_bound) + nested_cached.safe_names.difference_update(local_bound) + parameters = _function_parameter_names(statement) + nested_cached.bound_names.update(parameters) + nested_cached.safe_names.difference_update(parameters) + self._scan_block( + statement.body, + trusted_names=nested_trusted_names, + initial_bound_names=_function_parameter_names(statement), + cached_subprocess=nested_cached, + ) + releases_unsafe_value = ( + statement.name in bound_names and statement.name not in finalizer_safe_names + ) + if passive_header and not releases_unsafe_value: + facts.pop(statement.name, None) + else: + facts.clear() + finalizer_safe_names.clear() + trusted_names.clear() + unknown_unsafe_bindings[0] = True + bound_names.add(statement.name) + finalizer_safe_names.discard(statement.name) + trusted_names.discard(statement.name) + elif isinstance(statement, (ast.Import, ast.ImportFrom)): + imported_names = _direct_bound_names(statement) + releases_unsafe_value = any( + name in bound_names and name not in finalizer_safe_names + for name in imported_names + ) + facts.clear() + finalizer_safe_names.difference_update(imported_names) + bound_names.update(imported_names) + _update_trusted_names_from_import(statement, trusted_names) + imports_unknown_names = isinstance(statement, ast.ImportFrom) and any( + imported.name == "*" for imported in statement.names + ) + if releases_unsafe_value or imports_unknown_names: + unknown_unsafe_bindings[0] = True + if releases_unsafe_value or unknown_unsafe_bindings[0]: + trusted_names.clear() + elif isinstance(statement, ast.Assign): + self._scan_assignment( + list(statement.targets), + statement.value, + facts, + trusted_names, + bound_names, + finalizer_safe_names, + unknown_unsafe_bindings, + ) + elif isinstance(statement, ast.AnnAssign): + value = statement.value + target = statement.target + simple_target = isinstance(target, ast.Name) + releases_unsafe_value = ( + value is not None + and simple_target + and target.id in bound_names + and target.id not in finalizer_safe_names + and not (isinstance(value, ast.Name) and value.id == target.id) + ) + result_is_finalizer_safe = value is not None and _is_finalizer_safe_value( + value, + finalizer_safe_names, + ) + preserves_receiver_trust = ( + simple_target + and _annotation_is_passive(statement.annotation) + and ( + value is None + or _expression_preserves_receiver_trust( + value, + trusted_names, + finalizer_safe_names, + ) + ) + ) + if isinstance(value, ast.Call): + self._record_bound_shell_call(value, trusted_names) + if isinstance(value, ast.Call) and _is_direct_subprocess_call(value, trusted_names): + if _shell_argument_is_captured_before_effects(value): + self._inspect_call(value, facts) + facts.clear() + if not preserves_receiver_trust or releases_unsafe_value: + finalizer_safe_names.clear() + if value is not None: + bound_names.update(_direct_bound_names(statement)) + if ( + simple_target + and preserves_receiver_trust + and not releases_unsafe_value + and result_is_finalizer_safe + ): + finalizer_safe_names.add(target.id) + elif simple_target: + finalizer_safe_names.discard(target.id) + if not preserves_receiver_trust or releases_unsafe_value: + trusted_names.clear() + unknown_unsafe_bindings[0] = True + else: + trusted_names.difference_update( + _changed_direct_names([statement], trusted_names) + ) + elif isinstance(statement, (ast.AugAssign, ast.Delete)): + facts.clear() + finalizer_safe_names.clear() + bound_names.update(_direct_bound_names(statement)) + trusted_names.clear() + unknown_unsafe_bindings[0] = True + elif isinstance(statement, ast.Expr) and isinstance(statement.value, ast.Call): + call = statement.value + self._record_bound_shell_call(call, trusted_names) + direct_call = _is_direct_subprocess_call(call, trusted_names) + if direct_call and _shell_argument_is_captured_before_effects(call): + self._inspect_call(call, facts) + arguments_are_passive = _call_arguments_are_passive(call) + if direct_call and arguments_are_passive: + if not _call_arguments_are_protocol_safe(call, finalizer_safe_names): + facts.clear() + finalizer_safe_names.clear() + trusted_names.clear() + unknown_unsafe_bindings[0] = True + else: + facts.clear() + finalizer_safe_names.clear() + trusted_names.clear() + unknown_unsafe_bindings[0] = True + elif isinstance(statement, (ast.Global, ast.Nonlocal, ast.Pass)) or ( + isinstance(statement, ast.Expr) and isinstance(statement.value, ast.Constant) + ): + continue + elif isinstance(statement, ast.Expr): + preserves_receiver_trust = _expression_preserves_receiver_trust( + statement.value, + trusted_names, + finalizer_safe_names, + ) + bound_names.update(_direct_bound_names(statement)) + if preserves_receiver_trust: + trusted_names.difference_update( + _changed_direct_names([statement], trusted_names) + ) + else: + facts.clear() + finalizer_safe_names.clear() + trusted_names.clear() + unknown_unsafe_bindings[0] = True + elif isinstance(statement, ast.Assert): + expressions = [statement.test] + if statement.msg is not None: + expressions.append(statement.msg) + preserves_receiver_trust = all( + _expression_preserves_receiver_trust( + item, + trusted_names, + finalizer_safe_names, + ) + for item in expressions + ) and _is_finalizer_safe_value(statement.test, finalizer_safe_names) + facts.clear() + if not preserves_receiver_trust: + finalizer_safe_names.clear() + trusted_names.clear() + unknown_unsafe_bindings[0] = True + elif isinstance(statement, ast.ClassDef): + releases_unsafe_value = ( + statement.name in bound_names and statement.name not in finalizer_safe_names + ) + class_trusted_names = set(trusted_names) + passive_class_header = _class_header_is_passive(statement) + if not passive_class_header: + class_trusted_names.clear() + method_outer_trust = set( + trusted_names + if nested_function_trusted_names is None + else nested_function_trusted_names + ) + if nested_function_trusted_names is None: + method_outer_trust = { + name + for name in method_outer_trust + if last_invalidation_by_name.get(name, -1) <= index + } + if not passive_class_header: + method_outer_trust.clear() + method_trusted_names, method_trusted_at_call = _class_deferred_receiver_trust( + statement, + method_outer_trust, + ) + self._scan_block( + statement.body, + trusted_names=class_trusted_names, + nested_function_trusted_names=method_trusted_names, + nested_function_trusted_at_call=method_trusted_at_call, + cached_subprocess=self.cached_subprocess, + ) + facts.clear() + bound_names.update(_direct_bound_names(statement)) + unsafe_class_execution = ( + not passive_class_header + or releases_unsafe_value + or _class_body_has_eager_effects(statement) + ) + if unsafe_class_execution: + finalizer_safe_names.clear() + trusted_names.clear() + unknown_unsafe_bindings[0] = True + else: + finalizer_safe_names.discard(statement.name) + trusted_names.difference_update( + _changed_direct_names([statement], trusted_names) + ) + trusted_names.difference_update( + _class_body_changed_direct_names(statement, trusted_names) + ) + else: + facts.clear() + finalizer_safe_names.clear() + bound_names.update(_direct_bound_names(statement)) + trusted_names.clear() + unknown_unsafe_bindings[0] = True + + self.cached_subprocess.advance(statement) + + self.cached_subprocess = previous_cached + + def run(self, tree: ast.Module) -> list[AnalyzerFinding]: + self._scan_block(tree.body) + return sorted(self.findings, key=lambda finding: finding.location.start_line) + + +def bound_shell_metadata( + parsed: ParsedPythonFile, + file_path: str, + *, + check_runtime: Callable[[], None] | None = None, +) -> tuple[BoundShellMetadata, ...]: + """Return every supported bound call independently of the finding cap.""" + if parsed.tree is None: + return () + analyzer = _Analyzer( + file_path, + parsed, + emit_findings=False, + check_runtime=check_runtime, + ) + analyzer.run(parsed.tree) + return tuple(sorted(analyzer.bound_shell_metadata, key=lambda item: item.call_start)) + + +def direct_literal_metadata( + parsed: ParsedPythonFile, + file_path: str, + call_starts: set[int], +) -> dict[int, DirectLiteralMetadata]: + """Enrich only retained lexical owners without creating budgeted findings.""" + if parsed.tree is None or not call_starts: + return {} + locator = _Analyzer(file_path, parsed) + metadata: dict[int, DirectLiteralMetadata] = {} + for node in ast.walk(parsed.tree): + if not isinstance(node, ast.Call): + continue + call_start = locator._source_start(node) + if call_start is None: + continue + lookup_coordinates = {call_start} + function = node.func + if ( + isinstance(function, ast.Attribute) + and isinstance(function.value, ast.Name) + and _normalized_direct_name(function.value.id) == "subprocess" + and _normalized_direct_name(function.attr) == "popen" + ): + function_end = locator._source_end(function) + raw_function = locator._source_segment(function) + raw_method = re.search(r"(?P\w+)\s*$", raw_function) + if function_end is not None and raw_method is not None: + lookup_coordinates.add(function_end - len(raw_method.group("method"))) + retained_coordinates = lookup_coordinates.intersection(call_starts) + if not retained_coordinates: + continue + shell_keyword = next((item for item in node.keywords if item.arg == "shell"), None) + if shell_keyword is None: + continue + shell = shell_keyword.value + if not ( + isinstance(shell, ast.Constant) and type(shell.value) is bool and shell.value is True + ): + continue + canonical = locator._canonical_fingerprint(node, shell_keyword, shell) + shell_anchor = locator._source_start(shell_keyword) + if canonical is None or shell_anchor is None: + continue + fingerprint, normalized_view, _ = canonical + direct_metadata = DirectLiteralMetadata( + call_start=call_start, + call_end=locator._source_end(node), + shell_anchor=shell_anchor, + match_fingerprint=fingerprint, + normalized_view=normalized_view, + ) + for coordinate in retained_coordinates: + metadata[coordinate] = direct_metadata + return metadata + + +def bound_shell_finding_for_call( + file_path: str, + python_ast: ParsedPythonFile, + call: ast.Call, +) -> AnalyzerFinding | None: + """Build metadata for a retained owner after positive dataflow confirmation.""" + shell_keyword = next((item for item in call.keywords if item.arg == "shell"), None) + if shell_keyword is None or not isinstance(shell_keyword.value, ast.Name): + return None + analyzer = _Analyzer(file_path, python_ast) + analyzer._append_finding(call, shell_keyword, shell_keyword.value) + return analyzer.findings[0] if analyzer.findings else None + + +def bound_shell_call_analysis( + file_path: str, + python_ast: ParsedPythonFile, +) -> tuple[dict[BoundShellCallKey, bool], set[BoundShellCallKey], set[BoundShellCallKey]]: + """Separate trust, detections, and affirmative cached-slot replacements.""" + if python_ast.tree is None: + return {}, set(), set() + analyzer = _Analyzer(file_path, python_ast, emit_findings=False) + analyzer.run(python_ast.tree) + return ( + dict(analyzer.bound_shell_call_ownership), + set(analyzer.emitted_shell_calls), + {key for key, replaced in analyzer.cached_replacement_by_call.items() if replaced}, + ) + + +def bound_shell_call_state( + file_path: str, + python_ast: ParsedPythonFile, +) -> tuple[dict[BoundShellCallKey, bool], set[BoundShellCallKey]]: + """Return receiver trust separately from affirmative companion detections.""" + ownership, emitted, _ = bound_shell_call_analysis(file_path, python_ast) + return ownership, emitted + + +def bound_shell_call_ownership( + file_path: str, + python_ast: ParsedPythonFile, +) -> dict[BoundShellCallKey, bool]: + """Return supported bound-shell calls and whether their receiver is trusted.""" + if python_ast.tree is None: + return {} + analyzer = _Analyzer(file_path, python_ast, emit_findings=False) + analyzer.run(python_ast.tree) + return dict(analyzer.bound_shell_call_ownership) + + +def analyze( + content: str, + file_path: str, + file_type: str, + *, + python_ast: ParsedPythonFile | None = None, +) -> list[AnalyzerFinding]: + """Find straight-line truthy names passed to direct subprocess calls.""" + if file_type != "python": + return [] + parsed = python_ast or parse_python_source(content, file_path) + if parsed.tree is None: + return [] + return _Analyzer(file_path, parsed).run(parsed.tree) diff --git a/src/skillspector/nodes/analyzers/static_runner.py b/src/skillspector/nodes/analyzers/static_runner.py index 185170665..800376be7 100644 --- a/src/skillspector/nodes/analyzers/static_runner.py +++ b/src/skillspector/nodes/analyzers/static_runner.py @@ -29,12 +29,15 @@ from collections.abc import Callable, Iterator, Mapping from contextvars import ContextVar from dataclasses import dataclass, field +from inspect import getattr_static +from itertools import chain from typing import cast from skillspector.artifacts import ( ContentKind, SecurityTextView, _contains_default_ignorable, + binary_instruction_view, is_default_ignorable, security_text_views, ) @@ -55,10 +58,14 @@ observe_analyzer_findings, ) from skillspector.nodes.deduplicate import classification_metadata_key +from skillspector.oms import project_oms_content from skillspector.python_ast import ( MAX_PYTHON_AST_SOURCE_CHARS, ParsedPythonFile, + PythonSourceClassification, get_python_ast, + may_be_python_source, + resolve_python_source_classification, ) from skillspector.security_reconstruction import ( MAX_DECLARED_MARKER_RIGHT_CONTEXT_CHARS, @@ -90,6 +97,7 @@ ".md": "markdown", ".markdown": "markdown", ".py": "python", + ".pyw": "python", ".sh": "shell", ".bash": "shell", ".zsh": "shell", @@ -112,8 +120,25 @@ _WINDOW_OVERLAP_CHARS = 8192 _RAW_WINDOW_OWNED_CHARS = SECURITY_VIEW_WINDOW_CHARS - 2 * _WINDOW_OVERLAP_CHARS _VIEW_START_EVIDENCE = "_security_view_start" +_VIEW_ANCHOR_EVIDENCE = "_security_view_anchor" +_VIEW_ALTERNATE_START_EVIDENCE = "_security_view_alternate_start" +_VIEW_REACH_END_EVIDENCE = "_security_view_reach_end" +_VIEW_REPLACEMENT_START_LIMIT_EVIDENCE = "_security_view_replacement_start_limit" _SOURCE_START_EVIDENCE = "_security_source_start" _SOURCE_END_EVIDENCE = "_security_source_end" +_SOURCE_ANCHOR_EVIDENCE = "_security_source_anchor" +_SOURCE_ALTERNATE_START_EVIDENCE = "_security_source_alternate_start" +_SOURCE_REACH_END_EVIDENCE = "_security_source_reach_end" +_SOURCE_REPLACEMENT_START_LIMIT_EVIDENCE = "_security_source_replacement_start_limit" +_SOURCE_REPLACEMENT_RECOVERY_START_EVIDENCE = "_security_source_replacement_recovery_start" +_PRESERVE_SOURCE_START_EVIDENCE = "_security_preserve_source_start" +_ABSOLUTE_START_EVIDENCE = "_security_absolute_start" +_ABSOLUTE_ANCHOR_EVIDENCE = "_security_absolute_anchor" +_ABSOLUTE_ALTERNATE_START_EVIDENCE = "_security_absolute_alternate_start" +_ABSOLUTE_REACH_END_EVIDENCE = "_security_absolute_reach_end" +_ABSOLUTE_REPLACEMENT_START_LIMIT_EVIDENCE = "_security_absolute_replacement_start_limit" +_ABSOLUTE_REPLACEMENT_RECOVERY_START_EVIDENCE = "_security_absolute_replacement_recovery_start" +_ALTERNATE_MATCHED_TEXT_EVIDENCE = "_security_alternate_matched_text" _VIEW_ORIGIN_TAGS = frozenset({"normalized-view", "declared-marker-view"}) _CONTEXTUAL_TRIAGE_TAG = "contextual-triage" _ActiveSecurityView = tuple[SecurityTextView, str] @@ -147,6 +172,7 @@ # ordinary module-input ceiling. _CONTINUITY_SEPARATOR_CHARS = _WINDOW_OVERLAP_CHARS _CONTINUITY_CONTEXT_CHARS = 2048 +_CONTINUITY_RIGHT_CONTEXT_CHARS = _WINDOW_OVERLAP_CHARS _CONTINUITY_MAX_CHAIN_RUNS = 24 MAX_FINDINGS_PER_ARTIFACT = 10_000 MAX_FINDINGS_PER_ANALYZER = 10_000 @@ -251,6 +277,10 @@ def deduplicate_analyzer_findings( _LICENSE_OTHER_SUFFIXES = frozenset({".lesser"}) _ASCII_CONTINUITY_SEPARATOR_RUN = re.compile(r"[\s\x00-\x08\x0b\x0c\x0e-\x1f\x7f]+") +_RETAINED_CONTINUITY_NON_ASCII_WHITESPACE = re.compile(r"(?=[^\x00-\x7f])[^\S\x00-\x1f\x7f-\x9f]") +_RETAINED_CONTINUITY_ASCII_WHITESPACE = re.compile(r"[ \t]") +_RETAINED_CONTINUITY_REPLACEMENT = re.compile("\ufffd") +_RETAINED_CONTINUITY_LINE_BREAK = re.compile(r"\r\n|[\r\n\u2028\u2029]") _ASCII_NON_NEWLINE_WHITESPACE = re.compile(r"[ \t\r\f\v]") _PARAGRAPH_BOUNDARY = re.compile( rf"(?>{LOGICAL_LINE_BREAK.pattern})[ \t]*(?>{LOGICAL_LINE_BREAK.pattern})" @@ -398,7 +428,42 @@ def _window_view_with_markdown_context( offsets = array("I", (max(0, offset - prefix_length) for offset in range(len(view.text)))) else: offsets = array("I", (max(0, offset - prefix_length) for offset in view.source_offsets)) - return SecurityTextView(view.name, view.text, offsets) + return SecurityTextView( + view.name, + view.text, + offsets, + right_boundary_is_fixed=view.right_boundary_is_fixed, + right_boundary_recovery_start=( + max(0, view.right_boundary_recovery_start - prefix_length) + if view.right_boundary_recovery_start is not None + else None + ), + ) + + +def _with_fixed_right_boundary( + view: SecurityTextView, + is_fixed: bool, + recovery_start: int | None = None, +) -> SecurityTextView: + """Attach a scanner right edge and any overlapping recovery coordinate.""" + recovery_candidates = [ + candidate + for candidate in (view.right_boundary_recovery_start, recovery_start) + if candidate is not None + ] + merged_recovery = min(recovery_candidates, default=None) + if ( + not is_fixed or view.right_boundary_is_fixed + ) and merged_recovery == view.right_boundary_recovery_start: + return view + return SecurityTextView( + view.name, + view.text, + view.source_offsets, + right_boundary_is_fixed=(view.right_boundary_is_fixed or is_fixed), + right_boundary_recovery_start=merged_recovery, + ) def _markdown_context_prefix( @@ -480,7 +545,7 @@ def _normalize_license_line(line: str) -> str: def _infer_file_type(path: str) -> str: - """Infer file type from path (extension).""" + """Infer the declared file type from the path extension.""" idx = path.rfind(".") suffix = path[idx:].lower() if idx >= 0 else "" return FILE_TYPES.get(suffix, "other") @@ -633,6 +698,30 @@ def _uses_python_ast(module: object) -> bool: return getattr(module, "USES_PYTHON_AST", False) is True +def _explicit_analysis_hook(target: object, name: str) -> Callable | None: + """Require a declared hook rather than one manufactured by dynamic lookup.""" + if getattr_static(target, name, None) is None: + return None + hook = getattr(target, name, None) + return hook if callable(hook) else None + + +def _uses_python_source_type(module: object) -> bool: + """Return whether a module needs the artifact's Python execution type.""" + return ( + _uses_python_ast(module) + or _explicit_module_hook(module, "POSTPROCESS_USES_PYTHON_AST") is True + or getattr(module, "USES_PYTHON_SOURCE_TYPE", False) is True + ) + + +def _effective_module_file_type(path: str, module: object, *, python_source: bool) -> str: + """Resolve the file type consistently for every hook owned by one module.""" + if python_source and _uses_python_source_type(module): + return "python" + return _infer_file_type(path) + + def _requires_python_ast(pattern_modules: list) -> bool: """Return whether an analyzer or its postprocessor consumes the shared AST.""" return any(_uses_python_ast(module) for module in pattern_modules) or bool( @@ -641,16 +730,25 @@ def _requires_python_ast(pattern_modules: list) -> bool: ) +def _requires_python_source_type(pattern_modules: list) -> bool: + """Return whether any analyzer behavior depends on Python execution identity.""" + return any(_uses_python_source_type(module) for module in pattern_modules) + + def _python_ast_for_path( path: str, content: str, pattern_modules: list, python_ast_cache_key: str | None, + *, + python_source: bool | None = None, ) -> ParsedPythonFile | None: """Return the shared parse needed by analyzer or postprocessor hooks.""" if len(content) > MAX_FILE_CHARS or not _requires_python_ast(pattern_modules): return None - if _infer_file_type(path) != "python": + if python_source is None: + python_source = may_be_python_source(path, content) + if not python_source: return None return get_python_ast(python_ast_cache_key, content, path) @@ -683,6 +781,12 @@ def __init__( self.partial_findings = partial_findings or [] +class _StaticScanMetrics(dict[str, int | float]): + """Internal scan metrics with status that must never enter ledger fields.""" + + fully_scanned_aggregate_overflow: bool = False + + @dataclass class _FindingBudget: """Bound findings while modules construct and return their private results.""" @@ -743,10 +847,11 @@ def observe_emission(self) -> None: @dataclass(frozen=True) class _ContinuityView: - """One bounded cross-window projection with exact raw line locations.""" + """One bounded cross-window projection with exact raw coordinates.""" view: SecurityTextView source_lines: tuple[int, ...] + source_offsets: array[int] @dataclass(frozen=True) @@ -849,38 +954,61 @@ def _scan_path( finding_budget: _FindingBudget, python_ast_cache_key: str | None = None, python_ast: ParsedPythonFile | None = None, + python_source: bool | None = None, + *, + prepared_analyses: Mapping[int, object] | None = None, + source_view: SecurityTextView | None = None, + analysis_method: str = "analyze", ) -> tuple[list[Finding], _StaticResourceLimitError | None]: """Run pattern modules with construction, emission, and runtime guards.""" findings: list[Finding] = [] file_type = _infer_file_type(path) + if python_source is None: + python_source = may_be_python_source(path, content) content_lines = content.splitlines() normalized_license_lines = ( tuple(_normalize_license_line(line) for line in content_lines) if _is_license_basename(path, file_type) else None ) - if file_type == "python" and any(_uses_python_ast(module) for module in pattern_modules): + if python_source and any(_uses_python_ast(module) for module in pattern_modules): finding_budget.check_runtime() python_ast = python_ast or get_python_ast(python_ast_cache_key, content, path) finding_budget.check_runtime() line_starts = logical_line_starts(content) for module in pattern_modules: + module_file_type = _effective_module_file_type( + path, + module, + python_source=python_source, + ) module_finding_start = len(findings) occurrence_columns = _OccurrenceColumnResolver(content, line_starts) finding_budget.begin_module() try: with observe_analyzer_findings(finding_budget.observe_creation): - analyze_kwargs: dict[str, object] = { - "content": content, - "file_path": path, - "file_type": file_type, - } - if file_type == "python" and _uses_python_ast(module): - analyze_kwargs["python_ast"] = python_ast - if _uses_runtime_check(module): - analyze_kwargs["check_runtime"] = finding_budget.check_runtime - raw = module.analyze(**analyze_kwargs) + prepared = (prepared_analyses or {}).get(id(module)) + if prepared is not None: + raw = getattr(prepared, analysis_method)( + content=content, + file_path=path, + file_type=module_file_type, + source_view=source_view or SecurityTextView("raw", content), + ) + else: + analyze_kwargs: dict[str, object] = { + "content": content, + "file_path": path, + "file_type": module_file_type, + } + if module_file_type == "python" and _uses_python_ast(module): + analyze_kwargs["python_ast"] = python_ast + if _uses_runtime_check(module): + analyze_kwargs["check_runtime"] = finding_budget.check_runtime + if _explicit_module_hook(module, "ANALYZE_USES_POSTPROCESS") is True: + analyze_kwargs["defer_variable_reconciliation"] = True + raw = module.analyze(**analyze_kwargs) finding_budget.check_runtime() for af in raw: finding_budget.observe_emission() @@ -888,7 +1016,7 @@ def _scan_path( converted = _convert_analyzer_finding( af, path=path, - file_type=file_type, + file_type=module_file_type, content=content, content_lines=content_lines, normalized_license_lines=normalized_license_lines, @@ -908,7 +1036,7 @@ def _scan_path( converted = _convert_analyzer_finding( af, path=path, - file_type=file_type, + file_type=module_file_type, content=content, content_lines=content_lines, normalized_license_lines=normalized_license_lines, @@ -984,6 +1112,7 @@ def _extend_unique_findings( candidates: list[Finding], *, max_findings: int, + coalesce: Callable[[list[Finding]], list[Finding]] | None = None, ) -> _StaticResourceLimitError | None: """Append distinct final findings and enforce the user-visible output cap.""" for finding in candidates: @@ -992,7 +1121,7 @@ def _extend_unique_findings( continue seen.add(key) result.append(finding) - if len(result) > max_findings: + if coalesce is None and len(result) > max_findings: return _StaticResourceLimitError( LedgerReason.OUTPUT_LIMIT, { @@ -1000,6 +1129,16 @@ def _extend_unique_findings( "limit_findings": max_findings, }, ) + if len(result) > max_findings and coalesce is not None: + result[:] = _deduplicate_view_findings(coalesce(result)) + if len(result) > max_findings: + return _StaticResourceLimitError( + LedgerReason.OUTPUT_LIMIT, + { + "observed_findings": len(result), + "limit_findings": max_findings, + }, + ) return None @@ -1049,7 +1188,12 @@ def _scan_view_windows( finding_budget: _FindingBudget, python_ast_cache_key: str | None, *, + python_source: bool, source_text: str, + prepared_analyses: Mapping[int, object] | None = None, + source_view: SecurityTextView | None = None, + evidence_source_view: SecurityTextView | None = None, + analysis_method: str = "analyze", ) -> tuple[list[Finding], _StaticResourceLimitError | None]: """Scan one already-bounded view.""" view_token = _ACTIVE_SECURITY_VIEW.set((view, source_text)) @@ -1060,20 +1204,87 @@ def _scan_view_windows( pattern_modules, finding_budget, python_ast_cache_key, + python_source=python_source, + prepared_analyses=prepared_analyses, + source_view=source_view, + analysis_method=analysis_method, ) finally: _ACTIVE_SECURITY_VIEW.reset(view_token) + coordinate_view = evidence_source_view or view + + def source_boundary(derived_offset: int) -> int: + if coordinate_view.source_offsets is None: + return derived_offset + if derived_offset < len(coordinate_view.source_offsets): + return coordinate_view.source_offsets[derived_offset] + return coordinate_view.source_offsets[-1] + 1 if coordinate_view.source_offsets else 0 + for finding in findings: finding.evidence.pop(_SOURCE_START_EVIDENCE, None) local_start = finding.evidence.pop(_VIEW_START_EVIDENCE, None) if not isinstance(local_start, int) and finding.start_column is not None: local_start = _line_start_offset(view.text, finding.start_line) + finding.start_column if isinstance(local_start, int) and 0 <= local_start < len(view.text): - finding.evidence[_SOURCE_START_EVIDENCE] = view.source_offset(local_start) + finding.evidence[_SOURCE_START_EVIDENCE] = coordinate_view.source_offset(local_start) + local_anchor = finding.evidence.pop(_VIEW_ANCHOR_EVIDENCE, None) + if isinstance(local_anchor, int) and 0 <= local_anchor < len(view.text): + finding.evidence[_SOURCE_ANCHOR_EVIDENCE] = coordinate_view.source_offset(local_anchor) + local_alternate = finding.evidence.pop(_VIEW_ALTERNATE_START_EVIDENCE, None) + if isinstance(local_alternate, int) and 0 <= local_alternate < len(view.text): + finding.evidence[_SOURCE_ALTERNATE_START_EVIDENCE] = coordinate_view.source_offset( + local_alternate + ) + local_reach_end = finding.evidence.pop(_VIEW_REACH_END_EVIDENCE, None) + if isinstance(local_reach_end, int) and 0 <= local_reach_end <= len(view.text): + finding.evidence[_SOURCE_REACH_END_EVIDENCE] = source_boundary(local_reach_end) + local_replacement_start_limit = finding.evidence.pop( + _VIEW_REPLACEMENT_START_LIMIT_EVIDENCE, + None, + ) + replacement_width = ( + len(view.text) - local_replacement_start_limit + if isinstance(local_replacement_start_limit, int) + else 0 + ) + prospective_slice_boundary = ( + not view.right_boundary_is_fixed + and 0 < replacement_width <= len(view.text) + and len(view.text) < SECURITY_VIEW_WINDOW_CHARS + and len(view.text) + max(0, replacement_width - 1) > SECURITY_VIEW_WINDOW_CHARS + ) + if ( + (view.right_boundary_is_fixed or prospective_slice_boundary) + and isinstance(local_replacement_start_limit, int) + and 0 <= local_replacement_start_limit < len(view.text) + ): + if prospective_slice_boundary: + local_replacement_start_limit += SECURITY_VIEW_WINDOW_CHARS - len(view.text) + finding.evidence[_SOURCE_REPLACEMENT_START_LIMIT_EVIDENCE] = ( + coordinate_view.source_offset(local_replacement_start_limit) + ) + recovery_candidates = [ + candidate + for candidate in ( + view.right_boundary_recovery_start, + ( + view.source_offset(SECURITY_VIEW_WINDOW_CHARS - _WINDOW_OVERLAP_CHARS) + if prospective_slice_boundary + else None + ), + ) + if candidate is not None + ] + if recovery_candidates: + finding.evidence[_SOURCE_REPLACEMENT_RECOVERY_START_EVIDENCE] = min( + recovery_candidates + ) if finding.end_line is not None and finding.end_column is not None: local_end = _line_start_offset(view.text, finding.end_line) + finding.end_column if 0 < local_end <= len(view.text): - finding.evidence[_SOURCE_END_EVIDENCE] = view.source_offset(local_end - 1) + 1 + finding.evidence[_SOURCE_END_EVIDENCE] = ( + coordinate_view.source_offset(local_end - 1) + 1 + ) if view.name != "raw": for finding in findings: if "normalized-view" not in finding.tags: @@ -1088,21 +1299,192 @@ def _scan_view_windows( def _bounded_view_slices(view: SecurityTextView) -> Iterator[SecurityTextView]: """Split an expanded derived view before any pattern module sees it.""" if len(view.text) <= SECURITY_VIEW_WINDOW_CHARS: - yield view + exact_ceiling_recovery = ( + view.source_offset(SECURITY_VIEW_WINDOW_CHARS - _WINDOW_OVERLAP_CHARS) + if len(view.text) == SECURITY_VIEW_WINDOW_CHARS + else None + ) + yield _with_fixed_right_boundary( + view, + len(view.text) == SECURITY_VIEW_WINDOW_CHARS, + exact_ceiling_recovery, + ) return step = SECURITY_VIEW_WINDOW_CHARS - _WINDOW_OVERLAP_CHARS for start in range(0, len(view.text), step): end = min(len(view.text), start + SECURITY_VIEW_WINDOW_CHARS) - offsets = None if view.source_offsets is None else view.source_offsets[start:end] + has_following_slice = end < len(view.text) + fills_ceiling = end - start == SECURITY_VIEW_WINDOW_CHARS + slice_recovery = ( + view.source_offset(start + step) if has_following_slice or fills_ceiling else None + ) + inherited_recovery = ( + view.right_boundary_recovery_start if view.right_boundary_is_fixed else None + ) + recovery_candidates = [ + candidate for candidate in (slice_recovery, inherited_recovery) if candidate is not None + ] + offsets = ( + array("I", range(start, end)) + if view.source_offsets is None + else view.source_offsets[start:end] + ) yield SecurityTextView( name=view.name, text=view.text[start:end], source_offsets=offsets, + right_boundary_is_fixed=( + view.right_boundary_is_fixed or has_following_slice or fills_ceiling + ), + right_boundary_recovery_start=min(recovery_candidates, default=None), ) if end == len(view.text): break +@dataclass(frozen=True, kw_only=True) +class _AbsoluteSourceView(SecurityTextView): + """Resolve absolute coordinates only for findings that need a source lookup. + + This secondary view is used by prepared analysis and line restoration, never + by text transformations that inspect or slice ``source_offsets`` directly. + """ + + derived_view: SecurityTextView + parent_view: SecurityTextView | None + source_start: int + + def source_offset(self, derived_offset: int) -> int: + offsets = self.derived_view.source_offsets + size = len(self.text) if offsets is None else len(offsets) + if size == 0: + return 0 + # Materialized absolute maps clamp at the final character, including + # identity-derived maps whose own endpoint lookup permits len(text). + index = min(max(derived_offset, 0), size - 1) + offset = self.derived_view.source_offset(index) + if self.parent_view is not None: + offset = self.parent_view.source_offset(offset) + return self.source_start + offset + + +def _absolute_source_view( + view: SecurityTextView, + *, + source_start: int = 0, + parent: SecurityTextView | None = None, +) -> SecurityTextView: + """Compose one bounded view's coordinates with its whole-artifact origin.""" + if parent is None and source_start == 0: + return view + return _AbsoluteSourceView( + name=view.name, + text=view.text, + derived_view=view, + parent_view=parent, + source_start=source_start, + ) + + +def _whitespace_continuity_tokens( + content: str, + finding_budget: _FindingBudget, + *, + default_ignorables_as_separators: bool, +) -> Iterator[tuple[int, int, bool]]: + """Yield source spans, optionally interpreting pinned ignorables as separators. + + The alternative uses bounded, length-preserving chunks. Separator spans are + joined across chunk boundaries before projection, without a whole-file copy. + """ + chunk_size = 65_536 if default_ignorables_as_separators else max(1, len(content)) + pending_separator: tuple[int, int] | None = None + for chunk_start in range(0, len(content), chunk_size): + finding_budget.check_runtime() + chunk = content[chunk_start : chunk_start + chunk_size] + if default_ignorables_as_separators and _contains_default_ignorable(chunk): + chunk = "".join(" " if is_default_ignorable(ch) else ch for ch in chunk) + finding_budget.check_runtime() + for token in re.finditer(r"\s+|\S+", chunk): + start = chunk_start + token.start() + end = chunk_start + token.end() + if chunk[token.start()].isspace(): + pending_separator = ( + pending_separator[0] if pending_separator is not None else start, + end, + ) + continue + if pending_separator is not None: + yield *pending_separator, True + pending_separator = None + yield start, end, False + if pending_separator is not None: + yield *pending_separator, True + + +def _whitespace_continuity_views( + content: str, + finding_budget: _FindingBudget, + *, + default_ignorables_as_separators: bool = False, +) -> Iterator[SecurityTextView]: + """Project unbounded whitespace for explicitly opted-in analyzer methods. + + Each whitespace run becomes one separator. Logical line boundaries remain + boundaries, and offsets stay absolute. Ordinary pattern modules must never + receive this projection: it changes the meaning of bounded-gap expressions. + """ + finding_budget.check_runtime() + needs_projection = ( + _contains_default_ignorable(content) + if default_ignorables_as_separators + else re.search(r"\s{2,}", content) is not None + ) + finding_budget.check_runtime() + if not needs_projection: + # Single separators already fit in the ordinary window overlap. Avoid + # rebuilding and rescanning the entire artifact when nothing can shrink. + return + view_name = ( + "ignorable-separator-continuity" + if default_ignorables_as_separators + else "whitespace-continuity" + ) + parts: list[str] = [] + offsets = array("I") + size = 0 + fresh = False + for start, end, whitespace in _whitespace_continuity_tokens( + content, + finding_budget, + default_ignorables_as_separators=default_ignorables_as_separators, + ): + finding_budget.check_runtime() + while start < end: + finding_budget.check_runtime() + if whitespace: + parts.append("\n" if LOGICAL_LINE_BREAK.search(content, start, end) else " ") + offsets.append(start) + size += 1 + start = end + else: + stop = min(end, start + SECURITY_VIEW_WINDOW_CHARS - size) + parts.append(content[start:stop]) + offsets.extend(range(start, stop)) + size += stop - start + start = stop + fresh = True + if size == SECURITY_VIEW_WINDOW_CHARS: + text = "".join(parts) + yield SecurityTextView(view_name, text, offsets) + parts = [text[-_WINDOW_OVERLAP_CHARS:]] + offsets = offsets[-_WINDOW_OVERLAP_CHARS:] + size = _WINDOW_OVERLAP_CHARS + fresh = False + if fresh: + yield SecurityTextView(view_name, "".join(parts), offsets) + + def _is_continuity_separator(character: str) -> bool: """Return whether a character separates tokens in a security text view.""" return ( @@ -1114,16 +1496,78 @@ def _is_continuity_separator(character: str) -> bool: ) +def _continuity_runs_for_anchors( + content: str, + anchors: set[int], + check_runtime: Callable[[], None], +) -> dict[int, tuple[int, int]]: + """Find each anchor's nearest relevant long run in merged local ranges.""" + ordered_anchors = sorted(anchor for anchor in anchors if 0 <= anchor < len(content)) + if not ordered_anchors: + return {} + + search_ranges: list[tuple[int, int]] = [] + for anchor in ordered_anchors: + left = max(0, anchor - _CONTINUITY_RIGHT_CONTEXT_CHARS) + if search_ranges and left <= search_ranges[-1][1]: + search_ranges[-1] = (search_ranges[-1][0], anchor) + else: + search_ranges.append((left, anchor)) + + examined = 0 + + def is_separator(index: int) -> bool: + nonlocal examined + examined += 1 + if examined % _WINDOW_OVERLAP_CHARS == 0: + check_runtime() + return _is_continuity_separator(content[index]) + + runs: set[tuple[int, int]] = set() + for left, right in search_ranges: + # A merged search range may begin inside the only relevant long run. + # Extend through that run once so its full length remains observable. + while left > 0 and is_separator(left - 1): + left -= 1 + cursor = left + while cursor < right: + if not is_separator(cursor): + cursor += 1 + continue + run_start = cursor + cursor += 1 + while cursor < right and is_separator(cursor): + cursor += 1 + if cursor - run_start > _WINDOW_OVERLAP_CHARS: + runs.add((run_start, cursor)) + + check_runtime() + ordered_runs = sorted(runs, key=lambda run: run[1]) + run_ends = [run[1] for run in ordered_runs] + relevant: dict[int, tuple[int, int]] = {} + for anchor in ordered_anchors: + run_index = bisect_right(run_ends, anchor) - 1 + if run_index < 0: + continue + run = ordered_runs[run_index] + if anchor - run[1] < _CONTINUITY_RIGHT_CONTEXT_CHARS: + relevant[anchor] = run + return relevant + + def _continuity_separator_runs( content: str, finding_budget: _FindingBudget, + *, + search_end: int | None = None, ) -> Iterator[tuple[int, int]]: """Yield long separator runs without allocating a whole-file projection.""" + limit = len(content) if search_end is None else min(len(content), max(0, search_end)) if content.isascii(): # Keep ordinary source files on the regex engine's bounded C-level # fast path. Unicode category inspection below is reserved for input # that can actually contain normalized-away format characters. - for match in _ASCII_CONTINUITY_SEPARATOR_RUN.finditer(content): + for match in _ASCII_CONTINUITY_SEPARATOR_RUN.finditer(content, 0, limit): finding_budget.check_runtime() if match.end() - match.start() > _WINDOW_OVERLAP_CHARS: yield match.start(), match.end() @@ -1134,7 +1578,7 @@ def _continuity_separator_runs( # character accepted by ``_is_continuity_separator``. Keep that common # multilingual-text case on C-level predicates instead of walking every # code point in Python. - if ( + if limit == len(content) and ( content.isprintable() and _ASCII_CONTINUITY_SEPARATOR_RUN.search(content) is None and "\ufffd" not in content @@ -1144,7 +1588,8 @@ def _continuity_separator_runs( return run_start: int | None = None - for index, character in enumerate(content): + for index in range(limit): + character = content[index] if index % _WINDOW_OVERLAP_CHARS == 0: finding_budget.check_runtime() if _is_continuity_separator(character): @@ -1154,27 +1599,138 @@ def _continuity_separator_runs( if run_start is not None and index - run_start > _WINDOW_OVERLAP_CHARS: yield run_start, index run_start = None - if run_start is not None and len(content) - run_start > _WINDOW_OVERLAP_CHARS: - yield run_start, len(content) + if run_start is not None and limit - run_start > _WINDOW_OVERLAP_CHARS: + yield run_start, limit def _append_projected_piece( text_parts: list[str], source_lines: list[int], + source_offsets: array[int], piece: str, + source_start: int, source_line: int, ) -> int: - """Append one contiguous raw piece and extend its exact line projection.""" + """Append one contiguous raw piece and extend its exact projections.""" text_parts.append(piece) + source_offsets.extend(range(source_start, source_start + len(piece))) for _ in LOGICAL_LINE_BREAK.finditer(piece): source_line += 1 source_lines.append(source_line) return source_line +def _retained_continuity_separator( + content: str, + start: int, + end: int, +) -> tuple[str, int] | None: + """Return one representative retained by the same security-view classes.""" + for pattern in ( + _RETAINED_CONTINUITY_ASCII_WHITESPACE, + _RETAINED_CONTINUITY_NON_ASCII_WHITESPACE, + _RETAINED_CONTINUITY_REPLACEMENT, + ): + if match := pattern.search(content, start, end): + return match.group(0), match.start() + return None + + +def _anchored_continuity_view( + content: str, + anchor: int, + check_runtime: Callable[[], None], +) -> SecurityTextView | None: + """Project the long-separator chain immediately preceding *anchor*. + + This mirrors the ordinary continuity chain bound while searching backward + from one already-retained finding. It therefore avoids rediscovering every + unrelated separator in the artifact during output-limit finalization. + """ + anchor = min(max(0, anchor), len(content)) + search_end = anchor + reverse_runs: list[tuple[int, int]] = [] + examined = 0 + while search_end > 0 and len(reverse_runs) < _CONTINUITY_MAX_CHAIN_RUNS: + search_span = ( + _CONTINUITY_RIGHT_CONTEXT_CHARS + if not reverse_runs + # The forward producer compares the exclusive prior-run end with + # the next-run start. Include the prior run's final character when + # that gap is exactly the configured chain limit. + else _CONTINUITY_CONTEXT_CHARS + 1 + ) + search_start = max(0, search_end - search_span) + cursor = search_end + prior_run: tuple[int, int] | None = None + while cursor > search_start: + cursor -= 1 + examined += 1 + if examined % _WINDOW_OVERLAP_CHARS == 0: + check_runtime() + if not _is_continuity_separator(content[cursor]): + continue + run_end = cursor + 1 + while cursor > 0 and _is_continuity_separator(content[cursor - 1]): + cursor -= 1 + examined += 1 + if examined % _WINDOW_OVERLAP_CHARS == 0: + check_runtime() + if run_end - cursor > _WINDOW_OVERLAP_CHARS: + prior_run = (cursor, run_end) + break + if prior_run is None: + break + reverse_runs.append(prior_run) + search_end = prior_run[0] + check_runtime() + if not reverse_runs: + return None + + separator_runs = list(reversed(reverse_runs)) + left = max(0, separator_runs[0][0] - _CONTINUITY_CONTEXT_CHARS) + right = min(len(content), separator_runs[-1][1] + _CONTINUITY_RIGHT_CONTEXT_CHARS) + source_offsets = array("I") + text_parts: list[str] = [] + + def append(piece: str, source_start: int) -> None: + text_parts.append(piece) + source_offsets.extend(range(source_start, source_start + len(piece))) + + cursor = left + for run_start, run_end in separator_runs: + append(content[cursor:run_start], cursor) + run_length = run_end - run_start + if run_length <= _CONTINUITY_SEPARATOR_CHARS: + append(content[run_start:run_end], run_start) + else: + head_length = _CONTINUITY_SEPARATOR_CHARS // 2 + tail_length = _CONTINUITY_SEPARATOR_CHARS - head_length + head_end = run_start + head_length + tail_start = run_end - tail_length + append(content[run_start:head_end], run_start) + if newline := _RETAINED_CONTINUITY_LINE_BREAK.search(content, head_end, tail_start): + append(newline.group(0), newline.start()) + elif retained := _retained_continuity_separator(content, head_end, tail_start): + character, source_offset = retained + text_parts.append(character) + source_offsets.append(source_offset) + append(content[tail_start:run_end], tail_start) + cursor = run_end + append(content[cursor:right], cursor) + + projected = "".join(text_parts) + assert len(projected) <= SECURITY_VIEW_WINDOW_CHARS + assert len(source_offsets) == len(projected) + return SecurityTextView("anchored-continuity", projected, source_offsets) + + def _continuity_views( content: str, finding_budget: _FindingBudget, + *, + separator_search_end: int | None = None, + include_source_offsets: bool = False, ) -> Iterator[_ContinuityView]: """Build bounded neighborhoods that preserve lexical state across raw windows. @@ -1186,7 +1742,13 @@ def _continuity_views( map is constructed per view, so neither a whole-file normalized copy nor a whole-file offset table exists. """ - separator_runs = list(_continuity_separator_runs(content, finding_budget)) + separator_runs = list( + _continuity_separator_runs( + content, + finding_budget, + search_end=separator_search_end, + ) + ) previous_left = 0 previous_left_line = 1 for run_index, (run_start, _) in enumerate(separator_runs): @@ -1201,12 +1763,13 @@ def _continuity_views( last_run_index += 1 selected_runs = separator_runs[run_index : last_run_index + 1] left = max(0, run_start - _CONTINUITY_CONTEXT_CHARS) - right = min(len(content), selected_runs[-1][1] + _CONTINUITY_CONTEXT_CHARS) + right = min(len(content), selected_runs[-1][1] + _CONTINUITY_RIGHT_CONTEXT_CHARS) previous_left_line += sum( 1 for _ in LOGICAL_LINE_BREAK.finditer(content, previous_left, left) ) previous_left = left source_lines = [previous_left_line] + source_offsets = array("I") text_parts: list[str] = [] current_line = previous_left_line cursor = left @@ -1214,7 +1777,9 @@ def _continuity_views( current_line = _append_projected_piece( text_parts, source_lines, + source_offsets, content[cursor:selected_start], + cursor, current_line, ) run_length = selected_end - selected_start @@ -1222,7 +1787,9 @@ def _continuity_views( current_line = _append_projected_piece( text_parts, source_lines, + source_offsets, content[selected_start:selected_end], + selected_start, current_line, ) else: @@ -1233,33 +1800,57 @@ def _continuity_views( current_line = _append_projected_piece( text_parts, source_lines, + source_offsets, content[selected_start:head_end], + selected_start, current_line, ) skipped_newlines = sum( 1 for _ in LOGICAL_LINE_BREAK.finditer(content, head_end, tail_start) ) - if skipped_newlines: + retained_line_break = _RETAINED_CONTINUITY_LINE_BREAK.search( + content, head_end, tail_start + ) + if retained_line_break is not None: # Retain a line boundary so DOT-without-DOTALL and anchors # do not acquire semantics absent from the original source. - text_parts.append("\n") + line_break = retained_line_break.group(0) + text_parts.append(line_break) + source_offsets.extend( + range(retained_line_break.start(), retained_line_break.end()) + ) + if skipped_newlines: current_line += skipped_newlines - source_lines.append(current_line) - elif _ASCII_NON_NEWLINE_WHITESPACE.search(content, head_end, tail_start): - # Never let truncation erase a real word boundary and turn - # separated tokens into a normalized security match. - text_parts.append(" ") + if retained_line_break is not None: + source_lines.append(current_line) + retained_separator = ( + None + if retained_line_break is not None + else _retained_continuity_separator(content, head_end, tail_start) + ) + if retained_separator is not None: + # Preserve a representative which the normalized view + # retains. Keeping the original character also lets the + # compact view make the same contextual decision as it + # would over the complete separator run. + character, source_offset = retained_separator + text_parts.append(character) + source_offsets.append(source_offset) current_line = _append_projected_piece( text_parts, source_lines, + source_offsets, content[tail_start:selected_end], + tail_start, current_line, ) cursor = selected_end _append_projected_piece( text_parts, source_lines, + source_offsets, content[cursor:right], + cursor, current_line, ) @@ -1267,9 +1858,16 @@ def _continuity_views( # Context, the retained separators, and the bounded text between # chained runs remain below the ordinary module-input ceiling. assert len(projected) <= SECURITY_VIEW_WINDOW_CHARS + assert len(source_offsets) == len(projected) yield _ContinuityView( - view=SecurityTextView("continuity", projected), + view=SecurityTextView( + "continuity", + projected, + source_offsets if include_source_offsets else None, + right_boundary_is_fixed=right < len(content), + ), source_lines=tuple(source_lines), + source_offsets=source_offsets, ) @@ -1327,6 +1925,7 @@ def _restore_source_lines( view: SecurityTextView, window_start: int = 0, source_line_starts: tuple[int, ...] | None = None, + start_source_offsets: array[int] | None = None, ) -> None: """Map normalized/window-relative locations to raw whole-file coordinates.""" @@ -1377,6 +1976,62 @@ def source_end_offset(offset: int) -> int: ) finding.end_line, raw_end_column = source_position(raw_end) finding.end_column = raw_end_column if has_exact_end else None + source_anchor = finding.evidence.pop(_SOURCE_ANCHOR_EVIDENCE, None) + source_alternate_start = finding.evidence.pop(_SOURCE_ALTERNATE_START_EVIDENCE, None) + source_reach_end = finding.evidence.pop(_SOURCE_REACH_END_EVIDENCE, None) + source_replacement_start_limit = finding.evidence.pop( + _SOURCE_REPLACEMENT_START_LIMIT_EVIDENCE, + None, + ) + source_replacement_recovery_start = finding.evidence.pop( + _SOURCE_REPLACEMENT_RECOVERY_START_EVIDENCE, + None, + ) + preserve_start = finding.evidence.pop(_PRESERVE_SOURCE_START_EVIDENCE, None) is True + + def absolute_offset(source_offset: object) -> int | None: + if not isinstance(source_offset, int) or source_offset < 0: + return None + if start_source_offsets is None: + return window_start + source_offset + if source_offset < len(start_source_offsets): + return start_source_offsets[source_offset] + return None + + def absolute_boundary(source_offset: object) -> int | None: + if not isinstance(source_offset, int) or source_offset < 0: + return None + if start_source_offsets is None: + return window_start + source_offset + if source_offset < len(start_source_offsets): + return start_source_offsets[source_offset] + if source_offset == len(start_source_offsets) and start_source_offsets: + return start_source_offsets[-1] + 1 + return None + + if preserve_start: + absolute_start = absolute_offset(source_start) + if absolute_start is not None: + finding.evidence[_ABSOLUTE_START_EVIDENCE] = absolute_start + absolute_anchor = absolute_offset(source_anchor) + if absolute_anchor is not None: + finding.evidence[_ABSOLUTE_ANCHOR_EVIDENCE] = absolute_anchor + absolute_alternate_start = absolute_offset(source_alternate_start) + if absolute_alternate_start is not None: + finding.evidence[_ABSOLUTE_ALTERNATE_START_EVIDENCE] = absolute_alternate_start + absolute_reach_end = absolute_boundary(source_reach_end) + if absolute_reach_end is not None: + finding.evidence[_ABSOLUTE_REACH_END_EVIDENCE] = absolute_reach_end + absolute_replacement_start_limit = absolute_offset(source_replacement_start_limit) + if absolute_replacement_start_limit is not None: + finding.evidence[_ABSOLUTE_REPLACEMENT_START_LIMIT_EVIDENCE] = ( + absolute_replacement_start_limit + ) + absolute_replacement_recovery_start = absolute_offset(source_replacement_recovery_start) + if absolute_replacement_recovery_start is not None: + finding.evidence[_ABSOLUTE_REPLACEMENT_RECOVERY_START_EVIDENCE] = ( + absolute_replacement_recovery_start + ) def _scan_declared_marker_views( @@ -1388,7 +2043,10 @@ def _scan_declared_marker_views( owned_starts: tuple[int, ...], raw_starts: tuple[int, ...], source_context: _WindowSourceContext, + python_source: bool, + defer_projected_output_limit: bool, complete_context: bool, + prepared_analyses: Mapping[int, object] | None = None, ) -> tuple[list[Finding], bool, bool, _StaticResourceLimitError | None]: """Reconstruct marker payloads with directive-relative context windows.""" findings: list[Finding] = [] @@ -1408,6 +2066,35 @@ def check_runtime() -> None: bounded_parse_limited = False seen_views: set[tuple[str, int, int]] = set() seen_finding_counts: dict[tuple[object, ...], int] = {} + deferred_output_limit: _StaticResourceLimitError | None = None + + def defer_output_limit( + resource_limit: _StaticResourceLimitError, + *, + across_security_views: bool, + ) -> bool: + nonlocal deferred_output_limit + if not ( + defer_projected_output_limit + and across_security_views + and resource_limit.reason is LedgerReason.OUTPUT_LIMIT + ): + return False + if deferred_output_limit is None: + deferred_output_limit = resource_limit + else: + observed = max( + int(deferred_output_limit.metrics.get("observed_findings", 0)), + int(resource_limit.metrics.get("observed_findings", 0)), + ) + deferred_output_limit = _StaticResourceLimitError( + LedgerReason.OUTPUT_LIMIT, + { + "observed_findings": observed, + "limit_findings": finding_budget.max_findings, + }, + ) + return True for owned_start, raw_start in zip(owned_starts, raw_starts, strict=True): check_runtime() @@ -1432,8 +2119,10 @@ def check_runtime() -> None: check_runtime=check_runtime, ) ) + across_security_views = len(full_views) > 1 check_runtime() for full_view in full_views: + full_view_limited = False reconstruction = build_declared_marker_views( full_view, check_runtime=check_runtime, @@ -1443,12 +2132,18 @@ def check_runtime() -> None: ) projection_limited = projection_limited or reconstruction.limited for marker_view in reconstruction.views: - if not marker_view.source_offsets: + marker_offsets = marker_view.source_offsets + if not marker_offsets: continue + marker_view = _with_fixed_right_boundary( + marker_view, + reconstruction.limited + or (raw_end < len(content) and marker_offsets[-1] >= len(raw_window) - 1), + ) marker_key = ( marker_view.text, - raw_start + marker_view.source_offsets[0], - raw_start + marker_view.source_offsets[-1], + raw_start + marker_offsets[0], + raw_start + marker_offsets[-1], ) if marker_key in seen_views: continue @@ -1469,7 +2164,14 @@ def check_runtime() -> None: pattern_modules, view_budget, None, + python_source=python_source, source_text=raw_window, + prepared_analyses=prepared_analyses, + source_view=( + _absolute_source_view(view, source_start=raw_start) + if prepared_analyses + else None + ), ) _restore_source_lines( view_findings, @@ -1492,25 +2194,42 @@ def check_runtime() -> None: seen_finding_counts[key] = projection_count findings.append(finding) if len(findings) > finding_budget.max_findings: + output_limit = _StaticResourceLimitError( + LedgerReason.OUTPUT_LIMIT, + { + "observed_findings": len(findings), + "limit_findings": finding_budget.max_findings, + }, + ) + if not defer_output_limit( + output_limit, + across_security_views=across_security_views, + ): + return ( + findings, + projection_limited, + bounded_parse_limited, + output_limit, + ) + full_view_limited = True + break + if full_view_limited: + break + if resource_limit is not None: + if not defer_output_limit( + resource_limit, + across_security_views=across_security_views, + ): return ( findings, projection_limited, bounded_parse_limited, - _StaticResourceLimitError( - LedgerReason.OUTPUT_LIMIT, - { - "observed_findings": len(findings), - "limit_findings": finding_budget.max_findings, - }, - ), + resource_limit, ) - if resource_limit is not None: - return ( - findings, - projection_limited, - bounded_parse_limited, - resource_limit, - ) + full_view_limited = True + break + if full_view_limited: + break # Preserve any concrete marker-view evidence before asking # module-specific completeness hooks whether the reconstructed # payload exceeded a bounded parser contract. If that hook @@ -1529,6 +2248,14 @@ def check_runtime() -> None: ) ) + if deferred_output_limit is not None: + return ( + findings, + projection_limited, + bounded_parse_limited, + deferred_output_limit, + ) + if owned_end == len(content): break @@ -1545,21 +2272,108 @@ def _scan_all_views_detailed( timeout_seconds: float | None = None, started_at: float | None = None, python_ast: ParsedPythonFile | None = None, + python_source: bool | None = None, + readable_binary: bool = False, + _oms_projected: bool = False, ) -> tuple[list[Finding], LedgerReason | None, dict[str, int | float]]: """Scan bounded raw windows and return any limit with observed/limit metrics.""" started_at = time.monotonic() if started_at is None else started_at - ast_modules = [module for module in pattern_modules if _uses_python_ast(module)] - lexical_modules = [module for module in pattern_modules if not _uses_python_ast(module)] - if python_ast is None: - python_ast = _python_ast_for_path( + if readable_binary: + modules = [ + module + for module in pattern_modules + if getattr(module, "ANALYZER_ID", "") + in { + "static_patterns_prompt_injection", + "static_patterns_privilege_escalation", + "static_patterns_data_exfiltration", + } + ] + if not modules: + return [], None, {} + view, _ = binary_instruction_view(content) + findings, reason, metrics = _scan_all_views_detailed( path, - content, + view.text, + modules, + None, + max_findings=max_findings, + timeout_seconds=timeout_seconds, + started_at=started_at, + python_source=False, + ) + findings = [ + finding + for finding in findings + if finding.rule_id + in { + "P1", + "P3", + "P4", + "PE3", + "E1", + "E2", + "E3", + } + ] + _restore_source_lines( + findings, + raw_window=content, + window_line=1, + view=view, + source_line_starts=logical_line_starts(content), + ) + # Projection scope is recorded by build_context; actual scan limits remain partial. + return findings, reason, metrics + runtime_limit = MAX_STATIC_ANALYSIS_SECONDS_PER_ARTIFACT + if timeout_seconds is not None: + runtime_limit = min(runtime_limit, max(0.0, timeout_seconds)) + now = time.monotonic() + if now >= started_at + runtime_limit: + return ( + [], + LedgerReason.RUNTIME_LIMIT, + { + "observed_seconds": max(0.0, now - started_at), + "limit_seconds": runtime_limit, + }, + ) + projection = None if _oms_projected else project_oms_content(content) + if projection is not None and projection.view.name != "raw": + findings, reason, metrics = _scan_all_views_detailed( + path, + projection.view.text, pattern_modules, - python_ast_cache_key, + None, + max_findings=max_findings, + timeout_seconds=timeout_seconds, + started_at=started_at, + # This view is derived JSON metadata, not a Python execution surface. + python_source=False, + _oms_projected=True, ) + for finding in findings: + finding.start_line = finding.end_line = 1 + finding.start_column = finding.end_column = None + finding.evidence["oms_projection"] = ( + "Decoded content in the original DSSE bundle; line 1 identifies the carrier." + ) + finding.evidence[_SOURCE_START_EVIDENCE] = projection.view.source_offset(0) + finding.evidence.pop(_SOURCE_END_EVIDENCE, None) + return findings, reason, metrics + ast_modules = [module for module in pattern_modules if _uses_python_ast(module)] + lexical_modules = [module for module in pattern_modules if not _uses_python_ast(module)] + if python_source is None: + python_source = ( + may_be_python_source(path, content) + if _requires_python_source_type(pattern_modules) + else False + ) + python_ast_eligible = python_source and len(content) <= MAX_FILE_CHARS + if python_ast_eligible and _requires_python_ast(pattern_modules) and python_ast is None: + python_ast = get_python_ast(python_ast_cache_key, content, path) python_syntax_error = bool( - _infer_file_type(path) == "python" - and len(content) <= MAX_FILE_CHARS + python_ast_eligible and _requires_python_ast(pattern_modules) and python_ast is not None and python_ast.tree is None @@ -1577,6 +2391,19 @@ def _scan_all_views_detailed( clock=time.monotonic, ) marker_projection_limited = False + coalesce_hook = ( + _explicit_module_hook(pattern_modules[0], "coalesce_path_findings") + if pattern_modules + else None + ) + coalesce: Callable[[list[Finding]], list[Finding]] | None = ( + (lambda candidates: coalesce_hook(content, candidates)) if callable(coalesce_hook) else None + ) + retained_reconciliation_hook = ( + _explicit_module_hook(pattern_modules[0], "reconcile_retained_findings") + if pattern_modules + else None + ) modules_for_windows = lexical_modules or ([] if ast_modules else pattern_modules) bounded_parse_limited = False marker_owned_starts: tuple[int, ...] = () @@ -1585,6 +2412,76 @@ def _scan_all_views_detailed( raw_starts: tuple[int, ...] = () source_context: _WindowSourceContext | None = None whole_artifact_window = False + prepared_analyses: dict[int, object] = {} + deferred_output_limit: _StaticResourceLimitError | None = None + deferred_scan_incomplete = False + defers_mixed_output_limit = bool( + coalesce is not None + and ast_modules + and modules_for_windows + and python_ast_eligible + and python_ast is not None + and python_ast.tree is not None + ) + + def defer_output_limit( + resource_limit: _StaticResourceLimitError, + *, + across_security_views: bool = False, + scan_incomplete: bool = False, + ) -> bool: + """Delay caps until mixed producers or alternate views can reconcile.""" + nonlocal deferred_output_limit, deferred_scan_incomplete + defers_projected_output_limit = coalesce is not None and across_security_views + if not ( + (defers_mixed_output_limit or defers_projected_output_limit) + and resource_limit.reason is LedgerReason.OUTPUT_LIMIT + ): + return False + deferred_scan_incomplete = deferred_scan_incomplete or scan_incomplete + if deferred_output_limit is None: + deferred_output_limit = resource_limit + else: + observed = max( + int(deferred_output_limit.metrics.get("observed_findings", 0)), + int(resource_limit.metrics.get("observed_findings", 0)), + ) + deferred_output_limit = _StaticResourceLimitError( + LedgerReason.OUTPUT_LIMIT, + { + "observed_findings": observed, + "limit_findings": max_findings, + }, + ) + return True + + def reconciled_findings() -> list[Finding]: + """Coalesce mixed owners before selecting any early-return prefix.""" + candidates = coalesce(findings) if coalesce is not None else findings + return _deduplicate_view_findings(candidates) + + def reconciled_prefix() -> list[Finding]: + return reconciled_findings()[:max_findings] + + def limited_result( + resource_limit: _StaticResourceLimitError, + ) -> tuple[list[Finding], LedgerReason, dict[str, int | float]]: + """Finalize retained identity without discovering work beyond a hard cap.""" + if ( + resource_limit.reason is LedgerReason.OUTPUT_LIMIT + and callable(retained_reconciliation_hook) + and source_context is not None + ): + try: + retained_reconciliation_hook( + content, + reconciled_prefix(), + finding_budget.check_runtime, + source_context, + ) + except _StaticResourceLimitError as exc: + return reconciled_prefix(), exc.reason, exc.metrics + return reconciled_prefix(), resource_limit.reason, resource_limit.metrics if modules_for_windows: marker_owned_starts = tuple(range(0, max(1, len(content)), DECLARED_MARKER_OWNED_CHARS)) @@ -1610,6 +2507,15 @@ def _scan_all_views_detailed( ) try: finding_budget.check_runtime() + for module in modules_for_windows: + prepare = _explicit_analysis_hook(module, "prepare_analysis") + if prepare is not None: + prepared_analyses[id(module)] = prepare( + content=content, + file_type=_infer_file_type(path), + check_runtime=finding_budget.check_runtime, + ) + finding_budget.check_runtime() source_context = _build_window_source_context( path, content, @@ -1629,7 +2535,10 @@ def _scan_all_views_detailed( owned_starts=marker_owned_starts, raw_starts=marker_raw_starts, source_context=source_context, + python_source=python_source, + defer_projected_output_limit=coalesce is not None, complete_context=whole_artifact_window, + prepared_analyses=prepared_analyses, ) bounded_parse_limited = bounded_parse_limited or marker_bounded_parse_limited except _StaticResourceLimitError as exc: @@ -1638,26 +2547,22 @@ def _scan_all_views_detailed( seen_findings, exc.partial_findings, max_findings=max_findings, + coalesce=coalesce, ) - return ( - findings[:max_findings], - exc.reason, - exc.metrics, - ) + return limited_result(exc) unique_limit = _extend_unique_findings( findings, seen_findings, marker_findings, max_findings=max_findings, + coalesce=coalesce, ) if unique_limit is not None: - return findings[:max_findings], unique_limit.reason, unique_limit.metrics + if not defer_output_limit(unique_limit): + return limited_result(unique_limit) if resource_limit is not None: - return ( - _deduplicate_view_findings(findings)[:max_findings], - resource_limit.reason, - resource_limit.metrics, - ) + if not defer_output_limit(resource_limit, scan_incomplete=True): + return limited_result(resource_limit) # Window overlap cannot prove completeness for one shell command that # spans several otherwise ordinary windows. Run each module's bounded, @@ -1693,23 +2598,29 @@ def _scan_all_views_detailed( finding_budget, python_ast_cache_key, python_ast, + python_source=python_source, ) except _StaticResourceLimitError as exc: - return _deduplicate_view_findings(findings), exc.reason, exc.metrics + return reconciled_prefix(), exc.reason, exc.metrics unique_limit = _extend_unique_findings( findings, seen_findings, ast_findings, max_findings=max_findings, + coalesce=coalesce, ) if unique_limit is not None: - return findings[:max_findings], unique_limit.reason, unique_limit.metrics + if not defer_output_limit(unique_limit): + return limited_result(unique_limit) if resource_limit is not None: - return ( - _deduplicate_view_findings(findings)[:max_findings], - resource_limit.reason, - resource_limit.metrics, - ) + if defer_output_limit(resource_limit, scan_incomplete=True): + # AST and lexical producers can own the same logical finding, + # and AST traversal alone cannot decide the earliest public + # prefix. Retain its bounded prefix and let the lexical producer + # enter reconciliation before enforcing the shared cap. + pass + else: + return limited_result(resource_limit) if modules_for_windows: assert source_context is not None @@ -1717,7 +2628,7 @@ def _scan_all_views_detailed( now = time.monotonic() if now >= deadline: return ( - _deduplicate_view_findings(findings), + reconciled_prefix(), LedgerReason.RUNTIME_LIMIT, { "observed_seconds": max(0.0, now - started_at), @@ -1745,6 +2656,19 @@ def _scan_all_views_detailed( source_context.fence_states, source_context.fence_transitions, ) + outer_right_boundary_is_fixed = raw_end < len(content) or ( + whole_artifact_window and len(content) == SECURITY_VIEW_WINDOW_CHARS + ) + if raw_end < len(content): + next_owned_start = owned_start + _RAW_WINDOW_OWNED_CHARS + right_boundary_recovery_start = next_owned_start - raw_start + elif whole_artifact_window and len(content) > SECURITY_VIEW_WINDOW_CHARS - len("True"): + # Replacing a short terminal value can move an exact-ceiling + # artifact into the multi-window regime. Its next raw window + # owns calls from the standard owned boundary onward. + right_boundary_recovery_start = _RAW_WINDOW_OWNED_CHARS + else: + right_boundary_recovery_start = None try: full_views = security_text_views( context_prefix + raw_window, @@ -1752,12 +2676,18 @@ def _scan_all_views_detailed( ) except _StaticResourceLimitError as exc: return ( - _deduplicate_view_findings(findings)[:max_findings], + reconciled_prefix(), exc.reason, exc.metrics, ) + across_security_views = len(full_views) > 1 for full_view in full_views: full_view = _window_view_with_markdown_context(full_view, len(context_prefix)) + full_view = _with_fixed_right_boundary( + full_view, + outer_right_boundary_is_fixed, + right_boundary_recovery_start, + ) try: for module in modules_for_windows: exhaustion_hook = getattr( @@ -1767,11 +2697,16 @@ def _scan_all_views_detailed( ) if callable(exhaustion_hook) and full_view.name != "raw": finding_budget.check_runtime() + module_file_type = _effective_module_file_type( + path, + module, + python_source=python_source, + ) bounded_parse_limited = bounded_parse_limited or bool( exhaustion_hook( full_view.text, finding_budget.check_runtime, - file_type=_infer_file_type(path), + file_type=module_file_type, # A fragment cannot prove surrounding HTML, # container, or inline delimiter ownership. complete_context=whole_artifact_window, @@ -1779,7 +2714,7 @@ def _scan_all_views_detailed( ) except _StaticResourceLimitError as exc: return ( - _deduplicate_view_findings(findings)[:max_findings], + reconciled_prefix(), exc.reason, exc.metrics, ) @@ -1798,11 +2733,18 @@ def _scan_all_views_detailed( modules_for_windows, view_budget, None, + python_source=python_source, source_text=raw_window, + prepared_analyses=prepared_analyses, + source_view=( + _absolute_source_view(view, source_start=raw_start) + if prepared_analyses + else None + ), ) except _StaticResourceLimitError as exc: return ( - _deduplicate_view_findings(findings)[:max_findings], + reconciled_prefix(), exc.reason, exc.metrics, ) @@ -1824,7 +2766,23 @@ def _scan_all_views_detailed( and source_end > owned_source_start + _WINDOW_OVERLAP_CHARS ) if not starts_in_owned_range and not first_discoverable_in_this_window: - continue + alternate_start = finding.evidence.get( + _SOURCE_ALTERNATE_START_EVIDENCE + ) + if not ( + isinstance(alternate_start, int) + and owned_source_start <= alternate_start < owned_source_end + ): + continue + finding.evidence[_SOURCE_START_EVIDENCE] = alternate_start + finding.evidence[_SOURCE_ALTERNATE_START_EVIDENCE] = source_start + alternate_matched_text = finding.evidence.pop( + _ALTERNATE_MATCHED_TEXT_EVIDENCE, + None, + ) + if isinstance(alternate_matched_text, str): + finding.matched_text = alternate_matched_text + finding.finding = alternate_matched_text owned_findings.append(finding) view_findings = owned_findings _restore_source_lines( @@ -1840,15 +2798,27 @@ def _scan_all_views_detailed( seen_findings, view_findings, max_findings=max_findings, + coalesce=coalesce, ) if unique_limit is not None: - return findings[:max_findings], unique_limit.reason, unique_limit.metrics + if not defer_output_limit( + unique_limit, + across_security_views=across_security_views, + ): + return limited_result(unique_limit) if resource_limit is not None: - return ( - _deduplicate_view_findings(findings)[:max_findings], - resource_limit.reason, - resource_limit.metrics, - ) + if not defer_output_limit( + resource_limit, + across_security_views=across_security_views, + scan_incomplete=True, + ): + return limited_result(resource_limit) + if deferred_output_limit is not None and not defers_mixed_output_limit: + # Alternate projections of this bounded raw window can expose + # an earlier source occurrence than its raw view. Once every + # projection has reconciled, enforce the cap before advancing + # to another source window so retained evidence stays bounded. + return limited_result(deferred_output_limit) if owned_end == len(content): break @@ -1860,15 +2830,96 @@ def _scan_all_views_detailed( # all resource accounting remains on the same artifact budget. continuity_seen = {_continuity_finding_key(finding) for finding in findings} try: - for continuity in _continuity_views(content, finding_budget): - for full_view in security_text_views( + whitespace_modules = [ + module + for module in modules_for_windows + if _explicit_analysis_hook( + prepared_analyses.get(id(module)), "analyze_whitespace_continuity" + ) + is not None + ] + if whitespace_modules: + for projection in chain( + _whitespace_continuity_views(content, finding_budget), + _whitespace_continuity_views( + content, finding_budget, default_ignorables_as_separators=True + ), + ): + for full_view in security_text_views( + projection.text, + check_runtime=finding_budget.check_runtime, + ): + named_view = SecurityTextView( + f"{projection.name}-{full_view.name}", + full_view.text, + full_view.source_offsets, + ) + for view in _bounded_view_slices(named_view): + finding_budget.check_runtime() + source_view = _absolute_source_view(view, parent=projection) + view_budget = _FindingBudget( + max_findings=max(0, max_findings), + started_at=started_at, + deadline=deadline, + clock=finding_budget.clock, + ) + view_findings, resource_limit = _scan_view_windows( + path, + view, + whitespace_modules, + view_budget, + None, + python_source=python_source, + source_text=projection.text, + prepared_analyses=prepared_analyses, + source_view=source_view, + evidence_source_view=source_view, + analysis_method="analyze_whitespace_continuity", + ) + _restore_source_lines( + view_findings, + raw_window=content, + window_line=1, + view=source_view, + source_line_starts=source_context.line_starts, + ) + for finding in view_findings: + key = _continuity_finding_key(finding) + if key in continuity_seen: + continue + continuity_seen.add(key) + unique_limit = _extend_unique_findings( + findings, + seen_findings, + [finding], + max_findings=max_findings, + ) + if unique_limit is not None: + return ( + findings[:max_findings], + unique_limit.reason, + unique_limit.metrics, + ) + if resource_limit is not None: + return ( + _deduplicate_view_findings(findings)[:max_findings], + resource_limit.reason, + resource_limit.metrics, + ) + for continuity in _continuity_views( + content, finding_budget, include_source_offsets=bool(prepared_analyses) + ): + full_views = security_text_views( continuity.view.text, check_runtime=finding_budget.check_runtime, - ): + ) + across_security_views = len(full_views) > 1 + for full_view in full_views: named_view = SecurityTextView( name=f"continuity-{full_view.name}", text=full_view.text, source_offsets=full_view.source_offsets, + right_boundary_is_fixed=continuity.view.right_boundary_is_fixed, ) for view in _bounded_view_slices(named_view): finding_budget.check_runtime() @@ -1884,13 +2935,21 @@ def _scan_all_views_detailed( modules_for_windows, view_budget, None, + python_source=python_source, source_text=continuity.view.text, + prepared_analyses=prepared_analyses, + source_view=( + _absolute_source_view(view, parent=continuity.view) + if prepared_analyses + else None + ), ) _restore_source_lines( view_findings, raw_window=continuity.view.text, window_line=1, view=view, + start_source_offsets=continuity.source_offsets, ) _restore_continuity_lines( view_findings, @@ -1906,35 +2965,51 @@ def _scan_all_views_detailed( seen_findings, [finding], max_findings=max_findings, + coalesce=coalesce, ) if unique_limit is not None: - return ( - findings[:max_findings], - unique_limit.reason, - unique_limit.metrics, - ) + if not defer_output_limit( + unique_limit, + across_security_views=across_security_views, + ): + return limited_result(unique_limit) if resource_limit is not None: - return ( - _deduplicate_view_findings(findings)[:max_findings], - resource_limit.reason, - resource_limit.metrics, - ) + if not defer_output_limit( + resource_limit, + across_security_views=across_security_views, + scan_incomplete=True, + ): + return limited_result(resource_limit) + if deferred_output_limit is not None and not defers_mixed_output_limit: + return limited_result(deferred_output_limit) except _StaticResourceLimitError as exc: return ( - _deduplicate_view_findings(findings)[:max_findings], + reconciled_prefix(), exc.reason, exc.metrics, ) - deduplicated = _deduplicate_view_findings(findings) + deduplicated = reconciled_findings() if len(deduplicated) > max_findings: - return ( - deduplicated[:max_findings], - LedgerReason.OUTPUT_LIMIT, + metrics = _StaticScanMetrics( { "observed_findings": len(deduplicated), "limit_findings": max_findings, - }, + } + ) + metrics.fully_scanned_aggregate_overflow = not deferred_scan_incomplete + return ( + deduplicated if not deferred_scan_incomplete else deduplicated[:max_findings], + LedgerReason.OUTPUT_LIMIT, + metrics, + ) + if deferred_output_limit is not None: + metrics = _StaticScanMetrics(deferred_output_limit.metrics) + metrics.fully_scanned_aggregate_overflow = not deferred_scan_incomplete + return ( + deduplicated, + deferred_output_limit.reason, + metrics, ) return ( deduplicated, @@ -1961,6 +3036,7 @@ def _scan_all_views( timeout_seconds: float | None = None, started_at: float | None = None, python_ast: ParsedPythonFile | None = None, + python_source: bool | None = None, ) -> list[Finding]: findings, _, _ = _scan_all_views_detailed( path, @@ -1971,6 +3047,7 @@ def _scan_all_views( timeout_seconds=timeout_seconds, started_at=started_at, python_ast=python_ast, + python_source=python_source, ) return findings @@ -2052,6 +3129,17 @@ def run_static_patterns( file_cache = cast( dict[str, str], state.get("local_file_cache") or state.get("file_cache") or {} ) + raw_file_cache = cast(Mapping[str, bytes] | None, state.get("raw_file_cache")) + source_classifications = cast( + Mapping[str, PythonSourceClassification | str] | None, + state.get("python_source_classifications") + if "python_source_classifications" in state + else None, + ) + source_classification_limitations = cast( + Mapping[str, str], state.get("python_source_classification_limitations") or {} + ) + needs_python_source = _requires_python_source_type(pattern_modules) python_ast_cache_key = cast(str | None, state.get("python_ast_cache_key")) container_paths = { str(metadata.get("path", "")) @@ -2064,7 +3152,18 @@ def run_static_patterns( { str(item.get("path", "")) for item in raw_inventory - if isinstance(item, dict) and item.get("content_kind") == ContentKind.BINARY + if isinstance(item, dict) + and item.get("content_kind") == ContentKind.BINARY + and not item.get("readable_binary") + } + if isinstance(raw_inventory, list) + else set() + ) + readable_binary_paths = ( + { + str(item.get("path", "")) + for item in raw_inventory + if isinstance(item, dict) and item.get("readable_binary") } if isinstance(raw_inventory, list) else set() @@ -2078,7 +3177,13 @@ def run_static_patterns( if content is None: logger.debug("Skipping %s: no content in file_cache", path) continue - if path in binary_paths or (not binary_paths and _is_binary_file(path, content)): + if needs_python_source and path in source_classification_limitations: + continue + if path in binary_paths or ( + not raw_inventory + and path not in readable_binary_paths + and _is_binary_file(path, content) + ): continue remaining = MAX_FINDINGS_PER_ANALYZER - len(findings) if remaining <= 0: @@ -2087,11 +3192,24 @@ def run_static_patterns( shared_remaining = transitive_remaining_seconds(cast(SkillspectorState, state)) if shared_remaining is not None and shared_remaining <= 0: break + python_source = False + if needs_python_source: + source_classification = resolve_python_source_classification( + path, + content, + source_classifications=source_classifications, + raw_file_cache=raw_file_cache, + ) + python_source = source_classification is not PythonSourceClassification.NON_PYTHON + current_remaining = transitive_remaining_seconds(cast(SkillspectorState, state)) + if current_remaining is not None and current_remaining <= 0: + break python_ast = _python_ast_for_path( path, content, pattern_modules, python_ast_cache_key, + python_source=python_source, ) path_limit = min(MAX_FINDINGS_PER_ARTIFACT, remaining) path_findings, resource_limit, _ = _scan_all_views_detailed( @@ -2103,6 +3221,8 @@ def run_static_patterns( timeout_seconds=shared_remaining, started_at=path_started_at, python_ast=python_ast, + python_source=python_source, + readable_binary=path in readable_binary_paths, ) runtime_limit = MAX_STATIC_ANALYSIS_SECONDS_PER_ARTIFACT if shared_remaining is not None: @@ -2142,6 +3262,20 @@ def run_static_patterns_with_ledger( file_cache = cast( dict[str, str], state.get("local_file_cache") or state.get("file_cache") or {} ) + raw_file_cache = cast(Mapping[str, bytes] | None, state.get("raw_file_cache")) + source_classifications = cast( + Mapping[str, PythonSourceClassification | str] | None, + state.get("python_source_classifications") + if "python_source_classifications" in state + else None, + ) + source_classification_limitations = cast( + Mapping[str, str], state.get("python_source_classification_limitations") or {} + ) + source_decode_failures = cast( + Mapping[str, str], state.get("python_source_decode_failures") or {} + ) + needs_python_source = _requires_python_source_type(pattern_modules) python_ast_cache_key = cast(str | None, state.get("python_ast_cache_key")) container_paths = { str(metadata.get("path", "")) @@ -2168,7 +3302,23 @@ def run_static_patterns_with_ledger( ) else: artifact = inventory.get(path, {}) - if path not in container_paths and artifact.get("content_kind") == ContentKind.OPAQUE: + if path not in container_paths and path in source_classification_limitations: + event = ledger_event( + outcome=LedgerOutcome.PARTIAL, + phase="static", + analyzer_id=analyzer_id, + path=path, + reason=LedgerReason.RUNTIME_LIMIT, + ) + elif path not in container_paths and path in source_decode_failures: + event = ledger_event( + outcome=LedgerOutcome.PARTIAL, + phase="static", + analyzer_id=analyzer_id, + path=path, + reason=LedgerReason.PYTHON_SOURCE_DECODE_ERROR, + ) + elif path not in container_paths and artifact.get("content_kind") == ContentKind.OPAQUE: event = ledger_event( outcome=( LedgerOutcome.FAILED @@ -2180,7 +3330,11 @@ def run_static_patterns_with_ledger( path=path, reason=LedgerReason.OPAQUE_CONTENT, ) - elif path not in container_paths and artifact.get("content_kind") == ContentKind.BINARY: + elif ( + path not in container_paths + and artifact.get("content_kind") == ContentKind.BINARY + and not artifact.get("readable_binary") + ): referenced = bool(artifact.get("referenced")) event = ledger_event( outcome=LedgerOutcome.PARTIAL if referenced else LedgerOutcome.OUT_OF_SCOPE, @@ -2219,6 +3373,9 @@ def run_static_patterns_with_ledger( path_findings: list[Finding] resource_limit: LedgerReason | None resource_metrics: dict[str, int | float] + source_classification: PythonSourceClassification | None = None + fully_scanned_aggregate_overflow = False + postprocessed = False if shared_remaining is not None and shared_remaining <= 0: path_findings = [] resource_limit = LedgerReason.RUNTIME_LIMIT @@ -2228,11 +3385,36 @@ def run_static_patterns_with_ledger( } else: try: + python_source = False + if needs_python_source: + source_classification = resolve_python_source_classification( + path, + content, + source_classifications=source_classifications, + raw_file_cache=raw_file_cache, + ) + python_source = ( + source_classification is not PythonSourceClassification.NON_PYTHON + ) + current_remaining = transitive_remaining_seconds( + cast(SkillspectorState, state) + ) + if current_remaining is not None and current_remaining <= 0: + raise _StaticResourceLimitError( + LedgerReason.RUNTIME_LIMIT, + { + "observed_seconds": max( + 0.0, time.monotonic() - path_started_at + ), + "limit_seconds": max(0.0, shared_remaining or 0.0), + }, + ) python_ast = _python_ast_for_path( path, content, pattern_modules, python_ast_cache_key, + python_source=python_source, ) path_limit = min(MAX_FINDINGS_PER_ARTIFACT, remaining) path_findings, resource_limit, resource_metrics = _scan_all_views_detailed( @@ -2244,6 +3426,16 @@ def run_static_patterns_with_ledger( timeout_seconds=shared_remaining, started_at=path_started_at, python_ast=python_ast, + python_source=python_source, + readable_binary=bool(artifact.get("readable_binary")), + ) + fully_scanned_aggregate_overflow = bool( + resource_limit is LedgerReason.OUTPUT_LIMIT + and getattr( + resource_metrics, + "fully_scanned_aggregate_overflow", + False, + ) ) has_postprocessor = bool( pattern_modules @@ -2285,6 +3477,7 @@ def run_static_patterns_with_ledger( started_at=path_started_at, timeout_seconds=runtime_limit, ) + postprocessed = True observed_seconds = max(0.0, time.monotonic() - path_started_at) if observed_seconds >= runtime_limit: resource_limit = LedgerReason.RUNTIME_LIMIT @@ -2316,6 +3509,13 @@ def run_static_patterns_with_ledger( "observed_findings": observed_findings, "limit_findings": limit_findings, } + elif ( + postprocessed + and fully_scanned_aggregate_overflow + and resource_limit is LedgerReason.OUTPUT_LIMIT + ): + resource_limit = None + resource_metrics = {} except _StaticResourceLimitError as exc: path_findings = [] resource_limit = exc.reason @@ -2340,12 +3540,22 @@ def run_static_patterns_with_ledger( path_findings = path_findings[:remaining] resource_limit = LedgerReason.OUTPUT_LIMIT findings.extend(path_findings) - partial = resource_limit is not None or ( - _infer_file_type(path) == "python" + oversized_python = ( + source_classification is not None + and source_classification is not PythonSourceClassification.NON_PYTHON and len(content) > MAX_FILE_CHARS and _requires_python_ast(pattern_modules) ) - partial_reason = resource_limit or LedgerReason.SIZE_LIMIT + ambiguous_python = ( + source_classification is PythonSourceClassification.AMBIGUOUS + and needs_python_source + ) + partial = resource_limit is not None or oversized_python or ambiguous_python + partial_reason = ( + resource_limit + or (LedgerReason.SIZE_LIMIT if oversized_python else None) + or LedgerReason.PYTHON_SOURCE_AMBIGUOUS + ) event = ledger_event( outcome=LedgerOutcome.PARTIAL if partial else LedgerOutcome.COMPLETED, phase="static", diff --git a/src/skillspector/nodes/analyzers/static_yara.py b/src/skillspector/nodes/analyzers/static_yara.py index 5de066b1a..34519dc7e 100644 --- a/src/skillspector/nodes/analyzers/static_yara.py +++ b/src/skillspector/nodes/analyzers/static_yara.py @@ -28,6 +28,7 @@ import math import os import stat +import threading import time from collections.abc import Callable from contextvars import ContextVar @@ -46,6 +47,7 @@ InspectionLedgerEvent, LedgerOutcome, LedgerReason, + LedgerRecordType, analyzer_status_event, ledger_event, ) @@ -172,9 +174,90 @@ def _enforce_rule_load_deadline() -> None: _check_rule_load_budget(budget) -# Module-level cache keyed by a content hash of all rule directories. -_compiled_rules: yara.Rules | None = None -_rules_hash: str | None = None +@dataclass(frozen=True, slots=True) +class _RuleCacheEntry: + """One compiled rule set, the hash it came from, and its own dropped-file count. + + Frozen, and only ever published by replacing :data:`_rule_cache` wholesale, + so the three halves cannot drift apart. They used to be three independent + globals, and the non-populating paths of :func:`_load_rules` wrote the skip + count while leaving the compiled rules and their hash in place. A later + request for that stale hash then hit the cache and returned those rules + paired with the intervening load's count -- zero, when the intervening load + found no rule files at all -- so a rule set that had silently dropped a + detector reported a complete scan, which is the false-clean result #554 is + about. + """ + + rules: yara.Rules + rules_hash: str + skipped_count: int + + +# Module-level cache keyed by a content hash of all rule directories. ``None`` +# means nothing usable is cached; there is deliberately no way to represent a +# half-populated cache, so every non-populating load path simply clears it. +_rule_cache: _RuleCacheEntry | None = None + +# Not cache state: the skip count of whichever load most recently ran, published +# under ``_RULES_LOCK`` so :func:`load_rules_with_skips` can read it inside the +# same transaction that produced it. On a cache hit it is assigned *from the +# cache entry*, so it always describes the rules actually returned. +_rules_skipped_count: int = 0 + +# Reentrant so the load-and-read transaction in :func:`load_rules_with_skips` +# can hold it across its own call to :func:`_load_rules`. +_RULES_LOCK = threading.RLock() + + +class _RulesLockWithinDeadline: + """Hold :data:`_RULES_LOCK`, waiting no longer than the caller's rule-load deadline. + + An unconditional wait cannot honour :data:`_RULE_LOAD_DEADLINE`: a scan whose + budget has expired would sit behind an unrelated, slow rule load in another + MCP/graph request for as long as that load takes. The wait is bounded by + the workflow wall-clock deadline (waiting consumes no thread CPU, so the + active-processing allowance cannot bound it), and expiry raises the same + ``runtime_limit`` signal the loader already raises at its other deadline + checks, which :func:`node` reports as partial work. + + Without a deadline (direct callers outside :func:`node`) this blocks as it + always did. The lock is reentrant, so a thread that already holds it (the + nested :func:`_load_rules` call inside :func:`load_rules_with_skips`) + re-acquires it at once whatever time remains. + + A plain class rather than :func:`contextlib.contextmanager`: the generator + form re-raises a body exception by assigning its ``__traceback__``, which a + frozen, slotted :class:`_YaraRuleResourceLimitError` rejects with a + ``TypeError``, so every rule-load limit raised under the lock would surface + as a crash instead of its partial result. + """ + + __slots__ = () + + def __enter__(self) -> None: + budget = _RULE_LOAD_DEADLINE.get() + if budget is None: + _RULES_LOCK.acquire() + return + elapsed = max(0.0, time.monotonic() - budget.workflow_started_at) + remaining = budget.workflow_limit_seconds - elapsed + acquired = ( + _RULES_LOCK.acquire(timeout=remaining) + if remaining > 0 + else _RULES_LOCK.acquire(blocking=False) + ) + if not acquired: + raise _YaraRuleResourceLimitError( + LedgerReason.RUNTIME_LIMIT, + { + "observed_seconds": max(0.0, time.monotonic() - budget.workflow_started_at), + "limit_seconds": budget.workflow_limit_seconds, + }, + ) + + def __exit__(self, *exc_info: object) -> None: + _RULES_LOCK.release() def _collect_rule_files(*dirs: Path) -> list[Path]: @@ -334,13 +417,36 @@ def _read_rule_source(rule_file: Path, data: bytes | None = None) -> str: return base64.b64decode("".join(encoded_source.split())).decode("utf-8") +#: Cap on how much of a decode/compile error is echoed into logs. Rule sources +#: are attacker-influenced when ``--yara-rules-dir`` points at untrusted content, +#: and YARA syntax errors can quote the offending source line, so the reason is +#: truncated rather than passed through whole. +MAX_RULE_REJECTION_REASON_CHARS = 200 + + +def _bounded_rejection_reason(exc: Exception) -> str: + """Return a single-line, length-capped description of a rule rejection.""" + reason = " ".join(str(exc).split()) + if len(reason) > MAX_RULE_REJECTION_REASON_CHARS: + reason = f"{reason[:MAX_RULE_REJECTION_REASON_CHARS]}..." + return reason or exc.__class__.__name__ + + def _build_namespace_map( rule_files: list[Path], temp_dir: Path | None = None, *, raw_cache: dict[Path, bytes] | None = None, + namespace_files: dict[str, str] | None = None, ) -> tuple[dict[str, str], int]: - """Build a {namespace: source} dict and count malformed rule files.""" + """Build a {namespace: source} dict and count malformed rule files. + + If ``namespace_files`` is given it is populated with ``{namespace: filename}`` + so a later compile failure can name the file the operator has to fix -- a + namespace has its extension stripped, so it is not a usable filename on its + own. Passed in rather than returned to keep this function's two-value + signature, which existing callers and tests unpack directly. + """ del temp_dir sources: dict[str, str] = {} skipped = 0 @@ -351,22 +457,40 @@ def _build_namespace_map( ns = _rule_namespace(rf) if ns in sources: ns = f"{rf.parent.name}/{ns}" + if namespace_files is not None: + namespace_files[ns] = rf.name try: sources[ns] = _read_rule_source(rf, raw_cache[rf]) except (binascii.Error, UnicodeDecodeError, ValueError) as exc: skipped += 1 - logger.debug("%s: skipping malformed encoded rule %s: %s", ANALYZER_ID, rf, exc) + # WARNING, not DEBUG: a dropped rule silently removes a detector, so + # the operator has to be able to identify and repair the file from a + # default-level run (#554). The filename is named explicitly because + # the ledger event is scoped to the rule set, not to one file. + logger.warning( + "%s: rejected rule file %s (could not decode): %s", + ANALYZER_ID, + rf.name, + _bounded_rejection_reason(exc), + ) return sources, skipped -def _compile_rules(sources: dict[str, str]) -> tuple[yara.Rules | None, int]: +def _compile_rules( + sources: dict[str, str], + *, + namespace_files: dict[str, str] | None = None, +) -> tuple[yara.Rules | None, int]: """Compile YARA rules from a namespace map. Falls back to per-source compilation on error. + ``namespace_files`` maps namespace to filename so a rejection can name the + file the operator has to fix rather than its extension-stripped namespace. + Returns (compiled_rules, skipped_count). """ _enforce_rule_load_deadline() try: - compiled = yara.compile(sources=sources) + compiled = yara.compile(sources=sources, includes=False) _enforce_rule_load_deadline() return compiled, 0 except yara.SyntaxError: @@ -378,14 +502,21 @@ def _compile_rules(sources: dict[str, str]) -> tuple[yara.Rules | None, int]: for ns, source in sources.items(): _enforce_rule_load_deadline() try: - yara.compile(source=source) + yara.compile(source=source, includes=False) good[ns] = source except (yara.SyntaxError, yara.Error) as exc: skipped += 1 - logger.debug("%s: skipping %s: %s", ANALYZER_ID, ns, exc) + # WARNING for the same reason as the decode path above: without it a + # broken detector disappears with no default-level trace (#554). + logger.warning( + "%s: rejected rule file %s (could not compile): %s", + ANALYZER_ID, + (namespace_files or {}).get(ns, ns), + _bounded_rejection_reason(exc), + ) _enforce_rule_load_deadline() - compiled = yara.compile(sources=good) if good else None + compiled = yara.compile(sources=good, includes=False) if good else None _enforce_rule_load_deadline() return compiled, skipped @@ -394,38 +525,124 @@ def _load_rules(extra_dir: Path | None = None) -> yara.Rules | None: """Compile YARA rules from built-in and optional user-supplied directories. Results are cached at module level and reused if directory contents haven't changed. - """ - global _compiled_rules, _rules_hash # noqa: PLW0603 - dirs = [_BUILTIN_RULES_DIR] - if extra_dir and extra_dir.is_dir(): - dirs.append(extra_dir) - elif extra_dir: - logger.warning("%s: user rules directory %s does not exist", ANALYZER_ID, extra_dir) + Rule files that fail to decode (malformed base64) or fail to compile (YARA + syntax errors) are dropped from the active rule set. The count is recorded + in the module-level ``_rules_skipped_count`` (read via + :func:`rules_skipped_count`) rather than returned here, so this keeps its + original single-value signature and every existing + ``monkeypatch.setattr(static_yara, "_load_rules", ...)`` test double stays + valid; callers that care about the skip count must surface it themselves + or a scan can report ``completed``/SAFE while some of its own detections + never ran (#554). + + A successful load publishes rules, hash and count together as one + :class:`_RuleCacheEntry`, and every path that does not produce usable rules + clears that entry outright. Both halves matter: without the first a cache + hit could answer with another load's count, and without the second the + stale rules would stay reachable under their old hash. + + Callers should prefer :func:`load_rules_with_skips`, which returns both + halves as one value; reading the count separately after this returns is + racy across concurrent scans. + """ + global _rule_cache, _rules_skipped_count # noqa: PLW0603 + + with _RulesLockWithinDeadline(): + # Cleared up front so that a load which raises part way through cannot + # leave a previous load's total readable through + # :func:`rules_skipped_count`. Every return path below assigns its own. + # ``_rule_cache`` is deliberately *not* cleared here: an entry is + # self-consistent, so on an exception it stays a valid answer for its + # own hash rather than forcing a needless recompile. + _rules_skipped_count = 0 + + dirs = [_BUILTIN_RULES_DIR] + if extra_dir and extra_dir.is_dir(): + dirs.append(extra_dir) + elif extra_dir: + logger.warning("%s: user rules directory %s does not exist", ANALYZER_ID, extra_dir) + + rule_files = _collect_rule_files(*dirs) + if not rule_files: + logger.info("%s: no YARA rule files found", ANALYZER_ID) + # Non-populating: discard the entry instead of leaving the previous + # rules cached under their old hash. Keeping them would let the next + # request for that hash return them alongside this load's zero. + _rule_cache = None + return None - rule_files = _collect_rule_files(*dirs) - if not rule_files: - logger.info("%s: no YARA rule files found", ANALYZER_ID) - return None + raw_cache = _read_rule_bytes_cache(rule_files) + current_hash = _content_hash(rule_files, raw_cache) + cached = _rule_cache + if cached is not None and cached.rules_hash == current_hash: + # The count is taken from the entry, so it describes these rules and + # not whichever load happened to run in between. + _rules_skipped_count = cached.skipped_count + return cached.rules + + namespace_files: dict[str, str] = {} + sources, materialize_skipped = _build_namespace_map( + rule_files, raw_cache=raw_cache, namespace_files=namespace_files + ) + compiled, compile_skipped = _compile_rules(sources, namespace_files=namespace_files) + skipped = materialize_skipped + compile_skipped + _rules_skipped_count = skipped + + if compiled is None: + logger.warning("%s: failed to compile any YARA rules", ANALYZER_ID) + # Non-populating for the same reason as the no-rule-files path above. + _rule_cache = None + return None + + _rule_cache = _RuleCacheEntry( + rules=compiled, + rules_hash=current_hash, + skipped_count=skipped, + ) + loaded = len(sources) - compile_skipped + logger.info("%s: compiled %d YARA rule file(s) (%d skipped)", ANALYZER_ID, loaded, skipped) + return compiled + + +def load_rules_with_skips(extra_dir: Path | None = None) -> tuple[yara.Rules | None, int]: + """Load rules and return them with their own skip count, as one value. + + The two halves must be obtained in a single locked transaction. Reading the + count separately after :func:`_load_rules` returns lets two concurrent + MCP/graph scans interleave: scan A loads rule set A, scan B loads rule set B + and overwrites the module-level count, then scan A reads B's count. Scan A + would then run rules A while reporting B's skip total -- and if B skipped + nothing, A reports ``completed`` even though one of A's own rules was + dropped, which is exactly the false-clean result #554 is about. + + :func:`_load_rules` is called through the module global so existing + ``monkeypatch.setattr(static_yara, "_load_rules", ...)`` doubles still apply. + + The lock wait is bounded by the caller's rule-load deadline (see + :class:`_RulesLockWithinDeadline`), so a scan queued behind another + scan's slow load returns its ``runtime_limit`` result on time instead of + waiting the other load out. The snapshot stays atomic either way: rules + and count are still read inside one hold of the lock, or not at all. + """ + with _RulesLockWithinDeadline(): + rules = _load_rules(extra_dir) + return rules, _rules_skipped_count - raw_cache = _read_rule_bytes_cache(rule_files) - current_hash = _content_hash(rule_files, raw_cache) - if _compiled_rules is not None and _rules_hash == current_hash: - return _compiled_rules - sources, materialize_skipped = _build_namespace_map(rule_files, raw_cache=raw_cache) - compiled, compile_skipped = _compile_rules(sources) - skipped = materialize_skipped + compile_skipped +def rules_skipped_count() -> int: + """Return how many rule files the rules from the most recent load dropped. - if compiled is None: - logger.warning("%s: failed to compile any YARA rules", ANALYZER_ID) - return None + On a cache hit this is the cached entry's own count, not zero: the whole + point is that the number travels with the rules it describes, so a rule set + that dropped a detector keeps reporting it on every later cache hit. - _compiled_rules = compiled - _rules_hash = current_hash - loaded = len(sources) - compile_skipped - logger.info("%s: compiled %d YARA rule file(s) (%d skipped)", ANALYZER_ID, loaded, skipped) - return compiled + Retained for callers that already hold :data:`_RULES_LOCK` or run + single-threaded. Anything reading this straight after :func:`_load_rules` + should use :func:`load_rules_with_skips` instead. + """ + with _RULES_LOCK: + return _rules_skipped_count def _bounded_match_instances( @@ -916,7 +1133,9 @@ def _rule_limit_response( ) deadline_token = _RULE_LOAD_DEADLINE.set(load_budget) try: - rules = _load_rules(extra_dir) + # One transaction: the skip count must describe *these* rules, not + # whatever a concurrent scan loaded in between. + rules, rules_skipped = load_rules_with_skips(extra_dir) except _YaraRuleResourceLimitError as exc: return _rule_limit_response(exc.reason, dict(exc.metrics)) finally: @@ -1072,6 +1291,47 @@ def _rule_limit_response( ) logger.info("%s: %d findings", ANALYZER_ID, len(findings)) + if rules_skipped: + # A rule that fails to compile or decode is dropped from the active + # set with no per-file signal: every scanned component can still + # report COMPLETED, because the rule that would have flagged it + # simply never ran. Surface that as its own ledger event, scoped to + # the rule directory rather than a skill file, so it isn't silently + # absorbed into a clean-looking events list (#554). + events.append( + ledger_event( + # analyzer_id is deliberately omitted. ledger_event derives the + # work identity as ``analyzer_id or f"{record_type}:{phase}"``, + # so passing it would identify this event as + # ``static_yara`` + path -- identical to the planned work item + # for a *scanned component of the same name*. A skill file + # literally named ``yara_rules`` then collides with this event, + # both planned targets resolve to two matching events, and + # reconciliation raises a fatal ``unaccounted_work`` instead of + # the nonfatal partial scan this is meant to record. Falling + # back to ``rule_set:static`` makes the identity disjoint from + # every analyzer work item by construction, so no choice of + # filename can collide -- renaming the synthetic path alone + # would only move the collision to the next unlucky name. + outcome=LedgerOutcome.PARTIAL, + # RULE_SET, not SYSTEM: SYSTEM rows describe real artifacts + # and feed path-keyed accounting (per-component coverage and + # AE1 reference findings). This row describes the rule set, + # so finalization excludes it from that accounting by type, + # and a benign file named ``yara_rules`` cannot be charged + # with this partial outcome. + record_type=LedgerRecordType.RULE_SET, + phase="static", + # Not a scanned skill file: a report-safe label for the rule + # set. Ledger paths must be relative POSIX paths, and the real + # rules directory (builtin or --yara-rules-dir) is absolute, so + # it cannot be used here. + path="yara_rules/", + reason=LedgerReason.READ_ERROR, + observed_artifacts=rules_skipped, + limit_artifacts=0, + ) + ) if not events: status = analyzer_status_event( analyzer_id=ANALYZER_ID, diff --git a/src/skillspector/nodes/build_context.py b/src/skillspector/nodes/build_context.py index 929b89afc..81659a08d 100644 --- a/src/skillspector/nodes/build_context.py +++ b/src/skillspector/nodes/build_context.py @@ -39,13 +39,16 @@ ArtifactDisposition, ArtifactRecord, ContentKind, + binary_instruction_view, classify_artifact, decode_text, + promote_artifact_to_decoded_text, ) from skillspector.constants import ( MAX_ANALYZABLE_FILE_BYTES, MAX_FILE_BYTES, MAX_LLM_TRUNCATED_FILE_CHARS, + MODEL_CONFIG, build_model_config, ) from skillspector.input_handler import ( @@ -59,9 +62,10 @@ LedgerOutcome, LedgerReason, LedgerRecordType, + inspection_work_id, ledger_event, ) -from skillspector.llm_provenance import capture_llm_provenance +from skillspector.llm_provenance import capture_llm_provenance, capture_static_llm_provenance from skillspector.logging_config import get_logger from skillspector.nested_artifacts import ( expected_container_type, @@ -70,7 +74,14 @@ is_executable_content, is_zip_content, ) -from skillspector.python_ast import prewarm_python_ast_cache +from skillspector.oms import project_oms_content +from skillspector.python_ast import ( + PythonSourceClassification, + classify_python_source, + decode_python_source, + is_python_source, + prewarm_python_ast_cache, +) from skillspector.references import ( MAX_ACCEPTED_REFERENCES, MAX_RAW_REFERENCE_CANDIDATES, @@ -136,6 +147,7 @@ def _is_allowed_inactive_git_hook_template(path: str, probe: bytes) -> bool: ".md": "markdown", ".markdown": "markdown", ".py": "python", + ".pyw": "python", ".sh": "shell", ".bash": "shell", ".zsh": "shell", @@ -787,8 +799,19 @@ def _record_runtime_limit( ) -def _infer_file_type(path: str) -> str: - """Infer file type from path (extension).""" +def _infer_file_type( + path: str, + content: str | bytes | None = None, + *, + source_classification: PythonSourceClassification | None = None, +) -> str: + """Infer file type from path and bounded execution metadata.""" + if ( + source_classification is PythonSourceClassification.PYTHON + if source_classification is not None + else is_python_source(path, content) + ): + return "python" idx = path.rfind(".") suffix = path[idx:].lower() if idx >= 0 else "" return _FILE_TYPES.get(suffix, "other") @@ -812,7 +835,7 @@ def _decode_base64_json(value: object) -> dict[str, object] | None: def _is_valid_oms_signature_bytes(data: bytes) -> bool: - """Recognize the minimal OMS DSSE/in-toto structure from bounded bytes.""" + """Recognize OMS structure only; this does not verify or trust its contents.""" try: if len(data) > MAX_FILE_BYTES: return False @@ -885,6 +908,7 @@ def _build_component_metadata( raw_file_cache: Mapping[str, bytes], recognized_oms_signatures: frozenset[str] = frozenset(), *, + source_classifications: Mapping[str, PythonSourceClassification] | None = None, clock: Callable[[], float] = monotonic, started_at: float | None = None, deadline: float | None = None, @@ -910,8 +934,21 @@ def _expired(path: str) -> bool: if _expired(path): break full = skill_dir / path - file_type = "oms_signature" if path in recognized_oms_signatures else _infer_file_type(path) content = file_cache.get(path) + raw_content = raw_file_cache.get(path) if raw_file_cache is not None else None + source_content = raw_content if raw_content is not None else content + source_classification = ( + source_classifications.get(path) if source_classifications is not None else None + ) + file_type = ( + "oms_signature" + if path in recognized_oms_signatures + else _infer_file_type( + path, + source_content, + source_classification=source_classification, + ) + ) lines = ( len(content.splitlines()) if content is not None @@ -928,7 +965,9 @@ def _expired(path: str) -> bool: size_bytes = 0 mode = 0 data = raw_file_cache.get(path, b"") - executable = is_executable_content(path, data, mode) + executable = is_executable_content( + path, data, mode, complete_content=len(data) == size_bytes + ) if executable: has_executable = True component: dict[str, object] = { @@ -989,8 +1028,13 @@ def _mark_unanalyzed_executables( disposition is ArtifactDisposition.PARTIAL and len(raw) < max(0, int(artifact.get("size_bytes", 0))) ) + readable_executable = bool(artifact.get("readable_binary")) if content_kind is ContentKind.BINARY: - reason = LedgerReason.BINARY_CONTENT + reason = ( + LedgerReason.BINARY_EXECUTABLE_TEXT + if readable_executable + else LedgerReason.BINARY_CONTENT + ) elif disposition is ArtifactDisposition.OUT_OF_SCOPE: reason = LedgerReason.OPAQUE_CONTENT elif disposition is ArtifactDisposition.FAILED or incomplete_bytes: @@ -1019,19 +1063,29 @@ def _mark_unanalyzed_executables( "container_depth": component.get("container_depth", path.count("!/")), "outer_hidden": component.get("outer_hidden", hidden), "concealed_executable": True, - "excluded_from_analysis": True, + "excluded_from_analysis": not readable_executable, + "partially_analyzed_executable": readable_executable, "inherited_exclusion_reason": reason.value, "concealment_reasons": list(dict.fromkeys([*reason_list, reason.value])), } ) artifact.setdefault("reason", reason.value) + if readable_executable and artifact["disposition"] in { + ArtifactDisposition.ANALYZED, + ArtifactDisposition.OUT_OF_SCOPE, + }: + artifact["disposition"] = ArtifactDisposition.PARTIAL events.append( ledger_event( outcome=LedgerOutcome.PARTIAL, record_type=LedgerRecordType.SYSTEM, phase="coverage_policy", path=path, - reason=LedgerReason.EXCLUDED_EXECUTABLE_CONTENT, + reason=( + LedgerReason.BINARY_EXECUTABLE_TEXT + if readable_executable + else LedgerReason.EXCLUDED_EXECUTABLE_CONTENT + ), ) ) return events @@ -1353,7 +1407,12 @@ def runtime_exhausted( executable=executable, ) continue - executable = is_executable_content(path, probe, file_stat.st_mode) + executable = is_executable_content( + path, + probe, + file_stat.st_mode, + complete_content=len(probe) == size_bytes, + ) archive_candidate = is_zip_content(probe) expected_archive = expected_container_type(path) is not None if expected_archive and not archive_candidate: @@ -1907,8 +1966,10 @@ def _record_cache_runtime_limit( if ( provider_submission_allowed and not _is_hidden_path(path) - and artifact["content_kind"] == "text" + and (artifact["content_kind"] == "text" or artifact.get("readable_binary")) ): + if artifact.get("readable_binary"): + content = binary_instruction_view(content)[0].text if truncated: content = _llm_view_of_truncated_file( content, @@ -2404,6 +2465,15 @@ def _check_runtime() -> None: runtime_limit=runtime_limit, ) return {} + while content.startswith("\ufeff"): + # decode_text() decodes with plain "utf-8", which never strips a + # leading byte-order mark (only "utf-8-sig" does), so a BOM-prefixed + # SKILL.md leaves content[0] == "\ufeff" and the delimiter check below + # silently sees {} instead of the frontmatter. Strip leading BOMs here, + # scoped to delimiter detection — decode_text() itself stays untouched + # because P2/TP1/P9 treat U+FEFF as a hidden-character injection signal + # in file bodies. + content = content[1:] if not content.startswith("---"): return {} end_match = re.search(r"\n---\s*\n", content[3:]) @@ -2509,12 +2579,37 @@ def _check_runtime() -> None: return {} -def _unsupported_primary_bytes(artifact: ArtifactRecord, data: bytes) -> bool: +def _decodes_as_declared_python( + path: str, + data: bytes, + classification: PythonSourceClassification | None, +) -> bool: + """Return whether Python bytes decode under their declared PEP 263 encoding.""" + if classification is None: + classification = classify_python_source(path, data) + if classification is PythonSourceClassification.NON_PYTHON: + return False + try: + decode_python_source(data) + except Exception: + return False + return True + + +def _unsupported_primary_bytes( + artifact: ArtifactRecord, + data: bytes, + *, + python_classification: PythonSourceClassification | None, + deadline: float, +) -> bool: """Recognize opaque primary content without opening or expanding containers. ZIPs are handled separately by bounded nested inspection. Other archive headers and UTF-16/32 instructions must not count as decoded source text, even when their bytes happen to be valid UTF-8 (for example an ASCII TAR). + Python source is decoded by its PEP 263 declaration rather than as UTF-8, + so only a failed declared decode leaves non-UTF-8 Python unsupported. """ split_utf8 = False if not artifact["decodable"] and artifact["size_bytes"] > len(data): @@ -2538,12 +2633,22 @@ def _unsupported_primary_bytes(artifact: ArtifactRecord, data: bytes) -> bool: and sample[3:4] in b"123456789" and sample[4:10] in (b"1AY&SY", b"\x17rE8P\x90") ) - return ( + utf8_unsupported = ( (artifact["content_kind"] != ContentKind.TEXT and not split_utf8) # A bounded prefix can split a valid UTF-8 code point. Existing size # accounting already marks that scan partial; it is not proof that the # complete source uses an unsupported encoding. or (not artifact["decodable"] and not split_utf8) + ) + if ( + utf8_unsupported + # Post-cache Python decoding must not begin after the shared deadline. + and monotonic() < deadline + and _decodes_as_declared_python(artifact["path"], data, python_classification) + ): + utf8_unsupported = False + return ( + utf8_unsupported or sample.startswith((b"\xff\xfe", b"\xfe\xff", b"\x00\x00\xfe\xff")) or sample.startswith((b"\x1f\x8b", b"\xfd7zXZ\x00", b"7z\xbc\xaf\x27\x1c", b"Rar!\x1a\x07")) or is_tar @@ -2669,16 +2774,7 @@ def _record_processing_runtime(*, phase: str, path: str, now: float) -> None: elif signature_valid: recognized_oms_signature_paths.add(_OMS_SIGNATURE_PATH) recognized_oms_signatures = frozenset(recognized_oms_signature_paths) - signature_events = [ - ledger_event( - outcome=LedgerOutcome.OUT_OF_SCOPE, - record_type=LedgerRecordType.SCOPE_BOUNDARY, - phase="discovery", - path=path, - reason=LedgerReason.OMS_SIGNATURE, - ) - for path in sorted(recognized_oms_signatures) - ] + signature_events: list[InspectionLedgerEvent] = [] baseline_events = [ ledger_event( outcome=LedgerOutcome.OUT_OF_SCOPE, @@ -2689,12 +2785,6 @@ def _record_processing_runtime(*, phase: str, path: str, now: float) -> None: ) for path in sorted(selected_baselines) ] - for artifact in artifact_inventory: - if artifact["path"] in recognized_oms_signatures: - artifact["disposition"] = ArtifactDisposition.OUT_OF_SCOPE - artifact["reason"] = LedgerReason.OMS_SIGNATURE.value - llm_file_cache.pop(artifact["path"], None) - primary_path = next( (path for path in ("SKILL.md", "skill.md") if path in inventoried_components), None ) @@ -2857,8 +2947,6 @@ def _record_processing_runtime(*, phase: str, path: str, now: float) -> None: ordinary_components: list[str] = [] for path in cache_candidates: ordinary_artifact = inventory_by_path.get(path) - if path in recognized_oms_signatures: - continue if path in raw_file_cache or ( ordinary_artifact is not None and ordinary_artifact.get("disposition") @@ -2889,10 +2977,14 @@ def _record_processing_runtime(*, phase: str, path: str, now: float) -> None: nested_input_cache = {**raw_file_cache, **excluded_archive_cache} nested = inspect_nested_artifacts( skill_dir, - [ - *(path for path in ordinary_components if path in raw_file_cache), - *excluded_archive_cache, - ], + sorted( + [ + *(path for path in ordinary_components if path in raw_file_cache), + *excluded_archive_cache, + ], + # Expand real containers before another archive can claim their children. + key=lambda path: -path.count("!/"), + ), raw_file_cache=nested_input_cache, max_members=remaining_artifacts, max_uncompressed_bytes=remaining_bytes, @@ -3007,7 +3099,9 @@ def mark_excluded_nested_metadata( continue nested_exclusion_reason = excluded_archive_reasons[outer_path] nested_artifact["inherited_exclusion_reason"] = nested_exclusion_reason.value - if nested_artifact["disposition"] is ArtifactDisposition.ANALYZED: + if nested_artifact["disposition"] is ArtifactDisposition.ANALYZED or ( + nested_artifact.get("readable_binary") and not nested_artifact.get("reason") + ): nested_artifact["disposition"] = ArtifactDisposition.OUT_OF_SCOPE nested_artifact["reason"] = nested_exclusion_reason.value elif nested_artifact["disposition"] is ArtifactDisposition.OUT_OF_SCOPE: @@ -3102,33 +3196,91 @@ def mark_excluded_nested_metadata( limitation_reason=limitation_reason, ) + # Only an actual collision is fatal: ordinary directories may end in "!". + nested_paths = ( + {item["path"] for item in nested.artifact_inventory} + | set(nested.components) + | set(nested.file_cache) + | set(nested.raw_file_cache) + ) + reserved_paths = {item["path"] for item in artifact_inventory} & nested_paths + member_containers = { + str(item["path"]): str(item["outer_path"]) + for item in nested.metadata + if item["path"] in reserved_paths and item.get("outer_path") + } + for path in sorted(reserved_paths): + container = member_containers.get(path, path.rsplit("!/", 1)[0]) + event = ledger_event( + outcome=LedgerOutcome.PARTIAL, + record_type=LedgerRecordType.SYSTEM, + phase="nested_artifact_inspection", + path=container, + reason=LedgerReason.ARCHIVE_AMBIGUOUS_MEMBER_PATH, + ) + event["message"] += ( + f" Archive member file {path!r} was not analyzed because it collides with a disk file." + " Distinct descendant members may still be analyzed." + ) + prework_events.append(event) + disk_container_event_ids = {id(event) for event in nested.disk_container_events} + for event in nested.ledger_events: + path = event["path"] + if path in reserved_paths: + if id(event) in disk_container_event_ids: + continue + event["path"] = member_containers.get(path, path.rsplit("!/", 1)[0]) + event["work_id"] = inspection_work_id( + f"{event['record_type'].value}:{event['phase']}", event["path"], None, None + ) + event["message"] += ( + f" This applies to withheld archive member {path!r}, not the disk file." + ) + for artifact in artifact_inventory: + if artifact["path"] in reserved_paths: + artifact["disposition"] = ArtifactDisposition.FAILED + artifact["reason"] = LedgerReason.ARTIFACT_PATH_COLLISION.value + prework_events.append( + ledger_event( + outcome=LedgerOutcome.FAILED, + record_type=LedgerRecordType.SYSTEM, + phase="discovery", + path=artifact["path"], + reason=LedgerReason.ARTIFACT_PATH_COLLISION, + ) + ) + blocked_nested_paths = excluded_nested_components | reserved_paths + nested.metadata = [item for item in nested.metadata if item["path"] not in reserved_paths] + nested.python_source_classifications = { + path: classification + for path, classification in nested.python_source_classifications.items() + if path not in blocked_nested_paths + } ordinary_nested_components = [ - path for path in nested.components if path not in excluded_nested_components + path for path in nested.components if path not in blocked_nested_paths ] local_file_cache = dict(ordinary_file_cache) local_file_cache.update( - { - path: data - for path, data in nested.file_cache.items() - if path not in excluded_nested_components - } + {path: data for path, data in nested.file_cache.items() if path not in blocked_nested_paths} ) raw_file_cache.update( { path: data for path, data in nested.raw_file_cache.items() - if path not in excluded_nested_components + if path not in blocked_nested_paths } ) - artifact_inventory.extend(nested.artifact_inventory) + artifact_inventory.extend( + item for item in nested.artifact_inventory if item["path"] not in reserved_paths + ) for artifact in artifact_inventory: override = nested.inventory_overrides.get(artifact["path"]) - if override is not None: + if override is not None and artifact["disposition"] != ArtifactDisposition.FAILED: artifact["disposition"], artifact["reason"] = override inventory_by_path = {item["path"]: item for item in artifact_inventory} recognized_containers = frozenset(nested.outer_metadata) - primary_content_events: list[InspectionLedgerEvent] = [] + content_interpretation_events: list[InspectionLedgerEvent] = [] selected_primary = state.get("primary_file_path") for artifact in artifact_inventory: path = artifact["path"] @@ -3142,7 +3294,12 @@ def mark_excluded_nested_metadata( if not required or path in nested.recognized_zip_paths: continue data = raw_file_cache.get(path) - if data is None or not _unsupported_primary_bytes(artifact, data): + if data is None or not _unsupported_primary_bytes( + artifact, + data, + python_classification=nested.python_source_classifications.get(path), + deadline=processing_deadline, + ): continue # Explicit input and primary instructions cannot be passive exclusions. # Keep canonical bytes for byte-based analysis and source attribution, @@ -3151,7 +3308,7 @@ def mark_excluded_nested_metadata( artifact["disposition"] = ArtifactDisposition.FAILED artifact["reason"] = LedgerReason.UNSUPPORTED_PRIMARY_CONTENT.value llm_file_cache.pop(path, None) - primary_content_events.append( + content_interpretation_events.append( ledger_event( outcome=LedgerOutcome.FAILED, record_type=LedgerRecordType.SYSTEM, @@ -3174,11 +3331,257 @@ def mark_excluded_nested_metadata( ] ) ) - for path in [*recognized_containers, *recognized_oms_signatures]: + for path in recognized_containers: llm_file_cache.pop(path, None) + + postprocessing_events: list[InspectionLedgerEvent] = [] + runtime_limit = max(0.0, processing_deadline - processing_started) + + def _mark_runtime_partial(affected_paths: list[str], first_limited_path: str) -> None: + limited = False + for affected_path in affected_paths: + if affected_path == first_limited_path: + limited = True + if not limited: + continue + affected_artifact = inventory_by_path.get(affected_path) + if ( + affected_artifact is None + or affected_artifact.get("disposition") == ArtifactDisposition.FAILED + ): + continue + if affected_artifact.get("disposition") != ArtifactDisposition.PARTIAL: + affected_artifact["reason"] = LedgerReason.RUNTIME_LIMIT.value + affected_artifact["disposition"] = ArtifactDisposition.PARTIAL + + classification_events: list[InspectionLedgerEvent] = [] + source_classifications = dict(nested.python_source_classifications) + completed_source_classifications: set[str] = set() + source_decode_failures: dict[str, str] = {} + nested_metadata_by_path = { + str(metadata.get("path", "")): metadata for metadata in nested.metadata + } + classification_runtime_limitation: tuple[str, float] | None = None + for path in components: + now = monotonic() + if now >= processing_deadline: + classification_runtime_limitation = ( + path, + max(0.0, now - processing_started), + ) + break + content = local_file_cache.get(path) + raw_content = raw_file_cache.get(path) + source_classification = source_classifications.get(path) + if source_classification is None: + source_classification = classify_python_source( + path, + raw_content if raw_content is not None else content, + ) + source_classifications[path] = source_classification + if ( + source_classification is not PythonSourceClassification.NON_PYTHON + and raw_content is not None + ): + try: + decoded_python = decode_python_source(raw_content) + except Exception as exc: + source_decode_failures[path] = LedgerReason.PYTHON_SOURCE_DECODE_ERROR.value + classified_artifact = inventory_by_path.get(path) + if ( + classified_artifact is not None + and classified_artifact.get("disposition") != ArtifactDisposition.FAILED + ): + if ( + classified_artifact.get("disposition") != ArtifactDisposition.PARTIAL + or "reason" not in classified_artifact + ): + classified_artifact["reason"] = ( + LedgerReason.PYTHON_SOURCE_DECODE_ERROR.value + ) + classified_artifact["disposition"] = ArtifactDisposition.PARTIAL + local_file_cache.pop(path, None) + llm_file_cache.pop(path, None) + classification_events.append( + ledger_event( + outcome=LedgerOutcome.PARTIAL, + record_type=LedgerRecordType.SYSTEM, + phase="python_source_decoding", + path=path, + reason=LedgerReason.PYTHON_SOURCE_DECODE_ERROR, + error_class=type(exc).__name__, + ) + ) + else: + local_file_cache[path] = decoded_python + nested_metadata = nested_metadata_by_path.get(path) + if nested_metadata is not None: + nested_metadata["lines"] = len(decoded_python.splitlines()) + classified_artifact = inventory_by_path.get(path) + if classified_artifact is not None: + promote_artifact_to_decoded_text(classified_artifact) + disposition = classified_artifact.get("disposition") + artifact_reason = classified_artifact.get("reason") + bounded_provider_view = ( + disposition == ArtifactDisposition.PARTIAL + and artifact_reason + in { + LedgerReason.SIZE_LIMIT.value, + LedgerReason.TOTAL_BYTES_LIMIT.value, + } + ) + if not source_local_only and ( + path in llm_file_cache + or ( + path not in nested.file_cache + and path not in recognized_containers + and not _is_hidden_path(path) + and ( + disposition == ArtifactDisposition.ANALYZED or bounded_provider_view + ) + ) + ): + provider_content = decoded_python + if bounded_provider_view: + provider_content = _llm_view_of_truncated_file( + decoded_python, + total_size=max( + len(raw_content), + int(classified_artifact.get("size_bytes", 0)), + ), + read_bytes=len(raw_content), + ) + llm_file_cache[path] = _redact_for_external_model( + path, + provider_content, + ) + completed_source_classifications.add(path) + now = monotonic() + if now >= processing_deadline: + classification_runtime_limitation = ( + path, + max(0.0, now - processing_started), + ) + break + if source_classification is PythonSourceClassification.AMBIGUOUS: + classified_artifact = inventory_by_path.get(path) + if classified_artifact is not None and classified_artifact.get("disposition") not in { + ArtifactDisposition.FAILED, + ArtifactDisposition.OUT_OF_SCOPE, + }: + if classified_artifact.get("disposition") != ArtifactDisposition.PARTIAL: + classified_artifact["reason"] = LedgerReason.PYTHON_SOURCE_AMBIGUOUS.value + classified_artifact["disposition"] = ArtifactDisposition.PARTIAL + classification_events.append( + ledger_event( + outcome=LedgerOutcome.PARTIAL, + record_type=LedgerRecordType.SYSTEM, + phase="python_source_classification", + path=path, + reason=LedgerReason.PYTHON_SOURCE_AMBIGUOUS, + ) + ) + source_classification_limitations = { + path: LedgerReason.RUNTIME_LIMIT.value + for path in components + if path not in completed_source_classifications + } + for path in source_classification_limitations: + local_file_cache.pop(path, None) + llm_file_cache.pop(path, None) + if classification_runtime_limitation is not None: + path, elapsed = classification_runtime_limitation + _mark_runtime_partial(components, path) + classification_events.append( + ledger_event( + outcome=LedgerOutcome.PARTIAL, + record_type=LedgerRecordType.SYSTEM, + phase="python_source_classification", + path=path, + reason=LedgerReason.RUNTIME_LIMIT, + observed_seconds=elapsed, + limit_seconds=runtime_limit, + ) + ) + + # Python's successful source decode takes precedence over a magic prefix. + for artifact in artifact_inventory: + path = artifact["path"] + if ( + not artifact.get("readable_binary") + or artifact["content_kind"] != ContentKind.BINARY + or path in excluded_nested_components + ): + continue + _, projection_truncated = binary_instruction_view(local_file_cache.get(path, "")) + partial = bool(artifact.get("referenced")) + reason = LedgerReason.SIZE_LIMIT if projection_truncated else LedgerReason.OPAQUE_CONTENT + if artifact["disposition"] == ArtifactDisposition.ANALYZED: + artifact["disposition"] = ( + ArtifactDisposition.PARTIAL if partial else ArtifactDisposition.OUT_OF_SCOPE + ) + content_interpretation_events.append( + ledger_event( + outcome=LedgerOutcome.OUT_OF_SCOPE, + record_type=LedgerRecordType.SCOPE_BOUNDARY, + phase="cache", + path=path, + reason=LedgerReason.BINARY_TEXT_PROJECTION, + ) + ) + if partial: + content_interpretation_events.append( + ledger_event( + outcome=LedgerOutcome.PARTIAL, + record_type=LedgerRecordType.SYSTEM, + phase="cache", + path=path, + reason=reason, + ) + ) + + for path, content in local_file_cache.items(): + if not content.lstrip().startswith("{"): + continue + projection_started = monotonic() + if projection_started >= processing_deadline: + _record_processing_runtime( + phase="signature_projection", path=path, now=projection_started + ) + continue + projection = project_oms_content(content) + if projection is None: + continue + projection_finished = monotonic() + if projection_finished >= processing_deadline: + _record_processing_runtime( + phase="signature_projection", path=path, now=projection_finished + ) + continue + artifact = inventory_by_path.get(path) + bounded_provider_view = artifact is not None and ( + artifact.get("disposition") == ArtifactDisposition.PARTIAL + and artifact.get("reason") + in {LedgerReason.SIZE_LIMIT.value, LedgerReason.TOTAL_BYTES_LIMIT.value} + ) + if path in llm_file_cache and projection.view.name != "raw" and not bounded_provider_view: + llm_file_cache[path] = projection.view.text + if not projection.complete: + signature_events.append( + ledger_event( + outcome=LedgerOutcome.PARTIAL, + record_type=LedgerRecordType.SYSTEM, + phase="discovery", + path=path, + reason=LedgerReason.OMS_SIGNATURE, + ) + ) + if artifact is not None and artifact["disposition"] == ArtifactDisposition.ANALYZED: + artifact["disposition"] = ArtifactDisposition.PARTIAL + artifact["reason"] = LedgerReason.OMS_SIGNATURE.value + llm_components = sorted(llm_file_cache) file_cache = dict(llm_file_cache) - manifest_events: list[InspectionLedgerEvent] = [] manifest = _parse_manifest( skill_dir, @@ -3261,49 +3664,39 @@ def mark_excluded_nested_metadata( ) ) - disposition_by_path = {item["path"]: item["disposition"] for item in artifact_inventory} - for reference in references: - target = reference["target_path"] - if target and target in disposition_by_path: - reference["disposition"] = disposition_by_path[target] - - postprocessing_events: list[InspectionLedgerEvent] = [] - runtime_limit = max(0.0, processing_deadline - processing_started) - - def _mark_runtime_partial(affected_paths: list[str], first_limited_path: str) -> None: - limited = False - for affected_path in affected_paths: - if affected_path == first_limited_path: - limited = True - if not limited: - continue - affected_artifact = inventory_by_path.get(affected_path) - if ( - affected_artifact is None - or affected_artifact.get("disposition") == ArtifactDisposition.FAILED - ): - continue - if affected_artifact.get("disposition") != ArtifactDisposition.PARTIAL: - affected_artifact["reason"] = LedgerReason.RUNTIME_LIMIT.value - affected_artifact["disposition"] = ArtifactDisposition.PARTIAL - + python_components = [ + component + for component in components + if component in local_file_cache + and source_classifications.get(component, PythonSourceClassification.AMBIGUOUS) + is not PythonSourceClassification.NON_PYTHON + ] + python_component_set = set(python_components) ast_runtime_limitations: list[tuple[str, float]] = [] - python_ast_cache_key = prewarm_python_ast_cache( - components, - local_file_cache, - clock=monotonic, - started_at=processing_started, - deadline=processing_deadline, - runtime_limitations=ast_runtime_limitations, - ) + if classification_runtime_limitation is not None: + python_ast_cache_key = None + ast_runtime_limitations.append(classification_runtime_limitation) + else: + python_ast_cache_key = prewarm_python_ast_cache( + python_components, + local_file_cache, + raw_file_cache=raw_file_cache, + source_classifications=source_classifications, + clock=monotonic, + started_at=processing_started, + deadline=processing_deadline, + runtime_limitations=ast_runtime_limitations, + ) if ast_runtime_limitations: path, elapsed = ast_runtime_limitations[0] - python_components = [ + limited_index = components.index(path) + affected_python_components = [ component - for component in components - if component.lower().endswith(".py") and component in local_file_cache + for component in components[limited_index:] + if component in python_component_set ] - _mark_runtime_partial(python_components, path) + if affected_python_components: + _mark_runtime_partial(affected_python_components, affected_python_components[0]) postprocessing_events.append( ledger_event( outcome=LedgerOutcome.PARTIAL, @@ -3325,6 +3718,7 @@ def _mark_runtime_partial(affected_paths: list[str], first_limited_path: str) -> local_file_cache, raw_file_cache, recognized_oms_signatures, + source_classifications=source_classifications, clock=monotonic, started_at=processing_started, deadline=processing_deadline, @@ -3368,13 +3762,33 @@ def _mark_runtime_partial(affected_paths: list[str], first_limited_path: str) -> or any(bool(metadata.get("executable")) for metadata in excluded_component_metadata) ) - model_config = build_model_config() + # Post-cache classification, AST prewarm, and metadata work can still + # downgrade inventory rows. Project those final dispositions only after + # every mutation so reference coverage cannot remain falsely complete. + disposition_by_path = {item["path"]: item["disposition"] for item in artifact_inventory} + for reference in references: + target = reference["target_path"] + if target and target in disposition_by_path: + reference["disposition"] = disposition_by_path[target] + + use_llm = state.get("use_llm", True) + model_config = build_model_config() if use_llm else {} + llm_provenance = ( + capture_llm_provenance(model_config) + if use_llm + else capture_static_llm_provenance(MODEL_CONFIG) + ) result: dict[str, object] = { "components": components, "llm_components": llm_components, "file_cache": file_cache, "local_file_cache": local_file_cache, "raw_file_cache": raw_file_cache, + "python_source_classifications": { + path: classification.value for path, classification in source_classifications.items() + }, + "python_source_classification_limitations": source_classification_limitations, + "python_source_decode_failures": source_decode_failures, "llm_file_cache": llm_file_cache, "source_local_only": source_local_only, "artifact_inventory": artifact_inventory, @@ -3390,8 +3804,9 @@ def _mark_runtime_partial(affected_paths: list[str], first_limited_path: str) -> *reference_events, *cache_events, *nested.ledger_events, - *primary_content_events, + *content_interpretation_events, *excluded_nested_events, + *classification_events, *manifest_events, *structured_events, *postprocessing_events, @@ -3403,7 +3818,7 @@ def _mark_runtime_partial(affected_paths: list[str], first_limited_path: str) -> "manifest": manifest, "previous_manifest": None, "model_config": model_config, - "llm_provenance": capture_llm_provenance(model_config), + "llm_provenance": llm_provenance, "component_metadata": component_metadata, "has_executable_scripts": has_executable_scripts, "workflow_resource_budget": workflow_budget, diff --git a/src/skillspector/nodes/deduplicate.py b/src/skillspector/nodes/deduplicate.py index 35fac90ca..a928d1600 100644 --- a/src/skillspector/nodes/deduplicate.py +++ b/src/skillspector/nodes/deduplicate.py @@ -10,7 +10,11 @@ from hashlib import sha256 from skillspector.logging_config import get_logger -from skillspector.models import Finding +from skillspector.models import ( + OCCURRENCE_CODE_SNIPPET_KEY, + OCCURRENCE_FINDING_ID_KEY, + Finding, +) logger = get_logger(__name__) @@ -19,7 +23,14 @@ def _occurrences(finding: Finding) -> list[dict[str, object]]: if finding.occurrences: - return [dict(item) for item in finding.occurrences] + occurrences = [dict(item) for item in finding.occurrences] + for occurrence in occurrences: + occurrence.setdefault(OCCURRENCE_FINDING_ID_KEY, finding.finding_id) + occurrence.setdefault( + OCCURRENCE_CODE_SNIPPET_KEY, + finding.code_snippet or finding.context, + ) + return occurrences return [ { "file": finding.file, @@ -31,6 +42,8 @@ def _occurrences(finding: Finding) -> list[dict[str, object]]: "source_identity": finding.source_identity, "source_digest": finding.source_digest, "transitive_depth": finding.transitive_depth, + OCCURRENCE_FINDING_ID_KEY: finding.finding_id, + OCCURRENCE_CODE_SNIPPET_KEY: finding.code_snippet or finding.context, } ] @@ -141,7 +154,14 @@ def _output_key(finding: Finding) -> tuple[object, ...]: _finding_source_scope(finding), finding.fingerprint() or "", json.dumps( - finding.occurrences, + [ + { + key: value + for key, value in occurrence.items() + if key not in {OCCURRENCE_FINDING_ID_KEY, OCCURRENCE_CODE_SNIPPET_KEY} + } + for occurrence in finding.occurrences + ], ensure_ascii=False, separators=(",", ":"), sort_keys=True, @@ -177,21 +197,30 @@ def deduplicate(findings: list[Finding]) -> list[Finding]: _classification_metadata, ), group in groups.items(): representative = min(group, key=_representative_key) - occurrences = { - ( - str(occurrence.get("file", "")), - _line(occurrence.get("start_line"), 1), - occurrence.get("end_line"), - occurrence.get("start_column"), - occurrence.get("end_column"), - str(occurrence.get("source_identity") or finding.source_identity or ""), - str(occurrence.get("source_digest") or finding.source_digest or ""), - str(occurrence.get("source_url") or finding.source_url or ""), - _line(occurrence.get("transitive_depth"), finding.transitive_depth), - ) - for finding in group - for occurrence in _occurrences(finding) - } + occurrences: dict[tuple[object, ...], tuple[object, object]] = {} + for finding in sorted(group, key=_representative_key): + for occurrence in _occurrences(finding): + location = ( + str(occurrence.get("file", "")), + _line(occurrence.get("start_line"), 1), + occurrence.get("end_line"), + occurrence.get("start_column"), + occurrence.get("end_column"), + str(occurrence.get("source_identity") or finding.source_identity or ""), + str(occurrence.get("source_digest") or finding.source_digest or ""), + str(occurrence.get("source_url") or finding.source_url or ""), + _line(occurrence.get("transitive_depth"), finding.transitive_depth), + ) + occurrences.setdefault( + location, + ( + occurrence.get(OCCURRENCE_FINDING_ID_KEY, finding.finding_id), + occurrence.get( + OCCURRENCE_CODE_SNIPPET_KEY, + finding.code_snippet or finding.context, + ), + ), + ) ordered_occurrences = [ { "file": file, @@ -203,29 +232,34 @@ def deduplicate(findings: list[Finding]) -> list[Finding]: **({"source_digest": source_digest} if source_digest else {}), **({"source_url": source_url} if source_url else {}), **({"transitive_depth": transitive_depth} if transitive_depth else {}), + OCCURRENCE_FINDING_ID_KEY: report_finding_id, + OCCURRENCE_CODE_SNIPPET_KEY: report_code_snippet, } for ( - file, - start, - end, - start_column, - end_column, - source_identity, - source_digest, - source_url, - transitive_depth, + ( + file, + start, + end, + start_column, + end_column, + source_identity, + source_digest, + source_url, + transitive_depth, + ), + (report_finding_id, report_code_snippet), ) in sorted( - occurrences, + occurrences.items(), key=lambda item: ( - item[5], - item[6], - item[7], - item[8], - item[0], - item[1], - _line(item[2], item[1]), - _line(item[3], -1), - _line(item[4], -1), + item[0][5], + item[0][6], + item[0][7], + item[0][8], + item[0][0], + item[0][1], + _line(item[0][2], item[0][1]), + _line(item[0][3], -1), + _line(item[0][4], -1), ), ) ] diff --git a/src/skillspector/nodes/finalize_inspection_ledger.py b/src/skillspector/nodes/finalize_inspection_ledger.py index 18135d443..e9097ad1d 100644 --- a/src/skillspector/nodes/finalize_inspection_ledger.py +++ b/src/skillspector/nodes/finalize_inspection_ledger.py @@ -47,6 +47,7 @@ } _MAX_PASSIVE_PNG_CHUNKS = 4_096 _MAX_REFERENCE_REASONS = 16 +_REFERENCE_CAVEAT_REASONS = (LedgerReason.REFERENCE_MISSING, LedgerReason.REFERENCE_UNRESOLVED) _REFERENCE_LIMIT_FIELDS = frozenset( f"{prefix}_{unit}" for prefix in ("observed", "limit") @@ -531,6 +532,22 @@ def _reference_coverage_findings( else [] ) for event in ledger_events: + if event.get("record_type") == LedgerRecordType.RULE_SET: + # A rule set's path is a label, not an artifact: a real file that + # happens to share it was not inspected any less completely. + continue + # Reference resolution records a mention of a path that the bundle does + # not carry, or that matches more than one bundled file, as a partial + # event on the file that contains the mention. Neither leaves bytes of + # that file uninspected. The completeness ledger still reports them; + # they must not turn a reference to that file (such as `SKILL.md`) + # into AE1. + if ( + event.get("outcome") == LedgerOutcome.PARTIAL + and event.get("phase") == "reference_resolution" + and str(event.get("reason_code")) in _REFERENCE_CAVEAT_REASONS + ): + continue events_by_path.setdefault(str(event.get("path", "")), []).append(event) status_shape_complete, invalid_status_paths = _status_paths_with_incomplete_evidence( state.get("analyzer_status_events"), ledger_events @@ -741,7 +758,14 @@ def finalize_inspection_ledger(state: SkillspectorState) -> dict[str, object]: ) for (path, line), finding_ids in coverage_ids_by_line.items() ] + raw_meta_review_required = state.get("meta_review_required") + meta_review_required = ( + raw_meta_review_required + if isinstance(raw_meta_review_required, bool) + else bool(state.get("effective_finding_ids") or state.get("findings")) + ) merged_state = dict(state) + merged_state["meta_review_required"] = meta_review_required all_findings = [*(state.get("findings") or []), *coverage_findings] output_events: list[InspectionLedgerEvent] = [] finding_output_records = sum(max(1, len(finding.occurrences)) for finding in all_findings) @@ -808,6 +832,7 @@ def finalize_inspection_ledger(state: SkillspectorState) -> dict[str, object]: "execution_successful": completeness["execution_successful"], "findings": coverage_findings, "effective_finding_ids": effective_finding_ids, + "meta_review_required": meta_review_required, "inspection_ledger": [ *reference_events, *output_events, diff --git a/src/skillspector/nodes/meta_analyzer.py b/src/skillspector/nodes/meta_analyzer.py index 9124e4e60..1f34d2f1f 100644 --- a/src/skillspector/nodes/meta_analyzer.py +++ b/src/skillspector/nodes/meta_analyzer.py @@ -76,6 +76,7 @@ class MetaAnalyzerFinding(BaseModel): """A single finding evaluated by the meta-analyzer LLM (filter/enrich mode).""" + finding_id: str | None = Field(default=None, description="Exact static Finding ID.") pattern_id: str = Field(description="The static analysis pattern ID (e.g. E2, P1)") start_line: int | None = Field( default=None, @@ -96,7 +97,10 @@ class MetaAnalyzerFinding(BaseModel): @classmethod def _normalize_confidence(cls, v: object) -> float: # Accept 0-100 scale values from some models, then clamp into [0, 1]. - value = float(v) # type: ignore[arg-type] + try: + value = float(v) # type: ignore[arg-type] + except (TypeError, ValueError) as exc: + raise ValueError("confidence must be numeric") from exc if value > 2.0: value = value / 100.0 return min(1.0, max(0.0, value)) @@ -129,21 +133,17 @@ class MetaAnalyzerResult(BaseModel): def _parse_stringified_findings(cls, v: object) -> object: """LLMs sometimes return the findings array as a JSON string.""" if isinstance(v, str): - try: - parsed = json.loads(v) - except (json.JSONDecodeError, TypeError): - return [] - return parsed if isinstance(parsed, list) else [] + return json.loads(v) return v @field_validator("overall_assessment", mode="before") @classmethod def _parse_stringified_assessment(cls, v: object) -> object: - """LLMs sometimes return nested objects as JSON strings.""" + """Ignore optional prose summaries without losing valid finding verdicts.""" if isinstance(v, str): try: return json.loads(v) - except (json.JSONDecodeError, TypeError): + except json.JSONDecodeError: return None return v @@ -192,9 +192,8 @@ def _parse_stringified_assessment(cls, v: object) -> object: 4. Does the skill context make this more or less dangerous? (e.g., "cyanide" in a cooking skill = CRITICAL, in a chemistry education skill = maybe OK) -IMPORTANT: Include the start_line from each finding's Location field (the number -after the colon, e.g. for "Location: file.md:15" use start_line=15). This is -required to distinguish multiple findings with the same pattern ID in one file. +IMPORTANT: Copy each finding's Finding ID exactly and include its start_line. +The Finding ID distinguishes multiple findings that share a pattern ID and line. For findings you confirm as vulnerabilities, provide an explanation of WHY this is dangerous and remediation steps for HOW to fix the issue. @@ -235,7 +234,8 @@ def _format_findings_for_prompt(findings: list[Finding]) -> str: matched = redact_text(f.matched_text or f.message) ctx = redact_text(f.context or "") lines.append( - f"{i}. [{f.rule_id}] {message} ({f.severity})\n" + f"{i}. Finding ID: {f.finding_id}\n" + f" [{f.rule_id}] {message} ({f.severity})\n" f" Location: {loc}\n" f" Matched: {matched}\n" f" Context:\n " + "\n ".join(ctx.splitlines()) @@ -292,6 +292,96 @@ def _passthrough_with_defaults(findings: list[Finding]) -> list[Finding]: ] +def _review_evidence( + finding: Finding, outcome: str, item: dict[str, Any] | None = None +) -> dict[str, object]: + """Preserve original evidence while recording review metadata only.""" + evidence = {**finding.evidence, "llm_review_outcome": outcome} + if item is not None: + confidence = item.get("confidence") + if isinstance(confidence, (int, float)) and not isinstance(confidence, bool): + evidence["llm_review_confidence"] = confidence + return evidence + + +def _review_outcome(item: dict[str, Any]) -> str: + if not item.get("is_vulnerability", False): + return "disagreed" + return "low-confidence" if float(item.get("confidence", 0.7)) < 0.6 else "confirmed" + + +_REVIEW_OUTCOME_PRIORITY = {"disagreed": 0, "low-confidence": 1, "confirmed": 2} + + +def _prefer_assessment( + current: dict[str, Any] | None, + candidate: dict[str, Any], +) -> dict[str, Any]: + """Return the stronger assessment, preserving first input order on ties.""" + if current is None: + return candidate + current_priority = _REVIEW_OUTCOME_PRIORITY.get(_review_outcome(current), -1) + candidate_priority = _REVIEW_OUTCOME_PRIORITY.get(_review_outcome(candidate), -1) + return candidate if candidate_priority > current_priority else current + + +def _assessment_matches_finding( + item: dict[str, Any], + finding: Finding, + batch_file_path: str, +) -> bool: + """Return whether an id-bearing assessment actually describes *finding*.""" + pattern_id = item.get("pattern_id") + if not isinstance(pattern_id, str) or pattern_id != finding.rule_id: + return False + if str(item.get("_file", batch_file_path)) != finding.file: + return False + start_line = item.get("start_line") + if start_line is not None: + try: + if int(start_line) != finding.start_line: + return False + except (TypeError, ValueError): + return False + end_line = item.get("end_line") + if end_line is not None and finding.end_line is not None: + try: + if int(end_line) != finding.end_line: + return False + except (TypeError, ValueError): + return False + return True + + +def _mark_unconfirmed( + finding: Finding, outcome: str, item: dict[str, Any] | None = None +) -> Finding: + tags = list(finding.tags) + if "llm-unconfirmed" not in tags: + tags.append("llm-unconfirmed") + return replace( + finding, + remediation=finding.remediation or get_remediation(finding.rule_id), + tags=tags, + code_snippet=finding.code_snippet or finding.context, + evidence=_review_evidence(finding, outcome, item), + occurrences=list(finding.occurrences), + ) + + +def _mark_review_failed(finding: Finding) -> Finding: + """Record a failed review without conflating the outage with disagreement.""" + if finding.rule_id in _AUTHORITATIVE_DETERMINISTIC_RULES: + return finding + return replace( + finding, + remediation=finding.remediation or get_remediation(finding.rule_id), + code_snippet=finding.code_snippet or finding.context, + evidence=_review_evidence(finding, "failed"), + occurrences=list(finding.occurrences), + ) + + # --------------------------------------------------------------------------- # LLMMetaAnalyzer (filter / enrich mode) # --------------------------------------------------------------------------- @@ -371,50 +461,81 @@ def apply_filter( ) -> list[Finding]: """Enrich deterministic findings without letting LLM output suppress them. - Uses granular ``(file, rule_id, start_line, end_line)`` keying when the - LLM provides a ``start_line``, so multiple findings with the same - rule_id in one file are independently confirmed or rejected. ``end_line`` - is included in the key when provided but falls back to ``None`` so - callers that omit it still match. Falls back to coarse - ``(file, rule_id)`` keying for LLM responses that omit ``start_line``. + Exact ``finding_id`` values are validated against the submitted batch + and the item's rule/file/start location. Every item is also indexed by + granular ``(file, rule_id, start_line, end_line)`` when ``start_line`` is + present, falling back to ``(file, rule_id, start_line)`` or coarse + ``(file, rule_id)`` when needed. Duplicate assessments are merged + deterministically with confirmed > low-confidence > disagreed. Every deterministic finding remains in primary output. Unconfirmed findings receive an annotation tag; confirmed findings may gain an explanation or higher confidence, but are never downgraded. """ - _enrichment = tuple[str, str, float] - confirmed_granular: dict[tuple[str, str, int, int | None], _enrichment] = {} + assessments_by_id: dict[str, dict[str, Any]] = {} + confirmed_granular: dict[tuple[str, str, int, int | None], dict[str, Any]] = {} # Fallback index keyed without end_line (see lookup below). Issue #67. - confirmed_by_start: dict[tuple[str, str, int], _enrichment] = {} - confirmed_coarse: dict[tuple[str, str], _enrichment] = {} + confirmed_by_start: dict[tuple[str, str, int], dict[str, Any]] = {} + confirmed_coarse: dict[tuple[str, str], dict[str, Any]] = {} + exact_counts: dict[tuple[str, str, int, int | None], int] = {} + for finding in findings: + key = (finding.file, finding.rule_id, finding.start_line, finding.end_line) + exact_counts[key] = exact_counts.get(key, 0) + 1 for batch, llm_items in batch_results: + batch_findings_by_id = { + finding.finding_id: finding + for finding in batch.findings + if isinstance(finding.finding_id, str) and finding.finding_id + } for item in llm_items: + file_path = str(item.get("_file", batch.file_path)) pattern_id = item.get("pattern_id") - if not pattern_id or not item.get("is_vulnerability", False): - continue - conf = float(item.get("confidence", 0.7)) - if conf < 0.6: - continue - pattern_id = str(pattern_id) - explanation = (item.get("explanation") or "").strip() or get_explanation(pattern_id) - remediation = (item.get("remediation") or "").strip() or get_remediation(pattern_id) - file_path = item.get("_file", batch.file_path) - enrichment: _enrichment = (explanation, remediation, conf) + pattern_id = str(pattern_id) if pattern_id else "" start_line = item.get("start_line") - if start_line is not None: - end_line = item.get("end_line") - confirmed_granular[ - ( - file_path, - pattern_id, - int(start_line), - int(end_line) if end_line is not None else None, + end_line = item.get("end_line") + + # Index every item by location too, so a bad or absent id can + # still bind when the rule and location are unambiguous. + if pattern_id: + if start_line is not None: + try: + start_line_int = int(start_line) + end_line_int = int(end_line) if end_line is not None else None + except (TypeError, ValueError): + start_line_int = None + end_line_int = None + if start_line_int is not None: + granular_key = ( + file_path, + pattern_id, + start_line_int, + end_line_int, + ) + confirmed_granular[granular_key] = _prefer_assessment( + confirmed_granular.get(granular_key), item + ) + start_key = (file_path, pattern_id, start_line_int) + confirmed_by_start[start_key] = _prefer_assessment( + confirmed_by_start.get(start_key), item + ) + else: + coarse_key = (file_path, pattern_id) + confirmed_coarse[coarse_key] = _prefer_assessment( + confirmed_coarse.get(coarse_key), item ) - ] = enrichment - confirmed_by_start[(file_path, pattern_id, int(start_line))] = enrichment - else: - confirmed_coarse[(file_path, pattern_id)] = enrichment + + finding_id = item.get("finding_id") + if not isinstance(finding_id, str) or not finding_id: + continue + batch_finding = batch_findings_by_id.get(finding_id) + if batch_finding is None or not _assessment_matches_finding( + item, batch_finding, batch.file_path + ): + continue + assessments_by_id[finding_id] = _prefer_assessment( + assessments_by_id.get(finding_id), item + ) result: list[Finding] = [] for f in findings: @@ -425,38 +546,43 @@ def apply_filter( start_only_key = (f.file, f.rule_id, f.start_line, None) coarse_key = (f.file, f.rule_id) start_key = (f.file, f.rule_id, f.start_line) if f.start_line is not None else None - if exact_key in confirmed_granular: - expl, rem, conf = confirmed_granular[exact_key] - elif start_only_key in confirmed_granular: - expl, rem, conf = confirmed_granular[start_only_key] - elif f.end_line is None and start_key is not None and start_key in confirmed_by_start: - expl, rem, conf = confirmed_by_start[start_key] - elif coarse_key in confirmed_coarse: - expl, rem, conf = confirmed_coarse[coarse_key] - else: - unconfirmed_tags = list(f.tags) - if "llm-unconfirmed" not in unconfirmed_tags: - unconfirmed_tags.append("llm-unconfirmed") - result.append( - replace( - f, - remediation=f.remediation or get_remediation(f.rule_id), - tags=unconfirmed_tags, - code_snippet=f.code_snippet or f.context, - evidence=dict(f.evidence), - occurrences=list(f.occurrences), - ) - ) + assessment = assessments_by_id.get(f.finding_id) + if assessment is None: + if exact_counts.get(exact_key, 0) > 1: + assessment = None + elif exact_key in confirmed_granular: + assessment = confirmed_granular[exact_key] + elif start_only_key in confirmed_granular: + assessment = confirmed_granular[start_only_key] + elif ( + f.end_line is None and start_key is not None and start_key in confirmed_by_start + ): + assessment = confirmed_by_start[start_key] + else: + assessment = confirmed_coarse.get(coarse_key) + if assessment is None: + result.append(_mark_unconfirmed(f, "missing")) + continue + outcome = _review_outcome(assessment) + if outcome != "confirmed": + result.append(_mark_unconfirmed(f, outcome, assessment)) continue + explanation = (assessment.get("explanation") or "").strip() or get_explanation( + f.rule_id + ) + remediation = (assessment.get("remediation") or "").strip() or get_remediation( + f.rule_id + ) + confidence = float(assessment.get("confidence", 0.7)) result.append( replace( f, - message=expl, - confidence=max(f.confidence, conf), - remediation=rem, - explanation=expl, + message=explanation, + confidence=max(f.confidence, confidence), + remediation=remediation, + explanation=explanation, code_snippet=f.code_snippet or f.context, - evidence=dict(f.evidence), + evidence=_review_evidence(f, outcome, assessment), occurrences=list(f.occurrences), ) ) @@ -607,8 +733,10 @@ def meta_analyzer(state: SkillspectorState) -> MetaAnalyzerResponse: When ``use_llm`` is *True* and an LLM API key is configured (see ``llm_utils._resolve_llm_credentials``), each file that has at least one finding gets its own LLM call (or multiple calls if the file is too - large for the model's input budget). Findings are matched back by - ``(file, rule_id)`` so enrichment is precise. + large for the model's input budget). Findings are matched back by exact + ``finding_id`` when that id validates against the submitted batch, with + unambiguous ``(file, rule_id, start_line, end_line)`` location matching as + the fallback so enrichment stays precise. Falls back to default remediations when ``use_llm`` is *False* or when an LLM call fails. @@ -618,6 +746,7 @@ def meta_analyzer(state: SkillspectorState) -> MetaAnalyzerResponse: return { "findings": [], "effective_finding_ids": [], + "meta_review_required": False, "inspection_ledger": [], "analyzer_status_events": [ analyzer_status_event( @@ -640,6 +769,7 @@ def meta_analyzer(state: SkillspectorState) -> MetaAnalyzerResponse: response: MetaAnalyzerResponse = { "findings": findings, "effective_finding_ids": _effective_finding_ids(findings), + "meta_review_required": True, "inspection_ledger": events, "analyzer_status_events": [analyzer_status_for_events("meta_analyzer", events)], } @@ -659,6 +789,7 @@ def meta_analyzer(state: SkillspectorState) -> MetaAnalyzerResponse: return { "findings": filtered, "effective_finding_ids": _effective_finding_ids(filtered), + "meta_review_required": False, "inspection_ledger": [], "analyzer_status_events": [ analyzer_status_event( @@ -697,6 +828,7 @@ def meta_analyzer(state: SkillspectorState) -> MetaAnalyzerResponse: return { "findings": filtered_local, "effective_finding_ids": _effective_finding_ids(filtered_local), + "meta_review_required": False, "inspection_ledger": events, "analyzer_status_events": [analyzer_status_for_events("meta_analyzer", events)], } @@ -784,7 +916,7 @@ def meta_analyzer(state: SkillspectorState) -> MetaAnalyzerResponse: len(unanalysed), len({f.file for f in unanalysed}), ) - filtered.extend(_fallback_filtered(unanalysed)) + filtered.extend(_mark_review_failed(finding) for finding in unanalysed) filtered_local = _fallback_filtered(local_only_findings) filtered.extend(filtered_local) @@ -799,6 +931,7 @@ def meta_analyzer(state: SkillspectorState) -> MetaAnalyzerResponse: return { "findings": filtered, "effective_finding_ids": _effective_finding_ids(filtered), + "meta_review_required": True, "inspection_ledger": ledger_events, "analyzer_status_events": [status], "llm_call_log": [ @@ -814,6 +947,10 @@ def meta_analyzer(state: SkillspectorState) -> MetaAnalyzerResponse: if isinstance(e, LLMRuntimeLimitError): filtered = _passthrough_with_defaults(findings) eligible_ids = {finding.finding_id for finding in eligible_findings} + filtered = [ + _mark_review_failed(finding) if finding.finding_id in eligible_ids else finding + for finding in filtered + ] filtered_eligible = [ finding for finding in filtered if finding.finding_id in eligible_ids ] @@ -827,6 +964,7 @@ def meta_analyzer(state: SkillspectorState) -> MetaAnalyzerResponse: return { "findings": filtered, "effective_finding_ids": _effective_finding_ids(filtered), + "meta_review_required": True, "inspection_ledger": ledger_events, "analyzer_status_events": [ analyzer_status_for_events("meta_analyzer", ledger_events) @@ -847,6 +985,11 @@ def meta_analyzer(state: SkillspectorState) -> MetaAnalyzerResponse: raise logger.warning("LLM call failed, passing all findings through (fail-closed): %s", e) filtered = _passthrough_with_defaults(findings) + eligible_ids = {finding.finding_id for finding in eligible_findings} + filtered = [ + _mark_review_failed(finding) if finding.finding_id in eligible_ids else finding + for finding in filtered + ] filtered_local = [finding for finding in filtered if finding.finding_id in local_only_ids] if post_response_value_error: ledger_events, status = _meta_ledger_response( @@ -866,6 +1009,7 @@ def meta_analyzer(state: SkillspectorState) -> MetaAnalyzerResponse: return { "findings": filtered, "effective_finding_ids": _effective_finding_ids(filtered), + "meta_review_required": True, "inspection_ledger": ledger_events, "analyzer_status_events": [status], "llm_call_log": [llm_call_record("meta_analyzer", ok=False, error=str(e))], diff --git a/src/skillspector/nodes/report.py b/src/skillspector/nodes/report.py index d2e5ae9bd..484316fe4 100644 --- a/src/skillspector/nodes/report.py +++ b/src/skillspector/nodes/report.py @@ -27,26 +27,32 @@ from dataclasses import replace from datetime import UTC, datetime from hashlib import sha256 +from html import escape as escape_html from io import StringIO from typing import Literal, cast from rich.console import Console -from rich.markup import escape from rich.panel import Panel from rich.table import Table +from rich.text import Text from skillspector import __version__ as skillspector_version from skillspector.dependency_sources import redact_text from skillspector.inference_usage import sanitize_inference_usage from skillspector.inspection_ledger import ( MAX_FINDING_OUTPUT_RECORDS, + RULE_SET_SCOPE, AnalysisCompleteness, finalize_ledger, ) from skillspector.llm_provenance import sanitize_llm_provenance from skillspector.llm_utils import is_llm_available from skillspector.logging_config import get_logger -from skillspector.models import Finding +from skillspector.models import ( + OCCURRENCE_CODE_SNIPPET_KEY, + OCCURRENCE_FINDING_ID_KEY, + Finding, +) from skillspector.nodes.analyzers import ANALYZER_MODULES from skillspector.nodes.deduplicate import deduplicate from skillspector.python_ast import clear_python_ast_cache @@ -78,6 +84,7 @@ ) from skillspector.state import SkillspectorState from skillspector.suppression import Baseline, SuppressedFinding, partition_findings +from skillspector.terminal_text import visible_terminal_text logger = get_logger(__name__) @@ -141,6 +148,13 @@ def clean_evidence(value: object) -> object: evidence = { clean(str(key)) or "": clean_evidence(value) for key, value in finding.evidence.items() } + occurrences = [] + for raw in finding.occurrences: + occurrence = dict(raw) + report_snippet = occurrence.get(OCCURRENCE_CODE_SNIPPET_KEY) + if isinstance(report_snippet, str): + occurrence[OCCURRENCE_CODE_SNIPPET_KEY] = clean(report_snippet) + occurrences.append(occurrence) return replace( finding, message=clean(finding.message) or "", @@ -151,6 +165,23 @@ def clean_evidence(value: object) -> object: matched_text=clean(finding.matched_text), code_snippet=clean(finding.code_snippet), evidence=evidence, + occurrences=occurrences, + ) + + +def _finding_for_occurrence(finding: Finding, occurrence: Mapping[str, object]) -> Finding: + """Return report-only metadata bound to one compacted occurrence.""" + finding_id = occurrence.get(OCCURRENCE_FINDING_ID_KEY) + snippet = occurrence.get(OCCURRENCE_CODE_SNIPPET_KEY) + if not isinstance(finding_id, str) or not finding_id: + finding_id = finding.finding_id + if OCCURRENCE_CODE_SNIPPET_KEY not in occurrence: + return replace(finding, finding_id=finding_id) + return replace( + finding, + finding_id=finding_id, + code_snippet=snippet if isinstance(snippet, str) else None, + context=None, ) @@ -244,6 +275,7 @@ def _expand_occurrences(findings: list[Finding]) -> list[Finding]: } ] for occurrence in occurrences: + occurrence_finding = _finding_for_occurrence(finding, occurrence) start_value = occurrence.get("start_line", finding.start_line) start_line = start_value if isinstance(start_value, int) else finding.start_line end_value = occurrence.get("end_line") @@ -253,7 +285,7 @@ def _expand_occurrences(findings: list[Finding]) -> list[Finding]: depth_value = provenance.get("transitive_depth") expanded.append( replace( - finding, + occurrence_finding, file=str(occurrence.get("file", finding.file)), start_line=start_line, end_line=end_line, @@ -340,9 +372,10 @@ def _build_sarif_properties( finding: Finding, occurrence: Mapping[str, object] | None = None ) -> dict[str, object] | None: """Project selected finding metadata into a SARIF properties dictionary.""" - finding_dict = finding.to_dict() + occurrence_finding = _finding_for_occurrence(finding, occurrence or {}) + finding_dict = occurrence_finding.to_dict() metadata: dict[str, object] = { - "findingId": finding.finding_id, + "findingId": occurrence_finding.finding_id, "severity": finding_dict["severity"], "category": finding_dict["category"], "pattern": finding_dict["pattern"], @@ -472,7 +505,10 @@ def _risk_score_floor(finding: Finding) -> int: configured_floor = _RISK_SCORE_FLOORS_BY_RULE_ID.get(finding.rule_id, 0) if configured_floor: return configured_floor - if finding.rule_id == "SC9" and finding.evidence.get("excluded_from_analysis") is True: + if finding.rule_id == "SC9" and ( + finding.evidence.get("excluded_from_analysis") is True + or finding.evidence.get("partially_analyzed_executable") is True + ): return 51 if (finding.severity or "").upper() != "CRITICAL": return 0 @@ -572,7 +608,10 @@ def finding_source_scope(finding: Finding) -> str: default=0, ) if component_metadata and any( - component.get("excluded_from_analysis") is True + ( + component.get("excluded_from_analysis") is True + or component.get("partially_analyzed_executable") is True + ) and component.get("allowed_exclusion") is not True and ( component.get("executable") is True @@ -798,8 +837,11 @@ def notification_from_exception( path = str(exception.get("path", "")) start_line = exception.get("start_line") end_line = exception.get("end_line") + scope = exception.get("scope") locations = None - if path: + # A rule-set row's path is a label, not an artifact; giving it a + # physical location would attribute it to any real file of that name. + if path and not scope: region = ( SarifRegion( startLine=int(start_line), @@ -823,6 +865,8 @@ def notification_from_exception( } if exception.get("fatal") is not None: properties["fatal"] = bool(exception["fatal"]) + if scope: + properties["scope"] = str(scope) analyzers = exception.get("analyzers") if isinstance(analyzers, list): properties["analyzers"] = list(analyzers) @@ -921,6 +965,11 @@ def notification_from_exception( return rendered +def _terminal_text(value: object, style: str = "") -> Text: + """Keep dynamic text literal and render its non-printable characters visibly.""" + return Text(visible_terminal_text(str(value)), style=style) + + def _render_terminal_completeness( console: Console, completeness: Mapping[str, object], @@ -932,17 +981,30 @@ def _render_terminal_completeness( table.add_column("Metric", style="bold") table.add_column("Value") table.add_row("Execution", "successful" if execution_successful else "failed") - table.add_row("Status", str(completeness.get("status", "complete"))) - table.add_row("Coverage", f"{completeness.get('coverage_percent', 100.0)}%") - table.add_row("Fully inspected", str(completeness.get("fully_inspected_files", 0))) - table.add_row("Partially inspected", str(completeness.get("partially_inspected_files", 0))) - table.add_row("Entirely uninspected", str(completeness.get("entirely_uninspected_files", 0))) + table.add_row("Status", _terminal_text(completeness.get("status", "complete"))) + table.add_row("Coverage", _terminal_text(f"{completeness.get('coverage_percent', 100.0)}%")) + table.add_row("Fully inspected", _terminal_text(completeness.get("fully_inspected_files", 0))) + table.add_row( + "Partially inspected", _terminal_text(completeness.get("partially_inspected_files", 0)) + ) + table.add_row( + "Entirely uninspected", _terminal_text(completeness.get("entirely_uninspected_files", 0)) + ) + if completeness.get("exclude_patterns"): + table.add_row( + "Explicit exclusion patterns", + _terminal_text(", ".join(completeness["exclude_patterns"])), + ) + table.add_row( + "Excluded files (not inspected)", + _terminal_text(completeness.get("excluded_file_count", 0)), + ) console.print(table) def render_rows(title: str, rows: object) -> None: if not isinstance(rows, list) or not rows: return - console.print(f"[bold]{escape(title)}[/bold]") + console.print(_terminal_text(title, "bold")) for row in rows: if not isinstance(row, Mapping): continue @@ -951,9 +1013,12 @@ def render_rows(title: str, rows: object) -> None: end_line = row.get("end_line") if isinstance(start_line, int): location += f":{start_line}" + (f"-{end_line}" if end_line else "") + if row.get("scope") == RULE_SET_SCOPE: + # The path of a rule-set row is a label, not an artifact. + location = f"rule set {location}" reason = str(row.get("reason_code", row.get("status", "status"))) message = str(row.get("message", "")) - console.print(f" - {escape(reason)} {escape(location)}: {escape(message)}") + console.print(_terminal_text(f" - {reason} {location}: {message}")) render_rows("Scope exclusions", completeness.get("scope_exclusions")) render_rows("Ledger exceptions", completeness.get("ledger_exceptions")) @@ -962,7 +1027,7 @@ def render_rows(title: str, rows: object) -> None: if isinstance(limitations, list) and limitations: console.print("[bold]Limitations[/bold]") for limitation in limitations: - console.print(f" - {escape(str(limitation))}") + console.print(_terminal_text(f" - {limitation}")) def _format_terminal( @@ -983,9 +1048,9 @@ def _format_terminal( analysis_completeness: Mapping[str, object] | None = None, execution_successful: bool = True, ) -> str: - """Generate Rich terminal output and export as string.""" + """Generate styled Rich output with literal, control-safe dynamic text.""" suppressed = suppressed or [] - console = Console(record=True, force_terminal=True, width=80, file=StringIO()) + console = Console(record=True, force_terminal=True, width=80, file=StringIO(), emoji=False) skill_name = (manifest.get("name") or "unknown") if manifest else "unknown" source = skill_path or "" @@ -996,8 +1061,8 @@ def _format_terminal( subtitle=f"v{skillspector_version}", ) ) - console.print(f"\n[bold]Skill:[/bold] {skill_name}") - console.print(f"[bold]Source:[/bold] {source}") + console.print(Text.assemble("\n", ("Skill: ", "bold"), _terminal_text(skill_name))) + console.print(Text.assemble(("Source: ", "bold"), _terminal_text(source))) console.print(f"[bold]Scanned:[/bold] {datetime.now(UTC).strftime('%Y-%m-%d %H:%M:%S UTC')}") severity_colors = { @@ -1030,7 +1095,9 @@ def _format_terminal( lines = comp.get("lines", 0) exec_flag = comp.get("executable", False) exec_marker = "[yellow]Yes[/yellow]" if exec_flag else "No" - comp_table.add_row(path, typ, str(lines), exec_marker) + comp_table.add_row( + _terminal_text(path), _terminal_text(typ), _terminal_text(lines), exec_marker + ) if len(component_metadata) > 15: comp_table.add_row(f"... and {len(component_metadata) - 15} more", "", "", "") console.print(comp_table) @@ -1042,7 +1109,9 @@ def _format_terminal( console.print() console.print( Panel( - f"[bold]Degraded scan[/bold]\n{effective_degraded_notice}", + Text.assemble( + ("Degraded scan\n", "bold"), _terminal_text(effective_degraded_notice) + ), title="[bold red]WARNING[/bold red]", border_style="red", ) @@ -1052,24 +1121,50 @@ def _format_terminal( console.print("\n") console.print(f"[bold]Issues ({len(findings)})[/bold]\n") severity_icons = { - "LOW": "[green]LOW[/green]", - "MEDIUM": "[yellow]MEDIUM[/yellow]", - "HIGH": "[red]HIGH[/red]", - "CRITICAL": "[bold red]CRITICAL[/bold red]", + "LOW": Text("LOW", style="green"), + "MEDIUM": Text("MEDIUM", style="yellow"), + "HIGH": Text("HIGH", style="red"), + "CRITICAL": Text("CRITICAL", style="bold red"), } for f in findings: - icon = severity_icons.get((f.severity or "LOW").upper(), f.severity) - console.print(f" {icon}: {f.rule_id} - {f.message[:60]}...") + icon = severity_icons.get( + (f.severity or "LOW").upper(), _terminal_text(f.severity or "") + ) + console.print( + Text.assemble( + " ", + icon, + ": ", + _terminal_text(f.rule_id), + " - ", + _terminal_text(f.message[:60]), + "...", + ) + ) end = f"–{f.end_line}" if f.end_line and f.end_line != f.start_line else "" - console.print(f" [dim]Location:[/dim] {f.file}:{f.start_line}{end}") + console.print( + Text.assemble( + (" Location: ", "dim"), _terminal_text(f.file), f":{f.start_line}{end}" + ) + ) if f.source_url: - console.print(f" [dim]Source:[/dim] {f.source_url} (depth {f.transitive_depth})") + console.print( + Text.assemble( + (" Source: ", "dim"), + _terminal_text(f.source_url), + f" (depth {f.transitive_depth})", + ) + ) console.print(f" [dim]Confidence:[/dim] {f.confidence:.0%}") if f.remediation: - console.print(f" [dim]Remediation:[/dim] {(f.remediation or '')[:150]}...") + console.print( + Text.assemble( + (" Remediation: ", "dim"), _terminal_text(f.remediation[:150]), "..." + ) + ) if f.evidence: rendered = ", ".join(f"{key}={value}" for key, value in sorted(f.evidence.items())) - console.print(f" [dim]Evidence:[/dim] {rendered}") + console.print(Text.assemble((" Evidence: ", "dim"), _terminal_text(rendered))) console.print() else: console.print("\n[green]No security issues detected.[/green]\n") @@ -1079,11 +1174,16 @@ def _format_terminal( console.print(f"[bold]Structured Skill Summary ({len(structured_summaries)})[/bold]\n") for summary in structured_summaries: console.print( - f" [cyan]{summary.get('id', 'SSR-1')}[/cyan]: {summary.get('message', '')}" + Text.assemble( + " ", + _terminal_text(summary.get("id", "SSR-1"), "cyan"), + ": ", + _terminal_text(summary.get("message", "")), + ) ) file = _summary_display_value(summary.get("file")) if file: - console.print(f" [dim]File:[/dim] {file}") + console.print(Text.assemble((" File: ", "dim"), _terminal_text(file))) for key, label in ( ("protocol", "Protocol"), ("layout_kind", "Layout"), @@ -1095,7 +1195,7 @@ def _format_terminal( ): value = _summary_display_value(summary.get(key)) if value: - console.print(f" [dim]{label}:[/dim] {value}") + console.print(Text.assemble((f" {label}: ", "dim"), _terminal_text(value))) console.print() if suppressed: @@ -1106,7 +1206,16 @@ def _format_terminal( for sf in suppressed: f = sf.finding console.print( - f" [dim]- {f.rule_id} {f.file}:{f.start_line} (reason: {sf.reason})[/dim]" + Text.assemble( + " - ", + _terminal_text(f.rule_id), + " ", + _terminal_text(f.file), + f":{f.start_line} (reason: ", + _terminal_text(sf.reason), + ")", + style="dim", + ) ) else: console.print("[dim]Use --show-suppressed to list them.[/dim]") @@ -1353,9 +1462,26 @@ def _format_json( return json.dumps(data, indent=2) +def _markdown_plain_text(value: object) -> str: + """Remove display controls and keep a Markdown field on one line.""" + text = _clean_text(str(value)) or "" + return " ".join("".join(c for c in text if c.isprintable() or c.isspace()).splitlines()) + + def _markdown_cell(value: object) -> str: - """Render dynamic report text safely inside a Markdown table cell.""" - return str(value).replace("|", "\\|").replace("\n", " ") + """Render dynamic prose as literal text on one Markdown line.""" + text = escape_html(_markdown_plain_text(value), quote=False) + return re.sub(r"([\\`*_{}\[\]()#+.!|>~-])", r"\\\1", text) + + +def _markdown_code(value: object, *, table_cell: bool = False) -> str: + """Keep literal values inside their code spans and table cells.""" + text = _markdown_plain_text(value) + if table_cell: + text = text.replace("|", "\\|") + delimiter = "`" * (max((len(run) for run in re.findall(r"`+", text)), default=0) + 1) + padding = " " if text.startswith(("`", " ")) or text.endswith(("`", " ")) else "" + return f"{delimiter}{padding}{text}{padding}{delimiter}" def _render_markdown_completeness( @@ -1396,8 +1522,12 @@ def render_rows(title: str, rows: object) -> None: if isinstance(start_line, int): location += f":{start_line}" + (f"-{end_line}" if end_line else "") reason = row.get("reason_code", row.get("status", "status")) + location_cell = _markdown_code(location, table_cell=True) + if row.get("scope") == RULE_SET_SCOPE: + # The path of a rule-set row is a label, not an artifact. + location_cell = f"rule set {location_cell}" lines.append( - f"| {_markdown_cell(reason)} | `{_markdown_cell(location)}` | " + f"| {_markdown_cell(reason)} | {location_cell} | " f"{_markdown_cell(row.get('message', ''))} |" ) lines.append("") @@ -1438,8 +1568,8 @@ def _format_markdown( source = skill_path or "" lines.append("# SkillSpector Security Report\n") - lines.append(f"**Skill:** {skill_name} ") - lines.append(f"**Source:** `{source}` ") + lines.append(f"**Skill:** {_markdown_cell(skill_name)} ") + lines.append(f"**Source:** {_markdown_code(source)} ") lines.append(f"**Scanned:** {datetime.now(UTC).strftime('%Y-%m-%d %H:%M:%S UTC')} ") lines.append("") @@ -1447,15 +1577,15 @@ def _format_markdown( use_llm, llm_call_log or [] ) if effective_degraded_notice: - lines.append(f"> ⚠️ **Degraded scan:** {effective_degraded_notice}") + lines.append(f"> ⚠️ **Degraded scan:** {_markdown_cell(effective_degraded_notice)}") lines.append("") lines.append("## Risk Assessment\n") lines.append("| Metric | Value |") lines.append("|--------|-------|") lines.append(f"| Score | {risk_score}/100 |") - lines.append(f"| Severity | {risk_severity} |") - lines.append(f"| Recommendation | {risk_recommendation.replace('_', ' ')} |") + lines.append(f"| Severity | {_markdown_cell(risk_severity)} |") + lines.append(f"| Recommendation | {_markdown_cell(risk_recommendation.replace('_', ' '))} |") lines.append("") lines.append(f"## Components ({len(component_metadata)})\n") @@ -1467,17 +1597,19 @@ def _format_markdown( line_count = comp.get("lines", 0) exec_flag = comp.get("executable", False) exec_marker = "Yes" if exec_flag else "No" - lines.append(f"| `{path}` | {typ} | {line_count} | {exec_marker} |") + lines.append( + f"| {_markdown_code(path, table_cell=True)} | {_markdown_cell(typ)} | {line_count} | {exec_marker} |" + ) lines.append("") if structured_summaries: lines.append(f"## Structured Skill Summary ({len(structured_summaries)})\n") for summary in structured_summaries: - lines.append(f"### {summary.get('id', 'SSR-1')}\n") - lines.append(f"**Message:** {summary.get('message', '')} ") + lines.append(f"### {_markdown_cell(summary.get('id', 'SSR-1'))}\n") + lines.append(f"**Message:** {_markdown_cell(summary.get('message', ''))} ") file = _summary_display_value(summary.get("file")) if file: - lines.append(f"**File:** `{file}` ") + lines.append(f"**File:** {_markdown_code(file)} ") for key, label in ( ("protocol", "Protocol"), ("layout_kind", "Layout"), @@ -1489,7 +1621,7 @@ def _format_markdown( ): value = _summary_display_value(summary.get(key)) if value: - lines.append(f"**{label}:** {value} ") + lines.append(f"**{label}:** {_markdown_cell(value)} ") lines.append("") lines.append(f"## Issues ({len(findings)})\n") @@ -1500,23 +1632,23 @@ def _format_markdown( for f in findings: sev = (f.severity or "LOW").upper() emoji = severity_emoji.get(sev, "") - lines.append(f"### {emoji} {sev}: {f.rule_id}\n") + lines.append(f"### {emoji} {_markdown_cell(sev)}: {_markdown_cell(f.rule_id)}\n") end = f"–{f.end_line}" if f.end_line and f.end_line != f.start_line else "" - lines.append(f"**Location:** `{f.file}:{f.start_line}{end}` ") + lines.append(f"**Location:** {_markdown_code(f'{f.file}:{f.start_line}{end}')} ") if f.source_url: - lines.append(f"**Source:** `{f.source_url}` ") + lines.append(f"**Source:** {_markdown_code(f.source_url)} ") lines.append(f"**Transitive depth:** {f.transitive_depth} ") lines.append(f"**Confidence:** {f.confidence:.0%} ") lines.append("") - lines.append(f"**Message:** {f.message}") + lines.append(f"**Message:** {_markdown_cell(f.message)}") lines.append("") if f.remediation: - lines.append(f"**Remediation:** {f.remediation}") + lines.append(f"**Remediation:** {_markdown_cell(f.remediation)}") lines.append("") if f.evidence: lines.append("**Evidence:**") for key, evidence_value in sorted(f.evidence.items()): - lines.append(f"- **{key}:** `{evidence_value}`") + lines.append(f"- **{_markdown_cell(key)}:** {_markdown_code(evidence_value)}") lines.append("") lines.append("---\n") @@ -1530,8 +1662,9 @@ def _format_markdown( lines.append("|------|----------|--------|") for sf in suppressed: f = sf.finding - reason = sf.reason.replace("|", "\\|") - lines.append(f"| {f.rule_id} | `{f.file}:{f.start_line}` | {reason} |") + reason = _markdown_cell(sf.reason) + location = _markdown_code(f"{f.file}:{f.start_line}", table_cell=True) + lines.append(f"| {_markdown_cell(f.rule_id)} | {location} | {reason} |") lines.append("") else: lines.append("_Run with `--show-suppressed` to list them._\n") @@ -1838,6 +1971,7 @@ def report(state: SkillspectorState) -> dict[str, object]: "report_body": report_body, "filtered_findings": reported_findings, "suppressed_findings": suppressed, + "active_findings": active_findings, "execution_successful": execution_successful, "analysis_completeness": dict(analysis_completeness), "transitive_targets_scanned": transitive_targets_scanned, diff --git a/src/skillspector/nodes/resolve_input.py b/src/skillspector/nodes/resolve_input.py index 9ab554c6a..81249ace9 100644 --- a/src/skillspector/nodes/resolve_input.py +++ b/src/skillspector/nodes/resolve_input.py @@ -56,7 +56,10 @@ def resolve_input(state: SkillspectorState) -> dict[str, object]: workflow_budget = ensure_workflow_resource_budget(state) if input_path and isinstance(input_path, str) and input_path.strip(): - handler = InputHandler(transitive_budget=workflow_budget) + handler = InputHandler( + transitive_budget=workflow_budget, + allow_git_credentials=state.get("allow_git_credentials", True), + ) try: resolved, source_type = handler.resolve(input_path.strip()) temp_dir = handler.temp_dir_for_cleanup() diff --git a/src/skillspector/oms.py b/src/skillspector/oms.py new file mode 100644 index 000000000..98e53bc1e --- /dev/null +++ b/src/skillspector/oms.py @@ -0,0 +1,235 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Bounded, untrusted DSSE content projections; never signature verification.""" + +from __future__ import annotations + +import base64 +import binascii +import json +import re +from array import array +from dataclasses import dataclass + +from skillspector.artifacts import SecurityTextView + +_MAX_CHARS = 1_000_000 +_MAX_NODES = 10_000 +_PRINTABLE = re.compile(r"[^\x00-\x1f\x7f-\x9f\ufffd]{4,}") + + +@dataclass(frozen=True) +class OMSProjection: + view: SecurityTextView + complete: bool + + +def _object(pairs: list[tuple[str, object]]) -> dict[str, object]: + result: dict[str, object] = {} + for key, value in pairs: + if key in result: + raise ValueError("duplicate JSON key") + result[key] = value + return result + + +def _load(text: str) -> object: + value = json.loads(text, object_pairs_hook=_object) + pending = [(value, 0)] + count = 0 + while pending: + item, depth = pending.pop() + count += 1 + if count > _MAX_NODES or depth > 64: + raise ValueError("JSON projection limit") + if isinstance(item, dict): + pending.extend((child, depth + 1) for child in item.values()) + elif isinstance(item, list): + pending.extend((child, depth + 1) for child in item) + return value + + +def _decode(value: object) -> bytes: + if not isinstance(value, str): + raise ValueError("expected base64 string") + return base64.b64decode(value, validate=True) + + +def _printable(data: bytes) -> list[str]: + # Preserve readable Unicode and format characters for semantic/obfuscation + # analysis. Invalid bytes and control bytes separate runs rather than + # silently joining attacker-controlled fragments. + return _PRINTABLE.findall(data.decode("utf-8", errors="replace")) + + +def _raw_projection(content: str) -> OMSProjection | None: + recognized = "dsseEnvelope" in content + if not recognized: + # Recognition only: escaped field names also identify an oversized or + # duplicate-key bundle. Never use this permissive parse as analysis data. + try: + untrusted = json.loads(content) + recognized = isinstance(untrusted, dict) and "dsseEnvelope" in untrusted + except (ValueError, RecursionError): + pass + return OMSProjection(SecurityTextView("raw", content), False) if recognized else None + + +def _certificate(value: object, *, pem: bool = False) -> dict[str, object]: + from cryptography import x509 + from cryptography.hazmat.primitives.serialization import Encoding + + encoded = _decode(value) + cert = (x509.load_pem_x509_certificate if pem else x509.load_der_x509_certificate)(encoded) + if cert.public_bytes(Encoding.PEM if pem else Encoding.DER) != encoded: + raise ValueError("noncanonical certificate") + der = cert.public_bytes(Encoding.DER) + return { + "subject": cert.subject.rfc4514_string(), + "issuer": cert.issuer.rfc4514_string(), + "extensions": [str(extension.value) for extension in cert.extensions], + "printable_der": _printable(der), + } + + +def project_oms_content(content: str) -> OMSProjection | None: + """Decode untrusted DSSE content without allowing parser failures to escape. + + Readable content and unknown fields remain in the view. Unsupported or + over-limit content remains raw and partial; decoding never verifies a signer. + """ + if not content.lstrip().startswith("{"): + return None + if len(content) > _MAX_CHARS: + return _raw_projection(content) + try: + bundle = _load(content) + except (ValueError, RecursionError): + return _raw_projection(content) + if not isinstance(bundle, dict) or "dsseEnvelope" not in bundle: + return None + try: + return _project_bundle(bundle, content) + except Exception: + # All callers retain raw evidence and an explicit gap if a parser fails. + return OMSProjection(SecurityTextView("raw", content), False) + + +def _project_bundle(bundle: dict[str, object], content: str) -> OMSProjection: + envelope = bundle["dsseEnvelope"] + if not isinstance(envelope, dict): + return OMSProjection(SecurityTextView("raw", content), False) + complete = True + try: + payload = _load(_decode(envelope.get("payload")).decode("utf-8")) + if not isinstance(payload, (dict, list)): + raise ValueError("expected JSON payload") + envelope["payload"] = payload + except (ValueError, UnicodeError, binascii.Error, RecursionError): + complete = False + + def project_certificate(entry: object, key: str = "rawBytes", *, pem: bool = False) -> None: + nonlocal complete + try: + if not isinstance(entry, dict): + raise ValueError("expected certificate object") + entry[key] = _certificate(entry.get(key), pem=pem) + except Exception: + # cryptography exposes malformed-DER exceptions outside ValueError. + complete = False + + material = bundle.get("verificationMaterial") + if not isinstance(material, dict): + complete = False + else: + recognized_material = False + if "x509CertificateChain" in material: + recognized_material = True + chain = material["x509CertificateChain"] + certificates = chain.get("certificates") if isinstance(chain, dict) else None + if not isinstance(certificates, list) or not 0 < len(certificates) <= 64: + complete = False + else: + for entry in certificates: + project_certificate(entry) + if "certificate" in material: + recognized_material = True + project_certificate(material["certificate"]) + if "publicKey" in material: + recognized_material = True + key = material["publicKey"] + if not isinstance(key, dict) or not isinstance(key.get("hint"), str): + complete = False + if not recognized_material: + complete = False + + entries = material.get("tlogEntries", []) + if not isinstance(entries, list) or len(entries) > 64: + complete = False + else: + for entry in entries: + try: + if not isinstance(entry, dict): + raise ValueError("expected transparency log entry") + body = _load(_decode(entry.get("canonicalizedBody")).decode("utf-8")) + if not isinstance(body, dict): + raise ValueError("expected transparency log JSON object") + entry["canonicalizedBody"] = body + spec = body.get("spec") + if isinstance(spec, dict): + signatures = spec.get("signatures", []) + if not isinstance(signatures, list) or len(signatures) > 64: + complete = False + else: + for signature in signatures: + project_certificate(signature, "verifier", pem=True) + record = spec.get("hashedRekordV002") + signature = record.get("signature") if isinstance(record, dict) else None + verifier = ( + signature.get("verifier") if isinstance(signature, dict) else None + ) + if isinstance(verifier, dict) and "x509Certificate" in verifier: + project_certificate(verifier["x509Certificate"]) + except Exception: + complete = False + + try: + from cryptography.hazmat.primitives.asymmetric.utils import ( + decode_dss_signature, + encode_dss_signature, + ) + + signatures = envelope.get("signatures") + if not isinstance(signatures, list) or not 0 < len(signatures) <= 64: + complete = False + else: + for entry in signatures: + try: + if not isinstance(entry, dict): + raise ValueError("expected signature object") + der = _decode(entry.get("sig")) + r, s = decode_dss_signature(der) + if ( + not (0 < r.bit_length() <= 521 and 0 < s.bit_length() <= 521) + or encode_dss_signature(r, s) != der + ): + raise ValueError("noncanonical ECDSA signature") + entry["sig"] = {"printable_der": _printable(der)} + except Exception: + complete = False + except ImportError: + complete = False + + projected = json.dumps(bundle, ensure_ascii=False, indent=2) + # Keep real Unicode (including bidi controls) visible while escaping lone + # surrogates that cannot be passed to UTF-8 analyzers or provider clients. + projected = projected.encode("utf-8", "backslashreplace").decode("utf-8") + if len(projected) > _MAX_CHARS: + return OMSProjection(SecurityTextView("raw", content), False) + # Findings refer to the original carrier, not synthetic files or lines in + # decoded data. Their evidence explicitly labels this derived view. + return OMSProjection( + SecurityTextView("oms-decoded", projected, array("I", [0]) * len(projected)), + complete, + ) diff --git a/src/skillspector/providers/__init__.py b/src/skillspector/providers/__init__.py index 30ca116c5..23287a9c3 100644 --- a/src/skillspector/providers/__init__.py +++ b/src/skillspector/providers/__init__.py @@ -31,7 +31,7 @@ azure_openai → AzureOpenAIProvider (Azure OpenAI Service) openai_compatible → OpenAICompatibleProvider (Groq, Together AI, Mistral, etc.) claude_cli → ClaudeCLIProvider (local ``claude`` binary, no API key) - codex_cli → CodexCLIProvider (local ``codex`` binary, no API key) + codex_cli → CodexCLIProvider (registered but disabled: host-file reads) gemini_cli → GeminiCLIProvider (local ``gemini`` binary, no API key) opencode_cli → OpencodeCLIProvider (local ``opencode`` binary, no API key) antigravity_cli → AntigravityCLIProvider (local ``agy`` binary; registered @@ -78,6 +78,10 @@ ) +class UnknownProviderError(ValueError): + """The configured provider name does not identify a supported provider.""" + + def raise_no_llm_api_key_configured() -> NoReturn: """Raise the shared no-LLM-credentials error.""" raise ValueError(NO_LLM_API_KEY_MESSAGE) @@ -166,7 +170,7 @@ def _select_active_provider() -> LLMProvider: except ImportError: return NvBuildProvider() - raise ValueError( + raise UnknownProviderError( f"Unknown SKILLSPECTOR_PROVIDER: {name!r}. " "Expected one of: openai, anthropic, anthropic_proxy, bedrock, nv_build, " "ollama, azure_openai, openai_compatible, " @@ -309,6 +313,7 @@ def create_chat_model( "LLMProvider", "ModelMetadataProvider", "NO_LLM_API_KEY_MESSAGE", + "UnknownProviderError", "create_chat_model", "create_chat_model_with_provider", "get_active_provider", diff --git a/src/skillspector/providers/_agent_cli.py b/src/skillspector/providers/_agent_cli.py index 085280e66..123fbb7ab 100644 --- a/src/skillspector/providers/_agent_cli.py +++ b/src/skillspector/providers/_agent_cli.py @@ -26,8 +26,8 @@ adversarial skill content) is written to the process stdin, never injected into argv. - **Capability stripping** (per-binary): tools disabled, MCP disabled, - no extra directories, deny permission mode (claude); read-only sandbox - (codex). ``--dangerously-skip-permissions`` is NEVER used. + no extra directories, deny permission mode (claude). Codex is disabled + until a complete no-tools policy is verified. ``--dangerously-skip-permissions`` is NEVER used. - **Environment scrubbing**: API keys, SSH keys, cloud credentials, and other secrets are stripped from the child environment. - **Timeout enforcement**: the call raises ``TimeoutError`` rather than @@ -41,8 +41,8 @@ of capability removal). The JSON output envelope (``claude -p --output-format json``) is parsed -and the assistant text is returned. ``codex exec --json`` produces -JSONL events; the last assistant message is extracted. +and the assistant text is returned. The Codex JSONL parser is retained for +compatibility, but Codex inference is disabled before a subprocess is started. """ from __future__ import annotations @@ -75,35 +75,46 @@ MAX_STDERR_BYTES = 64_000 # stderr is only used for error snippets CLI_TIMEOUT_SECONDS = 300 # 5-minute per-call hard limit -# Environment variables that must NOT be forwarded to child processes. -# Includes API keys, cloud creds, SSH agent, and SkillSpector's own keys. -_SECRET_ENV_PREFIXES: tuple[str, ...] = ( - "ANTHROPIC_API_KEY", - "ANTHROPIC_PROXY_API_KEY", - "OPENAI_API_KEY", - "SKILLSPECTOR_API_KEYS", - "SKILLSPECTOR_COMPAT_API_KEY", - "NVIDIA_INFERENCE_KEY", - "NVIDIA_INFERENCE_METADATA_KEY", - "AWS_", - "AZURE_", - "GOOGLE_", - "GCLOUD_", - "GCP_", - "SSH_", - "GPG_", - "GITHUB_TOKEN", - "GITLAB_TOKEN", - "HUGGINGFACE_TOKEN", - "HF_TOKEN", - "COHERE_API_KEY", - "REPLICATE_API_TOKEN", - "MISTRAL_API_KEY", - "TOGETHER_API_KEY", - "GROQ_API_KEY", - "FIREWORKS_API_KEY", - "LANGCHAIN_API_KEY", - "LANGSMITH_API_KEY", +# Only runtime, network transport, and local-login discovery variables cross into agent processes. +# Do not inherit provider keys, arbitrary application variables, loader hooks, +# or CLI configuration overrides from the operator's environment. +_RUNTIME_ENV_NAMES = frozenset( + { + "PATH", + "PATHEXT", + "HOME", + "USER", + "LOGNAME", + "USERNAME", + "USERPROFILE", + "APPDATA", + "LOCALAPPDATA", + "SYSTEMROOT", + "WINDIR", + "TEMP", + "TMP", + "TMPDIR", + "XDG_DATA_HOME", + "CLAUDE_CONFIG_DIR", + "HTTP_PROXY", + "HTTPS_PROXY", + "ALL_PROXY", + "NO_PROXY", + "NODE_EXTRA_CA_CERTS", + "SSL_CERT_FILE", + "SSL_CERT_DIR", + "LANG", + "LANGUAGE", + "LC_ALL", + "LC_CTYPE", + "LC_COLLATE", + "LC_MESSAGES", + "LC_MONETARY", + "LC_NUMERIC", + "LC_TIME", + "TERM", + "NO_COLOR", + } ) @@ -117,18 +128,13 @@ class AgentCLIError(RuntimeError): def _scrub_env() -> dict[str, str]: - """Return a copy of ``os.environ`` with secret variables removed. + """Keep only runtime variables needed by the CLIs and their local logins. - Any variable whose name starts with a prefix in ``_SECRET_ENV_PREFIXES`` - is stripped. The resulting environment is passed to the subprocess. + Exact names (case-insensitive for Windows) fail closed for new secret names + and configuration overrides. Provider API keys are deliberately excluded: + agent CLI providers authenticate through the user's existing local login. """ - clean: dict[str, str] = {} - for key, val in os.environ.items(): - upper = key.upper() - if any(upper.startswith(p.upper()) for p in _SECRET_ENV_PREFIXES): - continue - clean[key] = val - return clean + return {key: val for key, val in os.environ.items() if key.upper() in _RUNTIME_ENV_NAMES} # --------------------------------------------------------------------------- @@ -276,60 +282,23 @@ def _parse_claude_output(raw: str) -> str: # --------------------------------------------------------------------------- -def _build_codex_argv(binary: str, model: str, max_output_tokens: int = 0) -> list[str]: - """Build the argv list for a capability-stripped ``codex exec`` call. - - Flags chosen (verified end-to-end against codex 0.139.0): - - ``exec`` - Non-interactive subcommand. With NO positional prompt, codex reads the - instructions from stdin — which is exactly where the runner pipes the - prompt. (Passing ``-`` makes the prompt literally ``"-"`` and demotes - the real content to a ```` block, so we do not pass it.) - - ``--json`` - Emit JSONL events to stdout, enabling structured parsing. - - ``--sandbox read-only`` - Most restrictive sandbox mode. Model-generated shell commands are - restricted to read-only filesystem access; no code execution. Unlike - claude/gemini (which block model tool use entirely), codex's strictest - mode still permits read-only filesystem *reads* by model-generated - commands. This is informational, not an exfil channel: the call runs in - an isolated empty temp CWD, output returns only to the operator's own - report, and there is no network egress path. - - ``--ephemeral`` - Do not persist session files to disk (no residue from the scan). - - ``--ignore-user-config`` - Ignore ``$CODEX_HOME/config.toml``; use only our explicit flags. +# Registered but disabled. Re-enabling requires a pinned CLI version and a +# verified policy preflight that denies every model-driven tool and host-file read. +_CODEX_DISABLED_REASON = ( + "codex_cli is disabled: its read-only sandbox still permits commands to read " + "host files, and SkillSpector has no verified deny-all-tools policy for Codex. " + "Use an HTTP API provider or another supported CLI provider instead." +) - ``--ignore-rules`` - Do not load user/project ``.rules`` files. - ``--model