From 749738782f5560c836a7f1d9586614ce15ca2d9d Mon Sep 17 00:00:00 2001 From: Rahul Krishna Date: Mon, 7 Sep 2026 23:05:30 -0400 Subject: [PATCH] feat(cli): --strict fails a run whose analysis degraded MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A run that loses the RTA overlay or the L4 semantic ddg exits 0 and says so only as a WARN on stderr. Interactively that is fine. In a pipeline it is not: `-a 2 --no-build` on an unbuilt project returns a call graph of declared edges only, and nothing in the exit code or the payload distinguishes it from a complete run. `--strict` turns any such degradation into a non-zero exit that names what was lost, and it fails before writing `analysis.json` so a caller cannot pick up a thin payload believing it is whole. Opt-in, and deliberately so. Degrading is a supported mode, not an oversight: `--no-build` on an unbuilt project still yields the tree, the declared call graph, the syntactic CFG/CDG/DDG and, at -a 4, the SDG vertices, param edges and summaries — everything except the WALA-derived overlays. Nine test call sites depend on that, including the container integration test, whose fixture (`mvnw-corrupt-test`) exists precisely to prove the analyzer survives a project whose build is broken. Defaulting to failure would delete a working mode to fix a reporting problem. Detection needs no changes to ScopeUtils, RtaCallGraph or WalaAnalysis: all three degradations are already visible where they are warned about, so the flag only had to collect them and check once, after every overlay pass has had its chance. The flag is an instance field, not another static, for the reason the `--schema` option already records: the pre-existing static options leak between CommandLine instances in one JVM. README gains the flag under a new FAQ entry, and says plainly that degrading is deliberate and exactly which overlays it costs. --- README.md | 23 ++++++++ src/main/java/com/ibm/cldk/CodeAnalyzer.java | 31 +++++++++++ .../com/ibm/cldk/CodeAnalyzerV2CliTest.java | 53 +++++++++++++++++++ 3 files changed, 107 insertions(+) diff --git a/README.md b/README.md index cc9905dd..656da17d 100644 --- a/README.md +++ b/README.md @@ -339,6 +339,29 @@ RUN_CONTAINER_TESTS=1 ./gradlew test `--no-build` and give the analyzer a real JDK (see the note in [Quick install](#quick-install) — the bundled `jdk4py` runtime has no `javac`). + Degrading here is deliberate, not a bug: you still get the tree, the declared call graph, the + syntactic CFG/CDG/DDG and (at `-a 4`) the SDG vertices, `param_in`/`param_out` and summaries. + Only the WALA-derived overlays — the RTA edges and the alias-aware `prov: ["points-to"]` ddg + edges — are missing. + +4. How do I tell a degraded run from a complete one in a script? + + Pass `--strict`. By default a degraded run exits **0** and reports the loss only as a `WARN` on + stderr, which is fine interactively and useless in a pipeline. `--strict` turns any missing + overlay into a non-zero exit, names what was lost, and writes no `analysis.json` — so a caller + cannot pick up a thin payload believing it is complete: + + ```console + $ codeanalyzer -i ./app -a 2 --no-build --strict + error: analysis degraded and --strict was requested: + - RTA overlay: no entrypoints; call graph is declared edges only + Build the project (or drop --no-build) to get these overlays, or rerun without --strict to + accept the degraded output. + ``` + + It is opt-in on purpose: degrading is a supported mode, and defaulting to failure would break + every caller who relies on it. + ## LICENSE ```LICENSE diff --git a/src/main/java/com/ibm/cldk/CodeAnalyzer.java b/src/main/java/com/ibm/cldk/CodeAnalyzer.java index a0a205f7..fc867c42 100644 --- a/src/main/java/com/ibm/cldk/CodeAnalyzer.java +++ b/src/main/java/com/ibm/cldk/CodeAnalyzer.java @@ -53,6 +53,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -159,6 +160,15 @@ public class CodeAnalyzer implements Runnable { + "suppress the L4 WALA build at --analysis-level 4: the semantic ddg still needs it.") private boolean noRta = false; + @Option(names = { + "--strict" }, description = "Fail with a non-zero exit when a requested analysis degrades " + + "(the RTA overlay or the L4 semantic ddg unavailable) instead of warning and " + + "continuing. Off by default: degrading is a supported mode, and this flag is for " + + "callers who need to detect it without parsing stderr.") + // Deliberately an INSTANCE field: the pre-existing options on this class are static, which leaks + // values between CommandLine instances in the same JVM. New flags do not add to that. + private boolean strict = false; + @Option(names = { "--external-calls" }, description = "Home out-of-project call targets as external_symbols at " + "--schema v2 --analysis-level 2. Off by default, matching v1's application-only call " @@ -481,6 +491,9 @@ private void analyzeV2() throws Exception { WalaAnalysis wala = null; // Run-scoped: the finders' failures accumulate across every file of one extraction. JEntrypointReport entrypointReport = new JEntrypointReport(); + // Overlays that were requested but could not be produced. Warned about either way; `--strict` + // is what turns them into a non-zero exit for a caller that cannot read stderr. + List degraded = new ArrayList<>(); try { modules = L1Extractor.extractAll( Paths.get(input), application, dependencyDir, cached, @@ -502,10 +515,17 @@ private void analyzeV2() throws Exception { wala = WalaAnalysis.of(input, deps, buildCommand).orElse(null); if (wala == null) { Log.warn("WALA L3 engine unavailable; emitting L2 declared edges only"); + degraded.add("--l3-engine wala: WALA could not build the call graph"); } rtaEndpoints = wala != null ? wala.rtaEndpoints() : java.util.List.of(); } else { rtaEndpoints = RtaCallGraph.endpoints(input, deps, buildCommand); + // An empty endpoint set means the RTA overlay contributed nothing — either the + // build produced no classes or the scope admitted none. Both leave the call graph + // with declared edges only, which is exactly what a strict caller wants to know. + if (rtaEndpoints.isEmpty()) { + degraded.add("RTA overlay: no entrypoints; call graph is declared edges only"); + } } } // Apply WALA L3 overlays while the dependency jars are still live (PDG/CFG need class files). @@ -528,6 +548,7 @@ private void analyzeV2() throws Exception { } else { Log.warn("L4 semantic ddg unavailable (WALA build failed); emitting the derived " + "SDG vertices and param edges only"); + degraded.add("L4 semantic ddg: WALA build failed, no points-to ddg edges"); } } } finally { @@ -588,6 +609,16 @@ private void analyzeV2() throws Exception { null, null, null, null, artifacts, dependencies); } + // Enforced here, after every overlay pass has had its chance and before anything is written: + // a strict run must not leave a half-written payload that looks like a complete one. + if (strict && !degraded.isEmpty()) { + throw new ParameterException(spec.commandLine(), + "error: analysis degraded and --strict was requested:\n - " + + String.join("\n - ", degraded) + + "\nBuild the project (or drop --no-build) to get these overlays, or rerun" + + " without --strict to accept the degraded output."); + } + // frameworks_detected is a union over the BUILT tree, so it must run after the modules are // final -- and it is correct on a warm cache for the same reason, where a tally kept during // the walk would not be. diff --git a/src/test/java/com/ibm/cldk/CodeAnalyzerV2CliTest.java b/src/test/java/com/ibm/cldk/CodeAnalyzerV2CliTest.java index 007333eb..5b6efcd2 100644 --- a/src/test/java/com/ibm/cldk/CodeAnalyzerV2CliTest.java +++ b/src/test/java/com/ibm/cldk/CodeAnalyzerV2CliTest.java @@ -294,6 +294,59 @@ void v2AtAnalysisLevelThreeEmitsDataflowOverlays(@TempDir Path tmp) throws IOExc assertTrue(json.contains("\"cfg\""), "level 3 lays the cfg overlay on callables"); } + @Test + void strictTurnsASilentDegradationIntoAFailure(@TempDir Path tmp) throws IOException { + // The complaint this flag answers: without it, `-a 2 --no-build` on an unbuilt project exits 0 + // with a declared-edges-only call graph and a WARN on stderr, which a scripted caller cannot + // tell apart from a complete run. + Path in = project(tmp.resolve("app")); + Path out = tmp.resolve("out"); + + assertNotEquals(0, run("-i", in.toString(), "-o", out.toString(), + "--schema", "v2", "-a", "2", "--no-build", "--strict"), + "--strict must fail when the RTA overlay could not be produced"); + } + + @Test + void strictIsOffByDefaultSoDegradingStaysASupportedMode(@TempDir Path tmp) throws IOException { + // Degrading is deliberate, not a bug: `--no-build` on an unbuilt project still yields the + // tree, the declared call graph and the syntactic overlays. --strict is opt-in precisely so + // this keeps working for everyone who relies on it. + Path in = project(tmp.resolve("app")); + Path out = tmp.resolve("out"); + + assertEquals(0, run("-i", in.toString(), "-o", out.toString(), + "--schema", "v2", "-a", "2", "--no-build"), + "without --strict the run must still exit 0 and emit declared edges"); + assertTrue(Files.exists(out.resolve("analysis.json")), + "the degraded run must still write its payload"); + } + + @Test + void strictPassesWhenNothingDegraded(@TempDir Path tmp) throws IOException { + // --strict must not fail a run that asked for nothing WALA-dependent: at -a 1 there is no + // overlay to lose, so the flag has nothing to complain about. + Path in = project(tmp.resolve("app")); + Path out = tmp.resolve("out"); + + assertEquals(0, run("-i", in.toString(), "-o", out.toString(), + "--schema", "v2", "-a", "1", "--no-build", "--strict"), + "--strict at -a 1 has no WALA overlay to lose and must pass"); + } + + @Test + void strictFailsBeforeWritingAMisleadingPayload(@TempDir Path tmp) throws IOException { + // A strict failure must not leave behind an analysis.json that looks complete — the whole + // point is that the caller cannot mistake a thin result for a full one. + Path in = project(tmp.resolve("app")); + Path out = tmp.resolve("out"); + + assertNotEquals(0, run("-i", in.toString(), "-o", out.toString(), + "--schema", "v2", "-a", "4", "--no-build", "--strict")); + assertFalse(Files.exists(out.resolve("analysis.json")), + "a --strict failure must not write a payload a caller could pick up anyway"); + } + @Test void v2WalaL3EngineDegradesClearlyWhenBuildAbsent(@TempDir Path tmp) throws IOException { // No class files present — WALA cannot build the call graph; must exit 0 with declared