fix: report incomplete batch gap-fill analysis

contrib/batch_scan/__init__.py

@@ -48,7 +48,7 @@
 from .api_pool import ApiKey, ApiKeyPool, PooledChatModel, create_api_key_pool_from_env
 from .detection import detect_language, detect_skill_language
 from .discovery import discover_skills
-from .gap_fill import GapFillAnalyzer, GapFillFinding, GapFillResult, run_gap_fill
+from .gap_fill import GapFillAnalyzer, GapFillError, GapFillFinding, GapFillResult, run_gap_fill
 from .runner import run_one
 
 __all__ = [
@@ -60,6 +60,7 @@
     "detect_skill_language",
     "discover_skills",
     "GapFillAnalyzer",
+    "GapFillError",
     "GapFillFinding",
     "GapFillResult",
     "is_language_compatible",

contrib/batch_scan/batch_scan.py

@@ -116,6 +116,7 @@ def _scan_skill(
     require_llm: bool,
     api_pool=None,
     verbose: bool = False,
+    timeout: float = 90,
 ) -> tuple[dict[str, object], str | None, str]:
     """Scan a single skill through the full pipeline.
 
@@ -139,6 +140,7 @@ def _scan_skill(
         detected_language=lang,
         apply_gap_fill=True,
         api_pool=api_pool,
+        timeout=timeout,
     )
 
     return entry, error_msg, rel_name
@@ -205,6 +207,7 @@ def _scan_skill_bounded(
     startup_timeout: float = 90, **options
 ) -> tuple[dict[str, object], str | None, str]:
     """Enforce the wall-clock limit on actual work, including local analysis."""
+    options["timeout"] = timeout
     owner = uuid4().hex
     if api_pool is not None:
         options["api_pool"] = SimpleNamespace(

contrib/batch_scan/docs/README.md

@@ -202,7 +202,8 @@ python -m contrib.batch_scan.batch_scan ./tests/fixtures/ -f json -o batch.json
 
 Key differences in batch output:
 - `scan_mode: "multilingual-enhanced"` — provenance marker
-- `enhancements.gap_fill_applied` — true if LLM gap-fill was used
+- `enhancements.gap_fill_applied` — true only if every gap-fill batch completed;
+  partial passes can still have `gap_fill_findings > 0`
 - `enhancements.english_keyword_rules_skipped` — count of static rules bypassed
 - `skill.language` — detected language tag
 
@@ -313,7 +314,19 @@ python -m contrib.batch_scan.batch_scan ./tests/fixtures/ --no-require-llm --no-
 |------|---------|
 | 0 | All safe (no HIGH/CRITICAL) |
 | 1 | ≥1 skill has HIGH or CRITICAL risk |
-| 2 | Scan errors occurred |
+| 2 | Scan errors or incomplete gap-fill occurred; `skills[].error` or `enhancements.gap_fill_error` explains the failure |
+
+
+Gap-fill uses the remaining per-skill budget after core analysis, leaving up to
+five seconds for reporting before the worker's 90-second hard limit. Its pooled
+calls receive that remaining budget; the provider pool otherwise uses its
+caller's timeout. Invalid responses can take up to four attempts within it.
+Failed setup and failed batches retain core findings and completed gap-fill
+findings. An incomplete pass records `enhancements.gap_fill_status: "incomplete"`,
+`gap_fill_error_reasons`, and `enhancements.gap_fill_error`, and exits 2 even when
+its ledger outcome is nonfatal. This deadline covers gap-fill only. Core analysis
+keeps its own budget and can reach the worker limit even when providers respect
+request timeouts. The worker limit also applies if a provider ignores its timeout.
 
 CI usage:
 
@@ -415,7 +428,7 @@ tests/
 ├── test_monkeypatch_fragility.py     ← Issue #2: 26 tests (guard verification)
 ├── tests-pro/
 │   ├── test_api_pool.py              ← Issue #3: 45 tests (acquire/backoff)
-│   ├── test_gap_fill.py              ← Issue #3: 41 tests (JSON parsing)
+│   ├── test_gap_fill.py              ← Issue #3: 29 tests (JSON parsing)
 │   ├── test_runner_patches.py        ← Issue #3: 24 tests (context manager)
 │   └── test_annotation.py            ← Issue #3: 10 tests (language compat)
 └── docs/

contrib/batch_scan/gap_fill.py

@@ -28,12 +28,21 @@
 
 from __future__ import annotations
 
+from collections.abc import Callable
 from typing import Literal
 
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, Field, ValidationError
 
 from skillspector.constants import MODEL_CONFIG
-from skillspector.llm_analyzer_base import LLMAnalyzerBase
+from skillspector.inspection_ledger import LedgerReason
+from skillspector.llm_analyzer_base import (
+    Batch,
+    BatchExecutionResult,
+    BatchFailure,
+    LLMRuntimeLimitError,
+    LLMAnalyzerBase,
+    _StructuredResponseValidationError,
+)
 from skillspector.logging_config import get_logger
 from skillspector.models import Finding
 
@@ -87,7 +96,16 @@ def to_finding(self, file: str) -> Finding:
 class GapFillResult(BaseModel):
     """Structured LLM response for the gap-fill analyzer."""
 
-    findings: list[GapFillFinding] = Field(default_factory=list)
+    findings: list[GapFillFinding]
+
+
+class GapFillError(RuntimeError):
+    """An incomplete pass, retaining successful batches for the report."""
+
+    def __init__(self, outcome: BatchExecutionResult):
+        self.outcome = outcome
+        self.findings = [finding for _, findings in outcome.successful for finding in findings]
+        super().__init__(f"Gap-fill analysis did not complete for {len(outcome.failures)} batch(es).")
 
 
 # ---------------------------------------------------------------------------
@@ -179,12 +197,12 @@ class GapFillAnalyzer(LLMAnalyzerBase):
     # response_format.  JSON is parsed manually in parse_response().
     response_schema: type | None = None
 
-    def __init__(self, language: str, model: str | None = None, api_pool: "ApiKeyPool | None" = None):
+    def __init__(self, language: str, model: str | None = None, api_pool: "ApiKeyPool | None" = None, *, timeout: float | None | Callable[[], float | None] = None):
         self.language = language
         resolved_model = model or MODEL_CONFIG.get("default", "gpt-5.4")
         # Inject language into the base prompt before passing to parent
         prompt = GAP_FILL_ANALYZER_PROMPT.format(language=language)
-        super().__init__(base_prompt=prompt, model=resolved_model)
+        super().__init__(base_prompt=prompt, model=resolved_model, timeout=timeout)
         # Wire multi-key pool into gap-fill LLM calls
         if api_pool:
             from .api_pool import PooledChatModel
@@ -209,38 +227,12 @@ def parse_response(self, response, batch):
         Because ``response_schema`` is ``None``, *response* is a raw string
         (not a Pydantic model).  We strip markdown code fences, parse JSON,
         validate with :class:`GapFillResult`, and filter to ``confidence >= 0.7``.
+        Invalid JSON or schema raises ``_StructuredResponseValidationError`` for retry.
         """
-        text = str(response).strip()
-
-        # Strip markdown code fences if present
-        if text.startswith("```"):
-            first_nl = text.find("\n")
-            if first_nl != -1:
-                text = text[first_nl + 1:]
-            if text.rstrip().endswith("```"):
-                text = text.rstrip()[:-3].rstrip()
-
-        # Parse JSON → Pydantic for validation
-        import json
-        try:
-            data = json.loads(text)
-        except json.JSONDecodeError as exc:
-            logger.warning(
-                "GapFillAnalyzer: invalid JSON for %s: %s",
-                batch.file_label,
-                exc,
-            )
-            return []
-
-        try:
-            result = GapFillResult.model_validate(data)
-        except Exception as exc:
-            logger.warning(
-                "GapFillAnalyzer: schema validation failed for %s: %s",
-                batch.file_label,
-                exc,
-            )
-            return []
+        if isinstance(response, GapFillResult):
+            result = response
+        else:
+            result = self._parse_json_response(response)
 
         findings: list[Finding] = []
         for item in result.findings:
@@ -256,9 +248,29 @@ def parse_response(self, response, batch):
             findings.append(item.to_finding(batch.file_path))
         return findings
 
+    @staticmethod
+    def _parse_json_response(response):
+        text = str(response).lstrip("\ufeff").strip()
+
+        # Strip markdown code fences if present
+        if text.startswith("```"):
+            first_nl = text.find("\n")
+            if first_nl != -1:
+                text = text[first_nl + 1:]
+            if text.rstrip().endswith("```"):
+                text = text.rstrip()[:-3].rstrip()
+
+        # Parse JSON → Pydantic for validation
+        import json
+        try:
+            data = json.loads(text)
+            return GapFillResult.model_validate(data)
+        except (json.JSONDecodeError, ValidationError) as exc:
+            raise _StructuredResponseValidationError from exc
+
 
 # ---------------------------------------------------------------------------
-# Backward-compatible entry point
+# Entry point preserving partial results
 # ---------------------------------------------------------------------------
 
 
@@ -267,6 +279,8 @@ def run_gap_fill(
     language: str,
     model: str | None = None,
     api_pool: "ApiKeyPool | None" = None,
+    *,
+    timeout: float | None | Callable[[], float | None] = None,
 ) -> list[Finding]:
     """Run a single targeted LLM pass covering the 8 gap-fill rules.
 
@@ -283,23 +297,50 @@ def run_gap_fill(
         Detected language string (``"zh"``, ``"ja"``, ``"ko"``, ``"en"``).
     model :
         Optional model override.  Falls back to the configured default.
+    timeout :
+        Remaining gap-fill budget in seconds, or a callable returning it before
+        each request. Does not set the core graph's analysis budget.
 
     Returns
     -------
     list[Finding]
         A (possibly empty) list of gap-fill findings.  Only findings with
         ``confidence >= 0.7`` are included.
+
+    Raises
+    ------
+    GapFillError
+        Any setup or batch failure. ``outcome`` contains per-batch evidence and
+        ``findings`` retains results from completed batches.
     """
     if not file_cache:
         return []
 
     try:
-        analyzer = GapFillAnalyzer(language=language, model=model, api_pool=api_pool)
+        analyzer = GapFillAnalyzer(language=language, model=model, api_pool=api_pool, timeout=timeout)
         batches = analyzer.get_batches(list(file_cache.keys()), file_cache)
-        results = analyzer.run_batches(batches, language=language)
-        return analyzer.collect_findings(results)
-    except ValueError:
-        raise
     except Exception as exc:
-        logger.warning("Gap-fill analysis failed: %s", exc)
-        return []
+        reason = LedgerReason.RUNTIME_LIMIT if isinstance(exc, LLMRuntimeLimitError) else LedgerReason.LLM_BATCH_FAILED
+        outcome = BatchExecutionResult(failures=[
+            BatchFailure(Batch(file_path=path, content=content), error_class=type(exc).__name__, reason=reason)
+            for path, content in file_cache.items()
+        ])
+        raise GapFillError(outcome) from exc
+    outcome = BatchExecutionResult()
+    for index, batch in enumerate(batches):
+        try:
+            completed = analyzer.run_batches_detailed([batch], language=language)
+        except Exception as exc:
+            # Core deliberately propagates some provider errors. Keep earlier
+            # results and record every batch that could not finish in this pass.
+            reason = LedgerReason.RUNTIME_LIMIT if isinstance(exc, LLMRuntimeLimitError) else LedgerReason.LLM_BATCH_FAILED
+            outcome.failures.extend(
+                BatchFailure(pending, error_class=type(exc).__name__, reason=reason)
+                for pending in batches[index:]
+            )
+            raise GapFillError(outcome) from exc
+        outcome.successful.extend(completed.successful)
+        outcome.failures.extend(completed.failures)
+    if outcome.failures:
+        raise GapFillError(outcome)
+    return analyzer.collect_findings(outcome.successful)

contrib/batch_scan/runner.py

@@ -34,23 +34,26 @@
 import json
 import os
 import shutil
+import time
 from datetime import UTC, datetime
 from pathlib import Path
 
 from pydantic import ValidationError
 
 from skillspector.graph import graph
+from skillspector.inspection_ledger import finalize_ledger
 from skillspector.llm_analyzer_base import (
     LLMAnalyzerBase,
     LLMAnalysisResult,
     _StructuredResponseValidationError,
+    ledger_events_for_batches,
 )
 from skillspector.logging_config import get_logger
 from skillspector.nodes.meta_analyzer import LLMMetaAnalyzer, MetaAnalyzerResult
 
 from .annotation import annotate_findings
 from .detection import detect_skill_language
-from .gap_fill import run_gap_fill
+from .gap_fill import GapFillError, run_gap_fill
 
 logger = get_logger(__name__)
 
@@ -725,6 +728,7 @@ def run_one(
     gap_fill_findings: int = 0,
     apply_gap_fill: bool = False,
     api_pool=None,
+    timeout: float = 90,
 ) -> tuple[dict[str, object], str | None]:
     """Scan a single skill through the full graph pipeline.
 
@@ -747,13 +751,20 @@ def run_one(
         Run non-English gap-fill against the graph's provider-eligible cache.
     api_pool :
         Optional API key pool for gap-fill calls.
+    timeout :
+        Per-skill worker budget in seconds. Gap-fill uses the time remaining
+        after core analysis, with reporting headroom. Core analysis keeps its
+        own budget and can still reach the worker's process timeout.
 
     Returns
     -------
     ``(entry, error_message_or_None)`` — on success *error_message*
-    is ``None``; on failure *entry* is a stub error entry and
-    *error_message* carries the exception text.
+    is ``None``. Gap-fill failure retains the core entry and adds incomplete
+    coverage and a machine-readable error. A failure before the core result
+    returns a stub error entry.
     """
+    # Leave time to serialize partial findings before the worker hard limit.
+    deadline = time.monotonic() + max(0, timeout - min(5.0, timeout / 10))
     result = None
     try:
         state = scan_state(skill_dir, use_llm=use_llm)
@@ -771,12 +782,38 @@ def run_one(
             gap_fill_findings=gap_fill_findings,
         )
         if apply_gap_fill and use_llm and detected_language != "en":
-            gap_findings = run_gap_fill(file_cache, detected_language, api_pool=api_pool)
+            gap_error = None
+            try:
+                gap_findings = run_gap_fill(
+                    file_cache, detected_language, api_pool=api_pool,
+                    timeout=lambda: deadline - time.monotonic(),
+                )
+            except GapFillError as exc:
+                gap_error = exc
+                gap_findings = exc.findings
             entry["issues"] = list(entry.get("issues", [])) + annotate_findings(
                 [finding.to_dict() for finding in gap_findings], detected_language
             )
-            entry["enhancements"]["gap_fill_applied"] = True
+            entry["enhancements"]["gap_fill_applied"] = bool(file_cache) and gap_error is None
             entry["enhancements"]["gap_fill_findings"] = len(gap_findings)
+            if gap_error is not None:
+                events, status = ledger_events_for_batches("gap_fill", gap_error.outcome)
+                completeness, _ = finalize_ledger({
+                    **result,
+                    "findings": [*(result.get("findings") or []), *gap_findings],
+                    "inspection_ledger": [*(result.get("inspection_ledger") or []), *events],
+                    "analyzer_status_events": [*(result.get("analyzer_status_events") or []), status],
+                })
+                entry["analysis_completeness"] = completeness
+                entry["execution_successful"] = completeness["execution_successful"]
+                if entry["risk_assessment"]["recommendation"] == "SAFE":
+                    entry["risk_assessment"]["recommendation"] = "CAUTION"
+                entry["enhancements"]["gap_fill_error"] = str(gap_error)
+                entry["enhancements"]["gap_fill_status"] = "incomplete"
+                entry["enhancements"]["gap_fill_error_reasons"] = sorted({
+                    failure.reason.value for failure in gap_error.outcome.failures
+                })
+                return entry, str(gap_error)
         return entry, None
     except Exception as exc:
         return entry_from_error(skill_dir, root, str(exc), detected_language), str(exc)