diff --git a/application/cmd/cre_main.py b/application/cmd/cre_main.py index 09abb66b4..e207e0479 100644 --- a/application/cmd/cre_main.py +++ b/application/cmd/cre_main.py @@ -911,6 +911,25 @@ def run(args: argparse.Namespace) -> None: # pragma: no cover sys.exit(1) return + if str(getattr(args, "ingest_github", "") or "").strip(): + import sys + + from application import sqla + from application.utils.harvester.github_url_ingest import ingest_github_url + + db_connect(args.cache_file) + counts = ingest_github_url( + str(args.ingest_github).strip(), + session=sqla.session, + cache_file=args.cache_file, + branch_override=(getattr(args, "ingest_branch", "") or "").strip() or None, + keep_all=bool(getattr(args, "ingest_keep_all", False)), + model_override=(getattr(args, "ingest_model", "") or "").strip() or None, + ) + if counts.errors or (counts.knowledge + counts.uncertain) == 0: + sys.exit(1) + return + if getattr(args, "run_noise_filter", False): import sys diff --git a/application/tests/harvester_test/chunk_merger_test.py b/application/tests/harvester_test/chunk_merger_test.py new file mode 100644 index 000000000..ddea7ad4a --- /dev/null +++ b/application/tests/harvester_test/chunk_merger_test.py @@ -0,0 +1,188 @@ +"""Tests for Module A.2 chunk merge profiles.""" + +from __future__ import annotations + +import unittest +from datetime import datetime, timezone + +from application.utils.harvester.chunk_merger import merge_chunks, requirement_ids +from application.utils.harvester.models import ( + ChunkInfo, + Document, + HeadingNode, + Locator, + SourceInfo, +) +from application.utils.harvester.schemas import ChunkingConfig + + +def _doc(text: str, headings: list[HeadingNode] | None = None) -> Document: + return Document( + schema_version="0.2.0", + artifact_id="art:OWASP/Test:x.md", + pipeline_run_id="test", + text=text, + source=SourceInfo( + type="github", + repository="OWASP/Test", + commit_sha="abc", + committed_at=datetime.now(timezone.utc), + ), + locator=Locator(kind="repo_path", id="x.md", path="x.md"), + heading_structure=headings or [], + ) + + +class ChunkMergerTests(unittest.TestCase): + def test_requirement_ids_extracted(self) -> None: + self.assertEqual( + requirement_ids("**2.1.1** Verify passwords"), frozenset({"2.1.1"}) + ) + self.assertIn("2.1.1", requirement_ids("V2.1.1 something")) + self.assertEqual( + requirement_ids("AC-2 Account Management"), frozenset({"ac-2"}) + ) + self.assertIn("a.5.1", requirement_ids("ISO A.5.1 policies")) + self.assertIn("3.4", requirement_ids("Req 3.4 Protect stored")) + self.assertEqual( + requirement_ids("Requirement 3.4.1 hashing"), frozenset({"3.4.1"}) + ) + + def test_requirements_does_not_merge_nist_or_iso_ids(self) -> None: + t1 = ("AC-2 Account Management. " * 20).strip() + t2 = ("AC-3 Access Enforcement. " * 20).strip() + text = t1 + "\n\n" + t2 + mid = len(t1) + 2 + chunks = [ + ChunkInfo(text=t1, start_char_idx=0, end_char_idx=len(t1)), + ChunkInfo(text=t2, start_char_idx=mid, end_char_idx=len(text)), + ] + headings = [ + HeadingNode(level=2, text="Access Control", start_line=1, end_line=99) + ] + doc = _doc(text, headings) + cfg = ChunkingConfig( + strategy="docling", + max_tokens=1200, + overlap_tokens=10, + merge_profile="requirements", + ) + out = merge_chunks(doc, chunks, cfg) + self.assertEqual(len(out), 2) + + def test_none_profile_is_noop(self) -> None: + text = "aaaa " * 50 + "\n\n" + "bbbb " * 50 + mid = text.index("bbbb") + chunks = [ + ChunkInfo(text=text[:mid], start_char_idx=0, end_char_idx=mid), + ChunkInfo(text=text[mid:], start_char_idx=mid, end_char_idx=len(text)), + ] + cfg = ChunkingConfig( + strategy="docling", max_tokens=1200, overlap_tokens=10, merge_profile="none" + ) + doc = _doc(text) + out = merge_chunks(doc, chunks, cfg) + self.assertEqual(len(out), 2) + + def test_requirements_does_not_merge_across_ids(self) -> None: + t1 = ("Verify 2.1.1 passwords are long. " * 20).strip() + t2 = ("Verify 2.1.2 passwords allow unicode. " * 20).strip() + text = t1 + "\n\n" + t2 + mid = len(t1) + 2 + chunks = [ + ChunkInfo(text=t1, start_char_idx=0, end_char_idx=len(t1)), + ChunkInfo(text=t2, start_char_idx=mid, end_char_idx=len(text)), + ] + headings = [ + HeadingNode( + level=2, text="V2.1 Password Security", start_line=1, end_line=99 + ) + ] + doc = _doc(text, headings) + cfg = ChunkingConfig( + strategy="docling", + max_tokens=1200, + overlap_tokens=10, + merge_profile="requirements", + ) + out = merge_chunks(doc, chunks, cfg) + self.assertEqual(len(out), 2) + + def test_narrative_merges_same_heading_crumbs(self) -> None: + # Same heading, tiny crumbs → one merged chunk under narrative profile. + a = "Short intro." + b = "More detail on the same topic." + text = a + "\n\n" + b + mid = len(a) + 2 + chunks = [ + ChunkInfo(text=a, start_char_idx=0, end_char_idx=len(a)), + ChunkInfo(text=b, start_char_idx=mid, end_char_idx=len(text)), + ] + headings = [ + HeadingNode(level=2, text="Password Storage", start_line=1, end_line=99) + ] + doc = _doc(text, headings) + cfg = ChunkingConfig( + strategy="docling", + max_tokens=1000, + overlap_tokens=10, + merge_profile="narrative", + ) + out = merge_chunks(doc, chunks, cfg) + self.assertEqual(len(out), 1) + self.assertIn("Short intro", out[0].text) + self.assertIn("More detail", out[0].text) + + def test_narrative_merges_h3_under_same_h2(self) -> None: + text = ( + "## Auth\n\n" + "### Intro\n\n" + "First crumb about auth.\n\n" + "### Detail\n\n" + "Second crumb still under Auth." + ) + # Approximate char ranges for two leaves under different ### but same ##. + first = "First crumb about auth." + second = "Second crumb still under Auth." + i1 = text.index(first) + i2 = text.index(second) + chunks = [ + ChunkInfo(text=first, start_char_idx=i1, end_char_idx=i1 + len(first)), + ChunkInfo(text=second, start_char_idx=i2, end_char_idx=i2 + len(second)), + ] + headings = [ + HeadingNode(level=2, text="Auth", start_line=1, end_line=20), + HeadingNode(level=3, text="Intro", start_line=3, end_line=6), + HeadingNode(level=3, text="Detail", start_line=8, end_line=20), + ] + doc = _doc(text, headings) + cfg = ChunkingConfig( + strategy="docling", + max_tokens=1000, + overlap_tokens=10, + merge_profile="narrative", + ) + out = merge_chunks(doc, chunks, cfg) + self.assertEqual(len(out), 1) + + def test_profile_defaults_on_config(self) -> None: + req = ChunkingConfig( + strategy="fixed_size", + max_tokens=500, + overlap_tokens=10, + merge_profile="requirements", + ) + self.assertTrue(req.split_on_requirement_id) + self.assertEqual(req.merge_min_tokens, 100) + nar = ChunkingConfig( + strategy="fixed_size", + max_tokens=500, + overlap_tokens=10, + merge_profile="narrative", + ) + self.assertFalse(nar.split_on_requirement_id) + self.assertEqual(nar.merge_min_tokens, 400) + + +if __name__ == "__main__": + unittest.main() diff --git a/application/tests/harvester_test/chunk_record_builder_test.py b/application/tests/harvester_test/chunk_record_builder_test.py index f128a19ab..d01882923 100644 --- a/application/tests/harvester_test/chunk_record_builder_test.py +++ b/application/tests/harvester_test/chunk_record_builder_test.py @@ -56,11 +56,46 @@ def test_builds_change_record_shaped_payload(self) -> None: self.assertEqual(record.source_repo, "OWASP/ASVS") self.assertEqual(record.locator_path, "README.md") self.assertEqual(record.span.heading_path, ["Root"]) + self.assertTrue(record.text.startswith("Standard: ASVS\n")) + self.assertIn("Source: README.md\n", record.text) + self.assertIn("Section: Root\n", record.text) + self.assertIn("First paragraph.", record.text) ChunkRecordValidator().validate(record) payload = ingest_record_to_payload(record) ChangeRecord.model_validate(payload) + def test_prefixes_section_id_from_filename(self) -> None: + text = "# Broken Access Control\n\nDeny by default." + document = Document( + schema_version="0.2.0", + artifact_id="art:B2/owasp_top10_2025:A01", + pipeline_run_id="run-1", + text=text, + source=SourceInfo( + type="github", + repository="B2/owasp_top10_2025", + commit_sha="abc1234", + committed_at=datetime(2026, 2, 1, 1, 0, 0, tzinfo=timezone.utc), + ), + locator=Locator( + kind="repo_path", + id="b2/owasp_top10_2025/A01.txt", + path="b2/owasp_top10_2025/A01.txt", + ), + heading_structure=[ + HeadingNode( + level=1, text="Broken Access Control", start_line=1, end_line=3 + ) + ], + ) + chunk = ChunkInfo(text=text, start_char_idx=0, end_char_idx=len(text)) + record = ChunkRecordBuilder().build(document, [chunk])[0] + self.assertIn("Standard: owasp_top10_2025", record.text) + self.assertIn("Version: 2025", record.text) + self.assertIn("Section-ID: A01", record.text) + self.assertIn("Section: Broken Access Control", record.text) + def test_indexes_multiple_chunks(self) -> None: text = "AAAA\n\nBBBB" document = self._document(text, []) @@ -74,6 +109,52 @@ def test_indexes_multiple_chunks(self) -> None: self.assertEqual(records[1].span.index, 1) self.assertEqual(records[1].chunk_id, "chk:art:OWASP/ASVS:README.md:1") + def test_requirement_extract_chunk_keeps_single_section_id(self) -> None: + """Do not re-scrape heading/path ids over a clean extractor prefix.""" + body = ( + "Source: V1.1.2\n" + "Section-ID: V1.1.2\n" + "Section: Verify that output encoding is applied.\n\n" + "Verify that output encoding is applied." + ) + document = Document( + schema_version="0.2.0", + artifact_id="art:OWASP/ASVS:0x10-V1.md", + pipeline_run_id="run-1", + text="# V1 Encoding\n\n## V1.1 Architecture\n\n" + body, + source=SourceInfo( + type="github", + repository="OWASP/ASVS", + commit_sha="abc1234deadbeef", + committed_at=datetime(2026, 2, 1, 1, 0, 0, tzinfo=timezone.utc), + ), + locator=Locator( + kind="repo_path", + id="0x10-V1.md", + path="5.0/en/0x10-V1-Encoding-and-Sanitization.md", + ), + heading_structure=[ + HeadingNode(level=1, text="V1 Encoding", start_line=1, end_line=10), + HeadingNode( + level=2, text="V1.1 Architecture", start_line=3, end_line=10 + ), + ], + ) + chunk = ChunkInfo(text=body, start_char_idx=0, end_char_idx=len(body)) + record = ChunkRecordBuilder().build(document, [chunk])[0] + sid_lines = [ + ln for ln in record.text.splitlines() if ln.startswith("Section-ID:") + ] + src_lines = [ln for ln in record.text.splitlines() if ln.startswith("Source:")] + # Do not prepend a second Source:/Section-ID: over extractor identity. + self.assertEqual(len(sid_lines), 1) + self.assertEqual(sid_lines[0], "Section-ID: V1.1.2") + self.assertEqual(len(src_lines), 1) + self.assertEqual(src_lines[0], "Source: V1.1.2") + self.assertNotIn("V1.1,", record.text) + self.assertNotIn("5.0,", record.text) + self.assertNotIn("0x10-V1.md", record.text) + if __name__ == "__main__": unittest.main() diff --git a/application/tests/harvester_test/config_loader_test.py b/application/tests/harvester_test/config_loader_test.py index f3dd1795a..14445b55c 100644 --- a/application/tests/harvester_test/config_loader_test.py +++ b/application/tests/harvester_test/config_loader_test.py @@ -99,9 +99,8 @@ def test_load_packaged_repos_yaml(self): config = load_repo_config(config_path) - self.assertEqual(len(config.repositories), 2) - self.assertEqual(config.repositories[0].id, "owasp-asvs") - self.assertEqual(config.repositories[1].id, "owasp-cheatsheets") + ids = [repo.id for repo in config.repositories] + self.assertEqual(ids, ["owasp-asvs", "owasp-aisvs", "owasp-cheatsheets"]) def test_empty_owner(self): config_path = FIXTURES_DIR / "empty_owner.yaml" diff --git a/application/tests/harvester_test/github_url_ingest_test.py b/application/tests/harvester_test/github_url_ingest_test.py new file mode 100644 index 000000000..56af0e699 --- /dev/null +++ b/application/tests/harvester_test/github_url_ingest_test.py @@ -0,0 +1,223 @@ +"""Unit tests for point-and-shoot GitHub URL ingest helpers.""" + +from __future__ import annotations + +import io +import tarfile +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +from application import create_app, sqla +from application.database.db import KnowledgeQueueItem +from application.utils.harvester import github_url_ingest as ingest +from application.utils.harvester.github_url_ingest import ( + format_knowledge_item, + parse_github_url, +) +from application.utils.noise_filter.queue_writer import write_verdicts +from application.utils.noise_filter.schemas import ChangeRecord, ClassifyResult + + +class ParseGitHubUrlTests(unittest.TestCase): + def test_owner_repo(self) -> None: + p = parse_github_url("https://github.com/OWASP/ASVS") + self.assertEqual(p.owner, "OWASP") + self.assertEqual(p.repo, "ASVS") + self.assertIsNone(p.branch) + self.assertIsNone(p.path_prefix) + + def test_git_suffix_and_www(self) -> None: + p = parse_github_url("https://www.github.com/OWASP/ASVS.git") + self.assertEqual(p.owner, "OWASP") + self.assertEqual(p.repo, "ASVS") + + def test_tree_branch_and_path(self) -> None: + p = parse_github_url("https://github.com/OWASP/ASVS/tree/master/5.0/en") + self.assertEqual(p.branch, "master") + self.assertEqual(p.path_prefix, "5.0/en") + + def test_rejects_non_github(self) -> None: + with self.assertRaises(ValueError): + parse_github_url("https://gitlab.com/OWASP/ASVS") + + def test_rejects_unsafe_owner_repo_and_branch(self) -> None: + for bad in ( + "https://github.com/OWASP/..", + "https://github.com/-bad/ASVS", + "https://github.com/OWASP/AS%56S", + "https://github.com/OWASP/ASVS/tree/a..b/x", + ): + with self.subTest(url=bad), self.assertRaises(ValueError): + parse_github_url(bad) + + +class FormatKnowledgeItemTests(unittest.TestCase): + def test_includes_path_and_label(self) -> None: + out = format_knowledge_item( + path="5.0/en/V1.md", + label="KNOWLEDGE", + confidence=0.91, + text="Verify that input is sanitized.", + reasoning="looks like a control", + ) + self.assertIn("path: 5.0/en/V1.md", out) + self.assertIn("label: KNOWLEDGE", out) + self.assertIn("Verify that input is sanitized.", out) + + +if __name__ == "__main__": + unittest.main() + + +def _tarball(path: Path, members: dict) -> None: + with tarfile.open(path, "w:gz") as tf: + for name, data in members.items(): + info = tarfile.TarInfo(name) + info.size = len(data) + tf.addfile(info, io.BytesIO(data)) + + +class DownloadTarballTests(unittest.TestCase): + def test_explicit_branch_does_not_fall_back(self) -> None: + requested = [] + + def fake_download(url: str, dest: Path) -> None: + requested.append(url) + raise OSError("404") + + with tempfile.TemporaryDirectory() as tmp, patch.object( + ingest, "_download_to_file", side_effect=fake_download + ): + with self.assertRaises(RuntimeError): + ingest._download_tarball( + "OWASP", + "ASVS", + "release", + Path(tmp) / "x.tgz", + allow_fallback=False, + ) + self.assertEqual(len(requested), 1) + self.assertTrue(requested[0].endswith("/tar.gz/refs/heads/release")) + + def test_default_branch_falls_back_to_main_and_master(self) -> None: + requested = [] + + def fake_download(url: str, dest: Path) -> None: + requested.append(url.rsplit("/", 1)[-1]) + raise OSError("404") + + with tempfile.TemporaryDirectory() as tmp, patch.object( + ingest, "_download_to_file", side_effect=fake_download + ): + with self.assertRaises(RuntimeError): + ingest._download_tarball("OWASP", "ASVS", "dev", Path(tmp) / "x.tgz") + self.assertEqual(requested, ["dev", "main", "master"]) + + +class IterMdLimitsTests(unittest.TestCase): + def test_oversized_member_is_skipped(self) -> None: + small = b"# Title\n" + b"security requirement text " * 5 + big = b"x" * (ingest.MAX_MD_MEMBER_BYTES + 1) + with tempfile.TemporaryDirectory() as tmp: + tgz = Path(tmp) / "a.tgz" + _tarball(tgz, {"repo-main/ok.md": small, "repo-main/big.md": big}) + found = ingest._iter_md(tgz, path_prefix=None) + self.assertEqual([rel for rel, _ in found], ["ok.md"]) + + def test_total_bytes_cap_stops_reading(self) -> None: + body = b"# T\n" + b"security requirement text " * 8 + with tempfile.TemporaryDirectory() as tmp, patch.object( + ingest, "MAX_MD_TOTAL_BYTES", len(body) + 1 + ): + tgz = Path(tmp) / "a.tgz" + _tarball(tgz, {f"repo-main/{i}.md": body for i in range(3)}) + found = ingest._iter_md(tgz, path_prefix=None) + self.assertEqual(len(found), 1) + + +def _queue_record(run_id: str, text: str) -> ChangeRecord: + return ChangeRecord.model_validate( + { + "schema_version": "0.2.0", + "chunk_id": f"chk-{run_id}", + "artifact_id": f"art:{run_id}", + "pipeline_run_id": run_id, + "text": text, + "span": {"index": 0, "total": 1, "heading_path": []}, + "source": { + "type": "github", + "repo": "OWASP/Test", + "commit_sha": "abc123", + "committed_at": "2026-07-17T00:00:00Z", + }, + "locator": {"kind": "repo_path", "id": "p.md", "path": "p.md"}, + } + ) + + +class IngestQueueScopingTests(unittest.TestCase): + def setUp(self) -> None: + self.app = create_app(mode="test") + self.ctx = self.app.app_context() + self.ctx.push() + sqla.create_all() + verdict = ClassifyResult(label="KNOWLEDGE", confidence=0.9, reasoning="r") + write_verdicts( + sqla.session, + [ + (_queue_record("harvest-1", "harvested requirement"), verdict, "h-a"), + ( + _queue_record("ingest-old", "older ingest requirement"), + verdict, + "h-b", + ), + ], + ) + sqla.session.commit() + self.tmp = tempfile.TemporaryDirectory() + + def tearDown(self) -> None: + self.tmp.cleanup() + sqla.session.remove() + sqla.drop_all() + self.ctx.pop() + + def _run_ids(self) -> set: + return {r.pipeline_run_id for r in KnowledgeQueueItem.query.all()} + + def test_failed_download_keeps_all_rows(self) -> None: + with patch.object(ingest, "TARBALL_DIR", Path(self.tmp.name)), patch.object( + ingest, "_download_to_file", side_effect=OSError("down") + ): + counts = ingest.ingest_github_url( + "https://github.com/OWASP/Test", + session=sqla.session, + cache_file="", + keep_all=True, + ) + self.assertEqual(counts.errors, 1) + self.assertEqual(self._run_ids(), {"harvest-1", "ingest-old"}) + + def test_successful_ingest_replaces_only_prior_ingest_rows(self) -> None: + body = ( + b"# Authentication\n\n" + b"Verify that all authentication controls fail securely and log failures.\n" + ) + + def fake_download(url: str, dest: Path) -> None: + _tarball(dest, {"Test-main/V1.md": body}) + + with patch.object(ingest, "TARBALL_DIR", Path(self.tmp.name)), patch.object( + ingest, "_download_to_file", side_effect=fake_download + ): + counts = ingest.ingest_github_url( + "https://github.com/OWASP/Test", + session=sqla.session, + cache_file="", + keep_all=True, + run_id="ingest-new", + ) + self.assertEqual(counts.errors, 0) + self.assertEqual(self._run_ids(), {"harvest-1", "ingest-new"}) diff --git a/application/tests/harvester_test/harvest_pipeline_test.py b/application/tests/harvester_test/harvest_pipeline_test.py index c21b615a1..f851bcf81 100644 --- a/application/tests/harvester_test/harvest_pipeline_test.py +++ b/application/tests/harvester_test/harvest_pipeline_test.py @@ -125,6 +125,56 @@ def test_emits_valid_change_records(self) -> None: } ) + def test_requirement_extract_auto_emits_section_id_chunks(self) -> None: + text = """\ +# V1 Encoding + +| # | Description | Level | +| :---: | :--- | :---: | +| **1.1.1** | Verify that input is decoded once. | 2 | +| **1.1.2** | Verify that output encoding is applied. | 2 | +| **1.1.3** | Verify that encoding preserves integrity. | 2 | +| **1.2.1** | Verify that SQL injection is blocked. | 1 | +| **1.2.2** | Verify that OS command injection is blocked. | 1 | +""" + document = Document( + schema_version="0.2.0", + artifact_id="art:OWASP/ASVS:v1.md", + pipeline_run_id="run-extract", + text=text, + source=SourceInfo( + type="github", + repository="OWASP/ASVS", + commit_sha="abc1234deadbeef", + committed_at=datetime(2026, 2, 1, tzinfo=timezone.utc), + ), + locator=Locator(kind="repo_path", id="v1.md", path="5.0/en/0x10-V1.md"), + heading_structure=[ + HeadingNode(level=1, text="V1 Encoding", start_line=1, end_line=10) + ], + ) + pipeline = DocumentChunkPipeline( + chunking=ChunkingConfig( + strategy="markdown_heading", + max_tokens=1200, + overlap_tokens=100, + merge_profile="requirements", + requirement_extract="auto", + ) + ) + records = pipeline.chunk(document) + self.assertGreaterEqual(len(records), 5) + joined = "\n".join(r.text for r in records) + self.assertIn("Section-ID: V1.1.2", joined) + self.assertIn("Section-ID: V1.2.2", joined) + for record in records: + sid_lines = [ + ln for ln in record.text.splitlines() if ln.startswith("Section-ID:") + ] + self.assertEqual(len(sid_lines), 1, msg=record.text[:240]) + self.assertRegex(sid_lines[0], r"^Section-ID: V\d+\.\d+\.\d+$") + self.assertNotIn("|", record.text.split("\n\n", 1)[-1]) + class OieOrchestratorTests(unittest.TestCase): def test_sequences_a_b_c_and_stops_on_a_error(self) -> None: diff --git a/application/tests/harvester_test/requirement_extractor_test.py b/application/tests/harvester_test/requirement_extractor_test.py new file mode 100644 index 000000000..b441e76fa --- /dev/null +++ b/application/tests/harvester_test/requirement_extractor_test.py @@ -0,0 +1,110 @@ +"""Tests for optional requirement matcher / extractor.""" + +from __future__ import annotations + +import unittest + +from application.utils.harvester.requirement_extractor import ( + extract_requirement_chunks, + extract_requirement_segments, + normalize_asvs_id, + requirements_needed, + should_extract_requirements, +) + +_ASVS_SNIPPET = """\ +# V1 Encoding and Sanitization + +## V1.1 Encoding and Sanitization Architecture + +| # | Description | Level | +| :---: | :--- | :---: | +| **1.1.1** | Verify that input is decoded into a canonical form only once. | 2 | +| **1.1.2** | Verify that the application performs output encoding and escaping. | 2 | +| **1.1.3** | Verify that output encoding preserves the integrity of data. | 2 | +| **1.2.1** | Verify that the application protects against SQL injection. | 1 | +| **1.2.2** | Verify that the application protects against OS command injection. | 1 | +""" + +_NARRATIVE = """\ +# Authorization Cheat Sheet + +This sheet describes how to design authorization for web apps. +It mentions version 1.2.3 only as an example release number once. +There are no Verify that requirement tables here. +""" + + +class RequirementExtractorTests(unittest.TestCase): + def test_normalize_asvs_id(self) -> None: + self.assertEqual(normalize_asvs_id("1.1.2"), "V1.1.2") + self.assertEqual(normalize_asvs_id("v1.1.2"), "V1.1.2") + self.assertEqual(normalize_asvs_id("V1.1.2"), "V1.1.2") + self.assertEqual(normalize_asvs_id("AC-2"), "AC-2") + + def test_requirements_needed_true_for_asvs_table(self) -> None: + self.assertTrue(requirements_needed(_ASVS_SNIPPET)) + + def test_requirements_needed_false_for_narrative(self) -> None: + self.assertFalse(requirements_needed(_NARRATIVE)) + + def test_should_extract_modes(self) -> None: + self.assertFalse(should_extract_requirements(_ASVS_SNIPPET, mode="off")) + self.assertTrue(should_extract_requirements(_ASVS_SNIPPET, mode="on")) + self.assertTrue(should_extract_requirements(_ASVS_SNIPPET, mode="auto")) + self.assertFalse(should_extract_requirements(_NARRATIVE, mode="auto")) + + def test_extract_table_segments(self) -> None: + segs = extract_requirement_segments(_ASVS_SNIPPET) + ids = [s.requirement_id for s in segs] + self.assertIn("V1.1.1", ids) + self.assertIn("V1.1.2", ids) + self.assertIn("V1.2.2", ids) + self.assertGreaterEqual(len(segs), 5) + + def test_extract_chunks_prefix_section_id(self) -> None: + chunks = extract_requirement_chunks(_ASVS_SNIPPET) + self.assertGreaterEqual(len(chunks), 5) + joined = "\n".join(c.text for c in chunks) + self.assertIn("Section-ID: V1.1.2", joined) + for c in chunks: + self.assertLess(c.start_char_idx, c.end_char_idx) + + def test_table_segment_is_prose_only_no_next_heading_bleed(self) -> None: + body = """\ +## V1.1 Encoding Architecture + +| # | Description | Level | +| :---: | :--- | :---: | +| **1.1.2** | Verify that the application performs output encoding. | 2 | +| **1.1.3** | Verify that output encoding preserves integrity. | 2 | + +## V1.2 Injection Prevention + +| # | Description | Level | +| :---: | :--- | :---: | +| **1.2.1** | Verify that the application protects against SQL injection. | 1 | +""" + segs = {s.requirement_id: s for s in extract_requirement_segments(body)} + self.assertIn("V1.1.3", segs) + # Must not pull the next subsection heading into the prior requirement. + self.assertNotIn("V1.2 Injection", segs["V1.1.3"].text) + self.assertNotIn("SQL injection", segs["V1.1.3"].text) + # Body is clean requirement prose (no raw table pipes). + self.assertIn("preserves integrity", segs["V1.1.3"].text) + self.assertNotIn("|", segs["V1.1.3"].text) + + def test_extract_chunk_single_section_id_line(self) -> None: + chunks = extract_requirement_chunks(_ASVS_SNIPPET) + v112 = next(c for c in chunks if "Section-ID: V1.1.2\n" in c.text) + # Exactly one Section-ID line, matching Source id (B2-shaped). + sid_lines = [ + ln for ln in v112.text.splitlines() if ln.startswith("Section-ID:") + ] + self.assertEqual(sid_lines, ["Section-ID: V1.1.2"]) + self.assertTrue(v112.text.startswith("Source: V1.1.2\n")) + self.assertNotIn("|", v112.text) + + +if __name__ == "__main__": + unittest.main() diff --git a/application/utils/harvester/chunk_merger.py b/application/utils/harvester/chunk_merger.py new file mode 100644 index 000000000..700764a28 --- /dev/null +++ b/application/utils/harvester/chunk_merger.py @@ -0,0 +1,166 @@ +"""Module A.2 — merge undersized / same-heading Docling leaves. + +One algorithm for all sources. Per-repo ``ChunkingConfig.merge_profile`` only +sets knobs (requirements vs narrative). Runs after primary chunking and before +``ChunkRecordBuilder``. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, List, Sequence + +from application.utils.harvester.chunk_record_builder import ChunkRecordBuilder +from application.utils.harvester.models import ChunkInfo, Document +from application.utils.harvester.requirement_ids import requirement_ids + +if TYPE_CHECKING: + from application.utils.harvester.schemas import ChunkingConfig + +CHARS_PER_TOKEN = 4 + + +def estimate_tokens(text: str) -> int: + return max(1, (len(text) + CHARS_PER_TOKEN - 1) // CHARS_PER_TOKEN) + + +def heading_path_for(document: Document, chunk: ChunkInfo) -> tuple[str, ...]: + path = ChunkRecordBuilder._heading_path_for_chunk(document, chunk) + return tuple(path) + + +def merge_heading_key( + document: Document, + chunk: ChunkInfo, + *, + profile: str, +) -> tuple[str, ...]: + """Heading identity used for A.2 merge boundaries. + + ``narrative`` keeps only heading levels ≤ 2 (``#`` / ``##``) so ``###`` + crumbs under the same section merge — cheat-sheet / sheet-level grain. + """ + if profile != "narrative": + return heading_path_for(document, chunk) + + start_line, _ = ChunkRecordBuilder._line_range( + document.text, + chunk.start_char_idx, + chunk.end_char_idx, + ) + active = [ + heading + for heading in document.heading_structure + if heading.level <= 2 and heading.start_line <= start_line <= heading.end_line + ] + active.sort(key=lambda heading: heading.start_line) + path: list[str] = [] + for heading in active: + while len(path) >= heading.level: + path.pop() + path.append(heading.text) + return tuple(path) + + +def merge_chunks( + document: Document, + chunks: Sequence[ChunkInfo], + config: "ChunkingConfig", +) -> List[ChunkInfo]: + """Merge adjacent leaves under the same heading within token caps. + + Boundaries that always flush the buffer: + - different merge-heading key (full path, or ``##`` grain for narrative) + - ``split_on_requirement_id`` and ``b`` introduces a new id not in the buffer + - ``tokens(buffer)+tokens(b) > merge_max_tokens`` + """ + if not chunks: + return [] + profile = getattr(config, "merge_profile", "none") or "none" + if profile == "none": + min_tokens = int(config.merge_min_tokens or 0) + if min_tokens <= 0: + return list(chunks) + + max_tokens = int(config.merge_max_tokens or config.max_tokens) + min_tokens = int(config.merge_min_tokens or 0) + split_req = bool(config.split_on_requirement_id) + + merged: List[ChunkInfo] = [] + buf: List[ChunkInfo] = [chunks[0]] + buf_heading = merge_heading_key(document, chunks[0], profile=profile) + buf_ids = set(requirement_ids(chunks[0].text)) + buf_tokens = estimate_tokens(chunks[0].text) + + def flush() -> None: + nonlocal buf, buf_heading, buf_ids, buf_tokens + if not buf: + return + merged.append(_join(buf)) + buf = [] + buf_heading = tuple() + buf_ids = set() + buf_tokens = 0 + + for chunk in chunks[1:]: + hp = merge_heading_key(document, chunk, profile=profile) + ids = requirement_ids(chunk.text) + tok = estimate_tokens(chunk.text) + + new_req_boundary = False + if split_req and ids: + if buf_ids and ids.isdisjoint(buf_ids): + new_req_boundary = True + elif not buf_ids and len(ids) > 1: + new_req_boundary = buf_tokens >= min_tokens + + fits = buf_tokens + tok <= max_tokens + same_heading = hp == buf_heading + + if same_heading and fits and not new_req_boundary: + buf.append(chunk) + buf_tokens += tok + buf_ids |= set(ids) + continue + + flush() + buf = [chunk] + buf_heading = hp + buf_ids = set(ids) + buf_tokens = tok + + flush() + + if min_tokens <= 0 or len(merged) < 2: + return merged + + tightened: List[ChunkInfo] = [merged[0]] + for chunk in merged[1:]: + prev = tightened[-1] + prev_hp = merge_heading_key(document, prev, profile=profile) + hp = merge_heading_key(document, chunk, profile=profile) + prev_tok = estimate_tokens(prev.text) + tok = estimate_tokens(chunk.text) + if ( + hp == prev_hp + and prev_tok < min_tokens + and prev_tok + tok <= max_tokens + and not ( + split_req + and requirement_ids(chunk.text) + and requirement_ids(prev.text) + and requirement_ids(chunk.text).isdisjoint(requirement_ids(prev.text)) + ) + ): + tightened[-1] = _join([prev, chunk]) + else: + tightened.append(chunk) + return tightened + + +def _join(parts: Sequence[ChunkInfo]) -> ChunkInfo: + text = "\n\n".join(p.text.strip() for p in parts if p.text.strip()) + return ChunkInfo( + text=text, + start_char_idx=parts[0].start_char_idx, + end_char_idx=parts[-1].end_char_idx, + ) diff --git a/application/utils/harvester/chunk_pipeline.py b/application/utils/harvester/chunk_pipeline.py index b9509e891..75fae02d2 100644 --- a/application/utils/harvester/chunk_pipeline.py +++ b/application/utils/harvester/chunk_pipeline.py @@ -5,14 +5,19 @@ from .chunk_record_builder import ChunkRecordBuilder from .chunk_record_validator import ChunkRecordValidator from .chunker import DocumentChunker +from .chunk_merger import merge_chunks from .models import Document, IngestChunkRecord +from .requirement_extractor import ( + extract_requirement_chunks, + should_extract_requirements, +) from .schemas import ChunkingConfig class DocumentChunkPipeline: """ - Runs config-driven chunking followed by structure-aware RFC - chunk-record construction and validation. + Runs config-driven chunking, optional requirement extract, optional A.2 + merge, then RFC chunk-record construction and validation. """ def __init__( @@ -22,12 +27,35 @@ def __init__( record_builder: ChunkRecordBuilder | None = None, validator: ChunkRecordValidator | None = None, ) -> None: + self._chunking = chunking self._chunker = chunker or DocumentChunker(chunking) self._record_builder = record_builder or ChunkRecordBuilder() self._validator = validator or ChunkRecordValidator() def chunk(self, document: Document) -> list[IngestChunkRecord]: - chunks = self._chunker.chunk(document.text, document=document) + mode = "off" + if self._chunking is not None: + mode = getattr(self._chunking, "requirement_extract", "off") or "off" + + extracted = None + if should_extract_requirements(document.text, mode=mode): + extracted = extract_requirement_chunks(document.text) + if extracted: + logger.info( + "requirement_extract=%s: %s segments for %s", + mode, + len(extracted), + document.artifact_id, + ) + + if extracted: + chunks = extracted + # Already requirement-grain — skip A.2 merge (would only re-glue). + else: + chunks = self._chunker.chunk(document.text, document=document) + if self._chunking is not None: + chunks = merge_chunks(document, chunks, self._chunking) + records = self._record_builder.build(document, chunks) for record in records: self._validator.validate(record) diff --git a/application/utils/harvester/chunk_record_builder.py b/application/utils/harvester/chunk_record_builder.py index 76fc1ff81..c0f917264 100644 --- a/application/utils/harvester/chunk_record_builder.py +++ b/application/utils/harvester/chunk_record_builder.py @@ -4,6 +4,7 @@ logger = get_logger(__name__) +import re from dataclasses import dataclass from datetime import datetime, timezone from typing import TYPE_CHECKING, Optional @@ -14,6 +15,20 @@ if TYPE_CHECKING: from .schemas import ChunkingConfig +# Section ids used by Top10 / API / LLM / K8s / ASVS / AISVS / agentic stubs. +_SECTION_ID_RE = re.compile( + r"\b(" + r"AAI\d{2}" # Agentic AI stub (before A\d{2}) + r"|A\d{2}" # OWASP Top 10 + r"|API\d{1,2}" # API Top 10 + r"|LLM\d{2}" # LLM Top 10 + r"|K\d{2}" # Kubernetes Top 10 + r"|AISVS\d{1,2}" # AISVS categories + r"|V?\d+\.\d+(?:\.\d+)?" # ASVS / numeric + r")\b", + re.IGNORECASE, +) + @dataclass(slots=True) class ChunkRecordBuilder: @@ -38,6 +53,9 @@ def build( committed_at_str = str(committed_at) records: list[IngestChunkRecord] = [] + source_name = self._source_filename(document) + standard = self._standard_name(document) + version = self._version_id(standard, document) for index, chunk in enumerate(chunks): heading_path = self._heading_path_for_chunk(document, chunk) start_line, end_line = self._line_range( @@ -51,7 +69,14 @@ def build( chunk_id=f"chk:{document.artifact_id}:{index}", artifact_id=document.artifact_id, pipeline_run_id=document.pipeline_run_id, - text=chunk.text, + text=self._prefixed_text( + chunk.text, + source_name=source_name, + standard=standard, + version=version, + heading_path=heading_path, + locator_path=document.locator.path or document.locator.id or "", + ), span=SpanInfo( heading_path=heading_path, start_line=start_line, @@ -72,6 +97,104 @@ def build( ) return records + @staticmethod + def _source_filename(document: Document) -> str: + path = (document.locator.path or document.locator.id or "").strip() + if path: + return path.rsplit("/", 1)[-1] + return document.artifact_id.rsplit(":", 1)[-1] + + @staticmethod + def _standard_name(document: Document) -> str: + repo = (document.source.repository or "").strip() + if "/" in repo: + return repo.rsplit("/", 1)[-1] + if repo: + return repo + art = document.artifact_id or "" + # art:OWASP/ASVS:path or art:B2/owasp_top10_2025:A01 + if art.startswith("art:"): + body = art[len("art:") :] + first = body.split(":", 1)[0] + if "/" in first: + return first.rsplit("/", 1)[-1] + return first + return "" + + @staticmethod + def _version_id(standard: str, document: Document) -> str: + """Year or version token for ``Version:`` (e.g. 2025, 4.0, 1.0).""" + blob = " ".join( + [ + standard or "", + document.source.repository or "", + document.artifact_id or "", + document.locator.path or "", + ] + ) + year = re.search(r"(20\d{2})", blob) + if year: + return year.group(1) + ver = re.search(r"\bv?(\d+\.\d+(?:\.\d+)?)\b", blob, re.I) + if ver: + return ver.group(1) + return "" + + @staticmethod + def _extract_section_ids(*parts: str) -> list[str]: + seen: set[str] = set() + out: list[str] = [] + for part in parts: + for match in _SECTION_ID_RE.findall(part or ""): + sid = match.upper() if match[0].isalpha() else match + if sid not in seen: + seen.add(sid) + out.append(sid) + return out + + @staticmethod + def _prefixed_text( + text: str, + *, + source_name: str, + standard: str, + version: str, + heading_path: list[str], + locator_path: str, + ) -> str: + """Enrich chunk text for Module C retrieval (standard + section identity). + + Requirement-extract chunks already embed ``Source:`` / ``Section-ID:`` + (see ``extract_requirement_chunks``). Do not prepend another pair — Module + C parsers read the first of each, so a filename Source would shadow the + real requirement id. + """ + body = text or "" + stripped = body.lstrip() + has_identity = stripped.startswith("Source:") or stripped.startswith( + "Section-ID:" + ) + lines: list[str] = [] + if standard: + lines.append(f"Standard: {standard}") + if version: + lines.append(f"Version: {version}") + if not has_identity: + section_ids = ChunkRecordBuilder._extract_section_ids( + source_name, + locator_path, + " ".join(heading_path), + body[:800], + ) + lines.append(f"Source: {source_name}") + if section_ids: + lines.append("Section-ID: " + ", ".join(section_ids[:6])) + if heading_path: + lines.append("Section: " + " > ".join(heading_path)) + lines.append("") + lines.append(body) + return "\n".join(lines) + @staticmethod def _heading_path_for_chunk( document: Document, @@ -101,11 +224,13 @@ def _line_range( start_char_idx: int, end_char_idx: int, ) -> tuple[int, int]: - if not 0 <= start_char_idx < end_char_idx <= len(text): - raise ValueError("Chunk character offsets are outside the source document") - start_line = text.count("\n", 0, start_char_idx) + 1 - end_position = end_char_idx - 1 - end_line = text.count("\n", 0, end_position) + 1 + n = len(text) + if n == 0: + return 1, 1 + start = max(0, min(int(start_char_idx), n - 1)) + end = max(start + 1, min(int(end_char_idx), n)) + start_line = text.count("\n", 0, start) + 1 + end_line = text.count("\n", 0, end - 1) + 1 return start_line, end_line diff --git a/application/utils/harvester/chunker.py b/application/utils/harvester/chunker.py index 778c145a0..9b93bc45d 100644 --- a/application/utils/harvester/chunker.py +++ b/application/utils/harvester/chunker.py @@ -15,6 +15,8 @@ class DocumentChunker: Strategies: - ``markdown_heading``: one chunk per heading section (plus preamble), then size-split oversized sections. + - ``docling``: LlamaIndex DoclingReader + DoclingNodeParser (HybridChunker + fallback) — structure/table-aware; preferred for ASVS-style MD and later PDF/DOCX. - ``fixed_size``: sliding windows by approximate token budget. - ``html_readability``: treated as fixed_size until a dedicated parser exists. """ @@ -34,6 +36,15 @@ def chunk(self, text: str, *, document: Document | None = None) -> list[ChunkInf return self._fixed_size(text, max_tokens=1200, overlap_tokens=100) strategy = config.strategy + if strategy == "docling": + from application.utils.harvester.docling_chunker import chunk_with_docling + + name = "document.md" + if document is not None and document.locator and document.locator.path: + name = document.locator.path + return chunk_with_docling( + text, document=document, config=config, source_name=name + ) if strategy == "markdown_heading" and document is not None: return self._markdown_heading(text, document, config) return self._fixed_size( diff --git a/application/utils/harvester/docling_chunker.py b/application/utils/harvester/docling_chunker.py new file mode 100644 index 000000000..8c83144e0 --- /dev/null +++ b/application/utils/harvester/docling_chunker.py @@ -0,0 +1,116 @@ +"""Docling + LlamaIndex chunking for Module A. + +``strategy: docling`` in ``ChunkingConfig`` routes here. LlamaIndex wraps +Docling's reader/node parser; we map nodes back to ``ChunkInfo`` so the rest of +the harvester (record builder → ``harvest_input``) stays unchanged. +""" + +from __future__ import annotations + +import logging +import tempfile +from pathlib import Path +from typing import TYPE_CHECKING, List, Optional + +from application.utils.harvester.models import ChunkInfo, Document + +if TYPE_CHECKING: + from application.utils.harvester.schemas import ChunkingConfig + +logger = logging.getLogger(__name__) + + +def chunk_with_docling( + text: str, + *, + document: Optional[Document] = None, + config: Optional["ChunkingConfig"] = None, + source_name: str = "document.md", +) -> List[ChunkInfo]: + """Structure-aware chunks via LlamaIndex DoclingReader + DoclingNodeParser. + + Falls back to Docling ``HybridChunker`` directly if the LlamaIndex path + fails (import/runtime). Offsets are best-effort against ``text``. + """ + if not text.strip(): + return [] + + try: + return _chunk_via_llamaindex(text, source_name=source_name) + except Exception: + logger.exception( + "LlamaIndex Docling path failed; falling back to HybridChunker" + ) + return _chunk_via_hybrid(text, source_name=source_name) + + +def _chunk_via_llamaindex(text: str, *, source_name: str) -> List[ChunkInfo]: + from llama_index.node_parser.docling import DoclingNodeParser + from llama_index.readers.docling import DoclingReader + + suffix = Path(source_name).suffix or ".md" + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / f"ingest{suffix}" + path.write_text(text, encoding="utf-8") + reader = DoclingReader(export_type=DoclingReader.ExportType.JSON) + docs = reader.load_data(str(path)) + nodes = DoclingNodeParser().get_nodes_from_documents(docs) + + return _nodes_to_chunk_infos(text, [n.get_content() for n in nodes]) + + +def _chunk_via_hybrid(text: str, *, source_name: str) -> List[ChunkInfo]: + from io import BytesIO + + from docling.chunking import HybridChunker + from docling.document_converter import DocumentConverter + from docling_core.types.io import DocumentStream + + stream = DocumentStream( + name=source_name if source_name.endswith(".md") else f"{source_name}.md", + stream=BytesIO(text.encode("utf-8")), + ) + dl_doc = DocumentConverter().convert(stream).document + chunker = HybridChunker() + pieces = [chunker.contextualize(c) for c in chunker.chunk(dl_doc=dl_doc)] + return _nodes_to_chunk_infos(text, pieces) + + +def _nodes_to_chunk_infos(source_text: str, pieces: List[str]) -> List[ChunkInfo]: + """Map chunk texts onto source offsets (search forward; tolerate serialization drift).""" + chunks: List[ChunkInfo] = [] + cursor = 0 + for piece in pieces: + body = (piece or "").strip() + if not body: + continue + # Prefer locating a distinctive mid substring in the source. + probe = body + if len(probe) > 80: + probe = body[len(body) // 4 : len(body) // 4 + 60] + idx = source_text.find(probe, cursor) + if idx < 0: + idx = source_text.find(probe) + if idx < 0: + # Synthetic span at cursor so the record builder still gets offsets. + start = min(cursor, len(source_text)) + end = min(start + max(len(body), 1), len(source_text)) or 1 + if start >= end and source_text: + start = max(0, len(source_text) - 1) + end = len(source_text) + chunks.append( + ChunkInfo( + text=body, start_char_idx=start, end_char_idx=max(end, start + 1) + ) + ) + cursor = end + continue + # Expand to cover the piece length when the probe matched inside source. + start = idx + end = min(len(source_text), start + len(body)) + # If contextualized text differs from source, keep piece text but clamp span. + if end <= start: + end = min(len(source_text), start + 1) + chunks.append(ChunkInfo(text=body, start_char_idx=start, end_char_idx=end)) + cursor = end + return chunks diff --git a/application/utils/harvester/git_repository_client.py b/application/utils/harvester/git_repository_client.py index c5c5f5e73..11fd10f28 100644 --- a/application/utils/harvester/git_repository_client.py +++ b/application/utils/harvester/git_repository_client.py @@ -64,9 +64,15 @@ def _clone_atomically(self) -> None: subprocess.run( [ "git", + "-c", + "core.hooksPath=/dev/null", "clone", + "--template=", "--branch", self.branch, + "--single-branch", + "--depth", + "1", self.repository_url, str(temp_path), ], @@ -74,6 +80,10 @@ def _clone_atomically(self) -> None: capture_output=True, text=True, timeout=300, + env={ + **os.environ, + "GIT_TERMINAL_PROMPT": "0", + }, ) if not self.is_valid_repository(temp_path): diff --git a/application/utils/harvester/github_url_ingest.py b/application/utils/harvester/github_url_ingest.py new file mode 100644 index 000000000..619bd177c --- /dev/null +++ b/application/utils/harvester/github_url_ingest.py @@ -0,0 +1,479 @@ +"""Point-and-shoot GitHub URL ingest: harvest → Module B → print knowledge. + +Not an eval scorer. Operational path for ``cre.py --ingest_github`` / +``make ingest ``. +""" + +from __future__ import annotations + +import dataclasses +import re +import sys +import tarfile +import urllib.request +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path, PurePosixPath +from typing import Any, List, Optional, Sequence, Tuple +from urllib.parse import quote, urlparse + +ROOT = Path(__file__).resolve().parents[3] +TARBALL_DIR = ROOT / "tmp" / "oie_ingest" / "tarballs" + +EXCLUDE = [ + "**/archive/**", + "**/.github/**", + "**/node_modules/**", + "**/LICENSE*", + "**/CHANGELOG*", + "**/CONTRIBUTING*", +] + +_GITHUB_HOSTS = {"github.com", "www.github.com"} + +_OWNER_RE = re.compile(r"^[A-Za-z0-9](?:[A-Za-z0-9-]{0,38})$") +_REPO_RE = re.compile(r"^[A-Za-z0-9._-]{1,100}$") +_BRANCH_RE = re.compile(r"^[A-Za-z0-9._/-]{1,200}$") + +DOWNLOAD_TIMEOUT_SECONDS = 30 +MAX_TARBALL_BYTES = 200 * 1024 * 1024 +MAX_MD_MEMBER_BYTES = 2 * 1024 * 1024 +MAX_MD_TOTAL_BYTES = 100 * 1024 * 1024 +MAX_TAR_MEMBERS = 20000 + + +def validate_branch(branch: str) -> str: + if not _BRANCH_RE.match(branch) or ".." in branch or branch.startswith("-"): + raise ValueError(f"invalid branch name {branch!r}") + return branch + + +@dataclass(frozen=True) +class ParsedGitHubUrl: + owner: str + repo: str + branch: Optional[str] = None + path_prefix: Optional[str] = None # e.g. 5.0/en from /tree/master/5.0/en + + +@dataclass +class IngestCounts: + files: int = 0 + chunks: int = 0 + knowledge: int = 0 + uncertain: int = 0 + noise: int = 0 + deduped: int = 0 + errors: int = 0 + + +def parse_github_url(url: str) -> ParsedGitHubUrl: + """Parse ``https://github.com/owner/repo[/tree/branch[/path…]]``.""" + raw = (url or "").strip() + if not raw: + raise ValueError("GitHub URL is empty") + if "://" not in raw: + raw = "https://" + raw + parsed = urlparse(raw) + host = (parsed.hostname or "").lower() + if host not in _GITHUB_HOSTS: + raise ValueError(f"only github.com URLs are supported (got host {host!r})") + parts = [p for p in parsed.path.strip("/").split("/") if p] + if len(parts) < 2: + raise ValueError("URL must look like https://github.com/owner/repo") + owner, repo = parts[0], parts[1] + if repo.endswith(".git"): + repo = repo[: -len(".git")] + if not _OWNER_RE.match(owner) or not _REPO_RE.match(repo) or repo in (".", ".."): + raise ValueError(f"invalid GitHub owner/repo {owner!r}/{repo!r}") + branch: Optional[str] = None + path_prefix: Optional[str] = None + if len(parts) >= 4 and parts[2] in ("tree", "blob"): + branch = validate_branch(parts[3]) + if len(parts) > 4: + path_prefix = "/".join(parts[4:]) + return ParsedGitHubUrl( + owner=owner, repo=repo, branch=branch, path_prefix=path_prefix + ) + + +def _match(path: str, patterns: Sequence[str]) -> bool: + posix = PurePosixPath(path) + expanded: List[str] = [] + for pat in patterns: + expanded.append(pat) + if "/**/" in pat: + expanded.append(pat.replace("/**/", "/")) + if pat.startswith("**/"): + expanded.append(pat[3:]) + for pat in expanded: + try: + if posix.match(pat): + return True + except ValueError: + pass + import fnmatch + + if fnmatch.fnmatch(path, pat): + return True + return False + + +def _download_to_file(url: str, dest: Path) -> None: + request = urllib.request.Request(url, headers={"User-Agent": "OpenCRE-ingest"}) + total = 0 + with urllib.request.urlopen( + request, timeout=DOWNLOAD_TIMEOUT_SECONDS + ) as resp, open(dest, "wb") as fh: + while True: + buf = resp.read(1 << 20) + if not buf: + break + total += len(buf) + if total > MAX_TARBALL_BYTES: + raise RuntimeError(f"tarball exceeds {MAX_TARBALL_BYTES} byte limit") + fh.write(buf) + + +def _download_tarball( + owner: str, + repo: str, + branch: str, + dest: Path, + *, + allow_fallback: bool = True, +) -> str: + dest.parent.mkdir(parents=True, exist_ok=True) + tried: List[str] = [] + candidates = [branch] + if allow_fallback: + for alt in ("main", "master"): + if alt not in candidates: + candidates.append(alt) + last_err: Optional[BaseException] = None + for try_branch in candidates: + url = ( + f"https://codeload.github.com/{quote(owner, safe='')}/" + f"{quote(repo, safe='')}/tar.gz/refs/heads/{quote(try_branch, safe='/')}" + ) + tried.append(try_branch) + print(f"GET {url}", file=sys.stderr, flush=True) + try: + _download_to_file(url, dest) + with tarfile.open(dest, "r:gz") as tf: + tf.next() + return try_branch + except Exception as exc: # noqa: BLE001 + last_err = exc + dest.unlink(missing_ok=True) + print(f" fail {try_branch}: {exc}", file=sys.stderr, flush=True) + raise RuntimeError( + f"could not download tarball for {owner}/{repo} " + f"(tried branches {tried}): {last_err}" + ) + + +def _iter_md(tgz: Path, *, path_prefix: Optional[str]) -> List[Tuple[str, str]]: + includes = ["**/*.md"] + if path_prefix: + pref = path_prefix.strip("/") + includes = [f"{pref}/**/*.md", f"{pref}/*.md"] + out: List[Tuple[str, str]] = [] + total_bytes = 0 + with tarfile.open(tgz, "r:gz") as tf: + for seen, member in enumerate(tf): + if seen >= MAX_TAR_MEMBERS: + break + if not member.isfile() or member.size > MAX_MD_MEMBER_BYTES: + continue + parts = Path(member.name).parts + if len(parts) < 2: + continue + rel = str(Path(*parts[1:])) + if not rel.lower().endswith(".md"): + continue + if _match(rel, EXCLUDE): + continue + if not _match(rel, includes): + continue + extracted = tf.extractfile(member) + if extracted is None: + continue + raw = extracted.read(MAX_MD_MEMBER_BYTES + 1) + if len(raw) > MAX_MD_MEMBER_BYTES: + continue + total_bytes += len(raw) + if total_bytes > MAX_MD_TOTAL_BYTES: + break + try: + text = raw.decode("utf-8") + except UnicodeDecodeError: + text = raw.decode("utf-8", errors="replace") + if len(text.strip()) < 40: + continue + out.append((rel, text)) + return out + + +def format_knowledge_item( + *, + path: str, + label: str, + confidence: float, + text: str, + reasoning: Optional[str] = None, + max_chars: int = 4000, +) -> str: + """Human-readable block for one knowledge chunk.""" + body = ( + text if len(text) <= max_chars else text[: max_chars - 20] + "\n… [truncated]" + ) + lines = [ + "=" * 72, + f"path: {path}", + f"label: {label} confidence: {confidence:.2f}", + ] + if reasoning: + lines.append(f"reason: {reasoning}") + lines.append("-" * 72) + lines.append(body) + lines.append("") + return "\n".join(lines) + + +def _keep_all_noise_filter(session: Any, pipeline_run_id: str, **_kwargs: Any) -> Any: + from application.database.db import HarvestInput + from application.utils.noise_filter.hashing import compute_content_hash + from application.utils.noise_filter.pipeline import RunSummary as BSummary + from application.utils.noise_filter.queue_writer import write_verdicts + from application.utils.noise_filter.schemas import ChangeRecord, ClassifyResult + + summary = BSummary(run_id=pipeline_run_id) + rows = ( + session.query(HarvestInput) + .filter_by(pipeline_run_id=pipeline_run_id, status="pending") + .all() + ) + summary.read = len(rows) + triples = [] + for row in rows: + try: + record = ChangeRecord.model_validate(row.payload) + except Exception: # noqa: BLE001 + summary.parse_errors += 1 + continue + verdict = ClassifyResult( + label="KNOWLEDGE", + confidence=1.0, + reasoning="ingest-keep-all", + ) + triples.append((record, verdict, compute_content_hash(record.text))) + row.status = "processed" + summary.kept_knowledge += 1 + stats = write_verdicts(session, triples) + summary.inserted = stats.inserted + summary.deduped = stats.deduped + session.commit() + summary.status = "ok" + return summary + + +def ingest_github_url( + url: str, + *, + session: Any, + cache_file: str, + branch_override: Optional[str] = None, + keep_all: bool = False, + model_override: Optional[str] = None, + run_id: Optional[str] = None, +) -> IngestCounts: + """Harvest ``url``, classify, print knowledge to stdout. Returns counts.""" + from application.database.db import HarvestInput, KnowledgeQueueItem + from application.utils.harvester.chunk_pipeline import DocumentChunkPipeline + from application.utils.harvester.harvest_writer import write_harvest_input + from application.utils.harvester.heading_extractor import HeadingExtractor + from application.utils.harvester.models import Document, Locator, SourceInfo + from application.utils.harvester.schemas import ChunkingConfig + from application.utils.noise_filter.pipeline import run_noise_filter + + parsed = parse_github_url(url) + _ = cache_file # reserved; session is already bound via db_connect + explicit_branch = branch_override or parsed.branch + branch_hint = validate_branch(explicit_branch) if explicit_branch else "main" + rid = run_id or ("ingest-" + datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")) + counts = IngestCounts() + + repo_key = f"{parsed.owner}/{parsed.repo}" + + tgz = TARBALL_DIR / f"{parsed.owner}_{parsed.repo}.tgz" + try: + used_branch = _download_tarball( + parsed.owner, + parsed.repo, + branch_hint, + tgz, + allow_fallback=not explicit_branch, + ) + except Exception as exc: # noqa: BLE001 + counts.errors += 1 + print(f"ERROR: {exc}", file=sys.stderr, flush=True) + return counts + + files = _iter_md(tgz, path_prefix=parsed.path_prefix) + counts.files = len(files) + if not files: + print( + "ERROR: no markdown files matched " f"(prefix={parsed.path_prefix!r})", + file=sys.stderr, + flush=True, + ) + counts.errors += 1 + return counts + + chunking = ChunkingConfig( + strategy="markdown_heading", + max_tokens=1200, + overlap_tokens=100, + merge_profile="requirements", + requirement_extract="auto", + ) + pipeline = DocumentChunkPipeline(chunking=chunking) + headings = HeadingExtractor() + committed_at = datetime.now(timezone.utc) + sha = "ingest000000000000000000000000000000001" + records = [] + for rel, text in files: + artifact_id = f"art:{repo_key}:{rel}" + doc = Document( + schema_version="0.2.0", + artifact_id=artifact_id, + pipeline_run_id=rid, + text=text, + heading_structure=headings.extract(text), + source=SourceInfo( + type="github", + repository=repo_key, + commit_sha=sha, + committed_at=committed_at, + ), + locator=Locator(kind="repo_path", id=rel, path=rel), + ) + records.extend(pipeline.chunk(doc)) + + # Earlier ingest rows for this repo would hide fresh output through the + # global content_hash dedupe. Only ingest-run rows are cleared (never rows + # from --run_harvester), and only once download and chunking succeeded. + deleted_kq = ( + session.query(KnowledgeQueueItem) + .filter( + KnowledgeQueueItem.source_repo == repo_key, + KnowledgeQueueItem.pipeline_run_id.like("ingest-%"), + ) + .delete(synchronize_session=False) + ) + session.query(HarvestInput).filter_by(pipeline_run_id=rid).delete( + synchronize_session=False + ) + written = write_harvest_input(session, rid, records) + session.commit() + if deleted_kq: + print( + f"cleared {deleted_kq} prior ingest knowledge_queue rows for {repo_key}", + file=sys.stderr, + flush=True, + ) + counts.chunks = written + print( + f"harvested {repo_key}@{used_branch}: files={len(files)} chunks={written} " + f"run_id={rid}", + file=sys.stderr, + flush=True, + ) + + if keep_all: + print( + "Module B: --ingest_keep_all (no LLM; all chunks → KNOWLEDGE)", + file=sys.stderr, + flush=True, + ) + b_summary = _keep_all_noise_filter(session, rid) + else: + from application.prompt_client.litellm_router import ( + has_credentials_for_model, + missing_credentials_hint, + ) + from application.utils.noise_filter.config_loader import load_config + + cfg = load_config() + if model_override and model_override.strip(): + cfg = dataclasses.replace(cfg, llm_model=model_override.strip()) + if not has_credentials_for_model(cfg.llm_model): + print( + "ERROR: " + missing_credentials_hint(cfg.llm_model), + file=sys.stderr, + flush=True, + ) + counts.errors += 1 + return counts + print( + f"Module B: noise filter via LiteLLM model={cfg.llm_model!r}…", + file=sys.stderr, + flush=True, + ) + b_summary = run_noise_filter(session, rid, config=cfg) + + counts.knowledge = getattr(b_summary, "kept_knowledge", 0) or 0 + counts.uncertain = getattr(b_summary, "kept_uncertain", 0) or 0 + counts.noise = getattr(b_summary, "dropped_noise", 0) or 0 + counts.deduped = getattr(b_summary, "deduped", 0) or 0 + + rows = ( + session.query(KnowledgeQueueItem) + .filter_by(pipeline_run_id=rid) + .order_by(KnowledgeQueueItem.locator_path, KnowledgeQueueItem.span_index) + .all() + ) + # If dedupe skipped inserts, fall back to any KQ rows we refreshed for repo. + if not rows: + rows = ( + session.query(KnowledgeQueueItem) + .filter(KnowledgeQueueItem.source_repo == repo_key) + .order_by(KnowledgeQueueItem.locator_path, KnowledgeQueueItem.span_index) + .all() + ) + + printed = 0 + for row in rows: + if row.llm_label not in ("KNOWLEDGE", "UNCERTAIN"): + continue + sys.stdout.write( + format_knowledge_item( + path=row.locator_path or row.artifact_id, + label=row.llm_label, + confidence=float(row.confidence or 0.0), + text=row.text or "", + reasoning=row.llm_reasoning, + ) + ) + printed += 1 + sys.stdout.flush() + + print( + f"files={counts.files} chunks={counts.chunks} " + f"knowledge={counts.knowledge} uncertain={counts.uncertain} " + f"noise={counts.noise} printed={printed} deduped={counts.deduped} " + f"run_id={rid}", + file=sys.stderr, + flush=True, + ) + return counts + + +__all__ = [ + "IngestCounts", + "ParsedGitHubUrl", + "format_knowledge_item", + "ingest_github_url", + "parse_github_url", +] diff --git a/application/utils/harvester/pipeline.py b/application/utils/harvester/pipeline.py index 756213554..76b6b91cd 100644 --- a/application/utils/harvester/pipeline.py +++ b/application/utils/harvester/pipeline.py @@ -74,7 +74,7 @@ def run_harvester( repos_path = Path(repos_yaml) if repos_yaml else DEFAULT_REPOS_YAML repos_file = load_repo_config(repos_path) - validate_repositories(repos_file.repositories) + validate_repositories(repos_file) checkpoint_store = CheckpointStore(session=session) builder = DocumentBuilder() diff --git a/application/utils/harvester/repos.yaml b/application/utils/harvester/repos.yaml index c448e5111..e90b6a454 100644 --- a/application/utils/harvester/repos.yaml +++ b/application/utils/harvester/repos.yaml @@ -7,16 +7,37 @@ repositories: branch: master paths: include: + # 4.0 kept for continuity; 5.0 is the primary full-pipeline target - "4.0/en/**/*.md" - + - "5.0/en/**/*.md" exclude: - "**/archive/**" - chunking: - strategy: markdown_heading + strategy: docling max_tokens: 1200 overlap_tokens: 100 + merge_profile: requirements + polling: + mode: incremental + interval_minutes: 60 + - id: owasp-aisvs + type: github + enabled: true + owner: OWASP + repo: AISVS + branch: main + paths: + include: + - "1.0/en/**/*.md" + exclude: + - "**/archive/**" + chunking: + strategy: docling + max_tokens: 1200 + overlap_tokens: 100 + merge_profile: requirements + requirement_extract: auto polling: mode: incremental interval_minutes: 60 @@ -24,19 +45,17 @@ repositories: - id: owasp-cheatsheets type: github enabled: true - owner: OWASP repo: CheatSheetSeries branch: master - paths: include: - "cheatsheets/**/*.md" - chunking: - strategy: markdown_heading + strategy: docling max_tokens: 1000 overlap_tokens: 100 + merge_profile: narrative polling: mode: incremental diff --git a/application/utils/harvester/requirement_extractor.py b/application/utils/harvester/requirement_extractor.py new file mode 100644 index 000000000..80d89de51 --- /dev/null +++ b/application/utils/harvester/requirement_extractor.py @@ -0,0 +1,217 @@ +"""Optional requirement matcher / extractor for Module A. + +When enabled (``ChunkingConfig.requirement_extract``), documents that look like +control catalogs (ASVS-style tables, NIST/ISO ids) are split into one chunk per +requirement. Narrative docs (cheat sheets, prose) are left alone under ``auto``. + +This is separate from A.2 ``merge_profile=requirements`` (merge fence). Extraction +produces requirement-grain *inputs*; the merger only avoids gluing different ids. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from typing import List, Optional, Sequence + +from application.utils.harvester.models import ChunkInfo +from application.utils.harvester.requirement_ids import requirement_ids + +# Table row: | **1.1.2** | Verify that … | 2 | +_TABLE_REQ_ROW_RE = re.compile( + r"^\|\s*\*?\*?V?(\d+\.\d+\.\d+)\*?\*?\s*\|\s*([^|]+?)\s*\|", + re.MULTILINE | re.IGNORECASE, +) +# Standalone heading-ish: ### V1.1.2 or **1.1.2** at line start +_LINE_REQ_RE = re.compile( + r"^(?:#{1,6}\s*)?\*?\*?V?(\d+\.\d+\.\d+)\*?\*?\b[^\n]*$", + re.MULTILINE | re.IGNORECASE, +) +_VERIFY_RE = re.compile(r"\bVerify that\b", re.IGNORECASE) + +# Auto gate: enough distinct 3-part decimal ids, or table rows + verify language. +_AUTO_MIN_DISTINCT_IDS = 5 +_AUTO_MIN_TABLE_ROWS = 3 + + +@dataclass(frozen=True) +class RequirementSegment: + """One extracted requirement with source offsets into the parent document.""" + + requirement_id: str # canonical display id, e.g. V1.1.2 or AC-2 + text: str + start_char_idx: int + end_char_idx: int + + +def normalize_asvs_id(raw: str) -> str: + """Map ``1.1.2`` / ``v1.1.2`` → ``V1.1.2``; leave other families unchanged.""" + s = (raw or "").strip() + if re.fullmatch(r"V?\d+\.\d+\.\d+", s, flags=re.IGNORECASE): + return "V" + s.lstrip("Vv") + return s + + +def requirements_needed(text: str) -> bool: + """Return True when the document looks like a requirement catalog. + + Heuristic only — used by ``requirement_extract=auto``. Prefer explicit + ``on`` / ``off`` in repo YAML when the answer is known. + """ + body = text or "" + if len(body) < 80: + return False + + table_rows = list(_TABLE_REQ_ROW_RE.finditer(body)) + if len(table_rows) >= _AUTO_MIN_TABLE_ROWS: + return True + + decimal_three = { + m.group(1) + for m in re.finditer(r"(?= _AUTO_MIN_DISTINCT_IDS and _VERIFY_RE.search(body): + return True + + # NIST / ISO dense catalogs (ids without ASVS tables). + ids = requirement_ids(body) + nist_iso = {i for i in ids if "-" in i or i.startswith("a.")} + if len(nist_iso) >= _AUTO_MIN_DISTINCT_IDS: + return True + + return False + + +def extract_requirement_segments(text: str) -> List[RequirementSegment]: + """Extract requirement-scoped segments from ``text``. + + Prefers markdown table rows (``| **1.1.2** | … |``). Falls back to + line-anchored ids. Returns [] when nothing usable is found (caller keeps + normal chunking). + """ + body = text or "" + if not body.strip(): + return [] + + table = _segments_from_table_rows(body) + if table: + return table + return _segments_from_line_ids(body) + + +def extract_requirement_chunks(text: str) -> List[ChunkInfo]: + """Build ``ChunkInfo`` list with B2-style Section-ID prefixes. + + Body is requirement prose only (no raw markdown table pipes) so Module C + retrieval is not biased by table chrome or neighboring section headings. + """ + out: List[ChunkInfo] = [] + for seg in extract_requirement_segments(text): + body = (seg.text or "").strip() + section_line = body.splitlines()[0][:200] if body else seg.requirement_id + prefixed = ( + f"Source: {seg.requirement_id}\n" + f"Section-ID: {seg.requirement_id}\n" + f"Section: {section_line}\n\n" + f"{body}" + ) + # Prefix is synthetic — keep offsets pointing at the original span so + # validators that check idx ordering still see start < end on body. + out.append( + ChunkInfo( + text=prefixed, + start_char_idx=seg.start_char_idx, + end_char_idx=max(seg.end_char_idx, seg.start_char_idx + 1), + ) + ) + return out + + +def should_extract_requirements( + text: str, + *, + mode: str, +) -> bool: + """Resolve ``off`` / ``auto`` / ``on`` against document text.""" + m = (mode or "off").strip().lower() + if m in ("0", "false", "no", "off", "none"): + return False + if m in ("1", "true", "yes", "on", "always"): + return True + if m == "auto": + return requirements_needed(text) + return False + + +def _segments_from_table_rows(body: str) -> List[RequirementSegment]: + matches = list(_TABLE_REQ_ROW_RE.finditer(body)) + if len(matches) < 2: + return [] + segs: List[RequirementSegment] = [] + for match in matches: + rid = normalize_asvs_id(match.group(1)) + desc = match.group(2).strip() + # One row only — do not span to the next requirement (that pulled in + # intervening ``## Vn.n`` headings and polluted Module C queries). + start = match.start() + end = match.end() + snippet = desc or f"{rid}".strip() + segs.append( + RequirementSegment( + requirement_id=rid, + text=snippet, + start_char_idx=start, + end_char_idx=max(end, start + 1), + ) + ) + return segs + + +def _segments_from_line_ids(body: str) -> List[RequirementSegment]: + matches = list(_LINE_REQ_RE.finditer(body)) + # Need several anchors or we risk chopping narrative headings. + if len(matches) < 2: + return [] + segs: List[RequirementSegment] = [] + for i, match in enumerate(matches): + rid = normalize_asvs_id(match.group(1)) + start = match.start() + end = matches[i + 1].start() if i + 1 < len(matches) else len(body) + snippet = body[start:end].strip() + if len(snippet) < 40: + continue + segs.append( + RequirementSegment( + requirement_id=rid, + text=snippet, + start_char_idx=start, + end_char_idx=end, + ) + ) + return segs + + +def _paragraph_end(body: str, from_idx: int) -> int: + """End of current table block or next blank line after ``from_idx``.""" + rest = body[from_idx:] + # Prefer end of contiguous table lines. + lines = rest.splitlines(keepends=True) + offset = from_idx + for line in lines: + if line.startswith("|") or not line.strip(): + offset += len(line) + if not line.strip() and offset > from_idx + 1: + return offset + continue + break + return min(len(body), max(from_idx + 1, offset)) + + +__all__ = [ + "RequirementSegment", + "extract_requirement_chunks", + "extract_requirement_segments", + "normalize_asvs_id", + "requirements_needed", + "should_extract_requirements", +] diff --git a/application/utils/harvester/requirement_ids.py b/application/utils/harvester/requirement_ids.py new file mode 100644 index 000000000..bde9cbb9b --- /dev/null +++ b/application/utils/harvester/requirement_ids.py @@ -0,0 +1,41 @@ +"""Requirement-id tokens used as Module A.2 merge fences. + +Not a splitter — ``chunk_merger`` flushes when adjacent leaves introduce a +disjoint id set. The ASVS-only decimal regex missed NIST ``AC-2``, ISO +``A.5.1``, and similar, so those controls glued together. +""" + +from __future__ import annotations + +import re +from typing import FrozenSet + +# Keep the original ASVS / OPC / PCI decimal capture (group 1). +_DECIMAL_ID_RE = re.compile( + r"(?:\*?\*?V)?(\d+\.\d+(?:\.\d+)?)(?:\*?\*)?\b", + re.IGNORECASE, +) +# NIST SP 800-53-style: AC-2, AU-12, AC-2(1). +_NIST_ID_RE = re.compile( + r"\b([A-Z]{1,4}-\d{1,3}(?:\(\d+\))?)\b", + re.IGNORECASE, +) +# ISO 27001 Annex A: A.5.1 / A.8.24. +_ISO_ID_RE = re.compile( + r"\b([A-Z]\.\d+(?:\.\d+)*)\b", + re.IGNORECASE, +) + +_PATTERNS = (_DECIMAL_ID_RE, _NIST_ID_RE, _ISO_ID_RE) + + +def requirement_ids(text: str) -> FrozenSet[str]: + """Return lowercased requirement ids found in ``text``.""" + found: set[str] = set() + for pattern in _PATTERNS: + for match in pattern.finditer(text or ""): + found.add(match.group(1).lower()) + return frozenset(found) + + +__all__ = ["requirement_ids"] diff --git a/application/utils/harvester/schemas.py b/application/utils/harvester/schemas.py index efa49dd8f..13f4e986b 100644 --- a/application/utils/harvester/schemas.py +++ b/application/utils/harvester/schemas.py @@ -24,7 +24,9 @@ class PathRules(BaseModel): class ChunkingConfig(BaseModel): model_config = ConfigDict(extra="forbid") - strategy: Literal["markdown_heading", "html_readability", "fixed_size"] = Field( + strategy: Literal[ + "markdown_heading", "html_readability", "fixed_size", "docling" + ] = Field( ..., description="Chunking strategy used for text segmentation", ) @@ -37,6 +39,69 @@ class ChunkingConfig(BaseModel): description="token overlap between adjacent chunks", ) + # A.2 merge — one algorithm; profiles only set defaults for these knobs. + merge_profile: Literal["none", "requirements", "narrative"] = Field( + default="none", + description=( + "A.2 merge profile: none=disabled; requirements=split on " + "requirement ids; narrative=cheat-sheet-like " + "(merge under the same heading up to max_tokens)." + ), + ) + merge_min_tokens: int | None = Field( + default=None, + ge=0, + description="Merge undersized siblings under the same heading (profile default if omitted).", + ) + merge_max_tokens: int | None = Field( + default=None, + gt=0, + description="Cap for a merged chunk (defaults to max_tokens).", + ) + split_on_requirement_id: bool | None = Field( + default=None, + description=( + "Do not merge across different requirement ids " + "(ASVS V2.1.1, NIST AC-2, ISO A.5.1, PCI 3.4, …)." + ), + ) + requirement_extract: Literal["off", "auto", "on"] = Field( + default="off", + description=( + "Optional requirement extractor: off=never; on=always try; " + "auto=only when requirements_needed(text) (ASVS-style tables / " + "dense control catalogs). When extraction yields segments, they " + "replace primary chunks for that document." + ), + ) + + @model_validator(mode="before") + @classmethod + def apply_merge_profile_defaults(cls, data: object) -> object: + """Fill merge knobs from ``merge_profile`` when left unset.""" + if not isinstance(data, dict): + return data + profile = data.get("merge_profile") or "none" + max_tokens = int(data.get("max_tokens") or 1200) + + def _fill(key: str, value: object) -> None: + if data.get(key) is None: + data[key] = value + + if profile == "none": + _fill("merge_min_tokens", 0) + _fill("merge_max_tokens", max_tokens) + _fill("split_on_requirement_id", False) + elif profile == "requirements": + _fill("merge_min_tokens", 100) + _fill("merge_max_tokens", max_tokens) + _fill("split_on_requirement_id", True) + else: # narrative + _fill("merge_min_tokens", 400) + _fill("merge_max_tokens", max_tokens) + _fill("split_on_requirement_id", False) + return data + @model_validator(mode="after") def overlap_must_be_less_than_max(self) -> "ChunkingConfig": if self.overlap_tokens >= self.max_tokens: @@ -44,6 +109,16 @@ def overlap_must_be_less_than_max(self) -> "ChunkingConfig": f"overlap_tokens ({self.overlap_tokens}) must be less than " f"max_tokens ({self.max_tokens})" ) + merge_max = ( + self.merge_max_tokens + if self.merge_max_tokens is not None + else self.max_tokens + ) + merge_min = self.merge_min_tokens if self.merge_min_tokens is not None else 0 + if merge_min > 0 and merge_min >= merge_max: + raise ValueError( + f"merge_min_tokens ({merge_min}) must be < merge_max_tokens ({merge_max})" + ) return self diff --git a/cre.py b/cre.py index d420bb48c..5407a93ec 100644 --- a/cre.py +++ b/cre.py @@ -375,6 +375,36 @@ def main() -> None: default="", help="optional path to repos.yaml for --run_harvester", ) + parser.add_argument( + "--ingest_github", + default="", + help=( + "Point-and-shoot: harvest a public GitHub repo URL, run Module B " + "noise filter, print KNOWLEDGE chunks to stdout" + ), + ) + parser.add_argument( + "--ingest_branch", + default="", + help="optional branch override for --ingest_github", + ) + parser.add_argument( + "--ingest_keep_all", + action="store_true", + help=( + "with --ingest_github: skip the LLM and treat every chunk as " + "KNOWLEDGE (offline dump; not a real classification test)" + ), + ) + parser.add_argument( + "--ingest_model", + default="", + help=( + "with --ingest_github: LiteLLM model id for Module B " + "(sets CRE_NOISE_FILTER_LLM_MODEL), e.g. openai/gpt-4o-mini, " + "anthropic/claude-haiku-4-5, gemini/gemini-2.5-flash" + ), + ) args = parser.parse_args() if args.export and not args.csv: @@ -383,6 +413,12 @@ def main() -> None: parser.error("--run_noise_filter requires --run_id ") if args.run_harvester and not args.run_id.strip(): parser.error("--run_harvester requires --run_id ") + if (args.ingest_branch or args.ingest_keep_all or args.ingest_model) and not str( + getattr(args, "ingest_github", "") or "" + ).strip(): + parser.error( + "--ingest_branch / --ingest_keep_all / --ingest_model require --ingest_github" + ) # The live queue path takes its rows from the DB, so a fixture path would be # silently ignored rather than doing what the caller plainly asked for. if args.librarian_source and args.run_id.strip(): diff --git a/scripts/ingest_github.sh b/scripts/ingest_github.sh new file mode 100755 index 000000000..b5f0d82a0 --- /dev/null +++ b/scripts/ingest_github.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# Point-and-shoot GitHub ingest (positional URL — Make cannot parse https:// goals). +# Usage: +# ./scripts/ingest_github.sh https://github.com/OWASP/ASVS +# ./scripts/ingest_github.sh https://github.com/OWASP/ASVS --ingest_keep_all +set -euo pipefail +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +cd "$ROOT" +if [[ $# -lt 1 ]]; then + echo "usage: $0 https://github.com/owner/repo [--ingest_model openai/gpt-4o-mini] [--ingest_keep_all] [--ingest_branch BRANCH]" >&2 + exit 2 +fi +URL="$1" +shift +if [[ -d ./venv ]]; then + # shellcheck disable=SC1091 + source ./venv/bin/activate +fi +export PYTHONPATH=. +exec python cre.py --ingest_github "$URL" "$@"