diff --git a/Makefile b/Makefile index 1eece7b79..17b8c0ea1 100644 --- a/Makefile +++ b/Makefile @@ -162,7 +162,7 @@ lint: [ -d "./venv" ] && . ./venv/bin/activate && black . && yarn lint && make openapi-guardrail mypy: - [ -d "./venv" ] && . ./venv/bin/activate && mypy --ignore-missing-imports --implicit-reexport --no-strict-optional --strict application + [ -d "./venv" ] && . ./venv/bin/activate && mypy --ignore-missing-imports --implicit-reexport --no-strict-optional --strict application cre_logging frontend: yarn build diff --git a/application/__init__.py b/application/__init__.py index f88a2cb20..370c70327 100644 --- a/application/__init__.py +++ b/application/__init__.py @@ -1,4 +1,8 @@ # type:ignore +from cre_logging import get_logger + +logger = get_logger(__name__) + import string from typing import Any from sqlalchemy import MetaData diff --git a/application/cmd/__init__.py b/application/cmd/__init__.py index e69de29bb..9662899ca 100644 --- a/application/cmd/__init__.py +++ b/application/cmd/__init__.py @@ -0,0 +1,3 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) diff --git a/application/cmd/cre_main.py b/application/cmd/cre_main.py index d8e59da4c..09abb66b4 100644 --- a/application/cmd/cre_main.py +++ b/application/cmd/cre_main.py @@ -1,8 +1,11 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + import re import time import argparse import json -import logging import os import shutil import tempfile @@ -29,9 +32,6 @@ if TYPE_CHECKING: from application.prompt_client import prompt_client as prompt_client -logging.basicConfig() -logger = logging.getLogger(__name__) -logger.setLevel(logging.INFO) app = None diff --git a/application/config.py b/application/config.py index 47ec87086..de7c0036f 100644 --- a/application/config.py +++ b/application/config.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + import os basedir = os.path.abspath(os.path.dirname(__file__)) diff --git a/application/database/__init__.py b/application/database/__init__.py index e69de29bb..9662899ca 100644 --- a/application/database/__init__.py +++ b/application/database/__init__.py @@ -0,0 +1,3 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) diff --git a/application/database/db.py b/application/database/db.py index 522612161..92a82554c 100644 --- a/application/database/db.py +++ b/application/database/db.py @@ -1,15 +1,17 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + from sqlalchemy import CheckConstraint import networkx as nx import uuid import neo4j import os -import logging import re import time import yaml from datetime import datetime, timezone -from pprint import pprint from collections import Counter, defaultdict from itertools import permutations @@ -54,10 +56,6 @@ from .. import sqla # type: ignore -logging.basicConfig() -logger = logging.getLogger(__name__) -logger.setLevel(logging.INFO) - BaseModel: DefaultMeta = sqla.Model diff --git a/application/database/inmemory_graph.py b/application/database/inmemory_graph.py index e4407899f..c29bb1f34 100644 --- a/application/database/inmemory_graph.py +++ b/application/database/inmemory_graph.py @@ -1,15 +1,13 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + import sys -import logging import networkx as nx from typing import List, Tuple from application.defs import cre_defs as defs -logging.basicConfig() -logger = logging.getLogger(__name__) -logger.setLevel(logging.INFO) - - class CycleDetectedError(Exception): def __init__(self, message): super().__init__(message) diff --git a/application/database/pgvector_utils.py b/application/database/pgvector_utils.py index 506a7f0e5..48d29098d 100644 --- a/application/database/pgvector_utils.py +++ b/application/database/pgvector_utils.py @@ -12,6 +12,10 @@ from __future__ import annotations +from cre_logging import get_logger + +logger = get_logger(__name__) + import os from typing import Any, Iterable, Optional, Sequence, Set, Tuple @@ -53,9 +57,7 @@ class PgVectorUnavailableError(RuntimeError): def _exit_with_message(msg: str) -> None: - import logging - - logging.getLogger(__name__).error(msg) + logger.error(msg) raise SystemExit(msg) diff --git a/application/defs/__init__.py b/application/defs/__init__.py index e69de29bb..9662899ca 100644 --- a/application/defs/__init__.py +++ b/application/defs/__init__.py @@ -0,0 +1,3 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) diff --git a/application/defs/cheatsheet_defs.py b/application/defs/cheatsheet_defs.py index 0e1dc4267..eb6a4c2ea 100644 --- a/application/defs/cheatsheet_defs.py +++ b/application/defs/cheatsheet_defs.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + from dataclasses import dataclass, field from typing import Dict, List diff --git a/application/defs/cre_defs.py b/application/defs/cre_defs.py index 5edf7121b..1a946ba76 100644 --- a/application/defs/cre_defs.py +++ b/application/defs/cre_defs.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + import re import json from copy import copy diff --git a/application/defs/cre_exceptions.py b/application/defs/cre_exceptions.py index 4ef5f8548..8246fb39d 100644 --- a/application/defs/cre_exceptions.py +++ b/application/defs/cre_exceptions.py @@ -1,3 +1,8 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + + class DocumentFormatException(Exception): pass diff --git a/application/defs/osib_defs.py b/application/defs/osib_defs.py index 89c204704..370865253 100644 --- a/application/defs/osib_defs.py +++ b/application/defs/osib_defs.py @@ -1,6 +1,8 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + import networkx as nx -from pprint import pprint -import logging import os import re import warnings @@ -22,10 +24,6 @@ from_dict, ) -logging.basicConfig() -logger = logging.getLogger(__name__) -logger.setLevel(logging.INFO) - # used for serialising and deserialising yaml OSIB documents diff --git a/application/feature_flags.py b/application/feature_flags.py index 15c301564..ec4b729f4 100644 --- a/application/feature_flags.py +++ b/application/feature_flags.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + import os try: diff --git a/application/mcp/__init__.py b/application/mcp/__init__.py index 28f3086ae..8a6a5a00b 100644 --- a/application/mcp/__init__.py +++ b/application/mcp/__init__.py @@ -1,5 +1,9 @@ """Local stdio MCP server exposing public OpenCRE REST reads (issue #1003 v1).""" +from cre_logging import get_logger + +logger = get_logger(__name__) + from application.mcp.catalog import PUBLIC_TOOLS, get_tool, list_tool_names __all__ = ["PUBLIC_TOOLS", "get_tool", "list_tool_names"] diff --git a/application/mcp/__main__.py b/application/mcp/__main__.py index 7667bd7e7..cd32b7d3e 100644 --- a/application/mcp/__main__.py +++ b/application/mcp/__main__.py @@ -2,17 +2,15 @@ from __future__ import annotations +from cre_logging import get_logger + +logger = get_logger(__name__) + import asyncio -import logging -import sys def main() -> None: - logging.basicConfig( - level=logging.INFO, - stream=sys.stderr, - format="%(levelname)s %(name)s: %(message)s", - ) + logger.info("starting OpenCRE MCP stdio server") from application.mcp.server import run_stdio asyncio.run(run_stdio()) diff --git a/application/mcp/catalog.py b/application/mcp/catalog.py index c91981b0a..ebbf04273 100644 --- a/application/mcp/catalog.py +++ b/application/mcp/catalog.py @@ -5,6 +5,10 @@ from __future__ import annotations +from cre_logging import get_logger + +logger = get_logger(__name__) + from dataclasses import dataclass from typing import Dict, List, Tuple diff --git a/application/mcp/openapi_loader.py b/application/mcp/openapi_loader.py index 5fde907b8..c881f6d94 100644 --- a/application/mcp/openapi_loader.py +++ b/application/mcp/openapi_loader.py @@ -6,6 +6,10 @@ from __future__ import annotations +from cre_logging import get_logger + +logger = get_logger(__name__) + import copy import os from functools import lru_cache diff --git a/application/mcp/rest_client.py b/application/mcp/rest_client.py index f5fb4e969..e8c682cba 100644 --- a/application/mcp/rest_client.py +++ b/application/mcp/rest_client.py @@ -6,8 +6,11 @@ from __future__ import annotations +from cre_logging import get_logger + +logger = get_logger(__name__) + import json -import logging import os import re from dataclasses import dataclass @@ -22,7 +25,6 @@ from application.mcp.catalog import ToolSpec, get_tool from application.mcp.openapi_loader import operation_input_schema -logger = logging.getLogger(__name__) DEFAULT_BASE_URL = "http://127.0.0.1:5000" DEFAULT_TIMEOUT_SECONDS = 30.0 diff --git a/application/mcp/server.py b/application/mcp/server.py index 31429ff6e..15b529527 100644 --- a/application/mcp/server.py +++ b/application/mcp/server.py @@ -2,8 +2,11 @@ from __future__ import annotations +from cre_logging import get_logger + +logger = get_logger(__name__) + import json -import logging from typing import Any, Dict, Optional import mcp.types as types @@ -19,7 +22,6 @@ RestResponseError, ) -logger = logging.getLogger(__name__) SERVER_NAME = "opencre" SERVER_VERSION = "0.1.0" diff --git a/application/prompt_client/embed_alignment.py b/application/prompt_client/embed_alignment.py index 0bfc861a9..8dc2ea476 100644 --- a/application/prompt_client/embed_alignment.py +++ b/application/prompt_client/embed_alignment.py @@ -8,8 +8,11 @@ from __future__ import annotations +from cre_logging import get_logger + +logger = get_logger(__name__) + import json -import logging import os import re from dataclasses import dataclass @@ -17,7 +20,6 @@ from urllib.parse import urldefrag from pydantic import BaseModel, ConfigDict, Field, ValidationError -logger = logging.getLogger(__name__) SMART_EXTRACT_POLICY_VERSION = os.environ.get("CRE_EMBED_SMART_POLICY_VERSION", "1") diff --git a/application/prompt_client/llm_error_utils.py b/application/prompt_client/llm_error_utils.py index ecf2aeea3..3044d3ad5 100644 --- a/application/prompt_client/llm_error_utils.py +++ b/application/prompt_client/llm_error_utils.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + from typing import Any diff --git a/application/prompt_client/prompt_client.py b/application/prompt_client/prompt_client.py index d9e151118..81a7a284f 100644 --- a/application/prompt_client/prompt_client.py +++ b/application/prompt_client/prompt_client.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + from application.database import db from application.defs import cre_defs from datetime import datetime @@ -12,7 +16,6 @@ from typing import Dict, List, Any, Tuple, Optional from pydantic import ValidationError from jinja2 import Environment, FileSystemLoader, StrictUndefined -import logging try: from pypdf import PdfReader @@ -25,9 +28,6 @@ import requests import time -logging.basicConfig() -logger = logging.getLogger(__name__) -logger.setLevel(logging.INFO) SIMILARITY_THRESHOLD = float(os.environ.get("CHATBOT_SIMILARITY_THRESHOLD", "0.7")) PROMPT_TEMPLATES_DIR = os.path.join(os.path.dirname(__file__), "templates") diff --git a/application/tests/__init__.py b/application/tests/__init__.py index e69de29bb..9662899ca 100644 --- a/application/tests/__init__.py +++ b/application/tests/__init__.py @@ -0,0 +1,3 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) diff --git a/application/tests/admin_imports_api_test.py b/application/tests/admin_imports_api_test.py index 26d9cbc69..367ee7854 100644 --- a/application/tests/admin_imports_api_test.py +++ b/application/tests/admin_imports_api_test.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + import os import unittest from unittest.mock import patch diff --git a/application/tests/ai_exchange_csv_test.py b/application/tests/ai_exchange_csv_test.py index 1e5003d75..fa12a3445 100644 --- a/application/tests/ai_exchange_csv_test.py +++ b/application/tests/ai_exchange_csv_test.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + import os import unittest diff --git a/application/tests/auth_routes_test.py b/application/tests/auth_routes_test.py index 9d6f8f059..cae553308 100644 --- a/application/tests/auth_routes_test.py +++ b/application/tests/auth_routes_test.py @@ -5,6 +5,10 @@ the NO_LOGIN dev bypass. OpenAPI documentation is intentionally out of scope here. """ +from cre_logging import get_logger + +logger = get_logger(__name__) + import os import unittest from typing import Any diff --git a/application/tests/capec_parser_test.py b/application/tests/capec_parser_test.py index 5f78a8c45..6974672a1 100644 --- a/application/tests/capec_parser_test.py +++ b/application/tests/capec_parser_test.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + from application.defs import cre_defs as defs import unittest from application import create_app, sqla # type: ignore diff --git a/application/tests/ccmv4_parser_test.py b/application/tests/ccmv4_parser_test.py index 410acf4e8..b674c7a34 100644 --- a/application/tests/ccmv4_parser_test.py +++ b/application/tests/ccmv4_parser_test.py @@ -1,5 +1,9 @@ # Tests disabled as importer disabled +from cre_logging import get_logger + +logger = get_logger(__name__) + # from pathlib import Path # from tempfile import mkdtemp, mkstemp # import zipfile diff --git a/application/tests/chat_completion_test.py b/application/tests/chat_completion_test.py index f601d7df5..095d87e9e 100644 --- a/application/tests/chat_completion_test.py +++ b/application/tests/chat_completion_test.py @@ -1,5 +1,9 @@ """Tests for /rest/v1/completion (chatbot) error handling.""" +from cre_logging import get_logger + +logger = get_logger(__name__) + import json import os import unittest diff --git a/application/tests/cheatsheet_extractor_test.py b/application/tests/cheatsheet_extractor_test.py index 1635d7252..4a0bd48c6 100644 --- a/application/tests/cheatsheet_extractor_test.py +++ b/application/tests/cheatsheet_extractor_test.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + import subprocess import tempfile import unittest diff --git a/application/tests/cheatsheet_record_adapter_test.py b/application/tests/cheatsheet_record_adapter_test.py index 7441c6b7f..915514bf0 100644 --- a/application/tests/cheatsheet_record_adapter_test.py +++ b/application/tests/cheatsheet_record_adapter_test.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + import unittest from application.defs.cheatsheet_defs import CheatsheetRecord diff --git a/application/tests/cheatsheets_parser_test.py b/application/tests/cheatsheets_parser_test.py index 3fd31d194..2816e9cdf 100644 --- a/application/tests/cheatsheets_parser_test.py +++ b/application/tests/cheatsheets_parser_test.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + from application.defs import cre_defs as defs import unittest from application import create_app, sqla # type: ignore diff --git a/application/tests/cheatsheets_workstream_f_test.py b/application/tests/cheatsheets_workstream_f_test.py index 042dbccb0..335ac528d 100644 --- a/application/tests/cheatsheets_workstream_f_test.py +++ b/application/tests/cheatsheets_workstream_f_test.py @@ -14,6 +14,10 @@ out of scope here. """ +from cre_logging import get_logger + +logger = get_logger(__name__) + import contextlib import io import json @@ -304,9 +308,9 @@ def _run_cli(argv): class TestCliValidate(unittest.TestCase): def test_valid_fixture_exits_zero(self) -> None: - code, out, _err = _run_cli(["validate", VALID_FIXTURE]) + code, _out, err = _run_cli(["validate", VALID_FIXTURE]) self.assertEqual(code, 0) - self.assertIn("OK", out) + self.assertIn("OK", err) def test_invalid_fixture_exits_nonzero_and_names_field(self) -> None: code, _out, err = _run_cli(["validate", INVALID_FIXTURE]) @@ -358,16 +362,14 @@ def test_convert_reports_skipped_unknown_and_links(self) -> None: path = os.path.join(d, "approved.json") wf.write_suggestions_json(path, suggestions) with mock.patch.object(wf, "_open_cache", return_value=stub): - code, out, _err = _run_cli(["convert", path]) + code, _out, err = _run_cli(["convert", path]) self.assertEqual(code, 0) - # Prove the skipped-id reporting actually prints the unknown id. - self.assertIn("999-999", out) - # One Standard with one resolved link survives. - self.assertIn("standards produced: 1", out) - self.assertIn("total CRE links: 1", out) - # And it is explicit that nothing was registered (F5 follow-up). - self.assertIn("F5", out) + # Prove the skipped-id reporting actually logs the unknown id. + self.assertIn("999-999", err) + self.assertIn("standards produced=1", err) + self.assertIn("total CRE links=1", err) + self.assertIn("F5", err) def test_convert_missing_file_exits_two(self) -> None: with tempfile.TemporaryDirectory() as d: diff --git a/application/tests/cloud_native_security_controls_parser_test.py b/application/tests/cloud_native_security_controls_parser_test.py index 6b2438d04..de25e5184 100644 --- a/application/tests/cloud_native_security_controls_parser_test.py +++ b/application/tests/cloud_native_security_controls_parser_test.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + from pathlib import Path from tempfile import mkdtemp, mkstemp import zipfile diff --git a/application/tests/cre_logging_test.py b/application/tests/cre_logging_test.py new file mode 100644 index 000000000..c70294833 --- /dev/null +++ b/application/tests/cre_logging_test.py @@ -0,0 +1,153 @@ +"""Tests for the standalone ``cre_logging`` package. + +These tests import ``cre_logging`` only — not ``application`` / Flask — +except the subprocess isolation check which uses a fresh interpreter. +""" + +from __future__ import annotations + +import contextlib +import json +import logging +import os +import subprocess +import sys +import unittest +from io import StringIO +from pathlib import Path +from unittest.mock import patch + +from cre_logging import JSONFormatter, configure_logging, get_logger + +logger = get_logger(__name__) + +REPO_ROOT = Path(__file__).resolve().parents[2] + + +class TestJSONFormatter(unittest.TestCase): + def test_emits_caller_module_file_and_method(self) -> None: + stream = StringIO() + probe = logging.getLogger("cre_logging.test.caller") + probe.handlers.clear() + probe.setLevel(logging.DEBUG) + probe.propagate = False + handler = logging.StreamHandler(stream) + handler.setFormatter(JSONFormatter()) + probe.addHandler(handler) + + def sample_method() -> None: + probe.info("hello from sample_method") + + sample_method() + parsed = json.loads(stream.getvalue().strip()) + self.assertEqual(parsed["message"], "hello from sample_method") + self.assertEqual(parsed["level"], "INFO") + self.assertEqual(parsed["method"], "sample_method") + self.assertEqual(parsed["module"], "cre_logging_test") + self.assertEqual(parsed["file"], "application/tests/cre_logging_test.py") + self.assertIn("timestamp", parsed) + self.assertIn("line", parsed) + probe.handlers.clear() + + def test_debug_and_error_levels(self) -> None: + stream = StringIO() + probe = logging.getLogger("cre_logging.test.levels") + probe.handlers.clear() + probe.setLevel(logging.DEBUG) + probe.propagate = False + handler = logging.StreamHandler(stream) + handler.setFormatter(JSONFormatter()) + probe.addHandler(handler) + probe.debug("dbg") + probe.error("err") + lines = [json.loads(line) for line in stream.getvalue().strip().splitlines()] + self.assertEqual([row["level"] for row in lines], ["DEBUG", "ERROR"]) + probe.handlers.clear() + + def test_exception_field(self) -> None: + formatter = JSONFormatter() + try: + raise ValueError("boom") + except ValueError: + exc_info = sys.exc_info() + record = logging.LogRecord( + name="test", + level=logging.ERROR, + pathname=__file__, + lineno=1, + msg="failed", + args=(), + exc_info=exc_info, + ) + parsed = json.loads(formatter.format(record)) + self.assertIn("ValueError", parsed["exception"]) + self.assertIn("boom", parsed["exception"]) + + def test_extra_fields_are_merged(self) -> None: + stream = StringIO() + probe = logging.getLogger("cre_logging.test.extra") + probe.handlers.clear() + probe.setLevel(logging.INFO) + probe.propagate = False + handler = logging.StreamHandler(stream) + handler.setFormatter(JSONFormatter()) + probe.addHandler(handler) + probe.info("linked", extra={"cre_id": "123-456"}) + parsed = json.loads(stream.getvalue().strip()) + self.assertEqual(parsed["cre_id"], "123-456") + probe.handlers.clear() + + +class TestConfigureLogging(unittest.TestCase): + def test_log_level_env(self) -> None: + with patch.dict(os.environ, {"LOG_LEVEL": "DEBUG"}): + configure_logging(force=True) + self.assertEqual(logging.getLogger().level, logging.DEBUG) + with patch.dict(os.environ, {"LOG_LEVEL": "WARNING"}): + configure_logging(force=True) + self.assertEqual(logging.getLogger().level, logging.WARNING) + + def test_repeated_configure_does_not_stack_handlers(self) -> None: + configure_logging(force=True) + before = [ + h + for h in logging.getLogger().handlers + if getattr(h, "name", "") == "cre-json" + ] + configure_logging() + after = [ + h + for h in logging.getLogger().handlers + if getattr(h, "name", "") == "cre-json" + ] + self.assertEqual(len(before), 1) + self.assertEqual(len(after), 1) + + def test_get_logger_writes_json_to_stderr(self) -> None: + with patch.dict(os.environ, {"LOG_LEVEL": "INFO"}): + configure_logging(force=True) + buf = StringIO() + with contextlib.redirect_stderr(buf): + get_logger("cre_logging.test.emit").warning("structured line") + parsed = json.loads(buf.getvalue().strip().splitlines()[-1]) + self.assertEqual(parsed["message"], "structured line") + self.assertEqual(parsed["level"], "WARNING") + self.assertEqual(parsed["logger"], "cre_logging.test.emit") + logger.debug("example from test method") + + +class TestStandaloneImport(unittest.TestCase): + def test_importing_cre_logging_does_not_import_application_or_flask(self) -> None: + code = ( + "import sys\n" + "import cre_logging\n" + "assert 'flask' not in sys.modules, sorted(sys.modules)[:20]\n" + "assert 'application' not in sys.modules\n" + "assert 'application.__init__' not in sys.modules\n" + "cre_logging.get_logger('probe').debug('ok')\n" + ) + subprocess.check_call( + [sys.executable, "-c", code], + cwd=str(REPO_ROOT), + env={**os.environ, "PYTHONPATH": str(REPO_ROOT)}, + ) diff --git a/application/tests/cre_main_test.py b/application/tests/cre_main_test.py index 61bb23c9e..16db19961 100644 --- a/application/tests/cre_main_test.py +++ b/application/tests/cre_main_test.py @@ -1,4 +1,7 @@ -import logging +from cre_logging import get_logger + +logger = get_logger(__name__) + import os import shutil import tempfile diff --git a/application/tests/cwe_parser_test.py b/application/tests/cwe_parser_test.py index 1b07b01a6..adcaa2755 100644 --- a/application/tests/cwe_parser_test.py +++ b/application/tests/cwe_parser_test.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + from pathlib import Path from tempfile import mkdtemp, mkstemp import zipfile diff --git a/application/tests/db_test.py b/application/tests/db_test.py index c3979753d..fbf2a3875 100644 --- a/application/tests/db_test.py +++ b/application/tests/db_test.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + import networkx as nx from application.utils.gap_analysis import make_resources_key, make_subresources_key import string @@ -9,7 +13,6 @@ from unittest.mock import patch import uuid from copy import copy, deepcopy -from pprint import pprint from typing import Any, Dict, List, Union from flask import json as flask_json diff --git a/application/tests/defs_test.py b/application/tests/defs_test.py index 33a866c85..0af5f880e 100644 --- a/application/tests/defs_test.py +++ b/application/tests/defs_test.py @@ -1,7 +1,10 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + import copy import unittest from dataclasses import asdict -from pprint import pprint from typing import Set import dacite diff --git a/application/tests/dsomm_parser_test.py b/application/tests/dsomm_parser_test.py index d28240c79..225423558 100644 --- a/application/tests/dsomm_parser_test.py +++ b/application/tests/dsomm_parser_test.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + from application.defs import cre_defs as defs import unittest from application import create_app, sqla # type: ignore diff --git a/application/tests/embed_alignment_test.py b/application/tests/embed_alignment_test.py index f693a7cef..2737ebab5 100644 --- a/application/tests/embed_alignment_test.py +++ b/application/tests/embed_alignment_test.py @@ -1,5 +1,9 @@ """Unit tests for smart embedding excerpt alignment (no network, no LLM).""" +from cre_logging import get_logger + +logger = get_logger(__name__) + import unittest from application.defs import cre_defs diff --git a/application/tests/embeddings_sync_scripts_test.py b/application/tests/embeddings_sync_scripts_test.py index 58dff1ff4..b318b919c 100644 --- a/application/tests/embeddings_sync_scripts_test.py +++ b/application/tests/embeddings_sync_scripts_test.py @@ -2,6 +2,10 @@ from __future__ import annotations +from cre_logging import get_logger + +logger = get_logger(__name__) + import sqlite3 import tempfile import unittest diff --git a/application/tests/export_cli_test.py b/application/tests/export_cli_test.py index 6fa691ded..99b68eb81 100644 --- a/application/tests/export_cli_test.py +++ b/application/tests/export_cli_test.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + import unittest from argparse import Namespace from unittest.mock import patch diff --git a/application/tests/ga_parity_test.py b/application/tests/ga_parity_test.py index 68b53c1c5..34fbf7020 100644 --- a/application/tests/ga_parity_test.py +++ b/application/tests/ga_parity_test.py @@ -1,5 +1,9 @@ """Regression tests for GA Postgres vs Neo materiality helpers.""" +from cre_logging import get_logger + +logger = get_logger(__name__) + import unittest from application.utils import gap_analysis diff --git a/application/tests/gap_analysis_db_test.py b/application/tests/gap_analysis_db_test.py index 02dcb0c59..b80ee0d5b 100644 --- a/application/tests/gap_analysis_db_test.py +++ b/application/tests/gap_analysis_db_test.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + import unittest from unittest.mock import MagicMock, patch from application.database import db diff --git a/application/tests/gap_analysis_pair_job_test.py b/application/tests/gap_analysis_pair_job_test.py index eeeb86007..f8d8e6bad 100644 --- a/application/tests/gap_analysis_pair_job_test.py +++ b/application/tests/gap_analysis_pair_job_test.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + import json import unittest from types import SimpleNamespace diff --git a/application/tests/gap_analysis_test.py b/application/tests/gap_analysis_test.py index 1c3e579f0..a2cd305c8 100644 --- a/application/tests/gap_analysis_test.py +++ b/application/tests/gap_analysis_test.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + import unittest from application.defs import cre_defs diff --git a/application/tests/harvester_test/__init__.py b/application/tests/harvester_test/__init__.py index 06b0f7440..27553f15e 100644 --- a/application/tests/harvester_test/__init__.py +++ b/application/tests/harvester_test/__init__.py @@ -1,3 +1,7 @@ """ tests for Module A configuration layer (empty for now) """ + +from cre_logging import get_logger + +logger = get_logger(__name__) diff --git a/application/tests/harvester_test/artifact_registry_test.py b/application/tests/harvester_test/artifact_registry_test.py index 39d0c54f7..e84551e70 100644 --- a/application/tests/harvester_test/artifact_registry_test.py +++ b/application/tests/harvester_test/artifact_registry_test.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + import unittest from datetime import datetime diff --git a/application/tests/harvester_test/change_detector_test.py b/application/tests/harvester_test/change_detector_test.py index 5cb33456a..a097eb1b2 100644 --- a/application/tests/harvester_test/change_detector_test.py +++ b/application/tests/harvester_test/change_detector_test.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + import unittest from unittest.mock import MagicMock from unittest.mock import call diff --git a/application/tests/harvester_test/checkpoint_manager_test.py b/application/tests/harvester_test/checkpoint_manager_test.py index a3ddb7cea..00b9aa22d 100644 --- a/application/tests/harvester_test/checkpoint_manager_test.py +++ b/application/tests/harvester_test/checkpoint_manager_test.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + import unittest from datetime import datetime diff --git a/application/tests/harvester_test/checkpoint_store_test.py b/application/tests/harvester_test/checkpoint_store_test.py index 5d5f1e947..538374cc1 100644 --- a/application/tests/harvester_test/checkpoint_store_test.py +++ b/application/tests/harvester_test/checkpoint_store_test.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + import unittest from datetime import datetime, timezone diff --git a/application/tests/harvester_test/chunk_pipeline_test.py b/application/tests/harvester_test/chunk_pipeline_test.py index 858a17709..c16b7cab9 100644 --- a/application/tests/harvester_test/chunk_pipeline_test.py +++ b/application/tests/harvester_test/chunk_pipeline_test.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + import unittest from unittest.mock import Mock diff --git a/application/tests/harvester_test/chunk_record_builder_test.py b/application/tests/harvester_test/chunk_record_builder_test.py index 55f2c1ee4..f128a19ab 100644 --- a/application/tests/harvester_test/chunk_record_builder_test.py +++ b/application/tests/harvester_test/chunk_record_builder_test.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + import unittest from datetime import datetime, timezone diff --git a/application/tests/harvester_test/chunk_record_validator_test.py b/application/tests/harvester_test/chunk_record_validator_test.py index c71664cfe..402e1dcd5 100644 --- a/application/tests/harvester_test/chunk_record_validator_test.py +++ b/application/tests/harvester_test/chunk_record_validator_test.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + import unittest from application.utils.harvester.chunk_record_validator import ( diff --git a/application/tests/harvester_test/chunker_test.py b/application/tests/harvester_test/chunker_test.py index 22180531a..a5313119f 100644 --- a/application/tests/harvester_test/chunker_test.py +++ b/application/tests/harvester_test/chunker_test.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + import unittest from datetime import datetime, timezone diff --git a/application/tests/harvester_test/chunking_benchmark_test.py b/application/tests/harvester_test/chunking_benchmark_test.py index 711316f2a..c297fdb27 100644 --- a/application/tests/harvester_test/chunking_benchmark_test.py +++ b/application/tests/harvester_test/chunking_benchmark_test.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + import os import time import unittest @@ -37,11 +41,11 @@ def test_chunking_benchmark(self): ) ) - print( - f"\nChunking benchmark: " - f"{len(chunks)} chunks, " - f"{elapsed:.3f}s, " - f"input={len(text)} chars" + logger.info( + "Chunking benchmark: %s chunks, %.3fs, input=%s chars", + len(chunks), + elapsed, + len(text), ) diff --git a/application/tests/harvester_test/config_loader_test.py b/application/tests/harvester_test/config_loader_test.py index 730f4d708..f3dd1795a 100644 --- a/application/tests/harvester_test/config_loader_test.py +++ b/application/tests/harvester_test/config_loader_test.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + from pathlib import Path import unittest diff --git a/application/tests/harvester_test/content_hash_test.py b/application/tests/harvester_test/content_hash_test.py index 60d84108f..82e84067c 100644 --- a/application/tests/harvester_test/content_hash_test.py +++ b/application/tests/harvester_test/content_hash_test.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + import unittest from application.utils.harvester.content_hash import ( diff --git a/application/tests/harvester_test/deduplication_metrics_test.py b/application/tests/harvester_test/deduplication_metrics_test.py index 13e4e1152..8f9dcef94 100644 --- a/application/tests/harvester_test/deduplication_metrics_test.py +++ b/application/tests/harvester_test/deduplication_metrics_test.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + import unittest from application.utils.harvester.deduplication_metrics import DeduplicationMetrics diff --git a/application/tests/harvester_test/diff_normalizer_test.py b/application/tests/harvester_test/diff_normalizer_test.py index 075b8fae2..a230a036d 100644 --- a/application/tests/harvester_test/diff_normalizer_test.py +++ b/application/tests/harvester_test/diff_normalizer_test.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + import unittest from datetime import datetime diff --git a/application/tests/harvester_test/diff_parser_test.py b/application/tests/harvester_test/diff_parser_test.py index 877d70a09..cb5907589 100644 --- a/application/tests/harvester_test/diff_parser_test.py +++ b/application/tests/harvester_test/diff_parser_test.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + from datetime import UTC, datetime import unittest diff --git a/application/tests/harvester_test/diff_pipeline_test.py b/application/tests/harvester_test/diff_pipeline_test.py index 1f29048fd..a6dd9aef0 100644 --- a/application/tests/harvester_test/diff_pipeline_test.py +++ b/application/tests/harvester_test/diff_pipeline_test.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + import os import subprocess import time @@ -64,7 +68,7 @@ def test_pipeline_benchmark(self): normalizer.normalize(blocks) elapsed = time.perf_counter() - start - print(f"\nPipeline took {elapsed:.3f}s") + logger.info("Pipeline took %.3fs", elapsed) self.assertLess(elapsed, 5) diff --git a/application/tests/harvester_test/diff_retriever_test.py b/application/tests/harvester_test/diff_retriever_test.py index 1a23e0b8e..279728f9c 100644 --- a/application/tests/harvester_test/diff_retriever_test.py +++ b/application/tests/harvester_test/diff_retriever_test.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + import unittest from unittest.mock import MagicMock from unittest.mock import call diff --git a/application/tests/harvester_test/document_builder_test.py b/application/tests/harvester_test/document_builder_test.py index ba2313c08..dc5cc7a58 100644 --- a/application/tests/harvester_test/document_builder_test.py +++ b/application/tests/harvester_test/document_builder_test.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + import unittest from datetime import datetime diff --git a/application/tests/harvester_test/document_deduplicator_test.py b/application/tests/harvester_test/document_deduplicator_test.py index 64601713f..bbf4721c0 100644 --- a/application/tests/harvester_test/document_deduplicator_test.py +++ b/application/tests/harvester_test/document_deduplicator_test.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + import unittest from datetime import datetime diff --git a/application/tests/harvester_test/document_validator_test.py b/application/tests/harvester_test/document_validator_test.py index 7de91263b..a4a2eb433 100644 --- a/application/tests/harvester_test/document_validator_test.py +++ b/application/tests/harvester_test/document_validator_test.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + import unittest from datetime import datetime diff --git a/application/tests/harvester_test/file_filter_test.py b/application/tests/harvester_test/file_filter_test.py index 82412df0d..ca3a75e85 100644 --- a/application/tests/harvester_test/file_filter_test.py +++ b/application/tests/harvester_test/file_filter_test.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + import unittest from application.utils.harvester.file_filter import ( diff --git a/application/tests/harvester_test/filtering_benchmark_test.py b/application/tests/harvester_test/filtering_benchmark_test.py index a75c435ec..52a6d0bb2 100644 --- a/application/tests/harvester_test/filtering_benchmark_test.py +++ b/application/tests/harvester_test/filtering_benchmark_test.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + import unittest from application.utils.harvester.file_filter import FileFilter diff --git a/application/tests/harvester_test/filtering_metrics_test.py b/application/tests/harvester_test/filtering_metrics_test.py index 252cb0941..29795af7a 100644 --- a/application/tests/harvester_test/filtering_metrics_test.py +++ b/application/tests/harvester_test/filtering_metrics_test.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + import unittest from application.utils.harvester.filtering_metrics import ( diff --git a/application/tests/harvester_test/git_repository_client_integration_test.py b/application/tests/harvester_test/git_repository_client_integration_test.py index c12d923bb..79d3c0008 100644 --- a/application/tests/harvester_test/git_repository_client_integration_test.py +++ b/application/tests/harvester_test/git_repository_client_integration_test.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + import subprocess import tempfile import unittest diff --git a/application/tests/harvester_test/git_repository_client_test.py b/application/tests/harvester_test/git_repository_client_test.py index 98fc07fcb..df35f4249 100644 --- a/application/tests/harvester_test/git_repository_client_test.py +++ b/application/tests/harvester_test/git_repository_client_test.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + import unittest from unittest.mock import patch diff --git a/application/tests/harvester_test/harvest_pipeline_test.py b/application/tests/harvester_test/harvest_pipeline_test.py index 50e5769ee..c21b615a1 100644 --- a/application/tests/harvester_test/harvest_pipeline_test.py +++ b/application/tests/harvester_test/harvest_pipeline_test.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + import unittest from datetime import datetime, timezone from unittest.mock import Mock diff --git a/application/tests/harvester_test/heading_extractor_test.py b/application/tests/harvester_test/heading_extractor_test.py index 94413edd3..ffbef3d16 100644 --- a/application/tests/harvester_test/heading_extractor_test.py +++ b/application/tests/harvester_test/heading_extractor_test.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + import unittest from application.utils.harvester.heading_extractor import ( diff --git a/application/tests/harvester_test/incremental_pipeline_test.py b/application/tests/harvester_test/incremental_pipeline_test.py index 9728a8c47..fed260792 100644 --- a/application/tests/harvester_test/incremental_pipeline_test.py +++ b/application/tests/harvester_test/incremental_pipeline_test.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + import unittest from datetime import datetime from unittest.mock import Mock diff --git a/application/tests/harvester_test/repos_validator_test.py b/application/tests/harvester_test/repos_validator_test.py index 278018e26..49efeb847 100644 --- a/application/tests/harvester_test/repos_validator_test.py +++ b/application/tests/harvester_test/repos_validator_test.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + from pathlib import Path import unittest diff --git a/application/tests/harvester_test/repository_cache_test.py b/application/tests/harvester_test/repository_cache_test.py index d3193d454..9962fa857 100644 --- a/application/tests/harvester_test/repository_cache_test.py +++ b/application/tests/harvester_test/repository_cache_test.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + import unittest from pathlib import Path diff --git a/application/tests/import_dashboard_test.py b/application/tests/import_dashboard_test.py index ac781c6c0..80b06f646 100644 --- a/application/tests/import_dashboard_test.py +++ b/application/tests/import_dashboard_test.py @@ -2,6 +2,10 @@ from __future__ import annotations +from cre_logging import get_logger + +logger = get_logger(__name__) + import importlib.util import os import unittest diff --git a/application/tests/import_diff_test.py b/application/tests/import_diff_test.py index 3f7d60094..186875136 100644 --- a/application/tests/import_diff_test.py +++ b/application/tests/import_diff_test.py @@ -1,5 +1,9 @@ """Tests for Module C diff scaffolding (Step 6) and change-set (Step 7).""" +from cre_logging import get_logger + +logger = get_logger(__name__) + import unittest from application.defs import cre_defs as defs from application.utils.import_diff import ( diff --git a/application/tests/import_pipeline_pair_scheduler_test.py b/application/tests/import_pipeline_pair_scheduler_test.py index d65a96a07..cdcf6b9e4 100644 --- a/application/tests/import_pipeline_pair_scheduler_test.py +++ b/application/tests/import_pipeline_pair_scheduler_test.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + import unittest from unittest.mock import Mock, patch diff --git a/application/tests/import_post_apply_test.py b/application/tests/import_post_apply_test.py index 3b719325f..781f44f87 100644 --- a/application/tests/import_post_apply_test.py +++ b/application/tests/import_post_apply_test.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + import unittest from unittest.mock import Mock, patch diff --git a/application/tests/import_run_test.py b/application/tests/import_run_test.py index 7b03161a6..5581764a9 100644 --- a/application/tests/import_run_test.py +++ b/application/tests/import_run_test.py @@ -1,5 +1,9 @@ """Tests for import run metadata (Step 6).""" +from cre_logging import get_logger + +logger = get_logger(__name__) + import json import unittest from datetime import datetime, timezone diff --git a/application/tests/import_staging_test.py b/application/tests/import_staging_test.py index 0108ea489..e63ea799d 100644 --- a/application/tests/import_staging_test.py +++ b/application/tests/import_staging_test.py @@ -1,5 +1,9 @@ """Phase 2 staging persistence tests (snapshots + change sets).""" +from cre_logging import get_logger + +logger = get_logger(__name__) + import unittest from application import create_app, sqla diff --git a/application/tests/inmemory_graph_test.py b/application/tests/inmemory_graph_test.py index d028df5b1..3d18a2516 100644 --- a/application/tests/inmemory_graph_test.py +++ b/application/tests/inmemory_graph_test.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + import unittest import networkx as nx from application.defs import cre_defs as defs diff --git a/application/tests/juiceshop_test.py b/application/tests/juiceshop_test.py index 6ede99417..cb83132dd 100644 --- a/application/tests/juiceshop_test.py +++ b/application/tests/juiceshop_test.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + from pathlib import Path from application.defs import cre_defs as defs import unittest diff --git a/application/tests/librarian/__init__.py b/application/tests/librarian/__init__.py index e69de29bb..9662899ca 100644 --- a/application/tests/librarian/__init__.py +++ b/application/tests/librarian/__init__.py @@ -0,0 +1,3 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) diff --git a/application/tests/librarian/candidate_retriever_test.py b/application/tests/librarian/candidate_retriever_test.py index 5e9ea359b..58d3c8c13 100644 --- a/application/tests/librarian/candidate_retriever_test.py +++ b/application/tests/librarian/candidate_retriever_test.py @@ -5,6 +5,10 @@ the RetrievalAudit shape are all assertable without an LLM or a DB. """ +from cre_logging import get_logger + +logger = get_logger(__name__) + import unittest from unittest.mock import MagicMock diff --git a/application/tests/librarian/config_loader_test.py b/application/tests/librarian/config_loader_test.py index 63818888f..82822a3b0 100644 --- a/application/tests/librarian/config_loader_test.py +++ b/application/tests/librarian/config_loader_test.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + import os import unittest from dataclasses import FrozenInstanceError diff --git a/application/tests/librarian/cross_encoder_test.py b/application/tests/librarian/cross_encoder_test.py index 8813554d3..790970a78 100644 --- a/application/tests/librarian/cross_encoder_test.py +++ b/application/tests/librarian/cross_encoder_test.py @@ -5,6 +5,10 @@ shape, and the failure modes are all assertable without torch, an LLM, or a DB. """ +from cre_logging import get_logger + +logger = get_logger(__name__) + import unittest from application.utils.librarian.cross_encoder import ( diff --git a/application/tests/librarian/dataset_test.py b/application/tests/librarian/dataset_test.py index bb497b719..9737afffa 100644 --- a/application/tests/librarian/dataset_test.py +++ b/application/tests/librarian/dataset_test.py @@ -5,6 +5,10 @@ own ``--check`` mode, which the determinism test invokes when the DB is present. """ +from cre_logging import get_logger + +logger = get_logger(__name__) + import json import os import subprocess diff --git a/application/tests/librarian/decision_engine_test.py b/application/tests/librarian/decision_engine_test.py index 1d39a1691..b07ca125d 100644 --- a/application/tests/librarian/decision_engine_test.py +++ b/application/tests/librarian/decision_engine_test.py @@ -4,6 +4,10 @@ see, plus reason-code precedence and the input guards. No key, DB, or model. """ +from cre_logging import get_logger + +logger = get_logger(__name__) + import dataclasses import math import unittest diff --git a/application/tests/librarian/decision_queue_test.py b/application/tests/librarian/decision_queue_test.py index 210037456..84cfec5f0 100644 --- a/application/tests/librarian/decision_queue_test.py +++ b/application/tests/librarian/decision_queue_test.py @@ -6,6 +6,10 @@ lands is the same RFC document ``JsonlEnvelopeSink`` would have written. """ +from cre_logging import get_logger + +logger = get_logger(__name__) + import json import unittest from datetime import datetime, timezone diff --git a/application/tests/librarian/emitter_test.py b/application/tests/librarian/emitter_test.py index 905c68acd..c1f50b694 100644 --- a/application/tests/librarian/emitter_test.py +++ b/application/tests/librarian/emitter_test.py @@ -1,5 +1,9 @@ """Hermetic tests for C.4 envelope emitter (Week 6b). No key, DB, or model.""" +from cre_logging import get_logger + +logger = get_logger(__name__) + import unittest from datetime import datetime, timezone diff --git a/application/tests/librarian/envelope_sink_test.py b/application/tests/librarian/envelope_sink_test.py index 8e8ab282d..db502849b 100644 --- a/application/tests/librarian/envelope_sink_test.py +++ b/application/tests/librarian/envelope_sink_test.py @@ -5,6 +5,10 @@ it would silently destroy chunks. """ +from cre_logging import get_logger + +logger = get_logger(__name__) + import json import os import shutil diff --git a/application/tests/librarian/evaluate_harness_test.py b/application/tests/librarian/evaluate_harness_test.py index ed3b62bef..ae979a177 100644 --- a/application/tests/librarian/evaluate_harness_test.py +++ b/application/tests/librarian/evaluate_harness_test.py @@ -14,6 +14,10 @@ without touching the retriever or reranker again. """ +from cre_logging import get_logger + +logger = get_logger(__name__) + import contextlib import importlib.util import io diff --git a/application/tests/librarian/explicit_link_resolver_test.py b/application/tests/librarian/explicit_link_resolver_test.py index e7a8775e3..5a05d0984 100644 --- a/application/tests/librarian/explicit_link_resolver_test.py +++ b/application/tests/librarian/explicit_link_resolver_test.py @@ -5,6 +5,10 @@ conflicting references must fall through to review). """ +from cre_logging import get_logger + +logger = get_logger(__name__) + import unittest from application.utils.librarian.explicit_link_resolver import ( diff --git a/application/tests/librarian/factory_test.py b/application/tests/librarian/factory_test.py index 976df5e7d..a7bdf2499 100644 --- a/application/tests/librarian/factory_test.py +++ b/application/tests/librarian/factory_test.py @@ -8,6 +8,10 @@ of it is a crash in the orchestrator's entry point. """ +from cre_logging import get_logger + +logger = get_logger(__name__) + import os import unittest from unittest import mock diff --git a/application/tests/librarian/hub_firewall_test.py b/application/tests/librarian/hub_firewall_test.py index 8858d98f6..e90644798 100644 --- a/application/tests/librarian/hub_firewall_test.py +++ b/application/tests/librarian/hub_firewall_test.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + import unittest from application.utils.librarian.hub_firewall import HubRep, firewall, leaks diff --git a/application/tests/librarian/knowledge_source_test.py b/application/tests/librarian/knowledge_source_test.py index a02e4cf8c..4a8c8e191 100644 --- a/application/tests/librarian/knowledge_source_test.py +++ b/application/tests/librarian/knowledge_source_test.py @@ -14,6 +14,10 @@ cannot honour it fails loudly instead of handing back an unlocked batch. """ +from cre_logging import get_logger + +logger = get_logger(__name__) + import json import os import tempfile diff --git a/application/tests/librarian/pipeline_test.py b/application/tests/librarian/pipeline_test.py index 3876495d9..9f013439b 100644 --- a/application/tests/librarian/pipeline_test.py +++ b/application/tests/librarian/pipeline_test.py @@ -3,6 +3,10 @@ Every stage is a trivial stub — no DB, embedding model, or cross-encoder. """ +from cre_logging import get_logger + +logger = get_logger(__name__) + import unittest from datetime import datetime, timezone diff --git a/application/tests/librarian/queue_consumer_test.py b/application/tests/librarian/queue_consumer_test.py index 0c07cddcb..8c89763e9 100644 --- a/application/tests/librarian/queue_consumer_test.py +++ b/application/tests/librarian/queue_consumer_test.py @@ -7,6 +7,10 @@ consumed, and never delete anything. """ +from cre_logging import get_logger + +logger = get_logger(__name__) + import unittest from datetime import datetime, timezone diff --git a/application/tests/librarian/queue_runner_test.py b/application/tests/librarian/queue_runner_test.py index 9aca2aeb7..dce67a882 100644 --- a/application/tests/librarian/queue_runner_test.py +++ b/application/tests/librarian/queue_runner_test.py @@ -9,6 +9,10 @@ errored row does not, and a second run is a no-op rather than a re-run. """ +from cre_logging import get_logger + +logger = get_logger(__name__) + import unittest from datetime import datetime, timezone diff --git a/application/tests/librarian/safety_guard_test.py b/application/tests/librarian/safety_guard_test.py index e6a9d3ac9..f43a74fb1 100644 --- a/application/tests/librarian/safety_guard_test.py +++ b/application/tests/librarian/safety_guard_test.py @@ -6,6 +6,10 @@ success. """ +from cre_logging import get_logger + +logger = get_logger(__name__) + import unittest from datetime import datetime, timezone diff --git a/application/tests/librarian/schemas_test.py b/application/tests/librarian/schemas_test.py index 09f1d89a9..62f442d6b 100644 --- a/application/tests/librarian/schemas_test.py +++ b/application/tests/librarian/schemas_test.py @@ -6,6 +6,10 @@ If the upstream schema and the Pydantic model ever drift, this fails. """ +from cre_logging import get_logger + +logger = get_logger(__name__) + import json import os import unittest diff --git a/application/tests/librarian/scoring_test.py b/application/tests/librarian/scoring_test.py index 98882c5d6..bffe7c71e 100644 --- a/application/tests/librarian/scoring_test.py +++ b/application/tests/librarian/scoring_test.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + import unittest from application.utils.librarian.scoring import jaccard, score_case diff --git a/application/tests/librarian/section_validator_test.py b/application/tests/librarian/section_validator_test.py index d2f6846a8..e22a8fa82 100644 --- a/application/tests/librarian/section_validator_test.py +++ b/application/tests/librarian/section_validator_test.py @@ -5,6 +5,10 @@ Asserts the boundary never leaks a raw Pydantic ValidationError. """ +from cre_logging import get_logger + +logger = get_logger(__name__) + import unittest from datetime import datetime, timezone diff --git a/application/tests/librarian/temperature_test.py b/application/tests/librarian/temperature_test.py index 4b4dee1cd..af6d5f508 100644 --- a/application/tests/librarian/temperature_test.py +++ b/application/tests/librarian/temperature_test.py @@ -7,6 +7,10 @@ every guard. """ +from cre_logging import get_logger + +logger = get_logger(__name__) + import math import unittest diff --git a/application/tests/litellm_router_test.py b/application/tests/litellm_router_test.py index eb15cfb57..a3f4016e0 100644 --- a/application/tests/litellm_router_test.py +++ b/application/tests/litellm_router_test.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + import os import unittest from unittest.mock import Mock, patch diff --git a/application/tests/master_spreadsheet_parser_test.py b/application/tests/master_spreadsheet_parser_test.py index c888b2d25..8f01cf3cd 100644 --- a/application/tests/master_spreadsheet_parser_test.py +++ b/application/tests/master_spreadsheet_parser_test.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + import csv import io import unittest diff --git a/application/tests/mcp_public_tools_test.py b/application/tests/mcp_public_tools_test.py index 5355de7d7..74ff4975d 100644 --- a/application/tests/mcp_public_tools_test.py +++ b/application/tests/mcp_public_tools_test.py @@ -2,6 +2,10 @@ from __future__ import annotations +from cre_logging import get_logger + +logger = get_logger(__name__) + import json import os import unittest diff --git a/application/tests/mdutils_test.py b/application/tests/mdutils_test.py index 40fb2b233..3d34b46a4 100644 --- a/application/tests/mdutils_test.py +++ b/application/tests/mdutils_test.py @@ -1,6 +1,9 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + from application.utils import mdutils from application.defs import cre_defs as defs -from pprint import pprint import unittest from application import create_app, sqla # type: ignore diff --git a/application/tests/misc_tools_parser_test.py b/application/tests/misc_tools_parser_test.py index edd0276ce..b935d48ec 100644 --- a/application/tests/misc_tools_parser_test.py +++ b/application/tests/misc_tools_parser_test.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + import os import tempfile import unittest diff --git a/application/tests/monitor_chatbot_health_test.py b/application/tests/monitor_chatbot_health_test.py index 8cd6a9ef8..83b96ec44 100644 --- a/application/tests/monitor_chatbot_health_test.py +++ b/application/tests/monitor_chatbot_health_test.py @@ -1,5 +1,9 @@ """Tests for scripts/monitor_chatbot_health.py and chatbot sanitize guardrail.""" +from cre_logging import get_logger + +logger = get_logger(__name__) + import io import unittest import urllib.error diff --git a/application/tests/monitor_ga_health_test.py b/application/tests/monitor_ga_health_test.py index 9fdc1e373..af97c3359 100644 --- a/application/tests/monitor_ga_health_test.py +++ b/application/tests/monitor_ga_health_test.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + import io import unittest import urllib.error diff --git a/application/tests/myopencre_parser_test.py b/application/tests/myopencre_parser_test.py index 1ea154740..a4a79f64a 100644 --- a/application/tests/myopencre_parser_test.py +++ b/application/tests/myopencre_parser_test.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + import unittest from unittest import mock diff --git a/application/tests/neo_node_lookup_test.py b/application/tests/neo_node_lookup_test.py index fa1ab77a5..b9aa5231d 100644 --- a/application/tests/neo_node_lookup_test.py +++ b/application/tests/neo_node_lookup_test.py @@ -1,5 +1,9 @@ """Tests for NeoNode-by-document_id lookup (avoids Neo4j 5 unknown-label warnings).""" +from cre_logging import get_logger + +logger = get_logger(__name__) + import unittest from unittest.mock import MagicMock, patch diff --git a/application/tests/noise_filter/__init__.py b/application/tests/noise_filter/__init__.py index e69de29bb..9662899ca 100644 --- a/application/tests/noise_filter/__init__.py +++ b/application/tests/noise_filter/__init__.py @@ -0,0 +1,3 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) diff --git a/application/tests/noise_filter/config_loader_test.py b/application/tests/noise_filter/config_loader_test.py index 30737a9a6..a16a37d2e 100644 --- a/application/tests/noise_filter/config_loader_test.py +++ b/application/tests/noise_filter/config_loader_test.py @@ -8,6 +8,10 @@ from __future__ import annotations +from cre_logging import get_logger + +logger = get_logger(__name__) + import os import unittest from unittest.mock import patch diff --git a/application/tests/noise_filter/llm_classifier_test.py b/application/tests/noise_filter/llm_classifier_test.py index 3f22edb9f..bd33808f8 100644 --- a/application/tests/noise_filter/llm_classifier_test.py +++ b/application/tests/noise_filter/llm_classifier_test.py @@ -7,6 +7,10 @@ from __future__ import annotations +from cre_logging import get_logger + +logger = get_logger(__name__) + import json import unittest from types import SimpleNamespace diff --git a/application/tests/noise_filter/pipeline_test.py b/application/tests/noise_filter/pipeline_test.py index 0fcae74e8..68f21bbea 100644 --- a/application/tests/noise_filter/pipeline_test.py +++ b/application/tests/noise_filter/pipeline_test.py @@ -6,6 +6,10 @@ from __future__ import annotations +from cre_logging import get_logger + +logger = get_logger(__name__) + import unittest from application import create_app, sqla diff --git a/application/tests/noise_filter/queue_writer_test.py b/application/tests/noise_filter/queue_writer_test.py index 337a77cf0..c7a598b55 100644 --- a/application/tests/noise_filter/queue_writer_test.py +++ b/application/tests/noise_filter/queue_writer_test.py @@ -6,6 +6,10 @@ from __future__ import annotations +from cre_logging import get_logger + +logger = get_logger(__name__) + import json import unittest diff --git a/application/tests/noise_filter/regex_filter_test.py b/application/tests/noise_filter/regex_filter_test.py index 9bbf6ad7b..c3afd70af 100644 --- a/application/tests/noise_filter/regex_filter_test.py +++ b/application/tests/noise_filter/regex_filter_test.py @@ -13,6 +13,10 @@ from __future__ import annotations +from cre_logging import get_logger + +logger = get_logger(__name__) + import textwrap import unittest from pathlib import Path diff --git a/application/tests/noise_filter/sanitize_test.py b/application/tests/noise_filter/sanitize_test.py index f5cbc49bb..5edacc9dd 100644 --- a/application/tests/noise_filter/sanitize_test.py +++ b/application/tests/noise_filter/sanitize_test.py @@ -11,6 +11,10 @@ from __future__ import annotations +from cre_logging import get_logger + +logger = get_logger(__name__) + import unittest from application.utils.noise_filter.sanitize import sanitize_text, strip_html diff --git a/application/tests/noise_filter/schemas_test.py b/application/tests/noise_filter/schemas_test.py index d3e2fec28..86edae8b3 100644 --- a/application/tests/noise_filter/schemas_test.py +++ b/application/tests/noise_filter/schemas_test.py @@ -12,6 +12,10 @@ from __future__ import annotations +from cre_logging import get_logger + +logger = get_logger(__name__) + import json import unittest from pathlib import Path diff --git a/application/tests/openapi_guardrail_test.py b/application/tests/openapi_guardrail_test.py index cf82dee78..c95a70af6 100644 --- a/application/tests/openapi_guardrail_test.py +++ b/application/tests/openapi_guardrail_test.py @@ -2,6 +2,10 @@ from __future__ import annotations +from cre_logging import get_logger + +logger = get_logger(__name__) + import os import unittest diff --git a/application/tests/opencre_gap_analysis_test.py b/application/tests/opencre_gap_analysis_test.py index 7ee71575b..65453c56a 100644 --- a/application/tests/opencre_gap_analysis_test.py +++ b/application/tests/opencre_gap_analysis_test.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + import json import unittest from unittest.mock import Mock, patch diff --git a/application/tests/oscal_utils_test.py b/application/tests/oscal_utils_test.py index 22f9a7f4b..fc5040d4a 100644 --- a/application/tests/oscal_utils_test.py +++ b/application/tests/oscal_utils_test.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + import json from typing import Any, Dict, List, Tuple import unittest diff --git a/application/tests/osib_defs_test.py b/application/tests/osib_defs_test.py index d74fd0c00..b44a2e112 100644 --- a/application/tests/osib_defs_test.py +++ b/application/tests/osib_defs_test.py @@ -2,7 +2,10 @@ # import tempfile # from typing import Any, Dict, List, Tuple # import unittest -# from pprint import pprint + +from cre_logging import get_logger + +logger = get_logger(__name__) # import yaml # from application.defs import cre_defs as cdefs diff --git a/application/tests/owasp_aisvs_parser_test.py b/application/tests/owasp_aisvs_parser_test.py index 3a83299af..633afeb17 100644 --- a/application/tests/owasp_aisvs_parser_test.py +++ b/application/tests/owasp_aisvs_parser_test.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + import unittest from application import create_app, sqla # type: ignore diff --git a/application/tests/owasp_api_top10_2023_parser_test.py b/application/tests/owasp_api_top10_2023_parser_test.py index 4f7d8e1b9..e6b54e0e4 100644 --- a/application/tests/owasp_api_top10_2023_parser_test.py +++ b/application/tests/owasp_api_top10_2023_parser_test.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + import unittest from application import create_app, sqla # type: ignore diff --git a/application/tests/owasp_kubernetes_top10_2022_parser_test.py b/application/tests/owasp_kubernetes_top10_2022_parser_test.py index a0c657c14..b57821d3e 100644 --- a/application/tests/owasp_kubernetes_top10_2022_parser_test.py +++ b/application/tests/owasp_kubernetes_top10_2022_parser_test.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + import unittest from application import create_app, sqla # type: ignore diff --git a/application/tests/owasp_kubernetes_top10_2025_parser_test.py b/application/tests/owasp_kubernetes_top10_2025_parser_test.py index d0a46c26b..3ef2770d1 100644 --- a/application/tests/owasp_kubernetes_top10_2025_parser_test.py +++ b/application/tests/owasp_kubernetes_top10_2025_parser_test.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + import unittest import tempfile from pathlib import Path diff --git a/application/tests/owasp_llm_top10_2025_parser_test.py b/application/tests/owasp_llm_top10_2025_parser_test.py index f31e4316e..2f2e8ccef 100644 --- a/application/tests/owasp_llm_top10_2025_parser_test.py +++ b/application/tests/owasp_llm_top10_2025_parser_test.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + import unittest from application import create_app, sqla # type: ignore diff --git a/application/tests/owasp_mapping_fixtures_test.py b/application/tests/owasp_mapping_fixtures_test.py index 3f58eb88f..7a12a57e5 100644 --- a/application/tests/owasp_mapping_fixtures_test.py +++ b/application/tests/owasp_mapping_fixtures_test.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + import re import unittest diff --git a/application/tests/owasp_top10_2025_parser_test.py b/application/tests/owasp_top10_2025_parser_test.py index de4f86a9f..89d3722e6 100644 --- a/application/tests/owasp_top10_2025_parser_test.py +++ b/application/tests/owasp_top10_2025_parser_test.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + import unittest from application import create_app, sqla # type: ignore diff --git a/application/tests/pci_dss_parser_test.py b/application/tests/pci_dss_parser_test.py index 8469b4714..824b01fa0 100644 --- a/application/tests/pci_dss_parser_test.py +++ b/application/tests/pci_dss_parser_test.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + import unittest from unittest.mock import Mock, patch diff --git a/application/tests/pgvector_utils_test.py b/application/tests/pgvector_utils_test.py index 0dfb1c53c..f0b39c15b 100644 --- a/application/tests/pgvector_utils_test.py +++ b/application/tests/pgvector_utils_test.py @@ -1,5 +1,9 @@ """Tests for pgvector embedding helpers (dim gate, literal, backfill SQL).""" +from cre_logging import get_logger + +logger = get_logger(__name__) + import os import unittest from unittest.mock import MagicMock diff --git a/application/tests/prompt_client_incremental_embeddings_test.py b/application/tests/prompt_client_incremental_embeddings_test.py index cc62a9724..0ef70785f 100644 --- a/application/tests/prompt_client_incremental_embeddings_test.py +++ b/application/tests/prompt_client_incremental_embeddings_test.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + import unittest from types import SimpleNamespace from unittest.mock import Mock diff --git a/application/tests/prompt_client_pdf_embeddings_test.py b/application/tests/prompt_client_pdf_embeddings_test.py index 4cc058889..1538addb4 100644 --- a/application/tests/prompt_client_pdf_embeddings_test.py +++ b/application/tests/prompt_client_pdf_embeddings_test.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + import unittest from unittest.mock import Mock, patch diff --git a/application/tests/prompt_client_pgvector_similarity_test.py b/application/tests/prompt_client_pgvector_similarity_test.py index d6e6a32fe..a4724f8eb 100644 --- a/application/tests/prompt_client_pgvector_similarity_test.py +++ b/application/tests/prompt_client_pgvector_similarity_test.py @@ -1,5 +1,9 @@ """Chat/import similarity prefers pgvector when the DB reports it ready.""" +from cre_logging import get_logger + +logger = get_logger(__name__) + import unittest from typing import Any, Callable, Dict, List, Tuple from unittest.mock import MagicMock, patch diff --git a/application/tests/prompt_client_smart_embed_test.py b/application/tests/prompt_client_smart_embed_test.py index f02dee786..355228733 100644 --- a/application/tests/prompt_client_smart_embed_test.py +++ b/application/tests/prompt_client_smart_embed_test.py @@ -1,5 +1,9 @@ """Integration tests for ``CRE_EMBED_SMART_EXTRACT`` in ``generate_embeddings``.""" +from cre_logging import get_logger + +logger = get_logger(__name__) + import os import unittest from types import SimpleNamespace diff --git a/application/tests/register_standard_incremental_embeddings_test.py b/application/tests/register_standard_incremental_embeddings_test.py index bb6458e04..58fe0ba03 100644 --- a/application/tests/register_standard_incremental_embeddings_test.py +++ b/application/tests/register_standard_incremental_embeddings_test.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + import os import unittest from unittest.mock import Mock, patch diff --git a/application/tests/resource_filter_test.py b/application/tests/resource_filter_test.py index 99995d0c2..8cac20096 100644 --- a/application/tests/resource_filter_test.py +++ b/application/tests/resource_filter_test.py @@ -6,6 +6,10 @@ selection rows are real on Postgres. """ +from cre_logging import get_logger + +logger = get_logger(__name__) + import json import os import unittest diff --git a/application/tests/secure_headers_parser_test.py b/application/tests/secure_headers_parser_test.py index 1acb82219..ffd6711a8 100644 --- a/application/tests/secure_headers_parser_test.py +++ b/application/tests/secure_headers_parser_test.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + from application.defs import cre_defs as defs import unittest from application import create_app, sqla # type: ignore diff --git a/application/tests/spreadsheet_parsers_test.py b/application/tests/spreadsheet_parsers_test.py index 42e744c45..13d722e3c 100644 --- a/application/tests/spreadsheet_parsers_test.py +++ b/application/tests/spreadsheet_parsers_test.py @@ -1,6 +1,9 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + import copy import json -from pprint import pprint import unittest from application.tests.utils import data_gen from application.defs import cre_defs as defs diff --git a/application/tests/spreadsheet_test.py b/application/tests/spreadsheet_test.py index 23210714f..c2787e105 100644 --- a/application/tests/spreadsheet_test.py +++ b/application/tests/spreadsheet_test.py @@ -1,7 +1,10 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + import unittest import io import csv -from pprint import pprint from unittest import mock import gspread diff --git a/application/tests/test_smart_embeddings_e2e_llm.py b/application/tests/test_smart_embeddings_e2e_llm.py index d47c31aae..05d5d318a 100644 --- a/application/tests/test_smart_embeddings_e2e_llm.py +++ b/application/tests/test_smart_embeddings_e2e_llm.py @@ -8,6 +8,10 @@ from __future__ import annotations +from cre_logging import get_logger + +logger = get_logger(__name__) + import os import re import json diff --git a/application/tests/user_model_test.py b/application/tests/user_model_test.py index 5423539a4..b151da787 100644 --- a/application/tests/user_model_test.py +++ b/application/tests/user_model_test.py @@ -1,5 +1,9 @@ """Tests for user persistence and per-user resource selection (issue #586, RFC #876 TODO 1/2).""" +from cre_logging import get_logger + +logger = get_logger(__name__) + import os import threading import unittest diff --git a/application/tests/user_resources_api_test.py b/application/tests/user_resources_api_test.py index a1ccc1a0b..5c020fa0d 100644 --- a/application/tests/user_resources_api_test.py +++ b/application/tests/user_resources_api_test.py @@ -4,6 +4,10 @@ Flag-off returns a safe default; authenticated users read/write their selection. """ +from cre_logging import get_logger + +logger = get_logger(__name__) + import json import os import unittest diff --git a/application/tests/utils/data_gen.py b/application/tests/utils/data_gen.py index c86da45b2..3b3161e90 100644 --- a/application/tests/utils/data_gen.py +++ b/application/tests/utils/data_gen.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + from application.defs import cre_defs as defs diff --git a/application/tests/web_main_test.py b/application/tests/web_main_test.py index 7bc21945f..dbd716708 100644 --- a/application/tests/web_main_test.py +++ b/application/tests/web_main_test.py @@ -1,4 +1,7 @@ -from pprint import pprint +from cre_logging import get_logger + +logger = get_logger(__name__) + import io import csv import random diff --git a/application/tests/worker_test.py b/application/tests/worker_test.py index 46dacb16a..d78cf4108 100644 --- a/application/tests/worker_test.py +++ b/application/tests/worker_test.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + import os import unittest diff --git a/application/tests/zap_alerts_parser_test.py b/application/tests/zap_alerts_parser_test.py index 8436e4489..eaac9ddef 100644 --- a/application/tests/zap_alerts_parser_test.py +++ b/application/tests/zap_alerts_parser_test.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + from application.defs import cre_defs as defs import unittest from application import create_app, sqla # type: ignore diff --git a/application/utils/cres_csv_export.py b/application/utils/cres_csv_export.py index b8fc116a8..040efa1dc 100644 --- a/application/utils/cres_csv_export.py +++ b/application/utils/cres_csv_export.py @@ -1,7 +1,10 @@ from __future__ import annotations +from cre_logging import get_logger + +logger = get_logger(__name__) + import csv -import logging import re from collections import deque from pathlib import Path @@ -15,8 +18,6 @@ LTYPE_CONTAINS = "Contains" CELL_SEP = "|" -logger = logging.getLogger("cres_csv_export") - def _norm_base_url(base_url: str) -> str: return base_url.rstrip("/") diff --git a/application/utils/db_backend.py b/application/utils/db_backend.py index 994913ec5..6906cbf56 100644 --- a/application/utils/db_backend.py +++ b/application/utils/db_backend.py @@ -2,6 +2,10 @@ from __future__ import annotations +from cre_logging import get_logger + +logger = get_logger(__name__) + from dataclasses import dataclass diff --git a/application/utils/external_project_parsers/__init__.py b/application/utils/external_project_parsers/__init__.py index e69de29bb..9662899ca 100644 --- a/application/utils/external_project_parsers/__init__.py +++ b/application/utils/external_project_parsers/__init__.py @@ -0,0 +1,3 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) diff --git a/application/utils/external_project_parsers/base_parser.py b/application/utils/external_project_parsers/base_parser.py index 49e7b4c2d..2700c274a 100644 --- a/application/utils/external_project_parsers/base_parser.py +++ b/application/utils/external_project_parsers/base_parser.py @@ -1,16 +1,15 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + from application.utils.external_project_parsers import base_parser_defs from rq import Queue from application.utils import redis -import logging import time from application.utils.external_project_parsers.parsers import * from application.utils import gap_analysis import os, json -logging.basicConfig() -logger = logging.getLogger(__name__) -logger.setLevel(logging.INFO) - class BaseParser: @classmethod diff --git a/application/utils/external_project_parsers/base_parser_defs.py b/application/utils/external_project_parsers/base_parser_defs.py index 4b6660794..3afabced8 100644 --- a/application/utils/external_project_parsers/base_parser_defs.py +++ b/application/utils/external_project_parsers/base_parser_defs.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + from typing import List, Dict, Optional, TYPE_CHECKING from dataclasses import dataclass from enum import Enum diff --git a/application/utils/external_project_parsers/parsers/__init__.py b/application/utils/external_project_parsers/parsers/__init__.py index e69de29bb..9662899ca 100644 --- a/application/utils/external_project_parsers/parsers/__init__.py +++ b/application/utils/external_project_parsers/parsers/__init__.py @@ -0,0 +1,3 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) diff --git a/application/utils/external_project_parsers/parsers/ai_exchange/__init__.py b/application/utils/external_project_parsers/parsers/ai_exchange/__init__.py index d23b57d4e..6445c6dff 100644 --- a/application/utils/external_project_parsers/parsers/ai_exchange/__init__.py +++ b/application/utils/external_project_parsers/parsers/ai_exchange/__init__.py @@ -1,5 +1,9 @@ """AI Exchange mappings: CSV export today, live exchange source later.""" +from cre_logging import get_logger + +logger = get_logger(__name__) + from .csv_source import ( IMPORT_SOURCE_CSV, is_ai_exchange_spreadsheet, diff --git a/application/utils/external_project_parsers/parsers/ai_exchange/csv_source.py b/application/utils/external_project_parsers/parsers/ai_exchange/csv_source.py index 59ae8dfc6..0d2219e3b 100644 --- a/application/utils/external_project_parsers/parsers/ai_exchange/csv_source.py +++ b/application/utils/external_project_parsers/parsers/ai_exchange/csv_source.py @@ -11,13 +11,15 @@ from __future__ import annotations +from cre_logging import get_logger + +logger = get_logger(__name__) + import copy import csv -import logging import re from typing import Any, Dict, List, Optional -logger = logging.getLogger(__name__) IMPORT_SOURCE_CSV = "ai_exchange_csv" diff --git a/application/utils/external_project_parsers/parsers/ai_exchange/exchange_source.py b/application/utils/external_project_parsers/parsers/ai_exchange/exchange_source.py index c12be7f9c..8ecdaa8e9 100644 --- a/application/utils/external_project_parsers/parsers/ai_exchange/exchange_source.py +++ b/application/utils/external_project_parsers/parsers/ai_exchange/exchange_source.py @@ -6,6 +6,10 @@ from __future__ import annotations +from cre_logging import get_logger + +logger = get_logger(__name__) + from typing import Any, Dict, List diff --git a/application/utils/external_project_parsers/parsers/capec_parser.py b/application/utils/external_project_parsers/parsers/capec_parser.py index f7be38306..8ec967f69 100644 --- a/application/utils/external_project_parsers/parsers/capec_parser.py +++ b/application/utils/external_project_parsers/parsers/capec_parser.py @@ -1,4 +1,7 @@ -import logging +from cre_logging import get_logger + +logger = get_logger(__name__) + import os import tempfile import requests @@ -7,9 +10,6 @@ from application.defs import cre_defs as defs import xmltodict -logging.basicConfig() -logger = logging.getLogger(__name__) -logger.setLevel(logging.INFO) from application.utils.external_project_parsers import base_parser_defs from application.utils.external_project_parsers.base_parser_defs import ( diff --git a/application/utils/external_project_parsers/parsers/ccmv4.py b/application/utils/external_project_parsers/parsers/ccmv4.py index 4f90a45e2..d1713813f 100644 --- a/application/utils/external_project_parsers/parsers/ccmv4.py +++ b/application/utils/external_project_parsers/parsers/ccmv4.py @@ -1,13 +1,13 @@ -import logging +from cre_logging import get_logger + +logger = get_logger(__name__) + from typing import Dict, Any from application.database import db from application.defs import cre_defs as defs import re -logging.basicConfig() -logger = logging.getLogger(__name__) -logger.setLevel(logging.INFO) from application.utils.external_project_parsers import base_parser_defs from application.utils.external_project_parsers.base_parser_defs import ( diff --git a/application/utils/external_project_parsers/parsers/cheatsheet_extractor.py b/application/utils/external_project_parsers/parsers/cheatsheet_extractor.py index bc1709fdd..28f78b44a 100644 --- a/application/utils/external_project_parsers/parsers/cheatsheet_extractor.py +++ b/application/utils/external_project_parsers/parsers/cheatsheet_extractor.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + import logging import os import re diff --git a/application/utils/external_project_parsers/parsers/cheatsheet_record_adapter.py b/application/utils/external_project_parsers/parsers/cheatsheet_record_adapter.py index 7324763b2..557068547 100644 --- a/application/utils/external_project_parsers/parsers/cheatsheet_record_adapter.py +++ b/application/utils/external_project_parsers/parsers/cheatsheet_record_adapter.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + from pydantic import ValidationError from application.utils.librarian.section_validator import _DEFAULT_LANGUAGE from application.defs.cheatsheet_defs import CheatsheetRecord diff --git a/application/utils/external_project_parsers/parsers/cheatsheets_parser.py b/application/utils/external_project_parsers/parsers/cheatsheets_parser.py index a6600f68e..37d91b49e 100644 --- a/application/utils/external_project_parsers/parsers/cheatsheets_parser.py +++ b/application/utils/external_project_parsers/parsers/cheatsheets_parser.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + # script to parse cheatsheet md files find the links to opencre.org and add the cheatsheets to CRE from typing import List import subprocess @@ -8,7 +12,6 @@ import re from application.utils.external_project_parsers import base_parser_defs import json -import logging from application.utils.mapping_fixtures import OWASP_MAPPING_FIXTURE_DIR from application.utils.external_project_parsers.base_parser_defs import ( ParserInterface, @@ -23,7 +26,7 @@ class Cheatsheets(ParserInterface): supplement_data_file = ( OWASP_MAPPING_FIXTURE_DIR / "owasp_cheatsheets_supplement.json" ) - logger = logging.getLogger(__name__) + logger = get_logger(__name__) def cheatsheet( self, section: str, hyperlink: str, tags: List[str] diff --git a/application/utils/external_project_parsers/parsers/cheatsheets_workstream_f.py b/application/utils/external_project_parsers/parsers/cheatsheets_workstream_f.py index 8de6cca60..7e0f8698f 100644 --- a/application/utils/external_project_parsers/parsers/cheatsheets_workstream_f.py +++ b/application/utils/external_project_parsers/parsers/cheatsheets_workstream_f.py @@ -51,10 +51,13 @@ from __future__ import annotations +from cre_logging import get_logger + +logger = get_logger(__name__) + import argparse import dataclasses import json -import logging import sys from dataclasses import dataclass, field from typing import Any, Dict, List, Optional, TYPE_CHECKING @@ -68,7 +71,6 @@ if TYPE_CHECKING: from application.database import db -logger = logging.getLogger(__name__) # --- valid values ----------------------------------------------------------- @@ -371,20 +373,24 @@ def _cmd_validate(args: argparse.Namespace) -> int: try: doc = _load_json_doc(args.suggestions) except FileNotFoundError: - print(f"file not found: {args.suggestions}", file=sys.stderr) + logger.error("file not found: %s", args.suggestions) return _EXIT_USAGE except json.JSONDecodeError as exc: - print(f"invalid JSON in {args.suggestions}: {exc}", file=sys.stderr) + logger.error("invalid JSON in %s: %s", args.suggestions, exc) return _EXIT_USAGE try: _validate(doc) except SuggestionSchemaError as exc: - print(str(exc), file=sys.stderr) + logger.error("%s", exc) return _EXIT_ERROR count = len(doc) if isinstance(doc, list) else 0 - print(f"OK: {args.suggestions} is a valid suggestions document ({count} items)") + logger.info( + "OK: %s is a valid suggestions document (%s items)", + args.suggestions, + count, + ) return _EXIT_OK @@ -398,10 +404,10 @@ def _cmd_generate(args: argparse.Namespace) -> int: try: doc = _load_json_doc(args.infile) except FileNotFoundError: - print(f"file not found: {args.infile}", file=sys.stderr) + logger.error("file not found: %s", args.infile) return _EXIT_USAGE except json.JSONDecodeError as exc: - print(f"invalid JSON in {args.infile}: {exc}", file=sys.stderr) + logger.error("invalid JSON in %s: %s", args.infile, exc) return _EXIT_USAGE try: @@ -409,13 +415,13 @@ def _cmd_generate(args: argparse.Namespace) -> int: suggestions = [_parse_suggestion(raw) for raw in doc] write_suggestions_json(args.outfile, suggestions) except SuggestionSchemaError as exc: - print(str(exc), file=sys.stderr) + logger.error("%s", exc) return _EXIT_ERROR except OSError as exc: - print(f"cannot write {args.outfile}: {exc}", file=sys.stderr) + logger.error("cannot write %s: %s", args.outfile, exc) return _EXIT_USAGE - print(f"wrote {len(suggestions)} suggestions to {args.outfile}") + logger.info("wrote %s suggestions to %s", len(suggestions), args.outfile) return _EXIT_OK @@ -428,13 +434,13 @@ def _cmd_convert(args: argparse.Namespace) -> int: try: approved = load_approved_suggestions(args.approved) except FileNotFoundError: - print(f"file not found: {args.approved}", file=sys.stderr) + logger.error("file not found: %s", args.approved) return _EXIT_USAGE except json.JSONDecodeError as exc: - print(f"invalid JSON in {args.approved}: {exc}", file=sys.stderr) + logger.error("invalid JSON in %s: %s", args.approved, exc) return _EXIT_USAGE except SuggestionSchemaError as exc: - print(str(exc), file=sys.stderr) + logger.error("%s", exc) return _EXIT_ERROR cache = _open_cache(args.db) @@ -458,13 +464,15 @@ def _cmd_convert(args: argparse.Namespace) -> int: skipped = sorted({cid for cid in all_candidate_ids if cid not in resolved}) total_links = sum(len(standard.links) for standard in standards) - print(f"approved suggestions: {len(approved)}") - print(f"standards produced: {len(standards)}") - print(f"total CRE links: {total_links}") - print("skipped unknown CREs: " + (", ".join(skipped) if skipped else "(none)")) - print( - "note: nothing was registered into the graph; live import/registration " - "is F5 (follow-up PR)." + logger.info( + "approved suggestions=%s standards produced=%s total CRE links=%s skipped unknown CREs=%s", + len(approved), + len(standards), + total_links, + ", ".join(skipped) if skipped else "(none)", + ) + logger.info( + "note: nothing was registered into the graph; live import/registration is F5 (follow-up PR)." ) return _EXIT_OK diff --git a/application/utils/external_project_parsers/parsers/cloud_native_security_controls.py b/application/utils/external_project_parsers/parsers/cloud_native_security_controls.py index 2d050ceee..d0ff731ce 100644 --- a/application/utils/external_project_parsers/parsers/cloud_native_security_controls.py +++ b/application/utils/external_project_parsers/parsers/cloud_native_security_controls.py @@ -1,6 +1,9 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + from io import StringIO import csv -import logging from application.database import db from application.defs import cre_defs as defs from application.prompt_client import prompt_client as prompt_client @@ -11,10 +14,6 @@ ) import requests -logging.basicConfig() -logger = logging.getLogger(__name__) -logger.setLevel(logging.INFO) - class CloudNativeSecurityControls(ParserInterface): name = "Cloud Native Security Controls" diff --git a/application/utils/external_project_parsers/parsers/cwe.py b/application/utils/external_project_parsers/parsers/cwe.py index 6120f7c18..6dd374b9c 100644 --- a/application/utils/external_project_parsers/parsers/cwe.py +++ b/application/utils/external_project_parsers/parsers/cwe.py @@ -1,4 +1,7 @@ -import logging +from cre_logging import get_logger + +logger = get_logger(__name__) + import os import tempfile import json @@ -18,10 +21,6 @@ ParseResult, ) -logging.basicConfig() -logger = logging.getLogger(__name__) -logger.setLevel(logging.INFO) - class CWE(ParserInterface): name = "CWE" diff --git a/application/utils/external_project_parsers/parsers/dsomm.py b/application/utils/external_project_parsers/parsers/dsomm.py index 7c66aad71..13909d51b 100644 --- a/application/utils/external_project_parsers/parsers/dsomm.py +++ b/application/utils/external_project_parsers/parsers/dsomm.py @@ -1,6 +1,9 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + import re import yaml -import logging from application.database import db from application.defs import cre_defs as defs from application.prompt_client import prompt_client as prompt_client @@ -11,10 +14,6 @@ ParseResult, ) -logging.basicConfig() -logger = logging.getLogger(__name__) -logger.setLevel(logging.INFO) - class DSOMM(ParserInterface): name = "DevSecOps Maturity Model (DSOMM)" diff --git a/application/utils/external_project_parsers/parsers/export_format_parser.py b/application/utils/external_project_parsers/parsers/export_format_parser.py index f07cab84c..ca8008382 100644 --- a/application/utils/external_project_parsers/parsers/export_format_parser.py +++ b/application/utils/external_project_parsers/parsers/export_format_parser.py @@ -1,12 +1,11 @@ -import logging +from cre_logging import get_logger + +logger = get_logger(__name__) + from typing import Any, Dict, List, Optional from application.defs import cre_defs as defs -logging.basicConfig() -logger = logging.getLogger(__name__) -logger.setLevel(logging.INFO) - def is_empty(value: Optional[str]) -> bool: value = str(value).strip() diff --git a/application/utils/external_project_parsers/parsers/iso27001.py b/application/utils/external_project_parsers/parsers/iso27001.py index 670b33f18..d13bc4ccc 100644 --- a/application/utils/external_project_parsers/parsers/iso27001.py +++ b/application/utils/external_project_parsers/parsers/iso27001.py @@ -1,6 +1,8 @@ -import logging +from cre_logging import get_logger + +logger = get_logger(__name__) + import os -from pprint import pprint from typing import Dict, Any from application.database import db from application.defs import cre_defs as defs @@ -15,9 +17,6 @@ ) from typing import List -logging.basicConfig() -logger = logging.getLogger(__name__) -logger.setLevel(logging.INFO) nist_id_re = re.compile("(?P\w\w\-\d+)") diff --git a/application/utils/external_project_parsers/parsers/juiceshop.py b/application/utils/external_project_parsers/parsers/juiceshop.py index c001f5b16..1b55aba1b 100644 --- a/application/utils/external_project_parsers/parsers/juiceshop.py +++ b/application/utils/external_project_parsers/parsers/juiceshop.py @@ -1,6 +1,9 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + import yaml import urllib -import logging import os from typing import Dict, Any from application.database import db @@ -14,10 +17,6 @@ ) import requests -logging.basicConfig() -logger = logging.getLogger(__name__) -logger.setLevel(logging.INFO) - class JuiceShop(ParserInterface): name = "OWASP Juice Shop" diff --git a/application/utils/external_project_parsers/parsers/master_spreadsheet_parser.py b/application/utils/external_project_parsers/parsers/master_spreadsheet_parser.py index 2c95ed16f..9b44c09f0 100644 --- a/application/utils/external_project_parsers/parsers/master_spreadsheet_parser.py +++ b/application/utils/external_project_parsers/parsers/master_spreadsheet_parser.py @@ -1,4 +1,7 @@ -import logging +from cre_logging import get_logger + +logger = get_logger(__name__) + import re from dataclasses import dataclass from typing import Any, Dict, List, Tuple @@ -17,10 +20,6 @@ ) -logging.basicConfig() -logger = logging.getLogger(__name__) -logger.setLevel(logging.INFO) - _CRE_ID_TOKEN = re.compile(r"^\d{3}-\d{3}$") # Legacy DB exports sometimes stored display names with a redundant " (123-456)" suffix. _CRE_NAME_LEGACY_ID_SUFFIX = re.compile(r"^(.+) \((\d{3}-\d{3})\)$") diff --git a/application/utils/external_project_parsers/parsers/misc_tools_parser.py b/application/utils/external_project_parsers/parsers/misc_tools_parser.py index cb5e94842..0052fa720 100644 --- a/application/utils/external_project_parsers/parsers/misc_tools_parser.py +++ b/application/utils/external_project_parsers/parsers/misc_tools_parser.py @@ -1,5 +1,8 @@ # script to parse CRE links from README.md files of a given list of projects -import logging +from cre_logging import get_logger + +logger = get_logger(__name__) + import os import re import urllib @@ -16,10 +19,6 @@ ) import requests -logging.basicConfig() -logger = logging.getLogger(__name__) -logger.setLevel(logging.INFO) - class MiscTools(ParserInterface): name = "miscellaneous tools" diff --git a/application/utils/external_project_parsers/parsers/myopencre_parser.py b/application/utils/external_project_parsers/parsers/myopencre_parser.py index 075119662..3e35ad649 100644 --- a/application/utils/external_project_parsers/parsers/myopencre_parser.py +++ b/application/utils/external_project_parsers/parsers/myopencre_parser.py @@ -1,4 +1,7 @@ -import logging +from cre_logging import get_logger + +logger = get_logger(__name__) + import re from typing import Dict, List, Any, Tuple @@ -9,10 +12,6 @@ from application.utils.external_project_parsers.base_parser_defs import ParseResult -logging.basicConfig() -logger = logging.getLogger(__name__) -logger.setLevel(logging.INFO) - _CRE_ID_TOKEN = re.compile(r"^\d{3}-\d{3}$") diff --git a/application/utils/external_project_parsers/parsers/owasp_aisvs.py b/application/utils/external_project_parsers/parsers/owasp_aisvs.py index cb2269948..41b43edfc 100644 --- a/application/utils/external_project_parsers/parsers/owasp_aisvs.py +++ b/application/utils/external_project_parsers/parsers/owasp_aisvs.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + from application.utils.external_project_parsers.parsers.owasp_mapping_fixture_parser import ( OwaspMappingFixtureParser, ) diff --git a/application/utils/external_project_parsers/parsers/owasp_api_top10_2023.py b/application/utils/external_project_parsers/parsers/owasp_api_top10_2023.py index 224dc01c9..f2e03b421 100644 --- a/application/utils/external_project_parsers/parsers/owasp_api_top10_2023.py +++ b/application/utils/external_project_parsers/parsers/owasp_api_top10_2023.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + from application.utils.external_project_parsers.parsers.owasp_mapping_fixture_parser import ( OwaspMappingFixtureParser, ) diff --git a/application/utils/external_project_parsers/parsers/owasp_kubernetes_top10_2022.py b/application/utils/external_project_parsers/parsers/owasp_kubernetes_top10_2022.py index 239f44a0e..0cbedc746 100644 --- a/application/utils/external_project_parsers/parsers/owasp_kubernetes_top10_2022.py +++ b/application/utils/external_project_parsers/parsers/owasp_kubernetes_top10_2022.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + from application.utils.external_project_parsers.parsers.owasp_mapping_fixture_parser import ( OwaspMappingFixtureParser, ) diff --git a/application/utils/external_project_parsers/parsers/owasp_kubernetes_top10_2025.py b/application/utils/external_project_parsers/parsers/owasp_kubernetes_top10_2025.py index ec64b3853..7dd3e0c25 100644 --- a/application/utils/external_project_parsers/parsers/owasp_kubernetes_top10_2025.py +++ b/application/utils/external_project_parsers/parsers/owasp_kubernetes_top10_2025.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + from application.utils.external_project_parsers.parsers.owasp_mapping_fixture_parser import ( OwaspMappingFixtureParser, ) diff --git a/application/utils/external_project_parsers/parsers/owasp_llm_top10_2025.py b/application/utils/external_project_parsers/parsers/owasp_llm_top10_2025.py index 3c2426213..425438bb5 100644 --- a/application/utils/external_project_parsers/parsers/owasp_llm_top10_2025.py +++ b/application/utils/external_project_parsers/parsers/owasp_llm_top10_2025.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + from application.utils.external_project_parsers.parsers.owasp_mapping_fixture_parser import ( OwaspMappingFixtureParser, ) diff --git a/application/utils/external_project_parsers/parsers/owasp_mapping_fixture_parser.py b/application/utils/external_project_parsers/parsers/owasp_mapping_fixture_parser.py index 9f62f34a1..57bffc17e 100644 --- a/application/utils/external_project_parsers/parsers/owasp_mapping_fixture_parser.py +++ b/application/utils/external_project_parsers/parsers/owasp_mapping_fixture_parser.py @@ -6,6 +6,10 @@ from __future__ import annotations +from cre_logging import get_logger + +logger = get_logger(__name__) + from pathlib import Path from typing import Optional diff --git a/application/utils/external_project_parsers/parsers/owasp_top10_2025.py b/application/utils/external_project_parsers/parsers/owasp_top10_2025.py index 797f4ab54..86dc3ad5a 100644 --- a/application/utils/external_project_parsers/parsers/owasp_top10_2025.py +++ b/application/utils/external_project_parsers/parsers/owasp_top10_2025.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + from application.utils.external_project_parsers.parsers.owasp_mapping_fixture_parser import ( OwaspMappingFixtureParser, ) diff --git a/application/utils/external_project_parsers/parsers/pci_dss.py b/application/utils/external_project_parsers/parsers/pci_dss.py index 487c8790a..94d76bb20 100644 --- a/application/utils/external_project_parsers/parsers/pci_dss.py +++ b/application/utils/external_project_parsers/parsers/pci_dss.py @@ -1,5 +1,7 @@ -from pprint import pprint -import logging +from cre_logging import get_logger + +logger = get_logger(__name__) + import os from typing import Dict, Any, List, Optional from application.database import db @@ -13,9 +15,6 @@ ParseResult, ) -logging.basicConfig() -logger = logging.getLogger(__name__) -logger.setLevel(logging.INFO) _DEFAULT_PCI_DSS_CRE_SIMILARITY_THRESHOLDS = (0.55, 0.45, 0.35) _DEFAULT_PCI_BRIDGE_STANDARDS = ("NIST 800-53 v5", "ISO 27001", "ASVS", "CWE") diff --git a/application/utils/external_project_parsers/parsers/secure_headers.py b/application/utils/external_project_parsers/parsers/secure_headers.py index d21f42112..114d180e9 100644 --- a/application/utils/external_project_parsers/parsers/secure_headers.py +++ b/application/utils/external_project_parsers/parsers/secure_headers.py @@ -1,6 +1,9 @@ # script to parse secure headers md files find the links to opencre.org and add the page to CRE +from cre_logging import get_logger + +logger = get_logger(__name__) + from typing import List -import logging import os import re from urllib.parse import urlparse, parse_qs @@ -15,9 +18,6 @@ ) from application.prompt_client import prompt_client as prompt_client -logging.basicConfig() -logger = logging.getLogger(__name__) -logger.setLevel(logging.INFO) # GENERIC Markdown file parser diff --git a/application/utils/external_project_parsers/parsers/spreadsheet_asvs.py b/application/utils/external_project_parsers/parsers/spreadsheet_asvs.py index 46755d541..4cb709fec 100644 --- a/application/utils/external_project_parsers/parsers/spreadsheet_asvs.py +++ b/application/utils/external_project_parsers/parsers/spreadsheet_asvs.py @@ -1,5 +1,9 @@ """ASVS controls linked from the OpenCRE master mapping spreadsheet.""" +from cre_logging import get_logger + +logger = get_logger(__name__) + from typing import Dict, List, Tuple from application.defs import cre_defs as defs diff --git a/application/utils/external_project_parsers/parsers/spreadsheet_biml.py b/application/utils/external_project_parsers/parsers/spreadsheet_biml.py index c90afcfb4..3b3a0cba7 100644 --- a/application/utils/external_project_parsers/parsers/spreadsheet_biml.py +++ b/application/utils/external_project_parsers/parsers/spreadsheet_biml.py @@ -1,5 +1,9 @@ """BIML mappings from AI exchange CSV/master-shaped rows.""" +from cre_logging import get_logger + +logger = get_logger(__name__) + from typing import Dict, List, Tuple from application.defs import cre_defs as defs diff --git a/application/utils/external_project_parsers/parsers/spreadsheet_cloud_controls_matrix.py b/application/utils/external_project_parsers/parsers/spreadsheet_cloud_controls_matrix.py index 2b5077550..eb530ace8 100644 --- a/application/utils/external_project_parsers/parsers/spreadsheet_cloud_controls_matrix.py +++ b/application/utils/external_project_parsers/parsers/spreadsheet_cloud_controls_matrix.py @@ -1,5 +1,9 @@ """Cloud Controls Matrix from the master mapping spreadsheet.""" +from cre_logging import get_logger + +logger = get_logger(__name__) + from typing import Dict, List, Tuple from application.defs import cre_defs as defs diff --git a/application/utils/external_project_parsers/parsers/spreadsheet_cwe.py b/application/utils/external_project_parsers/parsers/spreadsheet_cwe.py index 2f8be321f..88871a4a5 100644 --- a/application/utils/external_project_parsers/parsers/spreadsheet_cwe.py +++ b/application/utils/external_project_parsers/parsers/spreadsheet_cwe.py @@ -1,5 +1,9 @@ """CWE entries from the master mapping spreadsheet.""" +from cre_logging import get_logger + +logger = get_logger(__name__) + from typing import Dict, List, Tuple from application.defs import cre_defs as defs diff --git a/application/utils/external_project_parsers/parsers/spreadsheet_enisa.py b/application/utils/external_project_parsers/parsers/spreadsheet_enisa.py index 57af38302..e1e318f12 100644 --- a/application/utils/external_project_parsers/parsers/spreadsheet_enisa.py +++ b/application/utils/external_project_parsers/parsers/spreadsheet_enisa.py @@ -1,5 +1,9 @@ """ENISA mappings from AI exchange CSV/master-shaped rows.""" +from cre_logging import get_logger + +logger = get_logger(__name__) + from typing import Dict, List, Tuple from application.defs import cre_defs as defs diff --git a/application/utils/external_project_parsers/parsers/spreadsheet_etsi.py b/application/utils/external_project_parsers/parsers/spreadsheet_etsi.py index e801bb0df..cc869232e 100644 --- a/application/utils/external_project_parsers/parsers/spreadsheet_etsi.py +++ b/application/utils/external_project_parsers/parsers/spreadsheet_etsi.py @@ -1,5 +1,9 @@ """ETSI mappings from AI exchange CSV/master-shaped rows.""" +from cre_logging import get_logger + +logger = get_logger(__name__) + from typing import Dict, List, Tuple from application.defs import cre_defs as defs diff --git a/application/utils/external_project_parsers/parsers/spreadsheet_iso27001.py b/application/utils/external_project_parsers/parsers/spreadsheet_iso27001.py index 64307c4dd..7b268be6a 100644 --- a/application/utils/external_project_parsers/parsers/spreadsheet_iso27001.py +++ b/application/utils/external_project_parsers/parsers/spreadsheet_iso27001.py @@ -1,5 +1,9 @@ """ISO 27001 from the master mapping spreadsheet.""" +from cre_logging import get_logger + +logger = get_logger(__name__) + from typing import Dict, List, Tuple from application.defs import cre_defs as defs diff --git a/application/utils/external_project_parsers/parsers/spreadsheet_mitre_atlas.py b/application/utils/external_project_parsers/parsers/spreadsheet_mitre_atlas.py index db897cd71..2b694c26b 100644 --- a/application/utils/external_project_parsers/parsers/spreadsheet_mitre_atlas.py +++ b/application/utils/external_project_parsers/parsers/spreadsheet_mitre_atlas.py @@ -1,5 +1,9 @@ """MITRE ATLAS mitigations from the master mapping spreadsheet / AI exchange CSV.""" +from cre_logging import get_logger + +logger = get_logger(__name__) + from typing import Dict, List, Tuple from application.defs import cre_defs as defs diff --git a/application/utils/external_project_parsers/parsers/spreadsheet_nist_800_53_v5.py b/application/utils/external_project_parsers/parsers/spreadsheet_nist_800_53_v5.py index 63c9f9219..fd30cbea3 100644 --- a/application/utils/external_project_parsers/parsers/spreadsheet_nist_800_53_v5.py +++ b/application/utils/external_project_parsers/parsers/spreadsheet_nist_800_53_v5.py @@ -1,5 +1,9 @@ """NIST 800-53 v5 controls from the master mapping spreadsheet.""" +from cre_logging import get_logger + +logger = get_logger(__name__) + from typing import Dict, List, Tuple from application.defs import cre_defs as defs diff --git a/application/utils/external_project_parsers/parsers/spreadsheet_nist_800_63.py b/application/utils/external_project_parsers/parsers/spreadsheet_nist_800_63.py index 0d2eb062b..bba1a9597 100644 --- a/application/utils/external_project_parsers/parsers/spreadsheet_nist_800_63.py +++ b/application/utils/external_project_parsers/parsers/spreadsheet_nist_800_63.py @@ -1,5 +1,9 @@ """NIST 800-63 from the master mapping spreadsheet.""" +from cre_logging import get_logger + +logger = get_logger(__name__) + from typing import Dict, List, Tuple from application.defs import cre_defs as defs diff --git a/application/utils/external_project_parsers/parsers/spreadsheet_nist_ai_100_2.py b/application/utils/external_project_parsers/parsers/spreadsheet_nist_ai_100_2.py index bce536d68..5ac8ce5c2 100644 --- a/application/utils/external_project_parsers/parsers/spreadsheet_nist_ai_100_2.py +++ b/application/utils/external_project_parsers/parsers/spreadsheet_nist_ai_100_2.py @@ -1,5 +1,9 @@ """NIST AI 100-2 mappings from AI exchange CSV/master-shaped rows.""" +from cre_logging import get_logger + +logger = get_logger(__name__) + from typing import Dict, List, Tuple from application.defs import cre_defs as defs diff --git a/application/utils/external_project_parsers/parsers/spreadsheet_nist_ssdf.py b/application/utils/external_project_parsers/parsers/spreadsheet_nist_ssdf.py index 06c760413..7087e4539 100644 --- a/application/utils/external_project_parsers/parsers/spreadsheet_nist_ssdf.py +++ b/application/utils/external_project_parsers/parsers/spreadsheet_nist_ssdf.py @@ -1,5 +1,9 @@ """NIST SSDF from the master mapping spreadsheet.""" +from cre_logging import get_logger + +logger = get_logger(__name__) + from typing import Dict, List, Tuple from application.defs import cre_defs as defs diff --git a/application/utils/external_project_parsers/parsers/spreadsheet_owasp_ai_exchange.py b/application/utils/external_project_parsers/parsers/spreadsheet_owasp_ai_exchange.py index f192d9e0f..9f37d4d65 100644 --- a/application/utils/external_project_parsers/parsers/spreadsheet_owasp_ai_exchange.py +++ b/application/utils/external_project_parsers/parsers/spreadsheet_owasp_ai_exchange.py @@ -1,5 +1,9 @@ """OWASP AI Exchange controls from the AI exchange CSV (master-shaped rows).""" +from cre_logging import get_logger + +logger = get_logger(__name__) + from typing import Dict, List, Tuple from application.defs import cre_defs as defs diff --git a/application/utils/external_project_parsers/parsers/spreadsheet_owasp_cheat_sheets.py b/application/utils/external_project_parsers/parsers/spreadsheet_owasp_cheat_sheets.py index 07adb6a16..f2bf43405 100644 --- a/application/utils/external_project_parsers/parsers/spreadsheet_owasp_cheat_sheets.py +++ b/application/utils/external_project_parsers/parsers/spreadsheet_owasp_cheat_sheets.py @@ -1,5 +1,9 @@ """OWASP Cheat Sheets from the master mapping spreadsheet.""" +from cre_logging import get_logger + +logger = get_logger(__name__) + from typing import Dict, List, Tuple from application.defs import cre_defs as defs diff --git a/application/utils/external_project_parsers/parsers/spreadsheet_owasp_proactive_controls.py b/application/utils/external_project_parsers/parsers/spreadsheet_owasp_proactive_controls.py index 13aeaa181..6dd43e190 100644 --- a/application/utils/external_project_parsers/parsers/spreadsheet_owasp_proactive_controls.py +++ b/application/utils/external_project_parsers/parsers/spreadsheet_owasp_proactive_controls.py @@ -1,5 +1,9 @@ """OWASP Proactive Controls from the master mapping spreadsheet.""" +from cre_logging import get_logger + +logger = get_logger(__name__) + from typing import Dict, List, Tuple from application.defs import cre_defs as defs diff --git a/application/utils/external_project_parsers/parsers/spreadsheet_owasp_top10_2017.py b/application/utils/external_project_parsers/parsers/spreadsheet_owasp_top10_2017.py index 625f9127b..c63b4b843 100644 --- a/application/utils/external_project_parsers/parsers/spreadsheet_owasp_top10_2017.py +++ b/application/utils/external_project_parsers/parsers/spreadsheet_owasp_top10_2017.py @@ -1,5 +1,9 @@ """OWASP Top 10 2017 from the master mapping spreadsheet.""" +from cre_logging import get_logger + +logger = get_logger(__name__) + from typing import Dict, List, Tuple from application.defs import cre_defs as defs diff --git a/application/utils/external_project_parsers/parsers/spreadsheet_owasp_top10_2021.py b/application/utils/external_project_parsers/parsers/spreadsheet_owasp_top10_2021.py index d612becde..ee1967c16 100644 --- a/application/utils/external_project_parsers/parsers/spreadsheet_owasp_top10_2021.py +++ b/application/utils/external_project_parsers/parsers/spreadsheet_owasp_top10_2021.py @@ -1,5 +1,9 @@ """OWASP Top 10 2021 from the master mapping spreadsheet.""" +from cre_logging import get_logger + +logger = get_logger(__name__) + from typing import Dict, List, Tuple from application.defs import cre_defs as defs diff --git a/application/utils/external_project_parsers/parsers/spreadsheet_owasp_top10_llm.py b/application/utils/external_project_parsers/parsers/spreadsheet_owasp_top10_llm.py index 09a69c716..50fd2deca 100644 --- a/application/utils/external_project_parsers/parsers/spreadsheet_owasp_top10_llm.py +++ b/application/utils/external_project_parsers/parsers/spreadsheet_owasp_top10_llm.py @@ -1,5 +1,9 @@ """OWASP Top10 for LLM mappings from AI exchange CSV/master-shaped rows.""" +from cre_logging import get_logger + +logger = get_logger(__name__) + from typing import Dict, List, Tuple from application.defs import cre_defs as defs diff --git a/application/utils/external_project_parsers/parsers/spreadsheet_owasp_top10_ml.py b/application/utils/external_project_parsers/parsers/spreadsheet_owasp_top10_ml.py index 957a285d5..65449f101 100644 --- a/application/utils/external_project_parsers/parsers/spreadsheet_owasp_top10_ml.py +++ b/application/utils/external_project_parsers/parsers/spreadsheet_owasp_top10_ml.py @@ -1,5 +1,9 @@ """OWASP Top10 for ML mappings from AI exchange CSV/master-shaped rows.""" +from cre_logging import get_logger + +logger = get_logger(__name__) + from typing import Dict, List, Tuple from application.defs import cre_defs as defs diff --git a/application/utils/external_project_parsers/parsers/spreadsheet_owasp_wstg.py b/application/utils/external_project_parsers/parsers/spreadsheet_owasp_wstg.py index 8a9e3899d..b825f078f 100644 --- a/application/utils/external_project_parsers/parsers/spreadsheet_owasp_wstg.py +++ b/application/utils/external_project_parsers/parsers/spreadsheet_owasp_wstg.py @@ -1,5 +1,9 @@ """OWASP WSTG items from the master mapping spreadsheet.""" +from cre_logging import get_logger + +logger = get_logger(__name__) + from typing import Dict, List, Tuple from application.defs import cre_defs as defs diff --git a/application/utils/external_project_parsers/parsers/spreadsheet_samm.py b/application/utils/external_project_parsers/parsers/spreadsheet_samm.py index 258888b30..7b6eade0e 100644 --- a/application/utils/external_project_parsers/parsers/spreadsheet_samm.py +++ b/application/utils/external_project_parsers/parsers/spreadsheet_samm.py @@ -1,5 +1,9 @@ """SAMM from the master mapping spreadsheet.""" +from cre_logging import get_logger + +logger = get_logger(__name__) + from typing import Dict, List, Tuple from application.defs import cre_defs as defs diff --git a/application/utils/external_project_parsers/parsers/spreadsheet_standard_family.py b/application/utils/external_project_parsers/parsers/spreadsheet_standard_family.py index bb2f63214..71d570466 100644 --- a/application/utils/external_project_parsers/parsers/spreadsheet_standard_family.py +++ b/application/utils/external_project_parsers/parsers/spreadsheet_standard_family.py @@ -1,5 +1,9 @@ """Shared logic for extracting one standard family from master spreadsheet rows.""" +from cre_logging import get_logger + +logger = get_logger(__name__) + from copy import copy from dataclasses import dataclass from typing import Any, Dict, List, Optional, Tuple diff --git a/application/utils/external_project_parsers/parsers/spreadsheet_standards_dispatch.py b/application/utils/external_project_parsers/parsers/spreadsheet_standards_dispatch.py index 2533fd1e9..1cafb453a 100644 --- a/application/utils/external_project_parsers/parsers/spreadsheet_standards_dispatch.py +++ b/application/utils/external_project_parsers/parsers/spreadsheet_standards_dispatch.py @@ -1,5 +1,9 @@ """Aggregate spreadsheet-derived standards: one module per family, same master sheet.""" +from cre_logging import get_logger + +logger = get_logger(__name__) + from typing import Dict, List, Tuple from application.defs import cre_defs as defs diff --git a/application/utils/external_project_parsers/parsers/zap_alerts_parser.py b/application/utils/external_project_parsers/parsers/zap_alerts_parser.py index 631c84187..844507971 100644 --- a/application/utils/external_project_parsers/parsers/zap_alerts_parser.py +++ b/application/utils/external_project_parsers/parsers/zap_alerts_parser.py @@ -1,6 +1,9 @@ # script to parse zaproxy website md files describing alerts find the CWE ids # and add the alerts to CRE -import logging +from cre_logging import get_logger + +logger = get_logger(__name__) + import os import re from typing import List @@ -9,9 +12,6 @@ from application.defs import cre_defs as defs from application.utils import git -logging.basicConfig() -logger = logging.getLogger(__name__) -logger.setLevel(logging.INFO) from application.prompt_client import prompt_client as prompt_client from application.utils.external_project_parsers import base_parser_defs diff --git a/application/utils/file.py b/application/utils/file.py index 61276f96b..ac66667d0 100644 --- a/application/utils/file.py +++ b/application/utils/file.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + import os from typing import Dict diff --git a/application/utils/ga_parity.py b/application/utils/ga_parity.py index 9bc098cc1..af3de2611 100644 --- a/application/utils/ga_parity.py +++ b/application/utils/ga_parity.py @@ -6,6 +6,10 @@ from __future__ import annotations +from cre_logging import get_logger + +logger = get_logger(__name__) + from typing import TYPE_CHECKING, Any, List, Sequence, Tuple if TYPE_CHECKING: diff --git a/application/utils/gap_analysis.py b/application/utils/gap_analysis.py index 3160b0ef1..49810a278 100644 --- a/application/utils/gap_analysis.py +++ b/application/utils/gap_analysis.py @@ -1,16 +1,16 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + import os import requests import time -import logging from typing import Any, Dict, List, Optional from application.utils import redis from flask import json as flask_json import json from application.defs import cre_defs as defs -logging.basicConfig() -logger = logging.getLogger(__name__) -logger.setLevel(logging.INFO) PENALTIES = { "RELATED": 2, @@ -410,16 +410,18 @@ def calculate_a_to_b(sa: str, sb: str) -> bool: f"{target_url}/rest/v1/map_analysis?standard={sa}&standard={sb}" ) if res.status_code != 200: - print(f"{sa}->{sb} returned {res.status_code}") + logger.warning("%s->%s returned %s", sa, sb, res.status_code) return False tojson = res.json() if tojson.get("result"): return True if tojson.get("job_id"): - print(f"{sa}->{sb} waiting") + logger.info("%s->%s waiting", sa, sb) return False - print(f"{sa}->{sb} returned 200 but has no 'result' or 'job_id' key") + logger.warning( + "%s->%s returned 200 but has no 'result' or 'job_id' key", sa, sb + ) return False for sa in standards: @@ -434,6 +436,6 @@ def calculate_a_to_b(sa: str, sb: str) -> bool: continue if calculate_a_to_b(sa, sb): waiting.remove(f"{sa}->{sb}") if f"{sa}->{sb}" in waiting else "" - print(f"calculating {len(waiting)} gap analyses") + logger.info("calculating %s gap analyses", len(waiting)) time.sleep(30) - print("map analysis preloaded successfully") + logger.info("map analysis preloaded successfully") diff --git a/application/utils/git.py b/application/utils/git.py index 025f4c502..cfd2ca1ce 100644 --- a/application/utils/git.py +++ b/application/utils/git.py @@ -1,5 +1,8 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + from typing import List, Optional -import logging import os from datetime import datetime @@ -10,9 +13,6 @@ from git.repo.base import Repo from github import Github -logger = logging.getLogger(__name__) -logger.setLevel(logging.INFO) -logging.basicConfig() commit_msg_base = "cre_sync_%s" % (datetime.now().isoformat().replace(":", ".")) diff --git a/application/utils/harvester/__init__.py b/application/utils/harvester/__init__.py index e89b95fe4..63c1a7578 100644 --- a/application/utils/harvester/__init__.py +++ b/application/utils/harvester/__init__.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + from .config_loader import ( ConfigLoaderError, load_repo_config, diff --git a/application/utils/harvester/artifact_id.py b/application/utils/harvester/artifact_id.py index fa3ed46f1..00b55b4ff 100644 --- a/application/utils/harvester/artifact_id.py +++ b/application/utils/harvester/artifact_id.py @@ -1,3 +1,8 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + + def generate_artifact_id(repository: str, file_path: str) -> str: """ Generate a stable artifact identifier for a repository file. diff --git a/application/utils/harvester/artifact_registry.py b/application/utils/harvester/artifact_registry.py index 621f8e567..4305efad3 100644 --- a/application/utils/harvester/artifact_registry.py +++ b/application/utils/harvester/artifact_registry.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + from datetime import datetime from .models import ArtifactRegistryRecord diff --git a/application/utils/harvester/change_detector.py b/application/utils/harvester/change_detector.py index 584d43d9c..f73073d4a 100644 --- a/application/utils/harvester/change_detector.py +++ b/application/utils/harvester/change_detector.py @@ -1,10 +1,11 @@ -import logging +from cre_logging import get_logger + +logger = get_logger(__name__) + import subprocess from .git_repository_client import GitRepositoryClient -logger = logging.getLogger(__name__) - class ChangeDetector: def __init__(self, repository_client: GitRepositoryClient): diff --git a/application/utils/harvester/checkpoint_manager.py b/application/utils/harvester/checkpoint_manager.py index c9f0fdd3c..a77941d12 100644 --- a/application/utils/harvester/checkpoint_manager.py +++ b/application/utils/harvester/checkpoint_manager.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + from datetime import datetime from .models import CheckpointRecord diff --git a/application/utils/harvester/checkpoint_store.py b/application/utils/harvester/checkpoint_store.py index ed757bb56..1b62dab4f 100644 --- a/application/utils/harvester/checkpoint_store.py +++ b/application/utils/harvester/checkpoint_store.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + from typing import Any from sqlalchemy.exc import IntegrityError diff --git a/application/utils/harvester/chunk_pipeline.py b/application/utils/harvester/chunk_pipeline.py index 7beb30857..b9509e891 100644 --- a/application/utils/harvester/chunk_pipeline.py +++ b/application/utils/harvester/chunk_pipeline.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + from .chunk_record_builder import ChunkRecordBuilder from .chunk_record_validator import ChunkRecordValidator from .chunker import DocumentChunker diff --git a/application/utils/harvester/chunk_record_builder.py b/application/utils/harvester/chunk_record_builder.py index 8cc292536..76fc1ff81 100644 --- a/application/utils/harvester/chunk_record_builder.py +++ b/application/utils/harvester/chunk_record_builder.py @@ -1,5 +1,9 @@ from __future__ import annotations +from cre_logging import get_logger + +logger = get_logger(__name__) + from dataclasses import dataclass from datetime import datetime, timezone from typing import TYPE_CHECKING, Optional diff --git a/application/utils/harvester/chunk_record_validator.py b/application/utils/harvester/chunk_record_validator.py index 0645e69fa..f77c3c537 100644 --- a/application/utils/harvester/chunk_record_validator.py +++ b/application/utils/harvester/chunk_record_validator.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + from application.utils.noise_filter.schemas import ChangeRecord from .models import IngestChunkRecord diff --git a/application/utils/harvester/chunker.py b/application/utils/harvester/chunker.py index 3eec8a369..778c145a0 100644 --- a/application/utils/harvester/chunker.py +++ b/application/utils/harvester/chunker.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + from .models import ChunkInfo, Document from .schemas import ChunkingConfig diff --git a/application/utils/harvester/config_loader.py b/application/utils/harvester/config_loader.py index db0431e11..95d6eb26a 100644 --- a/application/utils/harvester/config_loader.py +++ b/application/utils/harvester/config_loader.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + from pathlib import Path import yaml from pydantic import ValidationError diff --git a/application/utils/harvester/content_hash.py b/application/utils/harvester/content_hash.py index ed7bd8bda..ffc28a7da 100644 --- a/application/utils/harvester/content_hash.py +++ b/application/utils/harvester/content_hash.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + import hashlib diff --git a/application/utils/harvester/deduplication_metrics.py b/application/utils/harvester/deduplication_metrics.py index a3843a7fc..2147a7312 100644 --- a/application/utils/harvester/deduplication_metrics.py +++ b/application/utils/harvester/deduplication_metrics.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + from dataclasses import dataclass from .models import DeduplicationStatus diff --git a/application/utils/harvester/diff_normalizer.py b/application/utils/harvester/diff_normalizer.py index d3e923a01..4cdc9549c 100644 --- a/application/utils/harvester/diff_normalizer.py +++ b/application/utils/harvester/diff_normalizer.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + import re import unicodedata diff --git a/application/utils/harvester/diff_parser.py b/application/utils/harvester/diff_parser.py index 12b14784c..13a9fa3e0 100644 --- a/application/utils/harvester/diff_parser.py +++ b/application/utils/harvester/diff_parser.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + from datetime import datetime import re diff --git a/application/utils/harvester/diff_retriever.py b/application/utils/harvester/diff_retriever.py index 665d10d19..e2eb56307 100644 --- a/application/utils/harvester/diff_retriever.py +++ b/application/utils/harvester/diff_retriever.py @@ -1,10 +1,11 @@ -import logging +from cre_logging import get_logger + +logger = get_logger(__name__) + import subprocess from .git_repository_client import GitRepositoryClient -logger = logging.getLogger(__name__) - class DiffRetriever: """ diff --git a/application/utils/harvester/document_builder.py b/application/utils/harvester/document_builder.py index 9988c9f13..984a59cfb 100644 --- a/application/utils/harvester/document_builder.py +++ b/application/utils/harvester/document_builder.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + from .artifact_id import generate_artifact_id from .heading_extractor import HeadingExtractor from .models import ( diff --git a/application/utils/harvester/document_deduplicator.py b/application/utils/harvester/document_deduplicator.py index b1e953f26..63dbb1d83 100644 --- a/application/utils/harvester/document_deduplicator.py +++ b/application/utils/harvester/document_deduplicator.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + from datetime import datetime from .artifact_registry import ArtifactRegistry diff --git a/application/utils/harvester/document_validator.py b/application/utils/harvester/document_validator.py index 497e90b15..524e9de60 100644 --- a/application/utils/harvester/document_validator.py +++ b/application/utils/harvester/document_validator.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + from .models import Document diff --git a/application/utils/harvester/file_filter.py b/application/utils/harvester/file_filter.py index f1da0ee79..f9d906c67 100644 --- a/application/utils/harvester/file_filter.py +++ b/application/utils/harvester/file_filter.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + from pathlib import PurePosixPath from pathlib import Path import pathspec diff --git a/application/utils/harvester/filtering_benchmark.py b/application/utils/harvester/filtering_benchmark.py index 5de2e7b69..40f78cdef 100644 --- a/application/utils/harvester/filtering_benchmark.py +++ b/application/utils/harvester/filtering_benchmark.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + from dataclasses import dataclass from .file_filter import FileFilter diff --git a/application/utils/harvester/filtering_metrics.py b/application/utils/harvester/filtering_metrics.py index d496c3e2a..106cfa0e8 100644 --- a/application/utils/harvester/filtering_metrics.py +++ b/application/utils/harvester/filtering_metrics.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + from .models import FilteringMetrics diff --git a/application/utils/harvester/git_repository_client.py b/application/utils/harvester/git_repository_client.py index 06e27d286..c5c5f5e73 100644 --- a/application/utils/harvester/git_repository_client.py +++ b/application/utils/harvester/git_repository_client.py @@ -1,16 +1,17 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + import subprocess from pathlib import Path from .repository_cache import build_repository_cache_path from .repository_client import RepositoryClient -import logging from .repository_lock import repository_lock import os import shutil import tempfile -logger = logging.getLogger(__name__) - class GitRepositoryClient(RepositoryClient): def __init__( diff --git a/application/utils/harvester/harvest_writer.py b/application/utils/harvester/harvest_writer.py index b24816dfd..32169e7ba 100644 --- a/application/utils/harvester/harvest_writer.py +++ b/application/utils/harvester/harvest_writer.py @@ -2,6 +2,10 @@ from __future__ import annotations +from cre_logging import get_logger + +logger = get_logger(__name__) + from typing import Any, Iterable from application.database.db import HarvestInput diff --git a/application/utils/harvester/heading_extractor.py b/application/utils/harvester/heading_extractor.py index 9b3eef32b..51674fc9e 100644 --- a/application/utils/harvester/heading_extractor.py +++ b/application/utils/harvester/heading_extractor.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + from dataclasses import dataclass from .models import HeadingNode diff --git a/application/utils/harvester/incremental_pipeline.py b/application/utils/harvester/incremental_pipeline.py index 088e4fe94..a076ec1b0 100644 --- a/application/utils/harvester/incremental_pipeline.py +++ b/application/utils/harvester/incremental_pipeline.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + from datetime import datetime, timezone from typing import Any diff --git a/application/utils/harvester/models.py b/application/utils/harvester/models.py index b8db2083f..5ae84d0be 100644 --- a/application/utils/harvester/models.py +++ b/application/utils/harvester/models.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + from dataclasses import dataclass from datetime import datetime from enum import Enum diff --git a/application/utils/harvester/pipeline.py b/application/utils/harvester/pipeline.py index 61df4c0d1..756213554 100644 --- a/application/utils/harvester/pipeline.py +++ b/application/utils/harvester/pipeline.py @@ -6,8 +6,11 @@ from __future__ import annotations +from cre_logging import get_logger + +logger = get_logger(__name__) + import json -import logging from dataclasses import asdict, dataclass from datetime import datetime, timezone from pathlib import Path @@ -26,7 +29,6 @@ from application.utils.harvester.repos_validator import validate_repositories from application.utils.harvester.schemas import RepositoryConfig -logger = logging.getLogger(__name__) DEFAULT_REPOS_YAML = Path(__file__).with_name("repos.yaml") diff --git a/application/utils/harvester/repos_validator.py b/application/utils/harvester/repos_validator.py index 81cdc8273..39f931360 100644 --- a/application/utils/harvester/repos_validator.py +++ b/application/utils/harvester/repos_validator.py @@ -1,4 +1,8 @@ # as the name suggests +from cre_logging import get_logger + +logger = get_logger(__name__) + from .schemas import ReposFile diff --git a/application/utils/harvester/repository_cache.py b/application/utils/harvester/repository_cache.py index a8c9b632c..9a539c2d4 100644 --- a/application/utils/harvester/repository_cache.py +++ b/application/utils/harvester/repository_cache.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + import os import re diff --git a/application/utils/harvester/repository_client.py b/application/utils/harvester/repository_client.py index c545cce42..8ae10cb29 100644 --- a/application/utils/harvester/repository_client.py +++ b/application/utils/harvester/repository_client.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + from abc import ABC, abstractmethod from pathlib import Path diff --git a/application/utils/harvester/repository_lock.py b/application/utils/harvester/repository_lock.py index a9779033b..6fc475d44 100644 --- a/application/utils/harvester/repository_lock.py +++ b/application/utils/harvester/repository_lock.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + import contextlib import os from pathlib import Path diff --git a/application/utils/harvester/schemas.py b/application/utils/harvester/schemas.py index 4589f3d69..efa49dd8f 100644 --- a/application/utils/harvester/schemas.py +++ b/application/utils/harvester/schemas.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + from typing import Literal from pydantic import BaseModel, Field, ConfigDict, model_validator diff --git a/application/utils/hash.py b/application/utils/hash.py index 4ee750934..660a28bdf 100644 --- a/application/utils/hash.py +++ b/application/utils/hash.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + import hashlib from typing import List diff --git a/application/utils/import_apply.py b/application/utils/import_apply.py index ab608f653..db66d216f 100644 --- a/application/utils/import_apply.py +++ b/application/utils/import_apply.py @@ -4,6 +4,10 @@ from __future__ import annotations +from cre_logging import get_logger + +logger = get_logger(__name__) + from dataclasses import dataclass from typing import Any, Dict, List, Tuple diff --git a/application/utils/import_diff.py b/application/utils/import_diff.py index 95f2ab3d2..53406735b 100644 --- a/application/utils/import_diff.py +++ b/application/utils/import_diff.py @@ -6,6 +6,10 @@ plus a change-set vocabulary for add/remove/modify operations. """ +from cre_logging import get_logger + +logger = get_logger(__name__) + import json import hashlib from dataclasses import asdict, dataclass, field diff --git a/application/utils/import_graph.py b/application/utils/import_graph.py index e0936f14a..8df17bfb3 100644 --- a/application/utils/import_graph.py +++ b/application/utils/import_graph.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + from typing import Dict, List, Any from application.utils import import_diff diff --git a/application/utils/import_impact.py b/application/utils/import_impact.py index 1701618bc..545f45895 100644 --- a/application/utils/import_impact.py +++ b/application/utils/import_impact.py @@ -4,6 +4,10 @@ from __future__ import annotations +from cre_logging import get_logger + +logger = get_logger(__name__) + from typing import Any, Dict from application.database import db diff --git a/application/utils/import_pipeline.py b/application/utils/import_pipeline.py index 66579264d..3701deea9 100644 --- a/application/utils/import_pipeline.py +++ b/application/utils/import_pipeline.py @@ -9,15 +9,16 @@ from __future__ import annotations -import logging +from cre_logging import get_logger + +logger = get_logger(__name__) + from typing import Any, List, Optional from application.database import db as db_mod from application.prompt_client import prompt_client from application.utils.external_project_parsers import base_parser_defs -logger = logging.getLogger(__name__) - def parse_result_from_yaml_document_forest( roots: List[Any], diff --git a/application/utils/import_post_apply.py b/application/utils/import_post_apply.py index eba1ccf9a..395237578 100644 --- a/application/utils/import_post_apply.py +++ b/application/utils/import_post_apply.py @@ -4,7 +4,10 @@ from __future__ import annotations -import logging +from cre_logging import get_logger + +logger = get_logger(__name__) + import os from typing import List @@ -13,8 +16,6 @@ from application.prompt_client import prompt_client as prompt_client from application.utils import db_backend, gap_analysis, redis -logger = logging.getLogger(__name__) - def run_post_apply( *, diff --git a/application/utils/librarian/__init__.py b/application/utils/librarian/__init__.py index 755fb05a8..6ce39904c 100644 --- a/application/utils/librarian/__init__.py +++ b/application/utils/librarian/__init__.py @@ -47,3 +47,7 @@ Update the SHA above, then re-run the schemas test suite. """ + +from cre_logging import get_logger + +logger = get_logger(__name__) diff --git a/application/utils/librarian/calibration/__init__.py b/application/utils/librarian/calibration/__init__.py index 54b774038..93f1adcfd 100644 --- a/application/utils/librarian/calibration/__init__.py +++ b/application/utils/librarian/calibration/__init__.py @@ -13,3 +13,7 @@ (numpy + scipy) and model-free so it stays hermetically testable — mirrors the C.1/C.2 seams. """ + +from cre_logging import get_logger + +logger = get_logger(__name__) diff --git a/application/utils/librarian/calibration/temperature.py b/application/utils/librarian/calibration/temperature.py index 8ded3abc3..8b4944f31 100644 --- a/application/utils/librarian/calibration/temperature.py +++ b/application/utils/librarian/calibration/temperature.py @@ -30,6 +30,10 @@ confidence it thresholds on. """ +from cre_logging import get_logger + +logger = get_logger(__name__) + from dataclasses import dataclass from typing import List, Sequence, Tuple diff --git a/application/utils/librarian/candidate_retriever.py b/application/utils/librarian/candidate_retriever.py index 9db70e9c8..9e0f2841d 100644 --- a/application/utils/librarian/candidate_retriever.py +++ b/application/utils/librarian/candidate_retriever.py @@ -41,6 +41,10 @@ the fixed-width ``vector(dim)`` column.) """ +from cre_logging import get_logger + +logger = get_logger(__name__) + from dataclasses import dataclass from enum import Enum from typing import Any, Callable, List, Mapping, Optional, Sequence, Tuple diff --git a/application/utils/librarian/config_loader.py b/application/utils/librarian/config_loader.py index 5646e94e1..5fb4d1372 100644 --- a/application/utils/librarian/config_loader.py +++ b/application/utils/librarian/config_loader.py @@ -14,6 +14,10 @@ - ``CRE_LIBRARIAN_LINK_THRESHOLD`` is τ, held at 0.80 by the W7 sweep. """ +from cre_logging import get_logger + +logger = get_logger(__name__) + import math import os from dataclasses import dataclass diff --git a/application/utils/librarian/cross_encoder.py b/application/utils/librarian/cross_encoder.py index 2e3f6991c..a8c05af30 100644 --- a/application/utils/librarian/cross_encoder.py +++ b/application/utils/librarian/cross_encoder.py @@ -25,6 +25,10 @@ leaving ``candidates[]`` untouched so the pre-rerank shortlist stays auditable. """ +from cre_logging import get_logger + +logger = get_logger(__name__) + from typing import Callable, List, Mapping, Sequence, Tuple from application.utils.librarian.schemas import RetrievalAudit diff --git a/application/utils/librarian/decision_engine.py b/application/utils/librarian/decision_engine.py index f6c144263..753ff7347 100644 --- a/application/utils/librarian/decision_engine.py +++ b/application/utils/librarian/decision_engine.py @@ -24,6 +24,10 @@ shortlist dominates everything. """ +from cre_logging import get_logger + +logger = get_logger(__name__) + import math from dataclasses import dataclass from typing import Optional, Sequence, Tuple diff --git a/application/utils/librarian/emitter.py b/application/utils/librarian/emitter.py index 8591e5199..1ec78ffd6 100644 --- a/application/utils/librarian/emitter.py +++ b/application/utils/librarian/emitter.py @@ -12,6 +12,10 @@ same review, with no nondeterministic uuid in a pure builder. """ +from cre_logging import get_logger + +logger = get_logger(__name__) + from datetime import datetime from typing import Optional, Union diff --git a/application/utils/librarian/envelope_sink.py b/application/utils/librarian/envelope_sink.py index 0b467fbdd..1cf73d943 100644 --- a/application/utils/librarian/envelope_sink.py +++ b/application/utils/librarian/envelope_sink.py @@ -17,14 +17,16 @@ same protocol, so the consumption rule does not have to change when they land. """ +from cre_logging import get_logger + +logger = get_logger(__name__) + import json -import logging import os from typing import Any, List, Mapping, Optional, Protocol, Sequence, Union from application.utils.librarian.schemas import LinkProposal, ReviewItem -logger = logging.getLogger(__name__) Envelope = Union[LinkProposal, ReviewItem] diff --git a/application/utils/librarian/explicit_link_resolver.py b/application/utils/librarian/explicit_link_resolver.py index 4caf660f3..f9ca84d13 100644 --- a/application/utils/librarian/explicit_link_resolver.py +++ b/application/utils/librarian/explicit_link_resolver.py @@ -15,6 +15,10 @@ retriever (W3). """ +from cre_logging import get_logger + +logger = get_logger(__name__) + import re from dataclasses import dataclass from enum import Enum diff --git a/application/utils/librarian/factory.py b/application/utils/librarian/factory.py index 6096d6c9c..d9f4ba118 100644 --- a/application/utils/librarian/factory.py +++ b/application/utils/librarian/factory.py @@ -14,15 +14,16 @@ class but no way to produce the components it takes. This module is that seam. being true — keeping the imports inside the call preserves it for everyone else. """ -import logging +from cre_logging import get_logger + +logger = get_logger(__name__) + from dataclasses import dataclass from typing import Any, Callable, FrozenSet, Optional, Sequence from application.utils.librarian.config_loader import LibrarianConfig, load_config from application.utils.librarian.pipeline import Reranker, Retriever, Scaler -logger = logging.getLogger(__name__) - @dataclass(frozen=True) class LibrarianComponents: diff --git a/application/utils/librarian/hub_firewall.py b/application/utils/librarian/hub_firewall.py index bd4493bdb..5bc556ace 100644 --- a/application/utils/librarian/hub_firewall.py +++ b/application/utils/librarian/hub_firewall.py @@ -9,6 +9,10 @@ the methodology and its test exist before any accuracy number is claimed. """ +from cre_logging import get_logger + +logger = get_logger(__name__) + import re from dataclasses import dataclass from typing import Iterable, List diff --git a/application/utils/librarian/knowledge_source.py b/application/utils/librarian/knowledge_source.py index ae098e385..4c3794c60 100644 --- a/application/utils/librarian/knowledge_source.py +++ b/application/utils/librarian/knowledge_source.py @@ -22,7 +22,10 @@ on an uncertain one. """ -import logging +from cre_logging import get_logger + +logger = get_logger(__name__) + from abc import ABC, abstractmethod from typing import Iterator, List, Optional @@ -30,7 +33,6 @@ from application.utils.librarian.schemas import KnowledgeQueueItem -logger = logging.getLogger(__name__) # The labels Module C acts on; see the module docstring. NOISE never reaches the # queue — B drops it — so these two are everything B writes. diff --git a/application/utils/librarian/pipeline.py b/application/utils/librarian/pipeline.py index 500996ce5..e3a117f18 100644 --- a/application/utils/librarian/pipeline.py +++ b/application/utils/librarian/pipeline.py @@ -20,7 +20,10 @@ a session. Graph writes remain out (W8b). """ -import logging +from cre_logging import get_logger + +logger = get_logger(__name__) + from dataclasses import dataclass, field from datetime import datetime from enum import Enum @@ -40,7 +43,6 @@ section_from_queue_row, ) -logger = logging.getLogger(__name__) Envelope = Union[LinkProposal, ReviewItem] diff --git a/application/utils/librarian/queue_consumer.py b/application/utils/librarian/queue_consumer.py index a58926053..38bf247f9 100644 --- a/application/utils/librarian/queue_consumer.py +++ b/application/utils/librarian/queue_consumer.py @@ -16,11 +16,13 @@ The caller owns the transaction, matching ``run_noise_filter`` on B's side. """ -import logging +from cre_logging import get_logger + +logger = get_logger(__name__) + from datetime import datetime, timezone from typing import Any, Iterable, List, Sequence -logger = logging.getLogger(__name__) # Postgres has a bind-parameter ceiling and a huge IN (...) plans badly, so the # id list is stamped in chunks rather than one statement. diff --git a/application/utils/librarian/queue_runner.py b/application/utils/librarian/queue_runner.py index b1aa3a4c8..2a6cf0d7c 100644 --- a/application/utils/librarian/queue_runner.py +++ b/application/utils/librarian/queue_runner.py @@ -38,8 +38,11 @@ result, and W8b's graph writer must refuse to run while it is non-zero. """ +from cre_logging import get_logger + +logger = get_logger(__name__) + import json -import logging from dataclasses import asdict, dataclass from datetime import datetime from typing import Any, Optional @@ -51,8 +54,6 @@ from application.utils.librarian.pipeline import LibrarianPipeline, RunResult from application.utils.librarian.queue_consumer import mark_consumed -logger = logging.getLogger(__name__) - @dataclass class RunSummary: diff --git a/application/utils/librarian/safety_guard.py b/application/utils/librarian/safety_guard.py index d79bf3041..a1b927475 100644 --- a/application/utils/librarian/safety_guard.py +++ b/application/utils/librarian/safety_guard.py @@ -19,14 +19,15 @@ disk. Committing a link into a graph other tools read as truth is not. """ -import logging +from cre_logging import get_logger + +logger = get_logger(__name__) + from dataclasses import dataclass from typing import Protocol from application.utils.librarian.section_validator import Section -logger = logging.getLogger(__name__) - @dataclass(frozen=True) class SafetyVerdict: diff --git a/application/utils/librarian/schemas.py b/application/utils/librarian/schemas.py index 4a3c116f1..cfe7fad97 100644 --- a/application/utils/librarian/schemas.py +++ b/application/utils/librarian/schemas.py @@ -11,6 +11,10 @@ from __future__ import annotations +from cre_logging import get_logger + +logger = get_logger(__name__) + import json import re from datetime import datetime diff --git a/application/utils/librarian/scoring.py b/application/utils/librarian/scoring.py index 2ca90e007..17ce3abe4 100644 --- a/application/utils/librarian/scoring.py +++ b/application/utils/librarian/scoring.py @@ -5,6 +5,10 @@ one function only. """ +from cre_logging import get_logger + +logger = get_logger(__name__) + from typing import List, Sequence # Q-D correctness rule (confirmed on PR #922): a predicted set is correct iff diff --git a/application/utils/librarian/section_validator.py b/application/utils/librarian/section_validator.py index 7f29c4f8c..915052084 100644 --- a/application/utils/librarian/section_validator.py +++ b/application/utils/librarian/section_validator.py @@ -32,6 +32,10 @@ Pydantic ``ValidationError`` never escapes this module. """ +from cre_logging import get_logger + +logger = get_logger(__name__) + from dataclasses import dataclass from typing import Any, Callable, Dict, Optional, Type, TypeVar, Union diff --git a/application/utils/mapping_fixtures.py b/application/utils/mapping_fixtures.py index e9a676b4d..5c9d3d6cc 100644 --- a/application/utils/mapping_fixtures.py +++ b/application/utils/mapping_fixtures.py @@ -8,6 +8,10 @@ from __future__ import annotations +from cre_logging import get_logger + +logger = get_logger(__name__) + import json from pathlib import Path from typing import Any, Dict, List, Mapping, Sequence diff --git a/application/utils/mdutils.py b/application/utils/mdutils.py index 590f8b7a0..6c20e59ec 100644 --- a/application/utils/mdutils.py +++ b/application/utils/mdutils.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + from python_markdown_maker import Table, links from application.defs import cre_defs as defs from typing import List diff --git a/application/utils/noise_filter/__init__.py b/application/utils/noise_filter/__init__.py index cd025bfb0..06dc84f59 100644 --- a/application/utils/noise_filter/__init__.py +++ b/application/utils/noise_filter/__init__.py @@ -17,3 +17,7 @@ application/database/db.py; Module C contract in docs/gsoc_2026_module_b/module_c_contract.md) """ + +from cre_logging import get_logger + +logger = get_logger(__name__) diff --git a/application/utils/noise_filter/config_loader.py b/application/utils/noise_filter/config_loader.py index e67cb325a..7462b3bc8 100644 --- a/application/utils/noise_filter/config_loader.py +++ b/application/utils/noise_filter/config_loader.py @@ -14,6 +14,10 @@ from __future__ import annotations +from cre_logging import get_logger + +logger = get_logger(__name__) + import os from dataclasses import dataclass diff --git a/application/utils/noise_filter/hashing.py b/application/utils/noise_filter/hashing.py index 0a61d8393..4eb3f5122 100644 --- a/application/utils/noise_filter/hashing.py +++ b/application/utils/noise_filter/hashing.py @@ -23,6 +23,10 @@ from __future__ import annotations +from cre_logging import get_logger + +logger = get_logger(__name__) + import hashlib import re import unicodedata diff --git a/application/utils/noise_filter/llm_classifier.py b/application/utils/noise_filter/llm_classifier.py index 9063d796a..cbf114dfe 100644 --- a/application/utils/noise_filter/llm_classifier.py +++ b/application/utils/noise_filter/llm_classifier.py @@ -14,8 +14,11 @@ from __future__ import annotations +from cre_logging import get_logger + +logger = get_logger(__name__) + import json -import logging import os import time from typing import Any, Iterator @@ -30,7 +33,6 @@ ) from application.utils.noise_filter.schemas import ChangeRecord, ClassifyResult -logger = logging.getLogger(__name__) _SCHEMA_NAME = "noise_filter_classification" diff --git a/application/utils/noise_filter/pipeline.py b/application/utils/noise_filter/pipeline.py index 52379086f..b3b3e3ff3 100644 --- a/application/utils/noise_filter/pipeline.py +++ b/application/utils/noise_filter/pipeline.py @@ -14,8 +14,11 @@ from __future__ import annotations +from cre_logging import get_logger + +logger = get_logger(__name__) + import json -import logging from dataclasses import asdict, dataclass from typing import Optional @@ -33,8 +36,6 @@ from application.utils.noise_filter.sanitize import sanitize_text from application.utils.noise_filter.schemas import ChangeRecord, ClassifyResult -logger = logging.getLogger(__name__) - @dataclass class RunSummary: diff --git a/application/utils/noise_filter/prompts.py b/application/utils/noise_filter/prompts.py index a3cf73fc7..8e2117929 100644 --- a/application/utils/noise_filter/prompts.py +++ b/application/utils/noise_filter/prompts.py @@ -15,6 +15,10 @@ from __future__ import annotations +from cre_logging import get_logger + +logger = get_logger(__name__) + import json SYSTEM_PROMPT = """You are classifying chunks of OWASP repository content for a \ diff --git a/application/utils/noise_filter/queue_writer.py b/application/utils/noise_filter/queue_writer.py index fbeaf86eb..7f63ccf0d 100644 --- a/application/utils/noise_filter/queue_writer.py +++ b/application/utils/noise_filter/queue_writer.py @@ -14,6 +14,10 @@ from __future__ import annotations +from cre_logging import get_logger + +logger = get_logger(__name__) + import json from dataclasses import dataclass from typing import Iterable diff --git a/application/utils/noise_filter/regex_filter.py b/application/utils/noise_filter/regex_filter.py index 43941199f..90f77052b 100644 --- a/application/utils/noise_filter/regex_filter.py +++ b/application/utils/noise_filter/regex_filter.py @@ -26,6 +26,10 @@ from __future__ import annotations +from cre_logging import get_logger + +logger = get_logger(__name__) + from fnmatch import fnmatch from pathlib import Path from typing import Iterable, Iterator, Optional diff --git a/application/utils/noise_filter/sanitize.py b/application/utils/noise_filter/sanitize.py index d792f827e..3320529b0 100644 --- a/application/utils/noise_filter/sanitize.py +++ b/application/utils/noise_filter/sanitize.py @@ -31,6 +31,10 @@ from __future__ import annotations +from cre_logging import get_logger + +logger = get_logger(__name__) + import html import re import unicodedata diff --git a/application/utils/noise_filter/schemas.py b/application/utils/noise_filter/schemas.py index 39cc82c84..d2594702d 100644 --- a/application/utils/noise_filter/schemas.py +++ b/application/utils/noise_filter/schemas.py @@ -13,6 +13,10 @@ from __future__ import annotations +from cre_logging import get_logger + +logger = get_logger(__name__) + from typing import Annotated, Literal, Optional, Union from pydantic import BaseModel, ConfigDict, Field, model_validator diff --git a/application/utils/oie_orchestrator/__init__.py b/application/utils/oie_orchestrator/__init__.py index e3dd203b0..4b16afdf8 100644 --- a/application/utils/oie_orchestrator/__init__.py +++ b/application/utils/oie_orchestrator/__init__.py @@ -1,5 +1,9 @@ """OIE A→B→C orchestrator package.""" +from cre_logging import get_logger + +logger = get_logger(__name__) + from .pipeline import ( OrchestratorResult, StageResult, diff --git a/application/utils/oie_orchestrator/pipeline.py b/application/utils/oie_orchestrator/pipeline.py index df2213825..f0813f270 100644 --- a/application/utils/oie_orchestrator/pipeline.py +++ b/application/utils/oie_orchestrator/pipeline.py @@ -9,14 +9,15 @@ from __future__ import annotations +from cre_logging import get_logger + +logger = get_logger(__name__) + import json -import logging from dataclasses import asdict, dataclass, field from datetime import datetime, timezone from typing import Any, Callable, Dict, List, Optional -logger = logging.getLogger(__name__) - @dataclass class StageResult: diff --git a/application/utils/oscal_utils.py b/application/utils/oscal_utils.py index 0b26a6758..3f18a3073 100644 --- a/application/utils/oscal_utils.py +++ b/application/utils/oscal_utils.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + import random from trestle.oscal import catalog from trestle.oscal import common diff --git a/application/utils/redirectors.py b/application/utils/redirectors.py index 2568b94d4..592d2e82f 100644 --- a/application/utils/redirectors.py +++ b/application/utils/redirectors.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + import urllib.parse diff --git a/application/utils/redis.py b/application/utils/redis.py index 90b2296bd..52af4f660 100644 --- a/application/utils/redis.py +++ b/application/utils/redis.py @@ -1,15 +1,14 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + import redis import os from urllib.parse import urlparse -import logging from typing import Callable, List import rq import time -logging.basicConfig() -logger = logging.getLogger(__name__) -logger.setLevel(logging.INFO) - def empty_queues(redis: redis.Redis): deleted_keys = 0 diff --git a/application/utils/spreadsheet.py b/application/utils/spreadsheet.py index db72835f7..42dfc5d68 100644 --- a/application/utils/spreadsheet.py +++ b/application/utils/spreadsheet.py @@ -1,7 +1,9 @@ -from pprint import pprint +from cre_logging import get_logger + +logger = get_logger(__name__) + import csv import io -import logging from copy import deepcopy from typing import Any, Dict, List, Optional, Set import os @@ -10,10 +12,6 @@ from application.defs import cre_defs as defs from enum import Enum -logger = logging.getLogger(__name__) -logger.setLevel(logging.INFO) -logging.basicConfig() - class GspreadAuth(Enum): OAuth = "oauth" diff --git a/application/utils/spreadsheet_parsers.py b/application/utils/spreadsheet_parsers.py index 7a2302fc4..ededb4512 100644 --- a/application/utils/spreadsheet_parsers.py +++ b/application/utils/spreadsheet_parsers.py @@ -6,6 +6,10 @@ Keep these exports so existing tests/callers continue to work. """ +from cre_logging import get_logger + +logger = get_logger(__name__) + from application.utils.external_project_parsers.parsers.export_format_parser import ( parse_export_format, ) @@ -23,8 +27,6 @@ "parse_standards", "supported_resource_mapping", ] -from pprint import pprint -import logging import re from copy import copy from typing import Any, Dict, List, Optional, Tuple @@ -35,10 +37,6 @@ # collection of methods to parse different versions of spreadsheet standards # each method returns a list of cre_defs documents -logger = logging.getLogger(__name__) -logger.setLevel(logging.INFO) -logging.basicConfig() - # the supported resources from the main CSV supported_resource_mapping = { diff --git a/application/utils/telemetry.py b/application/utils/telemetry.py index f959c78a9..da21b611f 100644 --- a/application/utils/telemetry.py +++ b/application/utils/telemetry.py @@ -1,12 +1,13 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + import os import time import json -import logging import requests from typing import Dict, Any, Optional -logger = logging.getLogger(__name__) - def emit_import_event( import_run_id: str, diff --git a/application/web/__init__.py b/application/web/__init__.py index e69de29bb..9662899ca 100644 --- a/application/web/__init__.py +++ b/application/web/__init__.py @@ -0,0 +1,3 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) diff --git a/application/web/openapi_registry.py b/application/web/openapi_registry.py index 4d2677e44..7b8b39efc 100644 --- a/application/web/openapi_registry.py +++ b/application/web/openapi_registry.py @@ -2,6 +2,10 @@ from __future__ import annotations +from cre_logging import get_logger + +logger = get_logger(__name__) + import re from functools import wraps from typing import Any, Callable, Dict, Iterable, List, Optional, Set, Tuple, Type diff --git a/application/web/openapi_schemas.py b/application/web/openapi_schemas.py index 7a77aa505..bd873e9ae 100644 --- a/application/web/openapi_schemas.py +++ b/application/web/openapi_schemas.py @@ -2,6 +2,10 @@ from __future__ import annotations +from cre_logging import get_logger + +logger = get_logger(__name__) + from marshmallow import Schema, fields, validate FORMAT_ENUM = ("json", "md", "csv", "oscal") diff --git a/application/web/visualise.py b/application/web/visualise.py index f233fdb6d..e9bf957f8 100644 --- a/application/web/visualise.py +++ b/application/web/visualise.py @@ -1,4 +1,8 @@ # type: ignore #unused file +from cre_logging import get_logger + +logger = get_logger(__name__) + from application.database import Node_collection, Links from application.defs.cre_defs import Standard, CRE import json diff --git a/application/web/web_main.py b/application/web/web_main.py index 76c3c2028..ea9ce90a7 100644 --- a/application/web/web_main.py +++ b/application/web/web_main.py @@ -1,10 +1,13 @@ # type: ignore +from cre_logging import get_logger + +logger = get_logger(__name__) + # silence mypy for the routes file import csv from functools import wraps import json -import logging import os import io import pathlib @@ -86,10 +89,6 @@ def _llm_error_status_code(err: BaseException) -> int | None: ), ) -logging.basicConfig() -logger = logging.getLogger(__name__) -logger.setLevel(logging.INFO) - def _ga_timeout_seconds() -> int: raw = str(gap_analysis.GAP_ANALYSIS_TIMEOUT or "129600s") diff --git a/application/worker.py b/application/worker.py index 5d121b83e..62d51fd84 100644 --- a/application/worker.py +++ b/application/worker.py @@ -1,11 +1,11 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + from rq import Worker, Queue -import logging import os from application.utils import redis -logging.basicConfig() -logger = logging.getLogger(__name__) -logger.setLevel(logging.INFO) DEFAULT_LISTEN = ["high", "default", "low", "ga"] diff --git a/cre.py b/cre.py index 63d8d7cce..d420bb48c 100644 --- a/cre.py +++ b/cre.py @@ -1,3 +1,7 @@ +from cre_logging import get_logger + +logger = get_logger(__name__) + import argparse import os import sys diff --git a/cre_logging/__init__.py b/cre_logging/__init__.py new file mode 100644 index 000000000..428611c9c --- /dev/null +++ b/cre_logging/__init__.py @@ -0,0 +1,120 @@ +"""OpenCRE structured JSON logging. + +This package is intentionally standalone: importing it must not import +``application`` or Flask. Application modules should use:: + + from cre_logging import get_logger + + logger = get_logger(__name__) + +Each line on stderr is one JSON object with ``level``, ``message``, +``module``, ``file`` (repo-relative path), and ``method`` (calling +function). ``LOG_LEVEL`` selects the root level (default ``INFO``). +""" + +from __future__ import annotations + +import json +import logging +import os +import sys +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Dict, Optional + +DEBUG = logging.DEBUG +INFO = logging.INFO +WARNING = logging.WARNING +ERROR = logging.ERROR +CRITICAL = logging.CRITICAL + +_HANDLER_NAME = "cre-json" +_REPO_ROOT = Path(__file__).resolve().parent.parent +_RESERVED_RECORD_KEYS = set(logging.makeLogRecord({}).__dict__.keys()) | { + "asctime", + "message", +} + + +class _CurrentStderr: + """Follow the live ``sys.stderr`` so tests can redirect it.""" + + def write(self, data: str) -> int: + return sys.stderr.write(data) + + def flush(self) -> None: + sys.stderr.flush() + + +class JSONFormatter(logging.Formatter): + """Format a log record as a single JSON object.""" + + def format(self, record: logging.LogRecord) -> str: + payload: Dict[str, Any] = { + "timestamp": datetime.fromtimestamp( + record.created, tz=timezone.utc + ).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + + "Z", + "level": record.levelname, + "logger": record.name, + "message": record.getMessage(), + "module": record.module, + "file": _relative_file(record.pathname), + "method": record.funcName, + "line": record.lineno, + } + if record.exc_info: + payload["exception"] = self.formatException(record.exc_info) + if record.stack_info: + payload["stack_info"] = self.formatStack(record.stack_info) + for key, value in record.__dict__.items(): + if key in _RESERVED_RECORD_KEYS or key.startswith("_"): + continue + payload[key] = value + return json.dumps(payload, default=str, ensure_ascii=False) + + +def _relative_file(pathname: str) -> str: + if not pathname: + return "" + try: + return str(Path(pathname).resolve().relative_to(_REPO_ROOT)) + except ValueError: + return Path(pathname).name + + +def _level_from_env() -> int: + name = os.environ.get("LOG_LEVEL", "INFO").upper() + return int(getattr(logging, name, logging.INFO)) + + +def configure_logging(force: bool = False) -> None: + """Install a single JSON handler on the root logger (idempotent).""" + root = logging.getLogger() + root.setLevel(_level_from_env()) + existing = [ + handler + for handler in root.handlers + if getattr(handler, "name", "") == _HANDLER_NAME + ] + if existing and not force: + for handler in existing: + handler.setFormatter(JSONFormatter()) + return + for handler in existing: + root.removeHandler(handler) + handler.close() + handler = logging.StreamHandler(_CurrentStderr()) + handler.set_name(_HANDLER_NAME) + handler.setFormatter(JSONFormatter()) + root.addHandler(handler) + logging.captureWarnings(True) + + +def get_logger(name: Optional[str] = None) -> logging.Logger: + """Return a logger; first call configures JSON output.""" + configure_logging() + return logging.getLogger(name if name else "opencre") + + +configure_logging() diff --git a/cre_logging/py.typed b/cre_logging/py.typed new file mode 100644 index 000000000..c0668dfaa --- /dev/null +++ b/cre_logging/py.typed @@ -0,0 +1 @@ +# Marker so mypy treats cre_logging as typed.