diff --git a/.cursor/skills/brevity-bot/SKILL.md b/.cursor/skills/brevity-bot/SKILL.md new file mode 100644 index 000000000..492a76109 --- /dev/null +++ b/.cursor/skills/brevity-bot/SKILL.md @@ -0,0 +1,56 @@ +--- +name: brevity-bot +description: >- + Review a diff like a long-tenured greybeard coworker: shorten AI blurbs, + simplify code, keep correctness/security/Bugbot fixes. Use after Bugbot and + Security reviews, or when asked for /review-brevity. +--- + +# Brevity bot + +You are the ponytailed, greybearded, Linux-using god of coworker who has been +with the company since the beginning. You can shorten or simplify any codebase. +Your job is to dramatically increase readability and quality and shorten AI +blurbs while preserving correctness, quality, and security. + +You simplify code **without dropping the fixes** from Bugbot or Security +review. If a check, error path, redaction, auth gate, or test exists because +those reviews required it, keep it. Prefer a shorter form of the same fix. + +## Diff contract (same as Bugbot) + +The parent gives: + +```text +Full Repository Path: +Diff: +``` + +Review that diff only. Do not invent drive-by refactors outside it. + +## What to cut + +- Restated comments, essay docstrings, "as an AI" padding, changelog comments +- Needless wrappers, duplicate branches, unused helpers, speculative abstraction +- Tests that only restate the same assertion five ways + +## What to keep + +- Auth, allowlists, redaction, abort codes, CSRF/login gates +- Bugbot/Security fixes (narrower code is fine; weaker checks are not) +- Names and structure that match this repo (`application/tests/`, Makefile) + +## Output + +Either: + +1. **Patches** — file path, short why, and the smaller replacement, or +2. **`NO_COMMENTS`** — nothing left to shorten without harming clarity, tests, + or security. + +End with exactly one of: + +- `VERDICT: NO_COMMENTS` +- `VERDICT: COMMENTS` (and the patch list) + +Do not praise the diff. Do not recap the plan. diff --git a/.cursor/skills/brevity-bot/examples.md b/.cursor/skills/brevity-bot/examples.md new file mode 100644 index 000000000..985c03e92 --- /dev/null +++ b/.cursor/skills/brevity-bot/examples.md @@ -0,0 +1,40 @@ +# Brevity bot fixtures + +## AI blurb → short + +Before: + +```python +def is_enabled() -> bool: + """Return True if the feature flag is enabled. + + This function checks the environment variable and compares it against + a set of truthy values so that operators can enable the feature. + """ + value = os.getenv("FLAG", "") + normalized = value.strip().lower() + if normalized in {"1", "true", "yes"}: + return True + return False +``` + +After: + +```python +def is_enabled() -> bool: + return os.getenv("FLAG", "").strip().lower() in {"1", "true", "yes"} +``` + +## Do not drop a security fix + +Security required redacting secrets. Illegal brevity (drops the fix): + +```python +return {"GEMINI_API_KEY": os.environ.get("GEMINI_API_KEY")} +``` + +Legal brevity (keeps redaction): + +```python +return {"GEMINI_API_KEY": "***"} if os.environ.get("GEMINI_API_KEY") else {} +``` diff --git a/.cursor/skills/review-brevity/SKILL.md b/.cursor/skills/review-brevity/SKILL.md new file mode 100644 index 000000000..5e373c940 --- /dev/null +++ b/.cursor/skills/review-brevity/SKILL.md @@ -0,0 +1,24 @@ +--- +name: review-brevity +description: Launch Brevity bot on the current diff after Bugbot and Security. +--- + +# Review Brevity + +When asked to run `/review-brevity` or Brevity bot: + +Launch **three** independent `generalPurpose` subagents (`run_in_background: false`) +with the contents of `.cursor/skills/brevity-bot/SKILL.md` as instructions. + +Prompt each with: + +```text +Full Repository Path: +Diff: branch changes +``` + +Default to `branch changes`. Use `uncommitted changes` only if asked. + +Collect the three `VERDICT:` lines. Majority `NO_COMMENTS` (2 of 3) is clean. +If majority is `COMMENTS`, apply agreed simplifications that do **not** revert +Bugbot or Security fixes, then re-run. diff --git a/.env.example b/.env.example index 7bee7e0ee..a2d010bfe 100644 --- a/.env.example +++ b/.env.example @@ -5,6 +5,10 @@ # Never commit your real `.env` file to version control. # ------------------------------------------------------------------------------ # Database +# Local Docker (make docker-postgres / make admin-local): +# postgresql://cre:password@127.0.0.1:5432/cre +# `make admin-local` starts that container when the URL is unreachable and +# never falls back to SQLite for the admin showcase. DEV_DATABASE_URL=postgresql://user:password@localhost:5432/opencre @@ -42,14 +46,12 @@ CRE_ENABLE_MYOPENCRE=false CRE_ENABLE_LOGIN=false # Experimental OIE OWASP metadata agent (chat + MCP only — never CRE graph UI). -# Off by default. When enabled, community/meta questions are answered from -# the local owasp_agent index instead of CRE RAG. Concepts/tags stay in -# OWASP_AGENT_DB and are reachable only from /completion and MCP owasp_meta_* tools. +# Off by default. Uses the main app Postgres (DEV_DATABASE_URL / DATABASE_URL) +# for the agent index + pgvector. Reachable from /completion and MCP owasp_meta_* tools. OWASP_AGENT_ENABLED=false # Nest REST API key (https://nest.owasp.org — create via GraphQL while logged in) NEST_API_KEY= NEST_API_BASE=https://nest.owasp.org/api/v0 -OWASP_AGENT_DB=tmp/owasp_agent.sqlite # Set to 1 to crawl all www-chapter / www-project repos during sync (slow). OWASP_AGENT_CRAWL_REPOS=0 diff --git a/.gitignore b/.gitignore index 7549096c3..3c7f6c1f7 100644 --- a/.gitignore +++ b/.gitignore @@ -49,6 +49,8 @@ v/ .cursor/* !.cursor/rules/ !.cursor/rules/** +!.cursor/skills/ +!.cursor/skills/** .claude/ ### Frontend @@ -67,6 +69,7 @@ standards_cache.sqlite ### Docs *.md !AGENTS.md +!.cursor/skills/**/*.md !docs/faq.md !docs/api/mcp.md !docs/Mid_eval_blog_gsoc2026/module_B_mideval_blog.md @@ -94,6 +97,8 @@ neo4j/ .mypy_cache tmp/ +import_telemetry.json + ### CREs dir cres/* ### Local project management tooling diff --git a/Makefile b/Makefile index bf4424d06..dbcb0ece1 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ .ONESHELL: -.PHONY: run test covers install-deps dev docker lint frontend clean all e2e e2e-db owasp-agent-sync owasp-agent-merge-concepts owasp-agent-synth-eval +.PHONY: run test covers install-deps dev dev-flask docker lint frontend clean all e2e e2e-db admin-local owasp-agent-sync owasp-agent-merge-concepts owasp-agent-synth-eval prod-run: gunicorn cre:app --log-file=- @@ -74,11 +74,19 @@ start-worker: . ./venv/bin/activate && FLASK_APP=`pwd`/cre.py python cre.py --start_worker upstream-sync: - . ./venv/bin/activate && python cre.py --upstream_sync + . ./venv/bin/activate &&\ + python cre.py --upstream_sync --cache_file "$(or $(DEV_DATABASE_URL),$(CRE_CACHE_FILE),$(CURDIR)/standards_cache.sqlite)" PORT?=5000 -dev-flask: +# Local app against Docker Postgres: migrate, upstream-sync if cre/node empty, Flask. +# Same entrypoint as admin showcase / OIE local (no SQLite). macOS AirPlay often +# holds :5000 — use PORT=5001 make dev if bind fails. +dev dev-flask admin-local: + bash ./scripts/run_admin_local.sh --port $(PORT) $(ADMIN_LOCAL_ARGS) + +# Bare Flask without Postgres bootstrap (legacy / CI helpers). Prefer `make dev`. +dev-flask-sqlite: . ./venv/bin/activate && INSECURE_REQUESTS=1 FLASK_APP=`pwd`/cre.py FLASK_CONFIG=development flask run --port $(PORT) dev-flask-docker: @@ -136,7 +144,14 @@ install-deps-typescript: install-deps: install-deps-python install-deps-typescript install-python: - virtualenv -p python3 venv + @if [ ! -d "./venv" ]; then \ + if command -v virtualenv >/dev/null 2>&1; then \ + virtualenv -p python3 venv; \ + else \ + echo "virtualenv not on PATH — using python3 -m venv"; \ + python3 -m venv venv; \ + fi; \ + fi . ./venv/bin/activate &&\ make install-deps-python &&\ playwright install # Python embeddings/scraping (prompt_client); NOT frontend e2e — keep when migrating to Cypress @@ -148,7 +163,9 @@ install-python: # `make install`. Rebuild with `make frontend` when you change the UI. install-typescript: install-deps-typescript -install: install-typescript install-python migrate-upgrade +# Deps + Docker Postgres schema + CRE upstream graph when empty (see run_admin_local.sh). +install: install-typescript install-python + bash ./scripts/run_admin_local.sh --migrate-only $(ADMIN_LOCAL_ARGS) docker-dev: docker build -f Dockerfile-dev -t opencre-dev:$(shell git rev-parse HEAD) . diff --git a/README.md b/README.md index 032a5f89c..8e2623b8a 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,7 @@ If you want to develop on OpenCRE or docker is not available in your environment ### Command Line To run outside of Docker you need to install OpenCRE. -To install this application you need Python 3.11.9, Node.js, Yarn Classic (the repository uses a Yarn v1 lockfile), and `virtualenv`. The Python version is pinned in `.python-version`; the repository does not currently pin a Node.js version. +To install this application you need Python 3.11.9, Node.js, Yarn Classic (the repository uses a Yarn v1 lockfile), and Docker (local Postgres). `virtualenv` is optional — `make install-python` falls back to `python3 -m venv`. The Python version is pinned in `.python-version`; the repository does not currently pin a Node.js version. Clone the repository: @@ -57,23 +57,18 @@ Clone the repository: git clone https://github.com/OWASP/OpenCRE ``` -`make install` creates and uses the project virtual environment. Do not create a second environment with `python3 -m venv`; install the prerequisites above, then let the Makefile manage `venv`. - -Install dependencies: +`make install` creates the project `venv` (if missing), installs deps, starts Docker Postgres when needed, migrates, and pulls the CRE graph from opencre.org when the local DB is empty. ```bash make install +make dev # same as make dev-flask / make admin-local ``` -`make install` creates the local schema as part of installation. Download the latest CRE graph from upstream with: - -```bash -make upstream-sync -``` +Then open http://127.0.0.1:5000/admin and http://127.0.0.1:5000/explorer. If port 5000 is busy (common on macOS with AirPlay Receiver), use `PORT=5001 make dev`. To reuse an existing DB without re-pulling the CRE graph: `SKIP_UPSTREAM_SYNC=1 PORT=5001 make dev`. For contributors, this is the supported local data bootstrap path. You do **not** need access to the internal OpenCRE Google Sheet to work on the -project. +project. From **Admin → Config**, use **Run golden-set harvest** and **Sync OWASP agent** (progress under Pipeline), or `make owasp-agent-sync` from the CLI. You can precompute local gap-analysis cache after imports with: @@ -81,12 +76,6 @@ You can precompute local gap-analysis cache after imports with: make backfill-gap-analysis ``` -To run CRE locally then you can do: - -```bash -make dev-flask -``` - To run the CLI application, you can run: ```bash diff --git a/application/config.py b/application/config.py index de7c0036f..1ae9f7918 100644 --- a/application/config.py +++ b/application/config.py @@ -4,9 +4,30 @@ import os +from application.utils.postgres_url import ( + is_postgres_url, + main_db_env_keys, + sqlalchemy_postgres_url, +) + basedir = os.path.abspath(os.path.dirname(__file__)) +def _sqlalchemy_uri_from_env(raw: str) -> str: + """Bind Postgres env URLs to psycopg2; leave sqlite and other schemes alone.""" + if is_postgres_url(raw): + return sqlalchemy_postgres_url(raw) + return raw + + +def _first_env_url(*keys: str) -> str: + for key in keys: + raw = (os.environ.get(key) or "").strip() + if raw: + return raw + return "" + + class Config: SQLALCHEMY_TRACK_MODIFICATIONS = False SQLALCHEMY_RECORD_QUERIES = False @@ -18,8 +39,8 @@ class DevelopmentConfig(Config): DEBUG = True ENVIRONMENT = "DEVELOPMENT" CACHE_TYPE = "SimpleCache" - SQLALCHEMY_DATABASE_URI = ( - os.environ.get("DEV_DATABASE_URL") + SQLALCHEMY_DATABASE_URI = _sqlalchemy_uri_from_env( + _first_env_url(*main_db_env_keys("development")) or f'sqlite:///{os.path.join(basedir, "../standards_cache.sqlite")}' ) @@ -35,8 +56,8 @@ class ProductionConfig(Config): ENVIRONMENT = "PRODUCTION" CACHE_TYPE = "SimpleCache" CACHE_DEFAULT_TIMEOUT = 3000 - SQLALCHEMY_DATABASE_URI = ( - os.environ.get("PROD_DATABASE_URL") + SQLALCHEMY_DATABASE_URI = _sqlalchemy_uri_from_env( + _first_env_url(*main_db_env_keys("production")) or f'sqlite:///{os.path.join(basedir, "../standards_cache.sqlite")}' ) @@ -46,11 +67,12 @@ class CMDConfig(Config): def __init__(self, db_uri: str): if "://" in db_uri: - # Heroku and some tools still emit ``postgres://``; SQLAlchemy 2 expects - # ``postgresql://`` for the psycopg dialect name. - if db_uri.startswith("postgres://"): - db_uri = "postgresql://" + db_uri[len("postgres://") :] - self.SQLALCHEMY_DATABASE_URI = db_uri + # Heroku still emits ``postgres://``; SQLAlchemy 2.1 maps bare + # ``postgresql://`` to psycopg v3 — pin psycopg2 (requirements.txt). + if is_postgres_url(db_uri): + self.SQLALCHEMY_DATABASE_URI = sqlalchemy_postgres_url(db_uri) + else: + self.SQLALCHEMY_DATABASE_URI = db_uri else: # Flask-SQLAlchemy 3+ resolves non-absolute sqlite URLs against # app.instance_path, not the shell cwd. CLI --cache_file should diff --git a/application/database/db.py b/application/database/db.py index 58de28853..c756482f9 100644 --- a/application/database/db.py +++ b/application/database/db.py @@ -263,6 +263,29 @@ class StagedChangeSet(BaseModel): # type: ignore created_at = sqla.Column(sqla.DateTime, nullable=False) +class IngestionTarget(BaseModel): # type: ignore + __tablename__ = "ingestion_target" + id = sqla.Column(sqla.String, primary_key=True, default=generate_uuid) + # oie_repo | import_source | owasp_agent (built-in, not persisted) + kind = sqla.Column(sqla.String, nullable=False) + name = sqla.Column(sqla.String, nullable=False) + spec_json = sqla.Column(sqla.Text, nullable=False, default="{}") + enabled = sqla.Column(sqla.Boolean, nullable=False, default=True) + created_at = sqla.Column(sqla.DateTime, nullable=False) + + +class AdminPipelineEvent(BaseModel): # type: ignore + __tablename__ = "admin_pipeline_event" + id = sqla.Column(sqla.String, primary_key=True, default=generate_uuid) + run_id = sqla.Column(sqla.String, nullable=False) + stage = sqla.Column(sqla.String, nullable=False) + status = sqla.Column(sqla.String, nullable=False) + detail = sqla.Column(sqla.Text, nullable=True) + created_at = sqla.Column(sqla.DateTime, nullable=False) + + __table_args__ = (sqla.Index("ix_admin_pipeline_event_run", "run_id"),) + + class User(BaseModel): # type: ignore """Persisted account identity for OIDC login (issue #586, RFC #876 TODO 1). @@ -738,12 +761,31 @@ def get_staged_change_set(*, run_id: str) -> Optional[StagedChangeSet]: ) +def ensure_staged_change_set( + *, + run_id: str, + changeset_json: str = "[]", + staging_status: str = "pending_review", +) -> StagedChangeSet: + """Return the staged set for ``run_id``, creating an empty one if missing.""" + cs = get_staged_change_set(run_id=run_id) + if cs: + return cs + return persist_staged_change_set( + run_id=run_id, + changeset_json=changeset_json, + has_conflicts=False, + staging_status=staging_status, + ) + + def update_staged_change_set( *, run_id: str, staging_status: Optional[str] = None, has_conflicts: Optional[bool] = None, apply_error: Optional[str] = None, + changeset_json: Optional[str] = None, ) -> Optional[StagedChangeSet]: cs = get_staged_change_set(run_id=run_id) if not cs: @@ -754,6 +796,8 @@ def update_staged_change_set( cs.has_conflicts = has_conflicts if apply_error is not None: cs.apply_error = apply_error + if changeset_json is not None: + cs.changeset_json = changeset_json sqla.session.add(cs) sqla.session.commit() return cs diff --git a/application/frontend/src/hooks/useCapabilities.ts b/application/frontend/src/hooks/useCapabilities.ts index 85f3cf140..53fde5d2b 100644 --- a/application/frontend/src/hooks/useCapabilities.ts +++ b/application/frontend/src/hooks/useCapabilities.ts @@ -5,11 +5,13 @@ import { useEnvironment } from './useEnvironment'; export type Capabilities = { myopencre: boolean; login: boolean; + admin?: boolean; }; const DEFAULT_CAPABILITIES: Capabilities = { myopencre: false, login: false, + admin: false, }; export const useCapabilities = () => { @@ -26,6 +28,7 @@ export const useCapabilities = () => { setCapabilities({ myopencre: Boolean(data?.myopencre), login: Boolean(data?.login), + admin: Boolean(data?.admin), }) ) .catch(() => setCapabilities(DEFAULT_CAPABILITIES)) diff --git a/application/frontend/src/pages/Admin/Admin.scss b/application/frontend/src/pages/Admin/Admin.scss new file mode 100644 index 000000000..2c13f79d1 --- /dev/null +++ b/application/frontend/src/pages/Admin/Admin.scss @@ -0,0 +1,94 @@ +.admin-page { + max-width: 1100px; + margin: 2rem auto 4rem; + padding: 0 1rem; +} + +.admin-tabs { + display: flex; + flex-wrap: wrap; + gap: 0.5rem; + margin: 1rem 0 1.5rem; +} + +.admin-tabs button { + cursor: pointer; +} + +.admin-table { + width: 100%; + border-collapse: collapse; +} + +.admin-table th, +.admin-table td { + border-bottom: 1px solid #ddd; + padding: 0.4rem 0.5rem; + text-align: left; + vertical-align: top; +} + +.admin-pre { + white-space: pre-wrap; + font-size: 0.85rem; + background: #f7f7f7; + padding: 0.75rem; + overflow: auto; +} + +.admin-help { + font-size: 0.85rem; + color: #555; +} + +.admin-yaml { + width: 100%; + min-height: 16rem; + font-family: monospace; + font-size: 0.85rem; + display: block; + margin: 0.4rem 0 0.75rem; + padding: 0.5rem; +} + +.admin-strip { + display: flex; + list-style: none; + padding: 0; + margin: 0.5rem 0 1rem; + gap: 0; +} + +.admin-strip-step { + flex: 1; + padding: 0.4rem 0.5rem; + text-align: center; + border: 1px solid #ddd; + background: #fafafa; +} + +.admin-strip-step.is-done { + background: #eef8ee; +} + +.admin-strip-step.is-current { + background: #e8f0fe; + font-weight: 600; +} + +.admin-strip-step.is-failed { + background: #fdeaea; +} + +.admin-map-op textarea { + width: 100%; + min-height: 6rem; + font-family: monospace; + margin-bottom: 0.4rem; +} + +.admin-page textarea { + width: 100%; + min-height: 4rem; + font-family: monospace; +} diff --git a/application/frontend/src/pages/Admin/Admin.test.tsx b/application/frontend/src/pages/Admin/Admin.test.tsx new file mode 100644 index 000000000..c6bd6f0df --- /dev/null +++ b/application/frontend/src/pages/Admin/Admin.test.tsx @@ -0,0 +1,729 @@ +import { fireEvent, render, waitFor } from '@testing-library/react'; +import React from 'react'; +import { MemoryRouter } from 'react-router-dom'; + +import { useCapabilities } from '../../hooks/useCapabilities'; +import { useUser } from '../../hooks/useUser'; +import { Admin } from './Admin'; + +jest.mock('../../hooks/useEnvironment', () => ({ + useEnvironment: () => ({ name: 'test', apiUrl: '/rest/v1' }), +})); +jest.mock('../../hooks/useUser'); +jest.mock('../../hooks/useCapabilities'); + +const mockUser = useUser as jest.Mock; +const mockCaps = useCapabilities as jest.Mock; + +function jsonRes(body: unknown, status = 200) { + return Promise.resolve({ + ok: status >= 200 && status < 300, + status, + statusText: 'OK', + text: () => Promise.resolve(JSON.stringify(body)), + json: () => Promise.resolve(body), + }); +} + +function loggedIn() { + mockUser.mockReturnValue({ + user: 'u', + isLoggedIn: true, + loading: false, + login: jest.fn(), + logout: jest.fn(), + }); + mockCaps.mockReturnValue({ + capabilities: { myopencre: true, login: true, admin: true }, + loading: false, + }); +} + +function mockRuns() { + return jsonRes({ runs: [{ id: 'r1', source: 'asvs', staging_status: 'pending_review' }] }); +} + +describe('Admin', () => { + beforeEach(() => { + mockCaps.mockReturnValue({ + capabilities: { myopencre: false, login: true, admin: false }, + loading: false, + }); + }); + + afterEach(() => jest.clearAllMocks()); + + it('hides admin tabs when import capability is off', () => { + loggedIn(); + mockCaps.mockReturnValue({ + capabilities: { myopencre: true, login: true, admin: false }, + loading: false, + }); + const { getByText, queryByText } = render( + + + + ); + expect(getByText(/CRE_ALLOW_IMPORT/)).toBeTruthy(); + expect(queryByText('Dashboard')).toBeNull(); + }); + + it('asks anonymous users to log in', () => { + mockUser.mockReturnValue({ + user: null, + isLoggedIn: false, + loading: false, + login: jest.fn(), + logout: jest.fn(), + }); + const { getByText } = render( + + + + ); + expect(getByText('Sign in to continue.')).toBeTruthy(); + }); + + it('loads import runs and links to MyOpenCRE', async () => { + loggedIn(); + (global as any).fetch = jest.fn((url: string) => { + if (String(url).includes('/admin/imports/runs')) { + return mockRuns(); + } + return jsonRes({}); + }); + const { getByText, findByText } = render( + + + + ); + fireEvent.click(getByText('Graph management')); + await findByText('asvs'); + fireEvent.click(getByText('MyOpenCRE')); + expect(getByText('Open MyOpenCRE').closest('a')?.getAttribute('href')).toBe('/myopencre'); + }); + + it('loads agent status on the dashboard', async () => { + loggedIn(); + (global as any).fetch = jest.fn((url: string) => { + if (String(url).includes('/admin/dashboard')) { + return jsonRes({ + running: [], + failed: [], + import_runs: [], + agent: { + enabled: true, + writes_cre_graph: false, + demo_path: '/chatbot', + db_url: 'postgresql://cre:***@127.0.0.1:5432/opencre', + db_env_key: 'DEV_DATABASE_URL', + package_present: false, + }, + }); + } + return jsonRes({}); + }); + const { getByText, findByText, queryByRole } = render( + + + + ); + await findByText(/writes CRE graph: false/); + expect(getByText(/Main DB \(DEV_DATABASE_URL\):/)).toBeTruthy(); + expect(getByText(/postgresql:\/\/cre:\*\*\*@127.0.0.1:5432\/opencre/)).toBeTruthy(); + expect(getByText('Open chat demo').closest('a')?.getAttribute('href')).toBe('/chatbot'); + expect(queryByRole('button', { name: 'OWASP agent' })).toBeNull(); + fireEvent.click(getByText('Job management and logs')); + expect(await findByText('New')).toBeTruthy(); + }); + + it('reviews links per run and loads the changeset graph', async () => { + loggedIn(); + (global as any).fetch = jest.fn((url: string, init?: RequestInit) => { + const u = String(url); + if (u.includes('/changeset/graph')) { + return jsonRes({ + nodes: [{ id: 'n1', label: 'ASVS 1.1', type: 'Standard', status: 'updated' }], + edges: [], + }); + } + if (u.endsWith('/links') && init?.method === 'POST') { + return jsonRes({ + run_id: 'r1', + links: [ + { + op_index: 0, + link_index: 0, + added: { op: 'add_control', name: 'ASVS', section: '1.1' }, + linked_to: { id: '123', name: 'Auth' }, + decision: 'approved', + }, + ], + }); + } + if (u.endsWith('/links')) { + return jsonRes({ + run_id: 'r1', + links: [ + { + op_index: 0, + link_index: 0, + added: { op: 'add_control', name: 'ASVS', section: '1.1' }, + linked_to: { id: '123', name: 'Auth' }, + decision: 'pending', + }, + ], + }); + } + if (u.includes('/changeset')) { + return jsonRes({ run_id: 'r1', changeset: [{ op: 'add_control', document: {} }] }); + } + if (u.includes('/admin/imports/runs')) { + return mockRuns(); + } + return jsonRes({}); + }); + const { getByText, findByText } = render( + + + + ); + fireEvent.click(getByText('Graph management')); + await findByText('asvs'); + fireEvent.click(getByText('Graph')); + expect(await findByText('ASVS 1.1')).toBeTruthy(); + fireEvent.click(getByText('Changeset')); + expect(await findByText('Added controls and links')).toBeTruthy(); + fireEvent.click(getByText('Approve')); + await waitFor(() => + expect((global as any).fetch).toHaveBeenCalledWith( + expect.stringContaining('/links'), + expect.objectContaining({ method: 'POST' }) + ) + ); + }); + + it('shows empty pipeline state', async () => { + loggedIn(); + (global as any).fetch = jest.fn((url: string) => { + if (String(url).includes('/admin/pipeline')) { + return jsonRes({ + import_runs: [], + events: [], + latest_strip: [ + { id: 'queued', label: 'Queued', state: 'current' }, + { id: 'pending_review', label: 'Review', state: 'idle' }, + ], + oie: { unconsumed: 0, recent: [], knowledge: [] }, + }); + } + return jsonRes({}); + }); + const { getByText, findByText } = render( + + + + ); + fireEvent.click(getByText('Pipeline')); + expect(await findByText('No pipeline runs yet.')).toBeTruthy(); + expect(getByText('Queued')).toBeTruthy(); + expect(getByText('New')).toBeTruthy(); + }); + + it('shows config restart instructions', async () => { + loggedIn(); + (global as any).fetch = jest.fn((url: string) => { + if (String(url).includes('/admin/config')) { + return jsonRes({ + writable: false, + restart_instructions: 'HTTP cannot change process env.', + config: [ + { + key: 'CRE_ALLOW_IMPORT', + value: '1', + help_text: 'kill', + help_url: 'https://example.test', + secret: false, + }, + ], + }); + } + if (String(url).includes('/admin/repos.yaml')) { + return jsonRes({ yaml: 'sources: []\n', source: 'repos.yaml:abc' }); + } + return jsonRes({}); + }); + const { getByText, findByText } = render( + + + + ); + fireEvent.click(getByText('Config')); + expect(await findByText('HTTP cannot change process env.')).toBeTruthy(); + expect(getByText('CRE_ALLOW_IMPORT')).toBeTruthy(); + }); + + it('runs golden-set harvest and OWASP agent sync from Config', async () => { + loggedIn(); + (global as any).fetch = jest.fn((url: string, init?: RequestInit) => { + const u = String(url); + if (u.includes('/admin/config') && (!init || init.method === 'GET' || !init.method)) { + return jsonRes({ + restart_instructions: 'HTTP cannot change process env.', + config: [], + }); + } + if (u.includes('/admin/repos.yaml')) { + return jsonRes({ yaml: 'sources: []\n', source: 'repos.yaml:abc' }); + } + if (u.includes('/admin/agent/status')) { + return jsonRes({ + enabled: true, + db_url: 'postgresql://cre:***@127.0.0.1:5432/cre', + counts: { projects: 1 }, + last_sync: '2026-01-01T00:00:00+00:00', + }); + } + if (u.includes('/admin/ingest/start')) { + return jsonRes({ source: 'repos.yaml:deadbeef', async: true, run_id: 'g1' }); + } + if (u.includes('/admin/agent/sync')) { + return jsonRes({ + run_id: 'a1b2c3d4-eeee', + async: true, + skip_nest: true, + source: 'owasp-agent-sync', + }); + } + return jsonRes({}); + }); + const { getByText, findByText } = render( + + + + ); + fireEvent.click(getByText('Config')); + expect(await findByText(/End-to-end bootstrap/)).toBeTruthy(); + fireEvent.click(getByText('Run golden-set harvest')); + await waitFor(() => + expect((global as any).fetch).toHaveBeenCalledWith( + expect.stringContaining('/admin/ingest/start'), + expect.objectContaining({ + method: 'POST', + body: JSON.stringify({ packaged: true }), + }) + ) + ); + expect(await findByText(/Golden-set harvest started/)).toBeTruthy(); + fireEvent.click(getByText('Sync OWASP agent')); + await waitFor(() => + expect((global as any).fetch).toHaveBeenCalledWith( + expect.stringContaining('/admin/agent/sync'), + expect.objectContaining({ method: 'POST' }) + ) + ); + expect(await findByText(/OWASP agent sync started/)).toBeTruthy(); + }); + + it('shows agent disabled when the flag is off', async () => { + loggedIn(); + (global as any).fetch = jest.fn((url: string) => { + if (String(url).includes('/admin/dashboard')) { + return jsonRes({ + running: [], + failed: [], + import_runs: [], + agent: { enabled: false, writes_cre_graph: false, db_url: null }, + }); + } + return jsonRes({}); + }); + const { findByText } = render( + + + + ); + expect(await findByText(/Enabled: false/)).toBeTruthy(); + }); + + it('shows pipeline error state', async () => { + loggedIn(); + (global as any).fetch = jest.fn((url: string) => { + if (String(url).includes('/admin/pipeline')) { + return jsonRes({ description: 'pipeline down' }, 500); + } + return jsonRes({}); + }); + const { getByText, findByText } = render( + + + + ); + fireEvent.click(getByText('Pipeline')); + expect(await findByText(/pipeline down/)).toBeTruthy(); + }); + + it('accepts and discards a staged run', async () => { + loggedIn(); + (global as any).fetch = jest.fn((url: string, init?: RequestInit) => { + const u = String(url); + if (u.endsWith('/accept') || u.endsWith('/discard') || u.includes('apply?dry_run')) { + return jsonRes({ run_id: 'r1', staging_status: 'ok' }); + } + if (u.includes('/impact')) { + return jsonRes({ + run_id: 'r1', + operation_count: 1, + impacted_standard_names: [], + impacted_cre_external_ids: [], + warnings: ['Skipped 1 operation(s) with empty standard keys'], + }); + } + if (u.includes('/admin/imports/runs')) { + return mockRuns(); + } + return jsonRes({}); + }); + const { getByText, findByText, findAllByText } = render( + + + + ); + fireEvent.click(getByText('Graph management')); + await findByText('asvs'); + fireEvent.click(getByText('Impact')); + await waitFor(() => + expect((global as any).fetch).toHaveBeenCalledWith( + expect.stringContaining('/impact'), + expect.objectContaining({ method: 'GET' }) + ) + ); + expect(await findAllByText(/empty standard keys/)).toBeTruthy(); + fireEvent.click(getByText('Accept')); + await waitFor(() => + expect((global as any).fetch).toHaveBeenCalledWith( + expect.stringContaining('/accept'), + expect.objectContaining({ method: 'POST' }) + ) + ); + expect(await findByText(/accept →/i)).toBeTruthy(); + fireEvent.click(getByText('Discard')); + await waitFor(() => + expect((global as any).fetch).toHaveBeenCalledWith( + expect.stringContaining('/discard'), + expect.objectContaining({ method: 'POST' }) + ) + ); + fireEvent.click(getByText('Dry-run')); + await waitFor(() => + expect((global as any).fetch).toHaveBeenCalledWith( + expect.stringContaining('apply?dry_run=1'), + expect.objectContaining({ method: 'POST' }) + ) + ); + }); + + it('surfaces a failed changeset load', async () => { + loggedIn(); + (global as any).fetch = jest.fn((url: string) => { + const u = String(url); + if (u.includes('/changeset')) { + return jsonRes({ description: 'run not found' }, 404); + } + if (u.includes('/admin/imports/runs')) { + return mockRuns(); + } + return jsonRes({}); + }); + const { getByText, findByText, queryByText } = render( + + + + ); + fireEvent.click(getByText('Graph management')); + await findByText('asvs'); + fireEvent.click(getByText('Changeset')); + expect(await findByText(/run not found/)).toBeTruthy(); + expect(queryByText('Added controls and links')).toBeNull(); + }); + + it('saves repos.yaml and expands an org from config', async () => { + loggedIn(); + let savedBody: any; + let expandBody: any; + (global as any).fetch = jest.fn((url: string, init?: RequestInit) => { + const u = String(url); + if (u.includes('/expand-org') && init?.method === 'POST') { + expandBody = JSON.parse(String(init.body || '{}')); + return jsonRes({ + yaml: 'sources:\n - github.com/OWASP/\n', + added: 1, + skipped: 0, + source_url: 'github.com/OWASP/', + source: 'repos.yaml:newhash12ab', + }); + } + if (u.includes('/admin/repos.yaml') && init?.method === 'PUT') { + savedBody = JSON.parse(String(init.body || '{}')); + return jsonRes({ + yaml: savedBody.yaml, + source: 'repos.yaml:savedhash12', + saved: true, + }); + } + if (u.includes('/admin/repos.yaml')) { + return jsonRes({ + yaml: 'repositories:\n - id: owasp-asvs\n', + source: 'repos.yaml:abc123abc123', + }); + } + if (u.includes('/admin/config')) { + return jsonRes({ config: [], restart_instructions: '' }); + } + return jsonRes({}); + }); + const { getByText, getByLabelText, getByPlaceholderText, findByText } = render( + + + + ); + fireEvent.click(getByText('Config')); + expect(await findByText('targets.yaml')).toBeTruthy(); + fireEvent.change(getByLabelText('repos.yaml'), { + target: { value: 'repositories:\n - id: custom\n' }, + }); + fireEvent.click(getByText('Save repos.yaml')); + expect(await findByText(/Saved repos.yaml:savedhash12/)).toBeTruthy(); + expect(savedBody.yaml).toContain('custom'); + fireEvent.change(getByPlaceholderText('GitHub org'), { target: { value: 'OWASP' } }); + fireEvent.click(getByText('Add org')); + expect(await findByText(/Saved source github.com\/OWASP/)).toBeTruthy(); + expect(expandBody.owner).toBe('OWASP'); + }); + + it('surfaces an inaccessible GitHub source immediately', async () => { + loggedIn(); + (global as any).fetch = jest.fn((url: string, init?: RequestInit) => { + const u = String(url); + if (u.includes('/expand-org') && init?.method === 'POST') { + return jsonRes({ description: 'GitHub source is not accessible: github.com/NoSuchOrgCcdd/' }, 400); + } + if (u.includes('/admin/repos.yaml')) { + return jsonRes({ yaml: 'sources: []\n', source: 'repos.yaml:abc123abc123' }); + } + if (u.includes('/admin/config')) { + return jsonRes({ config: [], restart_instructions: '' }); + } + return jsonRes({}); + }); + const { getByText, getByPlaceholderText, findByText } = render( + + + + ); + fireEvent.click(getByText('Config')); + await findByText('targets.yaml'); + fireEvent.change(getByPlaceholderText('GitHub org'), { target: { value: 'NoSuchOrgCcdd' } }); + fireEvent.click(getByText('Add org')); + expect(await findByText(/not accessible: github.com\/NoSuchOrgCcdd\//)).toBeTruthy(); + }); + + it('starts a packaged ingest from Pipeline New', async () => { + loggedIn(); + let startBody: any; + (global as any).fetch = jest.fn((url: string, init?: RequestInit) => { + const u = String(url); + if (u.includes('/admin/ingest/start') && init?.method === 'POST') { + startBody = JSON.parse(String(init.body || '{}')); + return jsonRes({ run_id: 'r-new', source: 'repos.yaml:abc', dry_run: true }); + } + if (u.includes('/admin/pipeline')) { + return jsonRes({ + import_runs: [], + events: [], + latest_strip: [{ id: 'queued', label: 'Queued', state: 'current' }], + oie: { unconsumed: 0, recent: [], knowledge: [] }, + }); + } + return jsonRes({}); + }); + const { getByText, findByText } = render( + + + + ); + fireEvent.click(getByText('Pipeline')); + await findByText('No pipeline runs yet.'); + fireEvent.click(getByText('New')); + fireEvent.click(getByText('Start')); + expect(await findByText(/Started repos.yaml:abc/)).toBeTruthy(); + expect(startBody.packaged).toBe(true); + }); + + it('rejects empty drop-last and empty add-target', async () => { + loggedIn(); + const fetchMock = jest.fn((url: string) => { + if (String(url).includes('/admin/imports/runs')) { + return jsonRes({ runs: [] }); + } + if (String(url).includes('/admin/repos.yaml')) { + return jsonRes({ yaml: 'sources: []\n', source: 'repos.yaml:abc' }); + } + if (String(url).includes('/admin/config')) { + return jsonRes({ config: [], restart_instructions: '' }); + } + return jsonRes({}); + }); + (global as any).fetch = fetchMock; + const { getByText, findByText } = render( + + + + ); + fireEvent.click(getByText('Graph management')); + expect(await findByText('Drop last')).toBeTruthy(); + fireEvent.click(getByText('Drop last')); + expect(await findByText(/source is required/)).toBeTruthy(); + fireEvent.click(getByText('Config')); + fireEvent.click(getByText('Add target')); + fireEvent.click(getByText('Add')); + expect(await findByText(/owner and repo are required/)).toBeTruthy(); + const posts = fetchMock.mock.calls.filter((call: unknown[]) => { + const init = call[1] as RequestInit | undefined; + return init?.method === 'POST'; + }); + expect(posts).toEqual([]); + }); + + it('surfaces a failed graph load', async () => { + loggedIn(); + (global as any).fetch = jest.fn((url: string) => { + const u = String(url); + if (u.includes('/changeset/graph')) { + return jsonRes({ description: 'graph unavailable' }, 500); + } + if (u.includes('/admin/imports/runs')) { + return mockRuns(); + } + return jsonRes({}); + }); + const { getByText, findByText } = render( + + + + ); + fireEvent.click(getByText('Graph management')); + await findByText('asvs'); + fireEvent.click(getByText('Graph')); + expect(await findByText(/graph unavailable/)).toBeTruthy(); + }); + + it('adds a target from the config lightbox', async () => { + loggedIn(); + let addBody: any; + let savedYaml: string | undefined; + (global as any).fetch = jest.fn((url: string, init?: RequestInit) => { + const u = String(url); + if (u.includes('/add-repo') && init?.method === 'POST') { + addBody = JSON.parse(String(init.body || '{}')); + return jsonRes({ + yaml: 'repositories:\n - id: owasp-asvs\n', + added: 'owasp-asvs', + source: 'repos.yaml:added12ab', + }); + } + if (u.includes('/admin/repos.yaml') && init?.method === 'PUT') { + savedYaml = JSON.parse(String(init.body || '{}')).yaml; + return jsonRes({ yaml: savedYaml, source: 'repos.yaml:saved12ab', saved: true }); + } + if (u.includes('/admin/repos.yaml')) { + return jsonRes({ yaml: 'sources: []\n', source: 'repos.yaml:abc' }); + } + if (u.includes('/admin/config')) { + return jsonRes({ config: [], restart_instructions: '' }); + } + return jsonRes({}); + }); + const { getByText, getByPlaceholderText, findByText } = render( + + + + ); + fireEvent.click(getByText('Config')); + await findByText('targets.yaml'); + fireEvent.click(getByText('Add target')); + fireEvent.change(getByPlaceholderText('OWASP'), { target: { value: 'OWASP' } }); + fireEvent.change(getByPlaceholderText('ASVS'), { target: { value: 'ASVS' } }); + fireEvent.click(getByText('Add')); + expect(await findByText(/Added target owasp-asvs/)).toBeTruthy(); + expect(addBody.owner).toBe('OWASP'); + expect(addBody.repo).toBe('ASVS'); + expect(addBody.cron).toBe('0 2 * * *'); + expect(savedYaml).toContain('owasp-asvs'); + }); + + it('starts a one-off single repository from Pipeline New', async () => { + loggedIn(); + let startBody: any; + (global as any).fetch = jest.fn((url: string, init?: RequestInit) => { + const u = String(url); + if (u.includes('/add-repo') && init?.method === 'POST') { + return jsonRes({ yaml: 'repositories:\n - id: one-off\n', added: 'one-off' }); + } + if (u.includes('/admin/ingest/start') && init?.method === 'POST') { + startBody = JSON.parse(String(init.body || '{}')); + return jsonRes({ run_id: 'r-one', source: 'one-off', dry_run: true }); + } + if (u.includes('/admin/pipeline')) { + return jsonRes({ + import_runs: [], + events: [], + latest_strip: [{ id: 'queued', label: 'Queued', state: 'current' }], + oie: { unconsumed: 0, recent: [], knowledge: [] }, + }); + } + return jsonRes({}); + }); + const { getByText, getByPlaceholderText, findByText } = render( + + + + ); + fireEvent.click(getByText('Pipeline')); + await findByText('No pipeline runs yet.'); + fireEvent.click(getByText('New')); + fireEvent.click(getByText('Single repository')); + fireEvent.change(getByPlaceholderText('OWASP'), { target: { value: 'OWASP' } }); + fireEvent.change(getByPlaceholderText('ASVS'), { target: { value: 'ASVS' } }); + fireEvent.click(getByText('Start')); + expect(await findByText(/Started one-off/)).toBeTruthy(); + expect(startBody.yaml).toContain('one-off'); + expect(startBody.name).toBe('owasp-asvs'); + }); + + it('shows an empty graph-management links table', async () => { + loggedIn(); + (global as any).fetch = jest.fn((url: string) => { + const u = String(url); + if (u.endsWith('/links')) { + return jsonRes({ run_id: 'r1', links: [] }); + } + if (u.includes('/changeset')) { + return jsonRes({ run_id: 'r1', changeset: [] }); + } + if (u.includes('/admin/imports/runs')) { + return mockRuns(); + } + return jsonRes({}); + }); + const { getByText, findByText } = render( + + + + ); + fireEvent.click(getByText('Graph management')); + await findByText('asvs'); + fireEvent.click(getByText('Changeset')); + expect(await findByText('No staged links on this run.')).toBeTruthy(); + }); +}); diff --git a/application/frontend/src/pages/Admin/Admin.tsx b/application/frontend/src/pages/Admin/Admin.tsx new file mode 100644 index 000000000..044bec5cf --- /dev/null +++ b/application/frontend/src/pages/Admin/Admin.tsx @@ -0,0 +1,1112 @@ +import './Admin.scss'; + +import React, { useCallback, useEffect, useState } from 'react'; +import { Link } from 'react-router-dom'; +import { Button, Form, Message, Modal, Header as SUIHeader } from 'semantic-ui-react'; + +import { useCapabilities, useEnvironment } from '../../hooks'; +import { useUser } from '../../hooks/useUser'; + +type Tab = 'dashboard' | 'config' | 'pipeline' | 'graph' | 'myopencre'; + +type RepoForm = { + id: string; + owner: string; + repo: string; + branch: string; + include: string; + exclude: string; + strategy: string; + max_tokens: string; + overlap_tokens: string; + mode: string; + interval_minutes: string; + cron: string; +}; + +function emptyRepo(): RepoForm { + return { + id: '', + owner: '', + repo: '', + branch: 'main', + include: '**/*.md', + exclude: '', + strategy: 'markdown_heading', + max_tokens: '1200', + overlap_tokens: '100', + mode: 'incremental', + interval_minutes: '60', + cron: '0 2 * * *', + }; +} + +function repoSpec(form: RepoForm) { + return { + id: form.id.trim() || `${form.owner.trim()}-${form.repo.trim()}`.toLowerCase(), + owner: form.owner.trim(), + repo: form.repo.trim(), + branch: form.branch.trim() || 'main', + include: form.include, + exclude: form.exclude, + strategy: form.strategy, + max_tokens: Number(form.max_tokens), + overlap_tokens: Number(form.overlap_tokens), + mode: form.mode, + interval_minutes: Number(form.interval_minutes), + cron: form.cron.trim(), + }; +} + +function adminOrigin(apiUrl: string): string { + if (apiUrl.startsWith('http')) { + return apiUrl.replace(/\/rest\/v1\/?$/, ''); + } + return ''; +} + +async function readJson(res: Response) { + const text = await res.text(); + try { + return JSON.parse(text); + } catch { + // Flask HTML aborts (legacy) — surface a short line, not the whole page. + const plain = text + .replace(/<[^>]+>/g, ' ') + .replace(/\s+/g, ' ') + .trim(); + const clipped = plain.slice(0, 240); + return { description: clipped || res.statusText, error: clipped || res.statusText }; + } +} + +function apiError(body: any, res: Response): string { + return body?.description || body?.error || res.statusText || 'Request failed'; +} + +export const Admin = () => { + const { apiUrl } = useEnvironment(); + const { isLoggedIn, loading, login } = useUser(); + const { capabilities, loading: capsLoading } = useCapabilities(); + const [tab, setTab] = useState('dashboard'); + const origin = adminOrigin(apiUrl); + + if (loading || capsLoading) { + return ( +
+ Admin +

Loading…

+
+ ); + } + + if (!isLoggedIn) { + return ( +
+ Admin +

Sign in to continue.

+ +
+ ); + } + + if (!capabilities?.admin) { + return ( +
+ Admin +

Admin APIs are off. Set CRE_ALLOW_IMPORT and restart the process.

+
+ ); + } + + return ( +
+ Admin +
+ {( + [ + ['dashboard', 'Dashboard'], + ['config', 'Config'], + ['pipeline', 'Pipeline'], + ['graph', 'Graph management'], + ['myopencre', 'MyOpenCRE'], + ] as [Tab, string][] + ).map(([id, label]) => ( + + ))} +
+ {tab === 'dashboard' && setTab('pipeline')} />} + {tab === 'config' && } + {tab === 'pipeline' && } + {tab === 'graph' && } + {tab === 'myopencre' && ( + + Open MyOpenCRE + + )} +
+ ); +}; + +function StageStrip({ steps }: { steps?: { id: string; label: string; state: string }[] }) { + if (!steps || !steps.length) return null; + return ( +
    + {steps.map((step) => ( +
  1. + {step.label} +
    2. + ))} +
+ ); +} + +function RepoFormFields({ form, setForm }: { form: RepoForm; setForm: (next: RepoForm) => void }) { + const set = + (key: keyof RepoForm) => + (e: React.ChangeEvent) => + setForm({ ...form, [key]: e.target.value }); + return ( +
+ + + + + + + + + + + + + + + + + + + + +