From c129a449d11e49596af36c7fabbb96e3ead36a63 Mon Sep 17 00:00:00 2001 From: EnRaiha <15997552+EnRaiha@users.noreply.github.com> Date: Tue, 22 Sep 2026 21:27:23 +0800 Subject: [PATCH 1/2] ci: resolve the sibling sources outside the workspace MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CI has failed on every run that reached the sibling build since the Origin crates adopted `workspace = true` inheritance, and the failure reads as the pull request's own breakage. It is neither upstream churn nor the PR: setup-workspace checks the siblings out under `.ci-sources/`, inside the Lite workspace. Cargo resolves a path dependency's inherited fields against the root manifest of the workspace it considers that dependency to be in — nested like that, Lite's root. `nodedb-array` asks its root for `nodedb-wal`, Lite's root defines none, and `cargo metadata` dies before a line of Lite compiles: error inheriting `nodedb-wal` from workspace root manifest's `workspace.dependencies.nodedb-wal` Outside the workspace tree — the layout local development uses, `../nodedb` — cargo walks up to the sibling's own root and the inheritance resolves. actions/checkout rejects a `path:` outside $GITHUB_WORKSPACE, so fetch the refs with git into $RUNNER_TEMP. With that fixed, make the remaining failure mode legible and early: - Parse both sibling workspaces after the patch table is written, so a genuinely inconsistent Origin ref fails with an annotation naming the ref, not a raw cargo error deep in the log. - Run the same suites nightly on main. The `run-ci` label gate keeps upstream churn out of PR runs, but then nothing reports it at all: the first run to see a broken Origin workspace is the next PR a maintainer labels. --- .github/actions/setup-workspace/action.yml | 77 ++++++++++++++++------ .github/workflows/ci.yml | 7 ++ 2 files changed, 64 insertions(+), 20 deletions(-) diff --git a/.github/actions/setup-workspace/action.yml b/.github/actions/setup-workspace/action.yml index bf75831..5785966 100644 --- a/.github/actions/setup-workspace/action.yml +++ b/.github/actions/setup-workspace/action.yml @@ -14,7 +14,8 @@ # `.cargo/config.toml`, which is gitignored and therefore absent on a runner. # So CI resolved against the registry and failed before compiling a single # line. This action reproduces the local patch on the runner: it checks the -# sibling sources out under `.ci-sources/` and writes the same patch table. +# sibling sources out outside the workspace, the layout local development uses +# (`../nodedb`), and writes the same patch table. # # `sources: registry` turns the whole thing into a no-op, for the release path # — a publish gate must validate against what crates.io actually serves, not @@ -40,24 +41,42 @@ inputs: runs: using: composite steps: - # `path:` must stay inside $GITHUB_WORKSPACE, so the siblings land under - # `.ci-sources/` rather than `../`. They are path dependencies, not - # workspace members, so `--workspace` lint/test still covers Lite only. - - name: Check out Origin workspace + # The siblings must live OUTSIDE $GITHUB_WORKSPACE. + # + # Cargo resolves a path dependency's `workspace = true` inheritance against + # the root manifest of the workspace it considers that dependency to be in. + # Nested inside the consumer's directory, that is the CONSUMER's root: + # `.ci-sources/nodedb/nodedb-array` inherited `nodedb-wal` from Lite's root + # manifest, which defines no such dependency, and the build died parsing + # the sibling. Outside the tree, cargo walks up to the sibling's own root — + # the layout local development uses (`../nodedb`). + # + # They stay path dependencies, not workspace members, so `--workspace` + # lint/test still covers Lite only. + # + # `actions/checkout` rejects a `path:` outside the workspace, so fetch with + # git. Fetching a ref covers branches, tags and SHAs alike. + - name: Check out the sibling sources if: inputs.sources == 'sibling' - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - with: - repository: NodeDB-Lab/nodedb - ref: ${{ inputs.nodedb-ref }} - path: .ci-sources/nodedb - - - name: Check out pagedb - if: inputs.sources == 'sibling' - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - with: - repository: NodeDB-Lab/pagedb - ref: ${{ inputs.pagedb-ref }} - path: .ci-sources/pagedb + shell: bash + env: + NODEDB_REF: ${{ inputs.nodedb-ref }} + PAGEDB_REF: ${{ inputs.pagedb-ref }} + run: | + set -euo pipefail + dest="$RUNNER_TEMP/ci-sources" + mkdir -p "$dest" + fetch() { + local repo="$1" ref="$2" dir="$dest/$3" + rm -rf "$dir" + mkdir -p "$dir" + git -C "$dir" init --quiet + git -C "$dir" remote add origin "https://github.com/NodeDB-Lab/$repo" + git -C "$dir" fetch --quiet --depth 1 origin "$ref" + git -C "$dir" checkout --quiet FETCH_HEAD + } + fetch nodedb "$NODEDB_REF" nodedb + fetch pagedb "$PAGEDB_REF" pagedb # Mirrors the `[patch.crates-io]` block in the gitignored local # `.cargo/config.toml`. Every shared crate is patched together, including @@ -69,7 +88,7 @@ runs: shell: bash run: | set -euo pipefail - origin="$GITHUB_WORKSPACE/.ci-sources/nodedb" + origin="$RUNNER_TEMP/ci-sources/nodedb" mkdir -p .cargo { echo "# Generated by .github/actions/setup-workspace — do not commit." @@ -78,7 +97,7 @@ runs: strict columnar sql array mem wal physical; do echo "nodedb-$crate = { path = \"$origin/nodedb-$crate\" }" done - echo "pagedb = { path = \"$GITHUB_WORKSPACE/.ci-sources/pagedb\" }" + echo "pagedb = { path = \"$RUNNER_TEMP/ci-sources/pagedb\" }" } > .cargo/config.toml cat .cargo/config.toml @@ -99,6 +118,24 @@ runs: done < <(grep -o 'path = "[^"]*"' .cargo/config.toml | sed 's/path = "//; s/"$//') exit "$missing" + # A workspace-level inconsistency in the sibling sources — a member + # inheriting a dependency the root manifest does not define — surfaces as a + # cargo error hundreds of lines into the first real command, where it reads + # as the pull request's own breakage. Parse the sibling workspaces here + # instead, so the failure names the upstream ref. `--no-deps` keeps this a + # manifest parse: no dependency resolution, no registry index. + - name: Preflight the sibling workspaces + if: inputs.sources == 'sibling' + shell: bash + run: | + set -euo pipefail + for manifest in "$RUNNER_TEMP/ci-sources/nodedb/Cargo.toml" "$RUNNER_TEMP/ci-sources/pagedb/Cargo.toml"; do + if ! cargo metadata --format-version=1 --no-deps --manifest-path "$manifest" > /dev/null; then + echo "::error::$manifest does not parse — the sibling sources at nodedb-ref=${{ inputs.nodedb-ref }} / pagedb-ref=${{ inputs.pagedb-ref }} are internally inconsistent. This is upstream state, not this pull request. Re-run once upstream settles." + exit 1 + fi + done + - name: Report registry mode if: inputs.sources != 'sibling' shell: bash diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index da5d0ef..7d416e6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,6 +15,11 @@ # # While the label is on, every subsequent push to the PR re-runs the suites. # +# A nightly `schedule` run probes the same suites against whatever the sibling +# refs currently point at, so upstream breakage surfaces on a quiet run of +# `main` instead of on the next PR a maintainer labels. GitHub sends failure +# notifications for a scheduled run to whoever last edited the cron line. +# # PRs build against the sibling nodedb/pagedb sources (the `sources` default); # the release path passes `registry` instead. See test.yml's header. @@ -25,6 +30,8 @@ on: branches: [main] types: [labeled, synchronize, reopened, ready_for_review] workflow_dispatch: + schedule: + - cron: "0 3 * * *" # 03:00 UTC daily concurrency: group: ci-${{ github.ref }} From be75fc4a459a28e40c7415f212240d21506e2e39 Mon Sep 17 00:00:00 2001 From: EnRaiha <15997552+EnRaiha@users.noreply.github.com> Date: Wed, 23 Sep 2026 00:55:31 +0800 Subject: [PATCH 2/2] ci: install the interop build deps and gate advisory ignores MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two failures keep the suite from running at all. The interop setup builds the Origin workspace with --all-features; rdkafka-sys compiles librdkafka from source and needs the libcurl and libsasl2 headers. Without them the setup script exits non-zero and nextest never runs — 0 of 1217 tests. Origin's test.yml has installed both since the interop suite landed; Lite's copy was missing them. The dependency audit then fails on three advisory ignores the sibling tree needs: loro-internal pulls im and its support crates, all archived upstream with no fixed version. Origin's deny.toml already carries the three with review-by dates; Lite's did not. The advisory gate is copied from Origin's scripts/ci so the review-by dates stay enforced on both sides. --- .github/workflows/test.yml | 8 +- deny.toml | 8 ++ scripts/ci/check_advisory_ignores.py | 195 +++++++++++++++++++++++++++ 3 files changed, 209 insertions(+), 2 deletions(-) create mode 100755 scripts/ci/check_advisory_ignores.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c2e9b65..accab36 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -59,7 +59,8 @@ jobs: run: | sudo apt-get update sudo apt-get install -y --no-install-recommends \ - cmake clang libclang-dev pkg-config protobuf-compiler perl + cmake clang libclang-dev pkg-config protobuf-compiler perl \ + libcurl4-openssl-dev libsasl2-dev - name: Set up workspace uses: ./.github/actions/setup-workspace with: @@ -98,6 +99,8 @@ jobs: -p nodedb-lite-ffi -E 'binary(abi_surface)'" exit 1 fi + - name: Advisory-ignore policy gate + run: python3 scripts/ci/check_advisory_ignores.py - name: Install cargo-deny uses: taiki-e/install-action@065d6a08a14e61e89fb0a4c10eecdbdef39c7d8e # v2 with: @@ -120,7 +123,8 @@ jobs: run: | sudo apt-get update sudo apt-get install -y --no-install-recommends \ - cmake clang libclang-dev pkg-config protobuf-compiler perl + cmake clang libclang-dev pkg-config protobuf-compiler perl \ + libcurl4-openssl-dev libsasl2-dev - name: Set up workspace uses: ./.github/actions/setup-workspace with: diff --git a/deny.toml b/deny.toml index b34cef5..09cef8c 100644 --- a/deny.toml +++ b/deny.toml @@ -30,6 +30,14 @@ ignore = [ "RUSTSEC-2024-0436", # paste; review-by: 2027-01-31 "RUSTSEC-2023-0089", # atomic-polyfill; review-by: 2027-01-31 "RUSTSEC-2021-0153", # encoding; review-by: 2027-01-31 + # loro-internal pulls im and its two support crates transitively. All + # three were archived upstream on 2026-05-03, so no fixed version exists + # and no known security impact applies. The maintained forks (imbl, + # imbl-sized-chunks) are loro's call, not ours. Revisit on each loro + # upgrade. + "RUSTSEC-2026-0247", # bitmaps; review-by: 2027-05-01 + "RUSTSEC-2026-0248", # im; review-by: 2027-05-01 + "RUSTSEC-2026-0251", # sized-chunks; review-by: 2027-05-01 ] [licenses] diff --git a/scripts/ci/check_advisory_ignores.py b/scripts/ci/check_advisory_ignores.py new file mode 100755 index 0000000..db89127 --- /dev/null +++ b/scripts/ci/check_advisory_ignores.py @@ -0,0 +1,195 @@ +#!/usr/bin/env python3 +"""Enforce time-bounded, individually reviewed RustSec advisory ignores.""" + +from __future__ import annotations + +import argparse +import re +import sys +from datetime import date +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +DEFAULT_CONFIG = ROOT / "deny.toml" +SECTION_RE = re.compile(r"^\s*\[advisories\]\s*(?:#.*)?$") +ANY_SECTION_RE = re.compile(r"^\s*\[[^]]+\]\s*(?:#.*)?$") +IGNORE_ASSIGN_RE = re.compile(r"^\s*ignore\s*=") +IGNORE_START_RE = re.compile(r"^\s*ignore\s*=\s*\[\s*(?:#.*)?$") +IGNORE_END_RE = re.compile(r"^\s*\]\s*(?:#.*)?$") +ENTRY_RE = re.compile( + r'^\s*"(?P[^"]*)"\s*,?\s*(?:#(?P.*))?$' +) +RUSTSEC_RE = re.compile(r"RUSTSEC-\d{4}-\d{4}\Z") +REVIEW_RE = re.compile(r"\breview-by\s*:\s*(\S+)") +DATE_RE = re.compile(r"\d{4}-\d{2}-\d{2}\Z") +WILDCARD_RE = re.compile(r"[\*?\[]") + + +def advisory_lines(text: str) -> tuple[list[tuple[int, str]], list[str]]: + """Return the ignore-list source lines from the advisories TOML table.""" + lines = text.splitlines() + errors: list[str] = [] + start = next((index for index, line in enumerate(lines) if SECTION_RE.match(line)), None) + if start is None: + return [], ["missing [advisories] table"] + + end = next( + (index for index in range(start + 1, len(lines)) if ANY_SECTION_RE.match(lines[index])), + len(lines), + ) + ignore_assignments = [ + index for index in range(start + 1, end) if IGNORE_ASSIGN_RE.match(lines[index]) + ] + if not ignore_assignments: + return [], errors + if len(ignore_assignments) != 1: + return [], ["[advisories].ignore is declared more than once"] + ignore_start = ignore_assignments[0] + if IGNORE_START_RE.match(lines[ignore_start]) is None: + return [], [ + "[advisories].ignore must place one advisory ID on each line " + "inside a multiline array" + ] + + entries: list[tuple[int, str]] = [] + for index in range(ignore_start + 1, end): + line = lines[index] + if IGNORE_END_RE.match(line): + return entries, errors + if not line.strip() or line.lstrip().startswith("#"): + continue + entries.append((index + 1, line)) + errors.append("[advisories].ignore is missing its closing bracket") + return entries, errors + + +def validate_text(text: str, today: date) -> list[str]: + """Validate a deny.toml source string and return stable, line-specific errors.""" + entries, errors = advisory_lines(text) + seen: set[str] = set() + + for line_number, line in entries: + match = ENTRY_RE.match(line) + if match is None: + errors.append(f"line {line_number}: malformed advisory ignore entry") + continue + + identifier = match.group("identifier") + comment = match.group("comment") or "" + if WILDCARD_RE.search(identifier): + errors.append(f"line {line_number}: blanket/wildcard advisory ignore is forbidden") + continue + if RUSTSEC_RE.fullmatch(identifier) is None: + errors.append( + f"line {line_number}: malformed advisory ID `{identifier}`; " + "expected RUSTSEC-YYYY-NNNN" + ) + continue + if identifier in seen: + errors.append(f"line {line_number}: duplicate advisory ID `{identifier}`") + seen.add(identifier) + + annotations = REVIEW_RE.findall(comment) + if len(annotations) != 1: + errors.append( + f"line {line_number}: `{identifier}` must have exactly one " + "review-by: YYYY-MM-DD annotation" + ) + continue + review_by = annotations[0] + if DATE_RE.fullmatch(review_by) is None: + errors.append( + f"line {line_number}: `{identifier}` has invalid review-by date `{review_by}`" + ) + continue + try: + review_date = date.fromisoformat(review_by) + except ValueError: + errors.append( + f"line {line_number}: `{identifier}` has invalid review-by date `{review_by}`" + ) + continue + if review_date < today: + errors.append( + f"line {line_number}: `{identifier}` review-by date `{review_by}` has expired" + ) + return errors + + +def fixture(*entries: str) -> str: + return "[advisories]\nignore = [\n" + "\n".join(entries) + "\n]\n" + + +def self_test() -> None: + today = date(2026, 1, 1) + fixtures = ( + ( + "positive", + fixture(' "RUSTSEC-2025-0134", # review-by: 2027-01-31'), + None, + ), + ( + "missing annotation", + fixture(' "RUSTSEC-2025-0134",'), + "must have exactly one", + ), + ( + "invalid annotation", + fixture(' "RUSTSEC-2025-0134", # review-by: 2027/01/31'), + "invalid review-by date", + ), + ( + "expired date", + fixture(' "RUSTSEC-2025-0134", # review-by: 2025-12-31'), + "has expired", + ), + ( + "duplicate ID", + fixture( + ' "RUSTSEC-2025-0134", # review-by: 2027-01-31', + ' "RUSTSEC-2025-0134", # review-by: 2027-01-31', + ), + "duplicate advisory ID", + ), + ( + "malformed ID", + fixture(' "RUSTSEC-2025-134", # review-by: 2027-01-31'), + "malformed advisory ID", + ), + ( + "wildcard ignore", + fixture(' "RUSTSEC-*", # review-by: 2027-01-31'), + "blanket/wildcard", + ), + ) + for name, source, expected_error in fixtures: + errors = validate_text(source, today) + if expected_error is None: + assert not errors, f"{name} fixture unexpectedly failed: {errors}" + else: + assert any(expected_error in error for error in errors), ( + f"{name} fixture did not fail with {expected_error!r}: {errors}" + ) + print("OK: advisory-ignore policy gate self-tests passed.") + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--self-test", action="store_true") + args = parser.parse_args() + if args.self_test: + self_test() + return 0 + + errors = validate_text(DEFAULT_CONFIG.read_text(encoding="utf-8"), date.today()) + if errors: + print("ERROR: advisory ignores must be individually time-bounded:", file=sys.stderr) + for error in errors: + print(f" {error}", file=sys.stderr) + return 1 + print("OK: advisory ignores are individually time-bounded.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())