From f2fd64fa3d5648210bea1c35ab313d4390d0239b Mon Sep 17 00:00:00 2001 From: Flotapponnier Date: Sun, 20 Sep 2026 20:41:24 +0200 Subject: [PATCH 1/2] crm: Search Console (service account, clicks/impressions/CTR/position, opportunities), Vercel log drain receiver (AI crawlers by bot and section, search bots, 404s, API and llms.txt callers, cache ratio), Core Web Vitals p75, time on page by section, searches with no result; site: not_found and search_no_result events Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01HJgbZCqjR4nvCfcJSzofbw --- crm/.env.example | 9 + crm/README.md | 36 ++- crm/app/actions/page.tsx | 18 ++ crm/app/api/ingest/vercel/route.ts | 32 +++ crm/app/audience/page.tsx | 90 +++++++ crm/app/crawlers/page.tsx | 188 +++++++++++++++ crm/app/search/page.tsx | 122 ++++++++++ crm/components/shell.tsx | 4 +- crm/instrumentation.node.ts | 31 +++ crm/instrumentation.ts | 20 +- crm/lib/gsc.ts | 131 ++++++++++ crm/lib/snapshot.ts | 26 +- crm/lib/traffic.ts | 73 +++++- crm/lib/vercel-logs.ts | 303 ++++++++++++++++++++++++ crm/proxy.ts | 3 +- crm/test/traffic.test.ts | 15 +- crm/test/vercel-logs.test.ts | 67 ++++++ src/app/not-found.tsx | 2 + src/components/not-found-ping.tsx | 17 ++ src/components/search/search-dialog.tsx | 8 + src/lib/analytics.ts | 6 +- 21 files changed, 1169 insertions(+), 32 deletions(-) create mode 100644 crm/app/api/ingest/vercel/route.ts create mode 100644 crm/app/crawlers/page.tsx create mode 100644 crm/app/search/page.tsx create mode 100644 crm/instrumentation.node.ts create mode 100644 crm/lib/gsc.ts create mode 100644 crm/lib/vercel-logs.ts create mode 100644 crm/test/vercel-logs.test.ts create mode 100644 src/components/not-found-ping.tsx diff --git a/crm/.env.example b/crm/.env.example index a855f73fb..2391cf0a5 100644 --- a/crm/.env.example +++ b/crm/.env.example @@ -16,3 +16,12 @@ REFRESH_MINUTES=15 SNAPSHOT_DIR=/data # Optional: Dune API key, to show credits left on the plan. DUNE_API_KEY= +# Optional: Vercel Log Drain (Settings → Log Drains, JSON, request logs) pointed at +# https:///api/ingest/vercel. Secret = the drain's custom secret (16+ chars), +# verify = the token Vercel shows for endpoint verification. +VERCEL_LOG_DRAIN_SECRET= +VERCEL_LOG_DRAIN_VERIFY= +# Optional: Google Search Console. A service account JSON (one line) added as a +# user of the property, and the property URL (sc-domain:openchainbench.com or https://openchainbench.com/). +GSC_SERVICE_ACCOUNT_JSON= +GSC_SITE_URL=sc-domain:openchainbench.com diff --git a/crm/README.md b/crm/README.md index 1cc9e047c..4e9c52498 100644 --- a/crm/README.md +++ b/crm/README.md @@ -10,8 +10,10 @@ One shared password, per-login sessions, one Railway service, no database. |---|---|---| | Overview | PostHog, blob, Prometheus | visitors, pageviews, sessions (7 d vs previous 7 d), **AI-referred visitors** (ChatGPT, Perplexity, Claude, Gemini, Copilot, …), search-referred visitors, 28 d daily series, 12 w weekly series with the AI share, channels, AI domains, sections | | Pages | PostHog | sections week over week, biggest gains and losses, top 100 pages (filter by section), entry pages | -| Audience | PostHog | new vs returning, bounce, countries, devices, UTM sources, referring domains with their channel | -| Actions | PostHog custom events | outbound clicks by destination host (visitors sent to providers), copies (endpoint, API URL, MCP, embed) per bench, search queries with the result picked | +| Audience | PostHog | new vs returning, bounce, Core Web Vitals p75 by device, time on page by section, countries, devices, UTM sources, referring domains with their channel | +| Actions | PostHog custom events | outbound clicks by destination host (visitors sent to providers), copies (endpoint, API URL, MCP, embed) per bench, searches with no result (content gaps), search queries with the result picked | +| Search | Google Search Console (service account) | clicks, impressions, CTR, position 7 d vs previous 7 d, 28 d series, opportunities (many impressions, CTR < 1 %, position ≤ 15), top pages and queries | +| Crawlers | Vercel Log Drain | AI crawler hits by bot (GPTBot, ClaudeBot, PerplexityBot, …) and the sections they read, search bots, human requests by section, who calls /api/stat, /api/citable, llms.txt, top 404 paths, cache hit ratio | | Data health | index blob, Prometheus, Dune | live / stale (> 24 h) / expired (> 7 d) benches per category, benches needing attention, scrape targets down, Dune credits and period end, the daily history kept on the volume | The site captures `$pageview`, `$pageleave` and three custom events @@ -29,10 +31,10 @@ from the sitemap before publishing, which is exactly what this page must show. PostHog allows **2400 query requests per hour per organisation**, shared by every key and every team member. This app never queries in the request path: -- a refresh runs a **fixed list of 15 HogQL queries**, one at a time +- a refresh runs a **fixed list of 19 HogQL queries**, one at a time (`lib/traffic.ts`), and writes a snapshot; pages read the snapshot; - the scheduler (`instrumentation.ts`) refreshes every `REFRESH_MINUTES` - (default 15): **60 queries per hour, 2.5 % of the organisation's budget**; + (default 15): **76 queries per hour, 3.2 % of the organisation's budget**; - the Refresh button is refused for 5 minutes after any refresh; - a local budget (`POSTHOG_HOURLY_BUDGET`, default 300 per rolling hour) is a second guard; a 429 or an exhausted budget stops the batch, the sections that @@ -93,4 +95,28 @@ and each layer gets its own module instance otherwise. 3. `pnpm test`: the query test checks every query stays scoped to `$pageview` on the production host. -Non-PostHog sources go in `lib/ocb.ts` and get a `step()` in `lib/snapshot.ts`. +Non-PostHog sources go in `lib/ocb.ts` (or their own module: `lib/gsc.ts`, +`lib/vercel-logs.ts`) and get a `step()` in `lib/snapshot.ts`. + +## Connecting Search Console + +1. Google Cloud console → a project → APIs → enable **Google Search Console API**. +2. IAM → Service accounts → create one, add a JSON key, download it. +3. Search Console → property `openchainbench.com` → Settings → Users and + permissions → add the service account e-mail (Full or Restricted, read is enough). +4. Railway service variables: `GSC_SERVICE_ACCOUNT_JSON` = the key file's + content on one line, `GSC_SITE_URL` = `sc-domain:openchainbench.com` (or the + URL-prefix property as listed in Search Console). Five API calls per refresh. + +## Connecting the Vercel Log Drain + +1. Railway variables: `VERCEL_LOG_DRAIN_SECRET` (random, 16+ chars), + `VERCEL_LOG_DRAIN_VERIFY` (the verification token Vercel displays when you add + the drain; set it, redeploy, then click Verify). +2. Vercel → team settings → **Log Drains** → Add: project `openchainbench-mobula`, + sources **Request logs** (proxy) only, format **JSON**, endpoint + `https:///api/ingest/vercel`, custom secret = the value above. +3. Requests are folded per UTC day into `/data/vercel/YYYY-MM-DD.json` + (counts only: bot names, sections, status codes, path families; no IPs, no + raw user agents). One log line per request; a busy day is a few thousand + POSTs of batched entries, negligible for the service. diff --git a/crm/app/actions/page.tsx b/crm/app/actions/page.tsx index 8384e0810..7251b2fae 100644 --- a/crm/app/actions/page.tsx +++ b/crm/app/actions/page.tsx @@ -114,6 +114,24 @@ export default async function ActionsPage({ searchParams }: { searchParams: Prom +
+

Searches with no result, 7 d (content gaps)

+ {(t.noResults ?? []).length > 0 ? ( + + + {(t.noResults ?? []).map((r) => ( + + + + + ))} + +
{r.query}{fmtInt(r.count)}
+ ) : ( + + )} +
+

What people search for, 7 d (a result was picked)

{searches.length > 0 ? ( diff --git a/crm/app/api/ingest/vercel/route.ts b/crm/app/api/ingest/vercel/route.ts new file mode 100644 index 000000000..1bf98a53e --- /dev/null +++ b/crm/app/api/ingest/vercel/route.ts @@ -0,0 +1,32 @@ +import { drainConfigured, ingestEntries, parseBody, verifySignature } from "@/lib/vercel-logs"; + +// Public route (proxy.ts lets it through): authenticated by Vercel's HMAC +// signature over the raw body, never by the session cookie. The +// verification handshake answers with the token Vercel expects. +export const dynamic = "force-dynamic"; +const MAX_BODY = 8 * 1024 * 1024; + +function verifyHeaders(): HeadersInit { + const v = process.env.VERCEL_LOG_DRAIN_VERIFY; + return v ? { "x-vercel-verify": v } : {}; +} + +export async function GET() { + return new Response("ok", { status: 200, headers: verifyHeaders() }); +} + +export async function HEAD() { + return new Response(null, { status: 200, headers: verifyHeaders() }); +} + +export async function POST(request: Request) { + if (!drainConfigured()) return new Response("drain not configured", { status: 503, headers: verifyHeaders() }); + const raw = await request.text(); + if (raw.length > MAX_BODY) return new Response("too large", { status: 413 }); + if (!verifySignature(raw, request.headers.get("x-vercel-signature"))) { + return new Response("bad signature", { status: 401, headers: verifyHeaders() }); + } + const entries = parseBody(raw); + const folded = ingestEntries(entries); + return Response.json({ received: entries.length, folded }, { headers: verifyHeaders() }); +} diff --git a/crm/app/audience/page.tsx b/crm/app/audience/page.tsx index b65dea648..f927424d3 100644 --- a/crm/app/audience/page.tsx +++ b/crm/app/audience/page.tsx @@ -1,7 +1,12 @@ import { Shell } from "@/components/shell"; import { Bars, Delta, Empty, fmtInt, fmtPct, Kpi } from "@/components/ui"; +import { SECTION_LABEL } from "@/lib/channels"; import { readSnapshot } from "@/lib/snapshot"; +const fmtMs = (v: number) => (v >= 1000 ? `${(v / 1000).toFixed(2)} s` : `${Math.round(v)} ms`); +const fmtSec = (v: number) => (v >= 60 ? `${Math.floor(v / 60)} min ${Math.round(v % 60)} s` : `${Math.round(v)} s`); +const vitalTone = (v: number, good: number, poor: number) => (v <= good ? "var(--good)" : v <= poor ? "var(--warn)" : "var(--bad)"); + export const dynamic = "force-dynamic"; const CHANNEL_LABEL = { ai: "AI", search: "Search", social: "Social", direct: "Direct", referral: "Referral", internal: "Internal" } as const; @@ -57,6 +62,91 @@ export default async function AudiencePage({ searchParams }: { searchParams: Pro
+
+
+

Core Web Vitals, 7 d (p75, by device)

+ {t.vitals && t.vitals.length > 0 ? ( + + + + + + + + + + + + + {t.vitals.map((v) => ( + + + + + + + + + ))} + +
DeviceSamplesLCPINPCLSFCP
{v.device}{fmtInt(v.samples)}{fmtMs(v.lcpP75)}{fmtMs(v.inpP75)}{v.clsP75.toFixed(3)}{fmtMs(v.fcpP75)}
+ ) : ( + + )} +

+ Google's thresholds: LCP 2.5 s, INP 200 ms, CLS 0.1 (good), 4 s / 500 ms / 0.25 (poor). Ranking signal on mobile. +

+
+
+

Time on page by section, 7 d (median of leaves)

+ {t.engaged && t.engaged.length > 0 ? ( + + + + + + + + + + + {t.engaged.map((e) => ( + + + + + + + ))} + +
SectionLeavesMedianp75
{SECTION_LABEL[e.section]}{fmtInt(e.leaves)}{fmtSec(e.medianSec)}{fmtSec(e.p75Sec)}
+ ) : ( + + )} +

Pages read longest (3+ leaves)

+ {t.engagedPages && t.engagedPages.length > 0 ? ( + + + {[...t.engagedPages] + .sort((a, b) => b.medianSec - a.medianSec) + .slice(0, 10) + .map((p) => ( + + + + + + ))} + +
+ {p.path} + {fmtInt(p.leaves)}{fmtSec(p.medianSec)}
+ ) : ( + + )} +
+
+

Referring domains, 7 d (direct and internal excluded)

{referrers.length > 0 ? ( diff --git a/crm/app/crawlers/page.tsx b/crm/app/crawlers/page.tsx new file mode 100644 index 000000000..fa961c684 --- /dev/null +++ b/crm/app/crawlers/page.tsx @@ -0,0 +1,188 @@ +import { Shell } from "@/components/shell"; +import { Bars, Delta, Empty, fmtInt, fmtPct, Kpi, Spark } from "@/components/ui"; +import { SECTION_LABEL, type Section } from "@/lib/channels"; +import { readSnapshot } from "@/lib/snapshot"; +import { drainConfigured, drainStats, readDays, type DayAggregate } from "@/lib/vercel-logs"; + +export const dynamic = "force-dynamic"; +const SITE = `https://${process.env.SITE_HOST ?? "openchainbench.com"}`; + +const sumMap = (days: DayAggregate[], pick: (d: DayAggregate) => Record) => { + const out: Record = {}; + for (const d of days) for (const [k, v] of Object.entries(pick(d))) out[k] = (out[k] ?? 0) + v; + return out; +}; +const sorted = (m: Record) => Object.entries(m).sort((a, b) => b[1] - a[1]); +const total = (m: Record) => Object.values(m).reduce((a, b) => a + b, 0); + +export default async function CrawlersPage({ searchParams }: { searchParams: Promise<{ refresh?: string }> }) { + const [snap, sp, days] = await Promise.all([readSnapshot(), searchParams, readDays(28)]); + const today = new Date().toISOString().slice(0, 10); + const last7 = days.filter((d) => d.day > new Date(Date.now() - 7 * 86_400_000).toISOString().slice(0, 10)); + const prev7 = days.filter((d) => d.day <= new Date(Date.now() - 7 * 86_400_000).toISOString().slice(0, 10) && d.day > new Date(Date.now() - 14 * 86_400_000).toISOString().slice(0, 10)); + const ai = sumMap(last7, (d) => d.aiBots); + const aiPrev = sumMap(prev7, (d) => d.aiBots); + const search = sumMap(last7, (d) => d.searchBots); + const searchPrev = sumMap(prev7, (d) => d.searchBots); + const cls = sumMap(last7, (d) => d.byClass); + const status = sumMap(last7, (d) => d.status); + const nf = sumMap(last7, (d) => d.notFound); + const cache = sumMap(last7, (d) => d.cache); + const aiSections = sumMap(last7, (d) => d.aiBySection); + const humanSections = sumMap(last7, (d) => d.humanBySection); + const api: Record> = {}; + for (const d of last7) for (const [fam, m] of Object.entries(d.api)) api[fam] = { ...(api[fam] ?? {}), ...Object.fromEntries(Object.entries(m).map(([k, v]) => [k, (api[fam]?.[k] ?? 0) + v])) }; + const series = (pick: (d: DayAggregate) => number) => { + const by = new Map(days.map((d) => [d.day, pick(d)])); + return Array.from({ length: 28 }, (_, i) => by.get(new Date(Date.now() - (27 - i) * 86_400_000).toISOString().slice(0, 10)) ?? 0); + }; + const cacheHits = (cache.HIT ?? 0) + (cache.STALE ?? 0) + (cache.PRERENDER ?? 0); + const cacheTotal = total(cache); + const nf404Posthog = snap.traffic.notFound ?? []; + const stats = drainStats(); + + return ( + + {!drainConfigured() || days.length === 0 ? ( +
+

Vercel request logs

+

+ {drainConfigured() + ? `Receiver armed, no request folded yet (${stats.received} entries received since start). Vercel needs the drain enabled: Team settings → Log Drains → Add, sources "Request logs", format JSON, endpoint https:///api/ingest/vercel, with the custom secret set as VERCEL_LOG_DRAIN_SECRET.` + : "Not connected. Set VERCEL_LOG_DRAIN_SECRET (16+ chars) and VERCEL_LOG_DRAIN_VERIFY on the Railway service, then add a Log Drain in the Vercel team settings (Request logs, JSON) pointing at /api/ingest/vercel on this host. AI crawlers, search bots, 404s and API hits appear here, per day, kept on the volume."} +

+
+ ) : null} + +
+ + + + 0 ? `cache hit ${fmtPct(cacheHits / cacheTotal)}` : undefined} /> +
+ +
+
+

AI crawler hits per day, 28 d

+ {days.length > 1 ? total(d.aiBots))} color="var(--good)" /> : } +

By bot, 7 d

+ {sorted(ai).length > 0 ? ( + + + {sorted(ai).map(([k, v]) => ( + + + + + + ))} + +
{k}{fmtInt(v)} + +
+ ) : ( + + )} +
+
+

What AI crawlers read, 7 d (sections)

+ {sorted(aiSections).length > 0 ? ( + ({ label: SECTION_LABEL[k as Section] ?? k, value: v }))} /> + ) : ( + + )} +

What humans read, 7 d (sections, server side)

+ {sorted(humanSections).length > 0 ? ( + ({ label: SECTION_LABEL[k as Section] ?? k, value: v }))} /> + ) : ( + + )} +
+
+ +
+
+

Machine surfaces, 7 d (who calls the API and llms.txt)

+ {Object.keys(api).length > 0 ? ( + + + + + + + + + + + + {Object.entries(api) + .sort((a, b) => total(b[1]) - total(a[1])) + .map(([fam, m]) => ( + + + + + + + + ))} + +
PathHumanAI botsSearch botsOther bots
{fam}{fmtInt(m.human)}{fmtInt(m.ai_bot)}{fmtInt(m.search_bot)}{fmtInt(m.other_bot)}
+ ) : ( + + )} +

Search bots, 7 d

+ {sorted(search).length > 0 ? ( + ({ label: k, value: v }))} /> + ) : ( + + )} +
+
+

Top 404 paths, 7 d (server logs)

+ {sorted(nf).length > 0 ? ( + + + {sorted(nf) + .slice(0, 25) + .map(([k, v]) => ( + + + + + ))} + +
+ {k} + {fmtInt(v)}
+ ) : ( + + )} +

404s seen by visitors, 7 d (browser event, with the referrer)

+ {nf404Posthog.length > 0 ? ( + + + {nf404Posthog.slice(0, 15).map((r) => ( + + + + + + ))} + +
+ {r.path} + + {r.topReferrer || "direct"} + {fmtInt(r.hits)}
+ ) : ( + + )} +
+
+

+ Today is {today}; the last day of the series is partial. Assets, /_next and /ingest are excluded; hosts other than {SITE.replace("https://", "")} are ignored. +

+ + ); +} diff --git a/crm/app/search/page.tsx b/crm/app/search/page.tsx new file mode 100644 index 000000000..68fbd1931 --- /dev/null +++ b/crm/app/search/page.tsx @@ -0,0 +1,122 @@ +import { Shell } from "@/components/shell"; +import { Delta, Empty, fmtInt, fmtPct, Kpi, Spark } from "@/components/ui"; +import { readSnapshot } from "@/lib/snapshot"; + +export const dynamic = "force-dynamic"; +const SITE = `https://${process.env.SITE_HOST ?? "openchainbench.com"}`; + +export default async function SearchPage({ searchParams }: { searchParams: Promise<{ refresh?: string }> }) { + const [snap, sp] = await Promise.all([readSnapshot(), searchParams]); + const g = snap.gsc; + const t = g?.totals; + + return ( + + {!g ? ( +
+

Google Search Console

+

+ Not connected. Create a Google Cloud service account, enable the Search Console API, add the account's e-mail as a user of the + property in Search Console, then set GSC_SERVICE_ACCOUNT_JSON (the key file on one line) and{" "} + GSC_SITE_URL on the Railway service. The refresh after that fills this page. +

+ {snap.status.gsc?.error && !snap.status.gsc.error.startsWith("GSC_SERVICE_ACCOUNT_JSON") && ( +

+ {snap.status.gsc.error} +

+ )} +
+ ) : ( + <> +

+ Search Console lags about three days. Window {g.window.start} → {g.window.end}, compared with {g.window.prevStart} → {g.window.prevEnd}. Property{" "} + {g.siteUrl}. +

+
+ + + + +
+ +
+
+

Daily clicks, 28 d

+ {g.daily.length > 1 ? d.clicks)} /> : } +
+
+

Daily impressions, 28 d

+ {g.daily.length > 1 ? d.impressions)} color="var(--good)" /> : } +
+
+ +
+

Opportunities: 50+ impressions, CTR under 1 %, position 15 or better (title and description work)

+ {g.opportunities.length > 0 ? ( + + ) : ( + + )} +
+ +
+
+

Top pages, 7 d

+ +
+
+

Top queries, 7 d

+ +
+
+ + )} + + ); +} + +function DimTable({ rows, link }: { rows: { key: string; clicks: number; impressions: number; ctr: number; position: number; prevClicks: number; prevImpressions: number }[]; link?: boolean }) { + if (rows.length === 0) return ; + return ( +
+ + + + + + + + + + + + + + {rows.map((r) => ( + + + + + + + + + + ))} + +
{link ? "Page" : "Query"}Clicksw/wImpr.w/wCTRPos.
+ {link ? ( + + {r.key} + + ) : ( + r.key + )} + {fmtInt(r.clicks)} + + {fmtInt(r.impressions)} + + {fmtPct(r.ctr, 1)}{r.position.toFixed(1)}
+
+ ); +} diff --git a/crm/components/shell.tsx b/crm/components/shell.tsx index 9c62c6a11..37ea73188 100644 --- a/crm/components/shell.tsx +++ b/crm/components/shell.tsx @@ -6,6 +6,8 @@ const NAV = [ ["/pages", "Pages"], ["/audience", "Audience"], ["/actions", "Actions"], + ["/search", "Search"], + ["/crawlers", "Crawlers"], ["/health", "Data health"], ] as const; @@ -19,7 +21,7 @@ export function fmtAge(min: number | null): string { export function Shell({ current, snapshot, refreshFlag, children }: { current: string; snapshot: Snapshot; refreshFlag?: string; children: React.ReactNode }) { const age = snapshotAgeMinutes(snapshot); - const errors = Object.entries(snapshot.status).filter(([, s]) => s.error); + const errors = Object.entries(snapshot.status).filter(([k, s]) => s.error && k !== "gsc"); const canRefresh = age == null || age >= MANUAL_COOLDOWN_MINUTES; return (
diff --git a/crm/instrumentation.node.ts b/crm/instrumentation.node.ts new file mode 100644 index 000000000..b558d81df --- /dev/null +++ b/crm/instrumentation.node.ts @@ -0,0 +1,31 @@ +/** + * Node-only half of instrumentation.ts (the file itself is compiled for + * both runtimes, so Node APIs cannot appear in it directly). + * The refresher: one interval per server process, a refresh at boot when the + * stored snapshot is older than the interval, then every REFRESH_MINUTES + * with a little jitter so two instances would not align on PostHog. + * Also flushes the log-drain accumulator on shutdown so a redeploy loses + * nothing. + */ +import { readSnapshot, refreshSnapshot, REFRESH_MINUTES, snapshotAgeMinutes } from "@/lib/snapshot"; +import { flush } from "@/lib/vercel-logs"; + +export async function start(): Promise { + for (const sig of ["SIGTERM", "SIGINT"] as const) { + process.once(sig, () => { + flush().finally(() => process.exit(0)); + }); + } + if (process.env.CRM_DISABLE_SCHEDULER === "1") return; + const snap = await readSnapshot(); + const age = snapshotAgeMinutes(snap); + const firstDelayMs = age == null || age >= REFRESH_MINUTES ? 5_000 : (REFRESH_MINUTES - age) * 60_000; + const tick = () => { + refreshSnapshot("scheduled").catch((e) => console.error("[scheduler]", e)); + }; + setTimeout(() => { + tick(); + setInterval(tick, REFRESH_MINUTES * 60_000 + Math.floor(Math.random() * 30_000)); + }, firstDelayMs).unref(); + console.log(`[scheduler] first refresh in ${Math.round(firstDelayMs / 1000)} s, then every ${REFRESH_MINUTES} min`); +} diff --git a/crm/instrumentation.ts b/crm/instrumentation.ts index f7323da51..974aca7d1 100644 --- a/crm/instrumentation.ts +++ b/crm/instrumentation.ts @@ -1,21 +1,5 @@ -/** - * The refresher. One interval per server process: a refresh at boot when - * the stored snapshot is older than the interval, then every REFRESH_MINUTES - * with a little jitter so two instances would not align on PostHog. - */ export async function register() { if (process.env.NEXT_RUNTIME !== "nodejs") return; - if (process.env.CRM_DISABLE_SCHEDULER === "1") return; - const { readSnapshot, refreshSnapshot, REFRESH_MINUTES, snapshotAgeMinutes } = await import("@/lib/snapshot"); - const snap = await readSnapshot(); - const age = snapshotAgeMinutes(snap); - const firstDelayMs = age == null || age >= REFRESH_MINUTES ? 5_000 : (REFRESH_MINUTES - age) * 60_000; - const tick = () => { - refreshSnapshot("scheduled").catch((e) => console.error("[scheduler]", e)); - }; - setTimeout(() => { - tick(); - setInterval(tick, REFRESH_MINUTES * 60_000 + Math.floor(Math.random() * 30_000)); - }, firstDelayMs).unref(); - console.log(`[scheduler] first refresh in ${Math.round(firstDelayMs / 1000)} s, then every ${REFRESH_MINUTES} min`); + const { start } = await import("./instrumentation.node"); + await start(); } diff --git a/crm/lib/gsc.ts b/crm/lib/gsc.ts new file mode 100644 index 000000000..bd40e4041 --- /dev/null +++ b/crm/lib/gsc.ts @@ -0,0 +1,131 @@ +/** + * Google Search Console, read with a service account (no OAuth dance): + * the account's email is added as a user of the property, the JSON key + * sits in GSC_SERVICE_ACCOUNT_JSON. Five Search Analytics calls per refresh + * (daily series, pages and queries for the last window and the previous + * one), well under the API's 1,200 calls per minute. + * + * Search Console data lags two to three days, so "last 7 days" here ends + * three days ago; the window is shown on the page. + */ +import { createSign } from "node:crypto"; +import { z } from "zod"; + +const SITE_URL = process.env.GSC_SITE_URL ?? "sc-domain:openchainbench.com"; +const DELAY_DAYS = 3; +const WINDOW_DAYS = 7; + +const saSchema = z.object({ client_email: z.string(), private_key: z.string(), token_uri: z.string().default("https://oauth2.googleapis.com/token") }); + +export function gscConfigured(): boolean { + return (process.env.GSC_SERVICE_ACCOUNT_JSON ?? "").length > 0; +} + +function b64url(s: Buffer | string): string { + return Buffer.from(s).toString("base64").replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/, ""); +} + +let tokenCache: { token: string; exp: number } | null = null; + +async function accessToken(): Promise { + if (tokenCache && tokenCache.exp > Date.now() + 60_000) return tokenCache.token; + const sa = saSchema.parse(JSON.parse(process.env.GSC_SERVICE_ACCOUNT_JSON ?? "{}")); + const now = Math.floor(Date.now() / 1000); + const header = b64url(JSON.stringify({ alg: "RS256", typ: "JWT" })); + const claims = b64url( + JSON.stringify({ iss: sa.client_email, scope: "https://www.googleapis.com/auth/webmasters.readonly", aud: sa.token_uri, iat: now, exp: now + 3600 }), + ); + const signer = createSign("RSA-SHA256"); + signer.update(`${header}.${claims}`); + const jwt = `${header}.${claims}.${b64url(signer.sign(sa.private_key))}`; + const res = await fetch(sa.token_uri, { + method: "POST", + headers: { "Content-Type": "application/x-www-form-urlencoded" }, + body: new URLSearchParams({ grant_type: "urn:ietf:params:oauth:grant-type:jwt-bearer", assertion: jwt }), + signal: AbortSignal.timeout(20_000), + }); + if (!res.ok) throw new Error(`gsc token ${res.status}: ${(await res.text()).slice(0, 200)}`); + const j = (await res.json()) as { access_token: string; expires_in: number }; + tokenCache = { token: j.access_token, exp: Date.now() + j.expires_in * 1000 }; + return j.access_token; +} + +const rowSchema = z.object({ keys: z.array(z.string()).optional(), clicks: z.number(), impressions: z.number(), ctr: z.number(), position: z.number() }); +const respSchema = z.object({ rows: z.array(rowSchema).optional() }); +type Row = z.infer; + +async function query(body: Record): Promise { + const token = await accessToken(); + const res = await fetch(`https://searchconsole.googleapis.com/webmasters/v3/sites/${encodeURIComponent(SITE_URL)}/searchAnalytics/query`, { + method: "POST", + headers: { Authorization: `Bearer ${token}`, "Content-Type": "application/json" }, + body: JSON.stringify({ type: "web", dataState: "final", ...body }), + signal: AbortSignal.timeout(30_000), + cache: "no-store", + }); + if (!res.ok) throw new Error(`gsc query ${res.status}: ${(await res.text()).slice(0, 200)}`); + return respSchema.parse(await res.json()).rows ?? []; +} + +const iso = (d: Date) => d.toISOString().slice(0, 10); +const daysAgo = (n: number, now: number) => iso(new Date(now - n * 86_400_000)); + +export type GscTotals = { clicks: number; impressions: number; ctr: number; position: number }; +export type GscDim = GscTotals & { key: string; prevClicks: number; prevImpressions: number; prevPosition: number }; +export type Gsc = { + siteUrl: string; + window: { start: string; end: string; prevStart: string; prevEnd: string }; + daily: (GscTotals & { date: string })[]; + totals: GscTotals & { prev: GscTotals }; + pages: GscDim[]; + queries: GscDim[]; + /** Pages with many impressions and a click-through below 1 % inside the top 15: the title/description work list. */ + opportunities: GscDim[]; +}; + +function totals(rows: Row[]): GscTotals { + const clicks = rows.reduce((a, r) => a + r.clicks, 0); + const impressions = rows.reduce((a, r) => a + r.impressions, 0); + const position = impressions > 0 ? rows.reduce((a, r) => a + r.position * r.impressions, 0) / impressions : 0; + return { clicks, impressions, ctr: impressions > 0 ? clicks / impressions : 0, position }; +} + +function joinWindows(cur: Row[], prev: Row[], strip: (k: string) => string): GscDim[] { + const prevBy = new Map(prev.map((r) => [r.keys?.[0] ?? "", r])); + return cur + .map((r) => { + const k = r.keys?.[0] ?? ""; + const p = prevBy.get(k); + return { key: strip(k), clicks: r.clicks, impressions: r.impressions, ctr: r.ctr, position: r.position, prevClicks: p?.clicks ?? 0, prevImpressions: p?.impressions ?? 0, prevPosition: p?.position ?? 0 }; + }) + .sort((a, b) => b.clicks - a.clicks || b.impressions - a.impressions); +} + +export async function loadGsc(now = Date.now()): Promise { + const end = daysAgo(DELAY_DAYS, now); + const start = daysAgo(DELAY_DAYS + WINDOW_DAYS - 1, now); + const prevEnd = daysAgo(DELAY_DAYS + WINDOW_DAYS, now); + const prevStart = daysAgo(DELAY_DAYS + 2 * WINDOW_DAYS - 1, now); + const seriesStart = daysAgo(DELAY_DAYS + 27, now); + const origin = SITE_URL.startsWith("sc-domain:") ? `https://${SITE_URL.slice("sc-domain:".length)}` : SITE_URL.replace(/\/$/, ""); + const stripOrigin = (k: string) => (k.startsWith(origin) ? k.slice(origin.length) || "/" : k); + + const daily = await query({ startDate: seriesStart, endDate: end, dimensions: ["date"], rowLimit: 60 }); + const pagesCur = await query({ startDate: start, endDate: end, dimensions: ["page"], rowLimit: 500 }); + const pagesPrev = await query({ startDate: prevStart, endDate: prevEnd, dimensions: ["page"], rowLimit: 500 }); + const queriesCur = await query({ startDate: start, endDate: end, dimensions: ["query"], rowLimit: 300 }); + const queriesPrev = await query({ startDate: prevStart, endDate: prevEnd, dimensions: ["query"], rowLimit: 300 }); + + const inWindow = (r: Row) => (r.keys?.[0] ?? "") >= start && (r.keys?.[0] ?? "") <= end; + const inPrev = (r: Row) => (r.keys?.[0] ?? "") >= prevStart && (r.keys?.[0] ?? "") <= prevEnd; + const pages = joinWindows(pagesCur, pagesPrev, stripOrigin); + return { + siteUrl: SITE_URL, + window: { start, end, prevStart, prevEnd }, + daily: daily.map((r) => ({ date: r.keys?.[0] ?? "", clicks: r.clicks, impressions: r.impressions, ctr: r.ctr, position: r.position })), + totals: { ...totals(daily.filter(inWindow)), prev: totals(daily.filter(inPrev)) }, + pages, + queries: joinWindows(queriesCur, queriesPrev, (k) => k), + opportunities: pages.filter((p) => p.impressions >= 50 && p.ctr < 0.01 && p.position <= 15).sort((a, b) => b.impressions - a.impressions).slice(0, 30), + }; +} diff --git a/crm/lib/snapshot.ts b/crm/lib/snapshot.ts index 256d6a6c8..9ca44ed9a 100644 --- a/crm/lib/snapshot.ts +++ b/crm/lib/snapshot.ts @@ -14,6 +14,7 @@ import path from "node:path"; import { budget, BudgetExhausted, HOURLY_BUDGET, posthogConfigured, RateLimited } from "@/lib/posthog"; import { loadBenchHealth, loadDuneUsage, loadHarnessHealth, type BenchHealth, type DuneUsage, type HarnessHealth } from "@/lib/ocb"; import { loadTrafficSection, TRAFFIC_SECTIONS, type Traffic } from "@/lib/traffic"; +import { gscConfigured, loadGsc, type Gsc } from "@/lib/gsc"; // 15 min by default: 15 queries per pass, 60 per hour, 2.5 % of PostHog's // organisation budget; the Railway cost does not move with this number, the @@ -36,13 +37,25 @@ export type Snapshot = { benches: BenchHealth | null; harness: HarnessHealth | null; dune: DuneUsage | null; + gsc: Gsc | null; status: Record; budget: { used: number; limit: number }; }; -export type HistoryLine = { day: string; visitors7d: number; pageviews7d: number; aiVisitors7d: number; searchVisitors7d: number; benches: number; stale: number; targetsDown: number }; +export type HistoryLine = { + day: string; + visitors7d: number; + pageviews7d: number; + aiVisitors7d: number; + searchVisitors7d: number; + benches: number; + stale: number; + targetsDown: number; + gscClicks7d?: number; + gscImpressions7d?: number; +}; -const EMPTY: Snapshot = { v: 1, refreshedAt: null, posthogConfigured: posthogConfigured(), traffic: {}, benches: null, harness: null, dune: null, status: {}, budget: { used: 0, limit: HOURLY_BUDGET } }; +const EMPTY: Snapshot = { v: 1, refreshedAt: null, posthogConfigured: posthogConfigured(), traffic: {}, benches: null, harness: null, dune: null, gsc: null, status: {}, budget: { used: 0, limit: HOURLY_BUDGET } }; function clampInt(raw: string | undefined, fallback: number, min: number, max: number): number { const n = Number.parseInt(raw ?? "", 10); @@ -118,6 +131,7 @@ async function appendHistory(s: Snapshot): Promise { benches: s.benches?.total ?? 0, stale: (s.benches?.stale ?? 0) + (s.benches?.expired ?? 0), targetsDown: s.harness?.down.length ?? 0, + ...(s.gsc ? { gscClicks7d: s.gsc.totals.clicks, gscImpressions7d: s.gsc.totals.impressions } : {}), }; await fs.mkdir(DIR, { recursive: true }); await fs.appendFile(HISTORY, `${JSON.stringify(line)}\n`); @@ -170,6 +184,14 @@ async function doRefresh(reason: string): Promise { await step("dune", async () => { next.dune = await loadDuneUsage(); }); + if (gscConfigured()) { + delete next.status.gsc; + await step("search-console", async () => { + next.gsc = await loadGsc(); + }); + } else { + next.status.gsc = { at: null, error: "GSC_SERVICE_ACCOUNT_JSON not set (Search Console panel empty)" }; + } if (posthogConfigured()) { // The "not configured" note from earlier refreshes must not outlive the fix. diff --git a/crm/lib/traffic.ts b/crm/lib/traffic.ts index 52048f5e8..39a93f0ac 100644 --- a/crm/lib/traffic.ts +++ b/crm/lib/traffic.ts @@ -1,6 +1,6 @@ /** * The PostHog side of the snapshot: one fixed list of HogQL queries per - * refresh (fifteen today), each mapped to a plain JSON section. Every query is + * refresh (nineteen today), each mapped to a plain JSON section. Every query is * scoped to the production host, so staging and localhost never count, and * to one named event: `$pageview` for the traffic sections, the three custom * events of src/lib/analytics.ts for the Actions sections (autocapture is off). @@ -16,6 +16,7 @@ const HOST_FILTER = `properties.$host = '${SITE_HOST}'`; const PV = `event = '$pageview' AND ${HOST_FILTER}`; // The site's custom events (src/lib/analytics.ts): outbound_click, copy, search. const CUSTOM = `event IN ('outbound_click', 'copy', 'search') AND ${HOST_FILTER}`; +const PL = `event = '$pageleave' AND ${HOST_FILTER}`; export type DailyPoint = { day: string; pageviews: number; visitors: number; sessions: number }; export type WeeklyPoint = { week: string; visitors: number; ai: number; search: number; pageviews: number }; @@ -27,6 +28,11 @@ export type ActionRow = { name: string; count: number; prevCount: number; visito export type OutboundRow = { host: string; clicks: number; prevClicks: number; visitors: number; topPage: string }; export type SearchRow = { query: string; count: number; kind: string; url: string }; export type CopyRow = { kind: string; value: string; bench: string; count: number }; +export type VitalsRow = { device: string; samples: number; lcpP75: number; inpP75: number; clsP75: number; fcpP75: number }; +export type EngagedRow = { section: Section; leaves: number; medianSec: number; p75Sec: number }; +export type EngagedPageRow = { path: string; section: Section; leaves: number; medianSec: number }; +export type NotFoundRow = { path: string; hits: number; visitors: number; topReferrer: string }; +export type NoResultRow = { query: string; count: number }; export type Traffic = { daily: DailyPoint[]; @@ -56,6 +62,11 @@ export type Traffic = { outbound: OutboundRow[]; searches: SearchRow[]; copies: CopyRow[]; + vitals: VitalsRow[]; + engaged: EngagedRow[]; + engagedPages: EngagedPageRow[]; + notFound: NotFoundRow[]; + noResults: NoResultRow[]; }; export const QUERIES = { @@ -148,6 +159,29 @@ export const QUERIES = { SELECT properties.kind AS kind, properties.value AS value, properties.bench AS bench, count() AS n FROM events WHERE event = 'copy' AND ${HOST_FILTER} AND timestamp >= now() - INTERVAL 7 DAY GROUP BY kind, value, bench ORDER BY n DESC LIMIT 40`, + vitals: () => ` + SELECT properties.$device_type AS device, count() AS samples, + quantile(0.75)(toFloat(properties.$web_vitals_LCP_value)) AS lcp, + quantile(0.75)(toFloat(properties.$web_vitals_INP_value)) AS inp, + quantile(0.75)(toFloat(properties.$web_vitals_CLS_value)) AS cls, + quantile(0.75)(toFloat(properties.$web_vitals_FCP_value)) AS fcp + FROM events WHERE event = '$web_vitals' AND ${HOST_FILTER} AND timestamp >= now() - INTERVAL 7 DAY + GROUP BY device ORDER BY samples DESC LIMIT 4`, + engaged: () => ` + SELECT properties.$prev_pageview_pathname AS path, count() AS leaves, + quantile(0.5)(toFloat(properties.$prev_pageview_duration)) AS med, + quantile(0.75)(toFloat(properties.$prev_pageview_duration)) AS p75 + FROM events WHERE ${PL} AND timestamp >= now() - INTERVAL 7 DAY + AND properties.$prev_pageview_duration IS NOT NULL AND toFloat(properties.$prev_pageview_duration) BETWEEN 0 AND 1800 + GROUP BY path ORDER BY leaves DESC LIMIT 1500`, + notFound: () => ` + SELECT properties.path AS path, count() AS hits, uniq(distinct_id) AS visitors, topK(1)(properties.referrer) AS ref + FROM events WHERE event = 'not_found' AND ${HOST_FILTER} AND timestamp >= now() - INTERVAL 7 DAY + GROUP BY path ORDER BY hits DESC LIMIT 40`, + noResults: () => ` + SELECT lower(properties.query) AS q, count() AS n + FROM events WHERE event = 'search_no_result' AND ${HOST_FILTER} AND timestamp >= now() - INTERVAL 7 DAY AND q != '' + GROUP BY q ORDER BY n DESC LIMIT 40`, } as const; export type TrafficSection = keyof typeof QUERIES; @@ -204,11 +238,48 @@ export async function loadTrafficSection(section: TrafficSection): Promise ({ query: str(r[0]), count: num(r[1]), kind: str(Array.isArray(r[2]) ? r[2][0] : r[2]), url: str(Array.isArray(r[3]) ? r[3][0] : r[3]) })) }; case "copies": return { copies: rows.map((r) => ({ kind: str(r[0]) || "other", value: str(r[1]), bench: str(r[2]), count: num(r[3]) })) }; + case "vitals": + return { vitals: rows.map((r) => ({ device: str(r[0]) || "unknown", samples: num(r[1]), lcpP75: num(r[2]), inpP75: num(r[3]), clsP75: num(r[4]), fcpP75: num(r[5]) })) }; + case "engaged": { + const pages = rows.map((r) => ({ path: str(r[0]) || "/", section: classifyPath(str(r[0])), leaves: num(r[1]), medianSec: num(r[2]), p75Sec: num(r[3]) })); + return { engagedPages: pages.filter((p) => p.leaves >= 3).slice(0, 40), engaged: engagedBySection(pages) }; + } + case "notFound": + return { notFound: rows.map((r) => ({ path: str(r[0]) || "/", hits: num(r[1]), visitors: num(r[2]), topReferrer: str(Array.isArray(r[3]) ? r[3][0] : r[3]) })) }; + case "noResults": + return { noResults: rows.map((r) => ({ query: str(r[0]), count: num(r[1]) })) }; case "engagement": return { engagement: { pagesPerSession: num(rows[0]?.[0]), bounceRate: num(rows[0]?.[1]), sessions: num(rows[0]?.[2]) } }; } } +/** Per-section engaged time: the leave-weighted median of the page medians + * (the exact section median would need every duration, 1500 page rows is the + * budget). Good enough to rank sections, labelled as such on the page. */ +export function engagedBySection(pages: { section: Section; leaves: number; medianSec: number; p75Sec: number }[]): EngagedRow[] { + const by = new Map(); + for (const p of pages) { + const cur = by.get(p.section) ?? { leaves: 0, meds: [], p75s: [] }; + cur.leaves += p.leaves; + cur.meds.push([p.medianSec, p.leaves]); + cur.p75s.push([p.p75Sec, p.leaves]); + by.set(p.section, cur); + } + const wmedian = (xs: [number, number][]) => { + const sorted = [...xs].sort((a, b) => a[0] - b[0]); + const total = sorted.reduce((a, x) => a + x[1], 0); + let acc = 0; + for (const [v, w] of sorted) { + acc += w; + if (acc >= total / 2) return v; + } + return sorted.at(-1)?.[0] ?? 0; + }; + return [...by.entries()] + .map(([section, v]) => ({ section, leaves: v.leaves, medianSec: wmedian(v.meds), p75Sec: wmedian(v.p75s) })) + .sort((a, b) => b.leaves - a.leaves); +} + function withShare(rows: { name: string; visitors: number }[]): NamedCount[] { const total = rows.reduce((a, r) => a + r.visitors, 0); return rows.map((r) => ({ ...r, share: total > 0 ? r.visitors / total : 0 })); diff --git a/crm/lib/vercel-logs.ts b/crm/lib/vercel-logs.ts new file mode 100644 index 000000000..5a83801b7 --- /dev/null +++ b/crm/lib/vercel-logs.ts @@ -0,0 +1,303 @@ +/** + * Vercel Log Drain receiver: the server side the browser SDK cannot see. + * AI crawlers (GPTBot, ClaudeBot, PerplexityBot, …), search bots, 404s, + * hits on /api/stat, /api/citable and llms.txt, cache hit ratio. + * + * Every request Vercel serves arrives as one JSON entry with a `proxy` + * block. Entries are folded into a per-UTC-day aggregate in memory and + * flushed to SNAPSHOT_DIR/vercel/YYYY-MM-DD.json once a minute (merged with + * what is already on disk, so a restart loses at most a minute). Nothing + * per-request is stored: no IPs, no full user agents. + * + * Setup (Vercel dashboard, team owner): Settings → Log Drains → add, + * source "Request logs" (proxy), format JSON, endpoint + * https:///api/ingest/vercel, custom secret = VERCEL_LOG_DRAIN_SECRET. + * Vercel verifies the endpoint once with `x-vercel-verify`; the value it + * expects back is VERCEL_LOG_DRAIN_VERIFY. + */ +import { createHmac, timingSafeEqual } from "node:crypto"; +import { promises as fs } from "node:fs"; +import path from "node:path"; +import { classifyPath, type Section } from "@/lib/channels"; + +const DIR = path.join(process.env.SNAPSHOT_DIR ?? path.join(process.cwd(), ".snapshots"), "vercel"); +const SITE_HOST = process.env.SITE_HOST ?? "openchainbench.com"; + +export const AI_BOTS: [RegExp, string][] = [ + [/GPTBot/i, "GPTBot"], + [/ChatGPT-User/i, "ChatGPT-User"], + [/OAI-SearchBot/i, "OAI-SearchBot"], + [/ClaudeBot|Claude-Web|anthropic-ai|Claude-User|Claude-SearchBot/i, "ClaudeBot"], + [/PerplexityBot|Perplexity-User/i, "PerplexityBot"], + [/Google-Extended/i, "Google-Extended"], + [/GoogleOther/i, "GoogleOther"], + [/Applebot-Extended/i, "Applebot-Extended"], + [/Bytespider/i, "Bytespider"], + [/CCBot/i, "CCBot"], + [/cohere-ai|cohere-training/i, "Cohere"], + [/Amazonbot/i, "Amazonbot"], + [/meta-externalagent|meta-externalfetcher|FacebookBot/i, "Meta"], + [/Diffbot/i, "Diffbot"], + [/YouBot/i, "YouBot"], + [/DuckAssistBot/i, "DuckAssistBot"], + [/MistralAI-User/i, "MistralAI"], + [/xAI-Grok|GrokBot/i, "Grok"], + [/Timpibot/i, "Timpibot"], + [/omgili|webzio/i, "Webz"], +]; +export const SEARCH_BOTS: [RegExp, string][] = [ + [/Googlebot|Google-InspectionTool|Storebot-Google|AdsBot-Google|Mediapartners-Google/i, "Googlebot"], + [/bingbot|BingPreview|msnbot/i, "Bingbot"], + [/DuckDuckBot/i, "DuckDuckBot"], + [/YandexBot|YandexImages/i, "Yandex"], + [/Baiduspider/i, "Baidu"], + [/Applebot/i, "Applebot"], + [/Slurp/i, "Yahoo"], + [/SeznamBot/i, "Seznam"], + [/PetalBot/i, "PetalBot"], +]; +const GENERIC_BOT = /bot|crawl|spider|slurp|fetch|curl\/|wget|python-requests|python-urllib|aiohttp|httpx|go-http-client|java\/|okhttp|axios|node-fetch|undici|libwww|scrapy|headless|phantom|lighthouse|pagespeed|uptime|monitor|pingdom|datadog|newrelic|ahrefs|semrush|mj12|dotbot|screaming/i; + +export type UaClass = { kind: "human" | "ai_bot" | "search_bot" | "other_bot"; name: string }; + +export function classifyUserAgent(ua: string | null | undefined): UaClass { + const s = ua ?? ""; + if (!s) return { kind: "other_bot", name: "empty" }; + for (const [re, name] of AI_BOTS) if (re.test(s)) return { kind: "ai_bot", name }; + for (const [re, name] of SEARCH_BOTS) if (re.test(s)) return { kind: "search_bot", name }; + if (GENERIC_BOT.test(s)) return { kind: "other_bot", name: "other" }; + return { kind: "human", name: "human" }; +} + +/** Paths the machine side of the site serves; everything else is "page". */ +export function apiFamily(p: string): string | null { + if (p.startsWith("/api/stat")) return "/api/stat"; + if (p.startsWith("/api/citable")) return "/api/citable"; + if (p === "/llms.txt" || p === "/llms-full.txt") return "/llms.txt"; + if (p.startsWith("/api/og")) return "/api/og"; + if (p.endsWith("sitemap.xml") || p === "/robots.txt") return "/sitemap+robots"; + if (p === "/rss.xml" || p === "/feed.json") return "/feeds"; + if (p.startsWith("/api/mcp") || p.startsWith("/mcp/")) return "/api/mcp"; + if (p.startsWith("/api/")) return "/api/other"; + return null; +} + +export type DayAggregate = { + day: string; + requests: number; + byClass: Record; + aiBots: Record; + searchBots: Record; + aiBySection: Record; + humanBySection: Record; + status: Record; + notFound: Record; + api: Record>; + cache: Record; + updatedAt: string; +}; + +export function emptyDay(day: string): DayAggregate { + return { day, requests: 0, byClass: {}, aiBots: {}, searchBots: {}, aiBySection: {}, humanBySection: {}, status: {}, notFound: {}, api: {}, cache: {}, updatedAt: new Date().toISOString() }; +} + +const inc = (m: Record, k: string, n = 1) => { + m[k] = (m[k] ?? 0) + n; +}; + +/** One Vercel drain entry (only the fields read here). */ +export type DrainEntry = { + id?: string; + requestId?: string; + timestamp?: number; + source?: string; + host?: string; + path?: string; + statusCode?: number; + proxy?: { + timestamp?: number; + path?: string; + host?: string; + userAgent?: string[] | string; + statusCode?: number; + vercelCache?: string; + pathType?: string; + }; +}; + +export function foldEntry(agg: DayAggregate, e: DrainEntry): boolean { + const p = e.proxy; + if (!p) return false; + const host = (p.host ?? e.host ?? "").toLowerCase(); + if (host && host !== SITE_HOST && host !== `www.${SITE_HOST}`) return false; + const rawPath = (p.path ?? e.path ?? "/").split("?")[0]; + if (rawPath.startsWith("/_next/") || rawPath.startsWith("/ingest/") || /\.(js|css|png|jpg|jpeg|svg|ico|woff2?|map|webp|avif|gif)$/i.test(rawPath)) return false; + const ua = Array.isArray(p.userAgent) ? p.userAgent.join(" ") : (p.userAgent ?? ""); + const cls = classifyUserAgent(ua); + const status = p.statusCode ?? e.statusCode ?? 0; + agg.requests += 1; + inc(agg.byClass, cls.kind); + inc(agg.status, String(status || "0")); + if (p.vercelCache) inc(agg.cache, p.vercelCache); + const fam = apiFamily(rawPath); + if (fam) { + agg.api[fam] ??= {}; + inc(agg.api[fam], cls.kind); + } + const section: Section = classifyPath(rawPath); + if (cls.kind === "ai_bot") { + inc(agg.aiBots, cls.name); + if (!fam) inc(agg.aiBySection, section); + } else if (cls.kind === "search_bot") { + inc(agg.searchBots, cls.name); + } else if (cls.kind === "human" && !fam) { + inc(agg.humanBySection, section); + } + if (status === 404 && Object.keys(agg.notFound).length < 500) inc(agg.notFound, rawPath.slice(0, 160)); + else if (status === 404) inc(agg.notFound, "(other)"); + agg.updatedAt = new Date().toISOString(); + return true; +} + +export function mergeDay(a: DayAggregate, b: DayAggregate): DayAggregate { + const out = emptyDay(a.day); + out.requests = a.requests + b.requests; + const mergeMap = (x: Record, y: Record) => { + const m: Record = { ...x }; + for (const [k, v] of Object.entries(y)) inc(m, k, v); + return m; + }; + out.byClass = mergeMap(a.byClass, b.byClass); + out.aiBots = mergeMap(a.aiBots, b.aiBots); + out.searchBots = mergeMap(a.searchBots, b.searchBots); + out.aiBySection = mergeMap(a.aiBySection, b.aiBySection); + out.humanBySection = mergeMap(a.humanBySection, b.humanBySection); + out.status = mergeMap(a.status, b.status); + out.notFound = mergeMap(a.notFound, b.notFound); + out.cache = mergeMap(a.cache, b.cache); + for (const fam of new Set([...Object.keys(a.api), ...Object.keys(b.api)])) out.api[fam] = mergeMap(a.api[fam] ?? {}, b.api[fam] ?? {}); + out.updatedAt = a.updatedAt > b.updatedAt ? a.updatedAt : b.updatedAt; + return out; +} + +// In-memory accumulator on globalThis (route handlers and the flusher can +// live in different bundler layers), plus a bounded request-id set so a +// request that produces two entries (proxy plus function log) counts once. +type State = { days: Map; seen: Set; seenOrder: string[]; flushTimer: ReturnType | null; received: number }; +const g = globalThis as unknown as { __ocbVercelLogs?: State }; +const state: State = (g.__ocbVercelLogs ??= { days: new Map(), seen: new Set(), seenOrder: [], flushTimer: null, received: 0 }); +const SEEN_MAX = 100_000; + +export function ingestEntries(entries: DrainEntry[], now = Date.now()): number { + let folded = 0; + for (const e of entries) { + const rid = e.requestId ?? e.id; + if (rid) { + if (state.seen.has(rid)) continue; + state.seen.add(rid); + state.seenOrder.push(rid); + if (state.seenOrder.length > SEEN_MAX) { + const old = state.seenOrder.splice(0, state.seenOrder.length - SEEN_MAX); + for (const o of old) state.seen.delete(o); + } + } + const ts = e.proxy?.timestamp ?? e.timestamp ?? now; + const day = new Date(ts).toISOString().slice(0, 10); + const agg = state.days.get(day) ?? emptyDay(day); + if (foldEntry(agg, e)) { + state.days.set(day, agg); + folded += 1; + } + } + state.received += entries.length; + armFlush(); + return folded; +} + +function armFlush(): void { + if (state.flushTimer) return; + state.flushTimer = setInterval(() => { + flush().catch((e) => console.warn("[vercel-logs] flush:", e)); + }, 60_000); + state.flushTimer.unref?.(); +} + +export async function flush(): Promise { + if (state.days.size === 0) return; + const pending = [...state.days.values()]; + state.days.clear(); + await fs.mkdir(DIR, { recursive: true }); + for (const agg of pending) { + const file = path.join(DIR, `${agg.day}.json`); + let onDisk: DayAggregate | null = null; + try { + onDisk = JSON.parse(await fs.readFile(file, "utf8")) as DayAggregate; + } catch { + // first write of the day + } + const merged = onDisk ? mergeDay(onDisk, agg) : agg; + const tmp = `${file}.${process.pid}.tmp`; + await fs.writeFile(tmp, JSON.stringify(merged)); + await fs.rename(tmp, file); + } +} + +export async function readDays(n = 28, now = Date.now()): Promise { + const out: DayAggregate[] = []; + for (let i = n - 1; i >= 0; i -= 1) { + const day = new Date(now - i * 86_400_000).toISOString().slice(0, 10); + let agg: DayAggregate | null = null; + try { + agg = JSON.parse(await fs.readFile(path.join(DIR, `${day}.json`), "utf8")) as DayAggregate; + } catch { + // no traffic recorded that day + } + const live = state.days.get(day); + if (agg && live) agg = mergeDay(agg, live); + else if (live) agg = live; + if (agg) out.push(agg); + } + return out; +} + +export function drainConfigured(): boolean { + return (process.env.VERCEL_LOG_DRAIN_SECRET ?? "").length >= 16; +} + +/** Vercel signs the raw body with HMAC-SHA1 under the drain secret (`x-vercel-signature`). */ +export function verifySignature(rawBody: string, signature: string | null): boolean { + const secret = process.env.VERCEL_LOG_DRAIN_SECRET ?? ""; + if (!signature || secret.length < 16) return false; + const expected = createHmac("sha1", secret).update(rawBody).digest("hex"); + const a = Buffer.from(expected); + const b = Buffer.from(signature.trim().toLowerCase()); + return a.length === b.length && timingSafeEqual(a, b); +} + +/** Vercel drains send either a JSON array or newline-delimited JSON. */ +export function parseBody(raw: string): DrainEntry[] { + const t = raw.trim(); + if (!t) return []; + if (t.startsWith("[")) { + try { + const arr = JSON.parse(t); + return Array.isArray(arr) ? (arr as DrainEntry[]) : []; + } catch { + return []; + } + } + const out: DrainEntry[] = []; + for (const line of t.split("\n")) { + if (!line.trim()) continue; + try { + out.push(JSON.parse(line) as DrainEntry); + } catch { + // skip a torn line + } + } + return out; +} + +export function drainStats(): { received: number; pendingDays: number } { + return { received: state.received, pendingDays: state.days.size }; +} diff --git a/crm/proxy.ts b/crm/proxy.ts index c6e6539f4..c5545e3e3 100644 --- a/crm/proxy.ts +++ b/crm/proxy.ts @@ -5,7 +5,8 @@ import { COOKIE, isValidSession } from "@/lib/auth"; // Static assets are excluded by the matcher. export async function proxy(request: NextRequest) { const { pathname } = request.nextUrl; - if (pathname === "/login" || pathname === "/api/login") return NextResponse.next(); + // /api/ingest/* is authenticated by the sender's signature (see the route). + if (pathname === "/login" || pathname === "/api/login" || pathname.startsWith("/api/ingest/")) return NextResponse.next(); const ok = await isValidSession(request.cookies.get(COOKIE)?.value); if (ok) return NextResponse.next(); if (pathname.startsWith("/api/")) return NextResponse.json({ error: "unauthorized" }, { status: 401 }); diff --git a/crm/test/traffic.test.ts b/crm/test/traffic.test.ts index e8cd0c735..8cfc9e741 100644 --- a/crm/test/traffic.test.ts +++ b/crm/test/traffic.test.ts @@ -1,16 +1,16 @@ import { describe, expect, test } from "bun:test"; -import { channelTotals, QUERIES, sectionTotals, sumWindow, TRAFFIC_SECTIONS } from "../lib/traffic"; +import { channelTotals, engagedBySection, QUERIES, sectionTotals, sumWindow, TRAFFIC_SECTIONS } from "../lib/traffic"; describe("queries", () => { test("every query is scoped to the production host and to a named event", () => { for (const name of TRAFFIC_SECTIONS) { const q = QUERIES[name](); expect(q).toContain("properties.$host = 'openchainbench.com'"); - expect(/event (= '\$pageview'|= '(outbound_click|search|copy)'|IN \('outbound_click', 'copy', 'search'\))/.test(q)).toBe(true); + expect(/event (= '\$pageview'|= '\$pageleave'|= '\$web_vitals'|= '(outbound_click|search|copy|not_found|search_no_result)'|IN \('outbound_click', 'copy', 'search'\))/.test(q)).toBe(true); } }); test("the refresh spends a bounded number of queries", () => { - expect(TRAFFIC_SECTIONS.length).toBeLessThanOrEqual(16); + expect(TRAFFIC_SECTIONS.length).toBeLessThanOrEqual(24); }); test("the weekly series and the totals embed the AI domain list", () => { expect(QUERIES.weekly()).toContain("'chatgpt.com'"); @@ -44,6 +44,15 @@ describe("aggregations", () => { expect(c[0]).toEqual({ channel: "direct", visitors: 20, prevVisitors: 25, domains: 1 }); expect(c[1]).toEqual({ channel: "ai", visitors: 5, prevVisitors: 1, domains: 2 }); }); + test("engagedBySection weights page medians by leaves", () => { + const rows = engagedBySection([ + { section: "rpc", leaves: 10, medianSec: 20, p75Sec: 60 }, + { section: "rpc", leaves: 90, medianSec: 5, p75Sec: 15 }, + { section: "compare", leaves: 1, medianSec: 100, p75Sec: 200 }, + ]); + expect(rows[0]).toEqual({ section: "rpc", leaves: 100, medianSec: 5, p75Sec: 15 }); + expect(rows[1].section).toBe("compare"); + }); test("sumWindow takes the last N days, with an offset", () => { const daily = [1, 2, 3, 4].map((i) => ({ day: `2026-09-0${i}`, pageviews: i, visitors: i, sessions: i })); expect(sumWindow(daily, 2).pageviews).toBe(7); diff --git a/crm/test/vercel-logs.test.ts b/crm/test/vercel-logs.test.ts new file mode 100644 index 000000000..ce9bd426b --- /dev/null +++ b/crm/test/vercel-logs.test.ts @@ -0,0 +1,67 @@ +import { describe, expect, test } from "bun:test"; +import { apiFamily, classifyUserAgent, emptyDay, foldEntry, mergeDay, parseBody } from "../lib/vercel-logs"; + +describe("classifyUserAgent", () => { + test("AI, search, other, human", () => { + expect(classifyUserAgent("Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.2; +https://openai.com/gptbot)")).toEqual({ kind: "ai_bot", name: "GPTBot" }); + expect(classifyUserAgent("Mozilla/5.0 (compatible; ClaudeBot/1.0; +claudebot@anthropic.com)").name).toBe("ClaudeBot"); + expect(classifyUserAgent("Mozilla/5.0 (compatible; PerplexityBot/1.0)").kind).toBe("ai_bot"); + expect(classifyUserAgent("Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)")).toEqual({ kind: "search_bot", name: "Googlebot" }); + expect(classifyUserAgent("Mozilla/5.0 (compatible; Google-Extended)").kind).toBe("ai_bot"); + expect(classifyUserAgent("curl/8.4.0").kind).toBe("other_bot"); + expect(classifyUserAgent("python-requests/2.31").kind).toBe("other_bot"); + expect(classifyUserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 14_5) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.5 Safari/605.1.15").kind).toBe("human"); + expect(classifyUserAgent("").kind).toBe("other_bot"); + }); +}); + +describe("apiFamily", () => { + test("machine paths", () => { + expect(apiFamily("/api/stat/bridge-fee")).toBe("/api/stat"); + expect(apiFamily("/llms-full.txt")).toBe("/llms.txt"); + expect(apiFamily("/sitemap.xml")).toBe("/sitemap+robots"); + expect(apiFamily("/benchmarks/bridge-fee")).toBeNull(); + }); +}); + +describe("foldEntry and mergeDay", () => { + const entry = (path: string, ua: string, status = 200, host = "openchainbench.com") => ({ + proxy: { path, userAgent: [ua], statusCode: status, host, vercelCache: "HIT", timestamp: 1_758_000_000_000 }, + }); + test("counts classes, sections, api families, 404s; skips assets and other hosts", () => { + const d = emptyDay("2026-09-20"); + expect(foldEntry(d, entry("/benchmarks/arbitrum-rpc", "GPTBot/1.2"))).toBe(true); + expect(foldEntry(d, entry("/api/stat/bridge-fee", "ClaudeBot/1.0"))).toBe(true); + expect(foldEntry(d, entry("/missing", "Mozilla/5.0 Safari/605.1.15", 404))).toBe(true); + expect(foldEntry(d, entry("/_next/static/x.js", "GPTBot"))).toBe(false); + expect(foldEntry(d, entry("/", "GPTBot", 200, "staging-openchainbench.vercel.app"))).toBe(false); + expect(foldEntry(d, { message: "no proxy" } as never)).toBe(false); + expect(d.requests).toBe(3); + expect(d.byClass).toEqual({ ai_bot: 2, human: 1 }); + expect(d.aiBots).toEqual({ GPTBot: 1, ClaudeBot: 1 }); + expect(d.aiBySection).toEqual({ rpc: 1 }); + expect(d.api["/api/stat"]).toEqual({ ai_bot: 1 }); + expect(d.notFound).toEqual({ "/missing": 1 }); + expect(d.status).toEqual({ "200": 2, "404": 1 }); + expect(d.cache).toEqual({ HIT: 3 }); + }); + test("mergeDay adds every map", () => { + const a = emptyDay("2026-09-20"); + const b = emptyDay("2026-09-20"); + foldEntry(a, entry("/x", "GPTBot")); + foldEntry(b, entry("/y", "GPTBot")); + foldEntry(b, entry("/api/citable", "curl/8")); + const m = mergeDay(a, b); + expect(m.requests).toBe(3); + expect(m.aiBots).toEqual({ GPTBot: 2 }); + expect(m.api["/api/citable"]).toEqual({ other_bot: 1 }); + }); +}); + +describe("parseBody", () => { + test("array and ndjson", () => { + expect(parseBody('[{"id":"1"},{"id":"2"}]').length).toBe(2); + expect(parseBody('{"id":"1"}\n{"id":"2"}\nnot json\n').length).toBe(2); + expect(parseBody("")).toEqual([]); + }); +}); diff --git a/src/app/not-found.tsx b/src/app/not-found.tsx index 354c07764..f840b0807 100644 --- a/src/app/not-found.tsx +++ b/src/app/not-found.tsx @@ -1,6 +1,7 @@ import type { Metadata } from "next"; import Link from "next/link"; import { ArrowLeft } from "lucide-react"; +import { NotFoundPing } from "@/components/not-found-ping"; export const metadata: Metadata = { title: "Page not found", @@ -20,6 +21,7 @@ export const metadata: Metadata = { export default function NotFound() { return (
+

404

This page does not exist. diff --git a/src/components/not-found-ping.tsx b/src/components/not-found-ping.tsx new file mode 100644 index 000000000..3bb710a3f --- /dev/null +++ b/src/components/not-found-ping.tsx @@ -0,0 +1,17 @@ +"use client"; + +import { useEffect } from "react"; +import { track } from "@/lib/analytics"; + +/** One `not_found` event per 404 render: the path that missed and where + * the visitor came from, so broken inbound links and dead internal links + * surface on the CRM instead of in a crawl report weeks later. */ +export function NotFoundPing() { + useEffect(() => { + track("not_found", { + path: window.location.pathname, + referrer: document.referrer ? new URL(document.referrer).host : "", + }); + }, []); + return null; +} diff --git a/src/components/search/search-dialog.tsx b/src/components/search/search-dialog.tsx index da1e457b1..ffe7388b6 100644 --- a/src/components/search/search-dialog.tsx +++ b/src/components/search/search-dialog.tsx @@ -261,6 +261,14 @@ export default function SearchDialog() { const isSearching = trimmed.length > 0; const showEmpty = isSearching && results.length === 0; + // A query that matched nothing, once the visitor stopped typing (800 ms), + // once per query: the content gaps the search box reveals. + useEffect(() => { + if (!showEmpty || indexStatus !== "ready" || trimmed.length < 3) return; + const t = window.setTimeout(() => track("search_no_result", { query: trimmed.slice(0, 80) }), 800); + return () => window.clearTimeout(t); + }, [showEmpty, indexStatus, trimmed]); + return (
(name: N, props: Extract["props"]): void { From 4f3380cfc7694f14bc63075aa187ad01bb135970 Mon Sep 17 00:00:00 2001 From: Flotapponnier Date: Sun, 20 Sep 2026 20:55:07 +0200 Subject: [PATCH 2/2] crm, review: 404 map capped on fold, merge and read; flush swaps the day out before writing and merges it back on failure; NEXT_MANUAL_SIG_HANDLE so the shutdown flush runs; content-length guard on the drain route; one status key for Search Console; page keys folded across www/http variants Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01HJgbZCqjR4nvCfcJSzofbw --- crm/Dockerfile | 4 ++ crm/app/api/ingest/vercel/route.ts | 2 + crm/app/search/page.tsx | 6 +-- crm/components/shell.tsx | 5 +- crm/instrumentation.node.ts | 8 ++- crm/lib/gsc.ts | 42 ++++++++++++--- crm/lib/snapshot.ts | 10 ++-- crm/lib/vercel-logs.ts | 83 +++++++++++++++++++++++------- crm/test/gsc.test.ts | 17 ++++++ crm/test/vercel-logs.test.ts | 18 ++++++- 10 files changed, 158 insertions(+), 37 deletions(-) create mode 100644 crm/test/gsc.test.ts diff --git a/crm/Dockerfile b/crm/Dockerfile index d2800ef3f..19639ed37 100644 --- a/crm/Dockerfile +++ b/crm/Dockerfile @@ -16,6 +16,10 @@ FROM node:22-alpine AS run WORKDIR /app ENV NODE_ENV=production ENV HOSTNAME=0.0.0.0 +# Next's standalone server exits on SIGTERM on its own; with this set it leaves +# the signal to instrumentation.node.ts, which flushes the log-drain +# accumulator to the volume before exiting. +ENV NEXT_MANUAL_SIG_HANDLE=1 COPY --from=build /app/.next/standalone ./ COPY --from=build /app/.next/static ./.next/static COPY --from=build /app/public ./public diff --git a/crm/app/api/ingest/vercel/route.ts b/crm/app/api/ingest/vercel/route.ts index 1bf98a53e..92d0c9672 100644 --- a/crm/app/api/ingest/vercel/route.ts +++ b/crm/app/api/ingest/vercel/route.ts @@ -21,6 +21,8 @@ export async function HEAD() { export async function POST(request: Request) { if (!drainConfigured()) return new Response("drain not configured", { status: 503, headers: verifyHeaders() }); + const declared = Number.parseInt(request.headers.get("content-length") ?? "", 10); + if (Number.isFinite(declared) && declared > MAX_BODY) return new Response("too large", { status: 413 }); const raw = await request.text(); if (raw.length > MAX_BODY) return new Response("too large", { status: 413 }); if (!verifySignature(raw, request.headers.get("x-vercel-signature"))) { diff --git a/crm/app/search/page.tsx b/crm/app/search/page.tsx index 68fbd1931..0a1e14354 100644 --- a/crm/app/search/page.tsx +++ b/crm/app/search/page.tsx @@ -1,6 +1,6 @@ import { Shell } from "@/components/shell"; import { Delta, Empty, fmtInt, fmtPct, Kpi, Spark } from "@/components/ui"; -import { readSnapshot } from "@/lib/snapshot"; +import { GSC_UNSET, readSnapshot } from "@/lib/snapshot"; export const dynamic = "force-dynamic"; const SITE = `https://${process.env.SITE_HOST ?? "openchainbench.com"}`; @@ -20,9 +20,9 @@ export default async function SearchPage({ searchParams }: { searchParams: Promi property in Search Console, then set GSC_SERVICE_ACCOUNT_JSON (the key file on one line) and{" "} GSC_SITE_URL on the Railway service. The refresh after that fills this page.

- {snap.status.gsc?.error && !snap.status.gsc.error.startsWith("GSC_SERVICE_ACCOUNT_JSON") && ( + {snap.status.gsc?.error && snap.status.gsc.error !== GSC_UNSET && (

- {snap.status.gsc.error} + Last attempt failed: {snap.status.gsc.error}

)}

diff --git a/crm/components/shell.tsx b/crm/components/shell.tsx index 37ea73188..dfa6edfbf 100644 --- a/crm/components/shell.tsx +++ b/crm/components/shell.tsx @@ -1,5 +1,5 @@ import Link from "next/link"; -import { MANUAL_COOLDOWN_MINUTES, REFRESH_MINUTES, snapshotAgeMinutes, type Snapshot } from "@/lib/snapshot"; +import { GSC_UNSET, MANUAL_COOLDOWN_MINUTES, REFRESH_MINUTES, snapshotAgeMinutes, type Snapshot } from "@/lib/snapshot"; const NAV = [ ["/", "Overview"], @@ -21,7 +21,8 @@ export function fmtAge(min: number | null): string { export function Shell({ current, snapshot, refreshFlag, children }: { current: string; snapshot: Snapshot; refreshFlag?: string; children: React.ReactNode }) { const age = snapshotAgeMinutes(snapshot); - const errors = Object.entries(snapshot.status).filter(([k, s]) => s.error && k !== "gsc"); + // A missing optional integration is a note on its own page, not a failure. + const errors = Object.entries(snapshot.status).filter(([, s]) => s.error && s.error !== GSC_UNSET); const canRefresh = age == null || age >= MANUAL_COOLDOWN_MINUTES; return (
diff --git a/crm/instrumentation.node.ts b/crm/instrumentation.node.ts index b558d81df..b75497e22 100644 --- a/crm/instrumentation.node.ts +++ b/crm/instrumentation.node.ts @@ -11,9 +11,15 @@ import { readSnapshot, refreshSnapshot, REFRESH_MINUTES, snapshotAgeMinutes } fr import { flush } from "@/lib/vercel-logs"; export async function start(): Promise { + // NEXT_MANUAL_SIG_HANDLE=1 (Dockerfile) keeps Next from exiting on its own + // signal handler before this flush has run. Hard stop after 8 s regardless. for (const sig of ["SIGTERM", "SIGINT"] as const) { process.once(sig, () => { - flush().finally(() => process.exit(0)); + const stop = setTimeout(() => process.exit(0), 8_000); + stop.unref(); + flush() + .catch((e) => console.warn("[shutdown] flush:", e)) + .finally(() => process.exit(0)); }); } if (process.env.CRM_DISABLE_SCHEDULER === "1") return; diff --git a/crm/lib/gsc.ts b/crm/lib/gsc.ts index bd40e4041..83285e0cc 100644 --- a/crm/lib/gsc.ts +++ b/crm/lib/gsc.ts @@ -83,6 +83,17 @@ export type Gsc = { opportunities: GscDim[]; }; +/** Page keys come back as absolute URLs; for a domain property Google reports + * every host and scheme (www, http). All of them map to the site path. */ +export function pageKeyStripper(siteUrl: string): (k: string) => string { + const domain = siteUrl.startsWith("sc-domain:") ? siteUrl.slice("sc-domain:".length) : new URL(siteUrl).host.replace(/^www\./, ""); + const re = new RegExp(`^https?://(?:[a-z0-9-]+\\.)*${domain.replace(/\./g, "\\.")}(?::\\d+)?`, "i"); + return (k: string) => { + const stripped = k.replace(re, ""); + return stripped === k ? k : stripped || "/"; + }; +} + function totals(rows: Row[]): GscTotals { const clicks = rows.reduce((a, r) => a + r.clicks, 0); const impressions = rows.reduce((a, r) => a + r.impressions, 0); @@ -90,13 +101,29 @@ function totals(rows: Row[]): GscTotals { return { clicks, impressions, ctr: impressions > 0 ? clicks / impressions : 0, position }; } +/** Rows keyed after stripping (www and http variants of a page fold into one), + * clicks and impressions summed, position impression-weighted. */ +function foldRows(rows: Row[], strip: (k: string) => string): Map { + const out = new Map(); + for (const r of rows) { + const k = strip(r.keys?.[0] ?? ""); + const cur = out.get(k) ?? { clicks: 0, impressions: 0, posW: 0 }; + cur.clicks += r.clicks; + cur.impressions += r.impressions; + cur.posW += r.position * r.impressions; + out.set(k, cur); + } + return new Map( + [...out.entries()].map(([k, v]) => [k, { clicks: v.clicks, impressions: v.impressions, ctr: v.impressions > 0 ? v.clicks / v.impressions : 0, position: v.impressions > 0 ? v.posW / v.impressions : 0 }]), + ); +} + function joinWindows(cur: Row[], prev: Row[], strip: (k: string) => string): GscDim[] { - const prevBy = new Map(prev.map((r) => [r.keys?.[0] ?? "", r])); - return cur - .map((r) => { - const k = r.keys?.[0] ?? ""; - const p = prevBy.get(k); - return { key: strip(k), clicks: r.clicks, impressions: r.impressions, ctr: r.ctr, position: r.position, prevClicks: p?.clicks ?? 0, prevImpressions: p?.impressions ?? 0, prevPosition: p?.position ?? 0 }; + const prevBy = foldRows(prev, strip); + return [...foldRows(cur, strip).entries()] + .map(([key, v]) => { + const p = prevBy.get(key); + return { key, ...v, prevClicks: p?.clicks ?? 0, prevImpressions: p?.impressions ?? 0, prevPosition: p?.position ?? 0 }; }) .sort((a, b) => b.clicks - a.clicks || b.impressions - a.impressions); } @@ -107,8 +134,7 @@ export async function loadGsc(now = Date.now()): Promise { const prevEnd = daysAgo(DELAY_DAYS + WINDOW_DAYS, now); const prevStart = daysAgo(DELAY_DAYS + 2 * WINDOW_DAYS - 1, now); const seriesStart = daysAgo(DELAY_DAYS + 27, now); - const origin = SITE_URL.startsWith("sc-domain:") ? `https://${SITE_URL.slice("sc-domain:".length)}` : SITE_URL.replace(/\/$/, ""); - const stripOrigin = (k: string) => (k.startsWith(origin) ? k.slice(origin.length) || "/" : k); + const stripOrigin = pageKeyStripper(SITE_URL); const daily = await query({ startDate: seriesStart, endDate: end, dimensions: ["date"], rowLimit: 60 }); const pagesCur = await query({ startDate: start, endDate: end, dimensions: ["page"], rowLimit: 500 }); diff --git a/crm/lib/snapshot.ts b/crm/lib/snapshot.ts index 9ca44ed9a..68d3945d0 100644 --- a/crm/lib/snapshot.ts +++ b/crm/lib/snapshot.ts @@ -16,6 +16,8 @@ import { loadBenchHealth, loadDuneUsage, loadHarnessHealth, type BenchHealth, ty import { loadTrafficSection, TRAFFIC_SECTIONS, type Traffic } from "@/lib/traffic"; import { gscConfigured, loadGsc, type Gsc } from "@/lib/gsc"; +export const GSC_UNSET = "GSC_SERVICE_ACCOUNT_JSON not set"; + // 15 min by default: 15 queries per pass, 60 per hour, 2.5 % of PostHog's // organisation budget; the Railway cost does not move with this number, the // container is always on and a pass is about 40 s of light CPU. @@ -185,12 +187,14 @@ async function doRefresh(reason: string): Promise { next.dune = await loadDuneUsage(); }); if (gscConfigured()) { - delete next.status.gsc; - await step("search-console", async () => { + await step("gsc", async () => { next.gsc = await loadGsc(); }); } else { - next.status.gsc = { at: null, error: "GSC_SERVICE_ACCOUNT_JSON not set (Search Console panel empty)" }; + // Unconfigured is a note, not a failure (the header filters this key); + // a previously loaded panel is dropped so it cannot go stale silently. + next.gsc = null; + next.status.gsc = { at: null, error: GSC_UNSET }; } if (posthogConfigured()) { diff --git a/crm/lib/vercel-logs.ts b/crm/lib/vercel-logs.ts index 5a83801b7..013aed07f 100644 --- a/crm/lib/vercel-logs.ts +++ b/crm/lib/vercel-logs.ts @@ -153,12 +153,28 @@ export function foldEntry(agg: DayAggregate, e: DrainEntry): boolean { } else if (cls.kind === "human" && !fam) { inc(agg.humanBySection, section); } - if (status === 404 && Object.keys(agg.notFound).length < 500) inc(agg.notFound, rawPath.slice(0, 160)); - else if (status === 404) inc(agg.notFound, "(other)"); + if (status === 404) { + inc(agg.notFound, rawPath.slice(0, 160)); + if (Object.keys(agg.notFound).length > NOT_FOUND_KEEP * 2) agg.notFound = capMap(agg.notFound, NOT_FOUND_KEEP); + } agg.updatedAt = new Date().toISOString(); return true; } +/** The map is bounded wherever it is built: live fold, merge on flush, merge on + * read. A scanner walking distinct paths otherwise grows a day file without + * limit. The top N by count survive, the rest is summed under "(other)". */ +export const NOT_FOUND_KEEP = 300; +export function capMap(m: Record, keep: number): Record { + const entries = Object.entries(m).filter(([k]) => k !== "(other)"); + if (entries.length <= keep) return m; + entries.sort((a, b) => b[1] - a[1]); + const out: Record = Object.fromEntries(entries.slice(0, keep)); + const rest = entries.slice(keep).reduce((a, [, v]) => a + v, 0) + (m["(other)"] ?? 0); + if (rest > 0) out["(other)"] = rest; + return out; +} + export function mergeDay(a: DayAggregate, b: DayAggregate): DayAggregate { const out = emptyDay(a.day); out.requests = a.requests + b.requests; @@ -173,7 +189,7 @@ export function mergeDay(a: DayAggregate, b: DayAggregate): DayAggregate { out.aiBySection = mergeMap(a.aiBySection, b.aiBySection); out.humanBySection = mergeMap(a.humanBySection, b.humanBySection); out.status = mergeMap(a.status, b.status); - out.notFound = mergeMap(a.notFound, b.notFound); + out.notFound = capMap(mergeMap(a.notFound, b.notFound), NOT_FOUND_KEEP); out.cache = mergeMap(a.cache, b.cache); for (const fam of new Set([...Object.keys(a.api), ...Object.keys(b.api)])) out.api[fam] = mergeMap(a.api[fam] ?? {}, b.api[fam] ?? {}); out.updatedAt = a.updatedAt > b.updatedAt ? a.updatedAt : b.updatedAt; @@ -183,9 +199,17 @@ export function mergeDay(a: DayAggregate, b: DayAggregate): DayAggregate { // In-memory accumulator on globalThis (route handlers and the flusher can // live in different bundler layers), plus a bounded request-id set so a // request that produces two entries (proxy plus function log) counts once. -type State = { days: Map; seen: Set; seenOrder: string[]; flushTimer: ReturnType | null; received: number }; +type State = { + days: Map; + seen: Set; + seenOrder: string[]; + flushTimer: ReturnType | null; + received: number; + flushing: Promise | null; + tmpCounter: number; +}; const g = globalThis as unknown as { __ocbVercelLogs?: State }; -const state: State = (g.__ocbVercelLogs ??= { days: new Map(), seen: new Set(), seenOrder: [], flushTimer: null, received: 0 }); +const state: State = (g.__ocbVercelLogs ??= { days: new Map(), seen: new Set(), seenOrder: [], flushTimer: null, received: 0, flushing: null, tmpCounter: 0 }); const SEEN_MAX = 100_000; export function ingestEntries(entries: DrainEntry[], now = Date.now()): number { @@ -222,23 +246,44 @@ function armFlush(): void { state.flushTimer.unref?.(); } -export async function flush(): Promise { +/** Writes the pending days to disk. A day leaves memory only once its file + * is renamed into place, so a failed write keeps the minute for the next + * pass; overlapping calls (interval plus shutdown) share one run. */ +export function flush(): Promise { + if (state.flushing) return state.flushing; + const run = doFlush().finally(() => { + state.flushing = null; + }); + state.flushing = run; + return run; +} + +async function doFlush(): Promise { if (state.days.size === 0) return; - const pending = [...state.days.values()]; - state.days.clear(); await fs.mkdir(DIR, { recursive: true }); - for (const agg of pending) { - const file = path.join(DIR, `${agg.day}.json`); - let onDisk: DayAggregate | null = null; + for (const day of [...state.days.keys()]) { + // Swap the live object out first: entries folded during the write land in + // a fresh aggregate, the one being written is never mutated. On a failed + // write it is merged back, so the minute is retried, never counted twice. + const pending = state.days.get(day)!; + state.days.delete(day); try { - onDisk = JSON.parse(await fs.readFile(file, "utf8")) as DayAggregate; - } catch { - // first write of the day + const file = path.join(DIR, `${day}.json`); + let onDisk: DayAggregate | null = null; + try { + onDisk = JSON.parse(await fs.readFile(file, "utf8")) as DayAggregate; + } catch { + // first write of the day + } + const merged = onDisk ? mergeDay(onDisk, pending) : pending; + const tmp = `${file}.${process.pid}.${++state.tmpCounter}.tmp`; + await fs.writeFile(tmp, JSON.stringify(merged)); + await fs.rename(tmp, file); + } catch (err) { + const live = state.days.get(day); + state.days.set(day, live ? mergeDay(live, pending) : pending); + throw err; } - const merged = onDisk ? mergeDay(onDisk, agg) : agg; - const tmp = `${file}.${process.pid}.tmp`; - await fs.writeFile(tmp, JSON.stringify(merged)); - await fs.rename(tmp, file); } } @@ -255,7 +300,7 @@ export async function readDays(n = 28, now = Date.now()): Promise { + test("domain property: every host and scheme of the domain", () => { + const strip = pageKeyStripper("sc-domain:openchainbench.com"); + expect(strip("https://openchainbench.com/benchmarks/bridge-fee")).toBe("/benchmarks/bridge-fee"); + expect(strip("https://www.openchainbench.com/rpc")).toBe("/rpc"); + expect(strip("http://openchainbench.com/")).toBe("/"); + expect(strip("https://staging.openchainbench.com/x")).toBe("/x"); + expect(strip("https://example.com/openchainbench.com")).toBe("https://example.com/openchainbench.com"); + }); + test("url-prefix property", () => { + const strip = pageKeyStripper("https://openchainbench.com/"); + expect(strip("https://openchainbench.com/compare/a-vs-b")).toBe("/compare/a-vs-b"); + }); +}); diff --git a/crm/test/vercel-logs.test.ts b/crm/test/vercel-logs.test.ts index ce9bd426b..4179bb188 100644 --- a/crm/test/vercel-logs.test.ts +++ b/crm/test/vercel-logs.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from "bun:test"; -import { apiFamily, classifyUserAgent, emptyDay, foldEntry, mergeDay, parseBody } from "../lib/vercel-logs"; +import { apiFamily, capMap, classifyUserAgent, emptyDay, foldEntry, mergeDay, NOT_FOUND_KEEP, parseBody } from "../lib/vercel-logs"; describe("classifyUserAgent", () => { test("AI, search, other, human", () => { @@ -58,6 +58,22 @@ describe("foldEntry and mergeDay", () => { }); }); +describe("capMap", () => { + test("keeps the top N and sums the rest under (other), in fold, merge and read", () => { + const m: Record = {}; + for (let i = 0; i < 1000; i += 1) m[`/scan/${i}`] = i % 7 === 0 ? 5 : 1; + const c = capMap(m, 10); + expect(Object.keys(c).length).toBe(11); + expect(c["(other)"]).toBe(Object.values(m).reduce((a, b) => a + b, 0) - 50); + const a = emptyDay("2026-09-20"); + const b = emptyDay("2026-09-20"); + for (let i = 0; i < 700; i += 1) foldEntry(a, { proxy: { path: `/a/${i}`, userAgent: ["curl/8"], statusCode: 404, host: "openchainbench.com" } }); + for (let i = 0; i < 700; i += 1) foldEntry(b, { proxy: { path: `/b/${i}`, userAgent: ["curl/8"], statusCode: 404, host: "openchainbench.com" } }); + expect(Object.keys(mergeDay(a, b).notFound).length).toBeLessThanOrEqual(NOT_FOUND_KEEP + 1); + expect(Object.values(mergeDay(a, b).notFound).reduce((x, y) => x + y, 0)).toBe(1400); + }); +}); + describe("parseBody", () => { test("array and ndjson", () => { expect(parseBody('[{"id":"1"},{"id":"2"}]').length).toBe(2);