From 6fb780355df80adc1d98d031cbaeef471e491009 Mon Sep 17 00:00:00 2001 From: Cameron Reeves Date: Thu, 8 Oct 2026 10:03:46 +1100 Subject: [PATCH 1/3] ci(e2e): run core in the stack and load pinned drivers from the build farm The e2e stack had no core, so nothing in it could run a driver and the specs that bind module state (the home availability panel, room check-in) could only be guarded. core is now part of the stack, started after init has created the tables it subscribes to, with its binaries coming from the PlaceOS build farm the way every deployment's do. The seed creates a repository row and driver rows for Place::Bookings and the demo calendar (PlaceOS/drivers#639), pinned to a commit so the farm builds each CPU architecture once, and waits until core holds the binaries before bring-up returns. Helpers for logic modules and module state go with it, for the specs that attach the drivers to rooms. The runner needs build.placeos.run and the drivers S3 bucket reachable. --- .github/workflows/e2e-advisory.yml | 3 +- e2e/README.md | 13 +- e2e/stack/SELF_HOSTED_RUNNER.md | 7 +- e2e/stack/docker-compose.yml | 40 +++- e2e/stack/up.sh | 9 + e2e/support/drivers/drivers.api.ts | 295 ++++++++++++++++++++++++++++ e2e/support/drivers/drivers.env.ts | 65 ++++++ e2e/support/drivers/drivers.seed.ts | 71 +++++++ e2e/support/seed.ts | 5 +- 9 files changed, 498 insertions(+), 10 deletions(-) create mode 100644 e2e/support/drivers/drivers.api.ts create mode 100644 e2e/support/drivers/drivers.env.ts create mode 100644 e2e/support/drivers/drivers.seed.ts diff --git a/.github/workflows/e2e-advisory.yml b/.github/workflows/e2e-advisory.yml index 6d9f817102..4ed62770dc 100644 --- a/.github/workflows/e2e-advisory.yml +++ b/.github/workflows/e2e-advisory.yml @@ -157,7 +157,8 @@ jobs: # Brings up an isolated PlaceOS stack (its own compose project, own # volumes, ports 9443/9080) and seeds it. Needs network access to # GitHub once, to clone PlaceOS/www-core into the www volume — that is - # where the platform /login page comes from. + # where the platform /login page comes from — and to the build farm, + # which core fetches the suite's driver binaries from. # # `--pull`: on this persistent machine `latest` otherwise stays # whatever was pulled the day the runner was set up. diff --git a/e2e/README.md b/e2e/README.md index eabbe1560a..8b8e40fa2c 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -50,10 +50,18 @@ e2e/stack/down.sh # stop (--volumes to wipe) ``` It is trimmed to what the e2e path exercises: postgres, elasticsearch, redis, -search-ingest, frontend-loader, auth, rest-api, staff-api, nginx, init. Dropped -from PlaceOS/local: core, edge, triggers, dispatch, source, influx, chronograf, +search-ingest, frontend-loader, auth, rest-api, staff-api, core, nginx, init. +Dropped from PlaceOS/local: edge, triggers, dispatch, source, influx, chronograf, mosquitto, minio and the loki/grafana profile — roughly half the containers. +core runs real drivers from PlaceOS/drivers, pinned to a commit in +`e2e/support/drivers/drivers.env.ts`. It compiles nothing itself: binaries come +from the PlaceOS build farm (build.placeos.run), which builds a commit it has not +seen for the CPU architecture on first request and serves a download after that. +`seed.ts` creates the driver rows and waits for core to hold the binaries, so a +spec never waits on the farm. To test a drivers branch against the suite, set +`E2E_DRIVERS_URI`, `E2E_DRIVERS_BRANCH` and `E2E_DRIVERS_COMMIT` before `up.sh`. + Two things a cold start taught us that a long-lived stack hides: - **`frontend-loader` is not optional**, even though the SPA is served by the dev @@ -238,6 +246,7 @@ CI-specific choices worth knowing: | Reclaim step | A self-hosted machine is not a fresh VM; a previous aborted run can leave the stack up or port 4214 held. | | No `vm.max_map_count` bump | Needed on GitHub-hosted Linux, meaningless on macOS — the value lives inside Colima's VM, which already sets it to 1048576. Restore it if reverting to `ubuntu-latest`. | | Network access to GitHub | `up.sh` clones `PlaceOS/www-core` into the `www` volume once — that is where the platform `/login` page comes from. | +| Network access to the build farm | core fetches driver binaries from build.placeos.run and the S3 bucket it hands out links to. A commit the farm has not built for the runner's architecture is compiled there on the first run that asks. | ## Booking specs: what the backend actually does diff --git a/e2e/stack/SELF_HOSTED_RUNNER.md b/e2e/stack/SELF_HOSTED_RUNNER.md index 9c158fc042..2a52a1bd5f 100644 --- a/e2e/stack/SELF_HOSTED_RUNNER.md +++ b/e2e/stack/SELF_HOSTED_RUNNER.md @@ -28,7 +28,8 @@ What it *does* need is outbound 443. Check that first: for host in github.com api.github.com codeload.github.com \ objects.githubusercontent.com release-assets.githubusercontent.com \ pipelines.actions.githubusercontent.com \ - registry-1.docker.io auth.docker.io production.cloudflare.docker.com; do + registry-1.docker.io auth.docker.io production.cloudflare.docker.com \ + build.placeos.run placeos-drivers.s3-ap-southeast-2.amazonaws.com; do printf '%-48s ' "$host" curl -sS -o /dev/null -m 10 -w '%{http_code}\n' "https://$host" 2>&1 | tail -1 done @@ -42,6 +43,10 @@ timeouts, not 200s. - **registry-1.docker.io / auth.docker.io / production.cloudflare.docker.com** — pulling the PlaceOS images. The one people forget; without it the stack cannot start. Every run pulls (`up.sh --pull`), so this needs to stay open, not just work once. +- **build.placeos.run / placeos-drivers.s3-ap-southeast-2.amazonaws.com** — the + build farm core fetches driver binaries from, and the bucket its download links + point at. The stack is torn down with its volumes after every run, so every run + downloads them again. > A slow link looks like a block. Colima's first VM image download failed here with a > timeout that read like a firewall; the host was fine, just slow. Retry before diff --git a/e2e/stack/docker-compose.yml b/e2e/stack/docker-compose.yml index 611b71004b..0754fae145 100644 --- a/e2e/stack/docker-compose.yml +++ b/e2e/stack/docker-compose.yml @@ -7,11 +7,10 @@ # with a running one. Everything here is project-scoped (no container_name, # no fixed network name), so it can run alongside a developer's own stack # without touching it. -# 2. It is what CI needs. This is a trimmed stack — only what the workplace e2e -# path actually exercises. Dropped from PlaceOS/local: core, edge, triggers, -# dispatch, source, frontend-loader, influx, chronograf, mosquitto, minio and -# the loki/promtail/grafana profile. That is roughly half the containers and -# most of the RAM. +# 2. It is what CI needs. This is a trimmed stack — only what the e2e path +# actually exercises. Dropped from PlaceOS/local: edge, triggers, dispatch, +# source, influx, chronograf, mosquitto, minio and the loki/promtail/grafana +# profile. That is roughly half the containers and most of the RAM. # # frontend-loader IS required, despite the SPA being served by the Angular dev # server. It does not only deploy app builds: on startup it clones @@ -20,6 +19,14 @@ # 404 at /login and the real-login spec dead-ends. (Consequence for CI: bringing # this stack up needs network access to GitHub once, to populate the volume.) # +# core runs the drivers. It compiles nothing itself: a driver binary comes from +# the PlaceOS build farm (`BUILD_URL`, build.placeos.run unless overridden with +# E2E_BUILD_URL), keyed by repository, branch, commit, file and CPU architecture, +# and lands in the `core-drivers` volume. The farm builds a key it has not seen +# on first request, which is why `e2e/support/drivers/drivers.env.ts` pins the +# commit. Bring-up needs the farm reachable, and the S3 bucket the farm hands +# out presigned links to (placeos-drivers.s3-ap-southeast-2.amazonaws.com). +# # Bring it up with ./up.sh — it generates secrets and seeds before returning. volumes: @@ -28,6 +35,7 @@ volumes: redis-data: nginx-data: www: + core-drivers: x-deployment-env: &deployment-env PLACE_LOG_FORMAT: JSON @@ -246,6 +254,28 @@ services: *search-ingest-client-env, ] + core: + image: placeos/core:${PLACEOS_TAG:-latest} + hostname: core # load-bearing: rest-api reaches it by the name it registers in redis + restart: always + # Reaps driver processes, which are children of core. + init: true + <<: *std-logging + depends_on: + postgres: { condition: service_healthy } + redis: { condition: service_started } + volumes: + - core-drivers:/app/bin/drivers + env_file: + - .secrets/.env.secret_key + ulimits: + nofile: 40000 + environment: + <<: [*deployment-env, *postgresdb-client-env, *redis-client-env] + BUILD_URL: ${E2E_BUILD_URL:-https://build.placeos.run} + # The User-Agent core sends the farm, so its logs can tell this stack apart. + CLUSTER_NAME: ${E2E_STACK_PROJECT:-placeos-e2e} + staff-api: image: placeos/staff-api:${PLACEOS_TAG:-latest} hostname: staff # load-bearing: nginx upstream is `staff` diff --git a/e2e/stack/up.sh b/e2e/stack/up.sh index e7f07da439..730e67edfe 100755 --- a/e2e/stack/up.sh +++ b/e2e/stack/up.sh @@ -131,6 +131,15 @@ echo " /login -> 200" step "seeding platform entities (init)" dc run --rm init start +# After init: core subscribes to the driver and module tables as it starts, and +# a core that comes up before the migrations have created them stops managing +# drivers for good while still answering HTTP. +step "starting core" +dc up -d core + +# The last step of the seed waits for core to hold the suite's driver binaries. +# A commit the build farm has already built for this CPU architecture is a +# download; one it has not takes a few minutes. step "seeding e2e fixtures" cd ../.. E2E_BACKEND_URL="https://localhost:${HTTPS_PORT}" bunx tsx e2e/support/seed.ts diff --git a/e2e/support/drivers/drivers.api.ts b/e2e/support/drivers/drivers.api.ts new file mode 100644 index 0000000000..e4246ce555 --- /dev/null +++ b/e2e/support/drivers/drivers.api.ts @@ -0,0 +1,295 @@ +/** + * Repositories, drivers and modules through the engine API. + * + * Needs ADMIN: only an admin may create any of these. + */ +import { APIRequestContext } from '@playwright/test'; +import { ENGINE_API } from '../api'; +import { DRIVERS_COMMIT, DRIVERS_REPOSITORY, DriverSpec } from './drivers.env'; + +function asList(body: unknown): any[] { + return Array.isArray(body) ? body : ((body as any)?.results ?? []); +} + +/** Does this error body mean the row is already there? */ +function alreadyExists(body: string): boolean { + return /already (exists|taken)|has already been taken|must be unique|should be unique|duplicate/i.test( + body, + ); +} + +async function getJson( + api: APIRequestContext, + path: string, + params?: Record, +) { + const res = await api.get(path, { params }); + if (!res.ok()) + throw new Error( + `GET ${path} failed: HTTP ${res.status()} ${await res.text()}`, + ); + return res.json(); +} + +const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); + +/** + * The repository row the e2e drivers are loaded from. + * + * `init` seeds its own `Drivers` row pointing at master; this one carries the + * branch the suite is pinned to and is left alone by everything else. + * `folder_name` is unique per repository type, so a second creator gets a + * uniqueness error rather than a second row. + */ +export async function ensureDriversRepository( + api: APIRequestContext, +): Promise<{ id: string; created: boolean }> { + const find = async () => + asList( + await getJson(api, `${ENGINE_API}/repositories`, { limit: '500' }), + ).find( + (r) => + r.folder_name === DRIVERS_REPOSITORY.folder_name && + `${r.repo_type}`.toLowerCase() === 'driver', + ); + + const found = await find(); + if (found) return { id: found.id, created: false }; + + const res = await api.post(`${ENGINE_API}/repositories`, { + data: { + ...DRIVERS_REPOSITORY, + repo_type: 'driver', + commit_hash: 'HEAD', + description: 'Drivers loaded by the e2e suite. Safe to delete.', + }, + }); + if (res.ok()) return { id: (await res.json()).id, created: true }; + + const body = await res.text(); + if (!alreadyExists(body)) { + throw new Error( + `create repository failed: HTTP ${res.status()} ${body}`, + ); + } + // The listing is search-backed and can lag a create by a moment. + for (let i = 0; i < 15; i++) { + const raced = await find(); + if (raced) return { id: raced.id, created: false }; + await sleep(2000); + } + throw new Error( + `repository ${DRIVERS_REPOSITORY.folder_name} exists but is not listed`, + ); +} + +/** + * A driver row for `spec` at the pinned commit, which is what makes core fetch + * (or have the farm build) its binary. + * + * Keyed on file, commit and repository: a row for another commit of the same + * file is a different driver, and nothing stops two rows for one file. + */ +export async function ensureDriver( + api: APIRequestContext, + repository_id: string, + spec: DriverSpec, +): Promise<{ id: string; created: boolean }> { + const existing = asList( + await getJson(api, `${ENGINE_API}/drivers`, { limit: '500' }), + ); + const found = existing.find( + (d) => + d.file_name === spec.file_name && + d.commit === DRIVERS_COMMIT && + d.repository_id === repository_id, + ); + if (found) return { id: found.id, created: false }; + + const res = await api.post(`${ENGINE_API}/drivers`, { + data: { + name: spec.name, + description: spec.description, + role: spec.role, + module_name: spec.module_name, + file_name: spec.file_name, + commit: DRIVERS_COMMIT, + repository_id, + }, + }); + if (!res.ok()) { + throw new Error( + `create driver ${spec.name} failed: HTTP ${res.status()} ${await res.text()}`, + ); + } + return { id: (await res.json()).id, created: true }; +} + +/** + * Wait until core holds a binary for the driver. + * + * `GET /drivers/:id/compiled` goes through core: 200 once the binary is on + * core's disk (core downloads it from the farm while answering), 404 while it is + * not, and 200 with `compilation_output` when the farm refused to build it. The + * driver row carries the same output, and is checked as well because core + * writes it there whether or not anyone asks. + */ +export async function waitForDriverCompiled( + api: APIRequestContext, + driver_id: string, + label: string, + timeoutMs: number, + onProgress: (message: string) => void = () => {}, +): Promise { + const deadline = Date.now() + timeoutMs; + const started = Date.now(); + let last = ''; + let reported = 0; + for (;;) { + const row = await getJson(api, `${ENGINE_API}/drivers/${driver_id}`); + if (row.compilation_output) { + throw new Error( + `${label} failed to build:\n${row.compilation_output}`, + ); + } + const res = await api.get( + `${ENGINE_API}/drivers/${driver_id}/compiled`, + ); + const body = await res.text(); + if (res.ok()) { + let parsed: any = null; + try { + parsed = body ? JSON.parse(body) : null; + } catch { + parsed = null; + } + if (parsed?.compilation_output) { + throw new Error( + `${label} failed to build:\n${parsed.compilation_output}`, + ); + } + return; + } + last = `HTTP ${res.status()} ${body.slice(0, 200)}`; + const elapsed = Math.round((Date.now() - started) / 1000); + if (elapsed - reported >= 30) { + reported = elapsed; + onProgress(`${label}: not loaded after ${elapsed}s (${last})`); + } + if (Date.now() > deadline) { + throw new Error( + `${label} was not loaded by core within ${Math.round(timeoutMs / 1000)}s ` + + `(last: ${last}). core fetches binaries from the build farm, which ` + + `compiles a commit it has not seen for this CPU architecture on first ` + + `request; that takes a few minutes and needs the farm reachable. ` + + `Check: docker compose -p placeos-e2e logs core`, + ); + } + await sleep(5000); + } +} + +export interface ModuleRow { + id: string; + driver_id: string; + control_system_id?: string; + running: boolean; + [k: string]: unknown; +} + +/** The modules in a system, straight from the database rather than search. */ +export async function modulesInSystem( + api: APIRequestContext, + control_system_id: string, +): Promise { + return asList( + await getJson(api, `${ENGINE_API}/modules`, { control_system_id }), + ); +} + +/** + * A running logic module of `driver_id` in the system. + * + * Creating a logic module adds it to the system's module list on the backend + * (`Module#add_logic_module`), so nothing has to patch the system. A module is + * created stopped; `start` flips `running`, and core launches it on that change. + */ +export async function ensureLogicModule( + api: APIRequestContext, + control_system_id: string, + driver_id: string, +): Promise<{ id: string; created: boolean }> { + let mod = (await modulesInSystem(api, control_system_id)).find( + (m) => m.driver_id === driver_id, + ); + let created = false; + if (!mod) { + const res = await api.post(`${ENGINE_API}/modules`, { + data: { driver_id, control_system_id }, + }); + if (!res.ok()) { + throw new Error( + `create module for ${driver_id} on ${control_system_id} failed: HTTP ` + + `${res.status()} ${await res.text()}`, + ); + } + mod = await res.json(); + created = true; + } + if (!mod!.running) { + const res = await api.post(`${ENGINE_API}/modules/${mod!.id}/start`); + if (!res.ok()) { + throw new Error( + `start module ${mod!.id} failed: HTTP ${res.status()} ${await res.text()}`, + ); + } + } + return { id: mod!.id, created }; +} + +/** + * A module's status variables, parsed. + * + * `GET /modules/:id/state` is the module's Redis hash, with every value a JSON + * document in a string. + */ +export async function moduleState( + api: APIRequestContext, + module_id: string, +): Promise> { + const raw = (await getJson( + api, + `${ENGINE_API}/modules/${module_id}/state`, + )) as Record; + const state: Record = {}; + for (const [key, value] of Object.entries(raw || {})) { + try { + state[key] = JSON.parse(value); + } catch { + state[key] = value; + } + } + return state; +} + +/** Wait until the module has published `key`, and return its value. */ +export async function waitForModuleState( + api: APIRequestContext, + module_id: string, + key: string, + timeoutMs: number, +): Promise { + const deadline = Date.now() + timeoutMs; + for (;;) { + const state = await moduleState(api, module_id); + if (key in state) return state[key]; + if (Date.now() > deadline) { + throw new Error( + `module ${module_id} never published "${key}" within ${Math.round(timeoutMs / 1000)}s. ` + + `It has: ${JSON.stringify(Object.keys(state))}. A module with no state at all ` + + `usually did not start: docker compose -p placeos-e2e logs core`, + ); + } + await sleep(2000); + } +} diff --git a/e2e/support/drivers/drivers.env.ts b/e2e/support/drivers/drivers.env.ts new file mode 100644 index 0000000000..a0f004c186 --- /dev/null +++ b/e2e/support/drivers/drivers.env.ts @@ -0,0 +1,65 @@ +/** + * The drivers the e2e stack loads, pinned. + * + * core compiles nothing itself. It asks the PlaceOS build farm (`BUILD_URL` in + * `e2e/stack/docker-compose.yml`, build.placeos.run by default) for a binary + * keyed by repository, branch, commit, file and CPU architecture, and the farm + * compiles on the first request for a key it has not seen, which takes a few + * minutes. Pinned to a commit, that happens once per architecture and every + * bring-up after it is a download. `HEAD` would make core force a rebuild on + * every bring-up (`core/src/placeos-core/driver_manager.cr`). + * + * To run the suite against a drivers branch, set E2E_DRIVERS_URI, + * E2E_DRIVERS_BRANCH and E2E_DRIVERS_COMMIT before `up.sh`. + */ +export const DRIVERS_REPOSITORY = { + name: 'E2E Drivers', + folder_name: 'e2e-drivers', + uri: process.env.E2E_DRIVERS_URI ?? 'https://github.com/placeos/drivers', + branch: process.env.E2E_DRIVERS_BRANCH ?? 'feat/place-demo-calendar', +}; + +/** + * PlaceOS/drivers at the branch above. The demo calendar is on that branch + * until its PR merges; then this becomes master and the squash commit. + */ +export const DRIVERS_COMMIT = + process.env.E2E_DRIVERS_COMMIT ?? + '635d0a1b4295f1132bf84c6326c68f8815a089b4'; + +export interface DriverSpec { + /** Shown in Backoffice; the module takes its name from `module_name`. */ + name: string; + /** Path inside the repository. */ + file_name: string; + /** The class's `generic_name`, which is what systems address it by. */ + module_name: string; + /** `PlaceDriverRole`; 99 is Logic, one module per system. */ + role: number; + description: string; +} + +/** What a bookable room runs so the apps see a live status for it. */ +export const ROOM_DRIVERS = { + /** In-memory calendar; nothing reaches a mailbox. */ + calendar: { + name: 'E2E Demo Calendar', + file_name: 'drivers/place/demo/calendar.cr', + module_name: 'Calendar', + role: 99, + description: 'Owned by the e2e suite. Safe to delete.', + }, + /** + * The room status driver the workplace app binds (`Bookings` / `status`). + * Polls `Calendar_1` in the same system, which is the module above. + */ + bookings: { + name: 'E2E Room Events', + file_name: 'drivers/place/bookings.cr', + module_name: 'Bookings', + role: 99, + description: 'Owned by the e2e suite. Safe to delete.', + }, +} satisfies Record; + +export type RoomDriver = keyof typeof ROOM_DRIVERS; diff --git a/e2e/support/drivers/drivers.seed.ts b/e2e/support/drivers/drivers.seed.ts new file mode 100644 index 0000000000..0d3379b6b6 --- /dev/null +++ b/e2e/support/drivers/drivers.seed.ts @@ -0,0 +1,71 @@ +/** + * Loading the suite's drivers into the stack. + * + * Run from `e2e/support/seed.ts` at bring-up, so the wait for the build farm + * happens once there rather than inside a spec's timeout. Idempotent: rows are + * found before they are created, and a driver core already holds returns at + * once. + */ +import { APIRequestContext } from '@playwright/test'; +import { + ensureDriver, + ensureDriversRepository, + waitForDriverCompiled, +} from './drivers.api'; +import { + DRIVERS_COMMIT, + DRIVERS_REPOSITORY, + ROOM_DRIVERS, + RoomDriver, +} from './drivers.env'; + +/** + * How long bring-up waits for core to hold every driver binary. + * + * A commit the farm has already built for this CPU architecture is a download + * of a few seconds. One it has not is a compile of a few minutes per driver. + */ +export const DRIVER_LOAD_TIMEOUT_MS = 15 * 60 * 1000; + +export interface SeededDrivers { + repository_id: string; + drivers: Record; +} + +export async function ensureDrivers( + api: APIRequestContext, + log: (message: string) => void = () => {}, +): Promise { + const repository = await ensureDriversRepository(api); + const drivers = {} as SeededDrivers['drivers']; + for (const key of Object.keys(ROOM_DRIVERS) as RoomDriver[]) { + drivers[key] = await ensureDriver( + api, + repository.id, + ROOM_DRIVERS[key], + ); + } + await Promise.all( + (Object.keys(ROOM_DRIVERS) as RoomDriver[]).map((key) => + waitForDriverCompiled( + api, + drivers[key].id, + `${ROOM_DRIVERS[key].name} (${ROOM_DRIVERS[key].file_name} @ ${DRIVERS_COMMIT.slice(0, 7)})`, + DRIVER_LOAD_TIMEOUT_MS, + log, + ), + ), + ); + return { repository_id: repository.id, drivers }; +} + +/** One line for the bring-up log: where the drivers came from. */ +export function describeDrivers(seeded: SeededDrivers): string { + const names = (Object.keys(ROOM_DRIVERS) as RoomDriver[]) + .map( + (key) => + `${ROOM_DRIVERS[key].module_name}${seeded.drivers[key].created ? '+' : ''}`, + ) + .join(', '); + return `${DRIVERS_REPOSITORY.uri}@${DRIVERS_REPOSITORY.branch} ${DRIVERS_COMMIT.slice(0, 7)}: ${names}`; +} diff --git a/e2e/support/seed.ts b/e2e/support/seed.ts index 2538f4571e..2833212ed1 100644 --- a/e2e/support/seed.ts +++ b/e2e/support/seed.ts @@ -26,11 +26,12 @@ import { } from './env'; import { mintToken, clientId, redirectUriFor } from './auth'; import { ENGINE_API, STAFF_API } from './api'; +import { describeDrivers, ensureDrivers } from './drivers/drivers.seed'; /** The app `init` guarantees exists — our way in before anything else is registered. */ const BOOTSTRAP_APP_URL = `${BACKEND_URL}/backoffice`; -async function adminApi(): Promise { +export async function adminApi(): Promise { const admin = roleFor('admin'); const mint = await mintToken(BACKEND_URL, BOOTSTRAP_APP_URL, admin.email, admin.password); return pwRequest.newContext({ @@ -285,6 +286,8 @@ export async function seed(): Promise { ` staff users ${users.created.length} created, ${users.present.length} present ` + `(${WORKERS} workers: ${staffEmail(0)} … ${staffEmail(WORKERS - 1)})`, ); + const drivers = await ensureDrivers(api, (message) => console.log(` drivers ${message}`)); + console.log(` drivers ${describeDrivers(drivers)}`); } finally { await api.dispose(); } From 4dc3b39f397a185433fbde85c17c07f3f0a9af71 Mon Sep 17 00:00:00 2001 From: Cameron Reeves Date: Thu, 8 Oct 2026 10:10:48 +1100 Subject: [PATCH 2/3] ci(e2e): let the driver load wait outlive a slow binary download --- e2e/support/drivers/drivers.api.ts | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/e2e/support/drivers/drivers.api.ts b/e2e/support/drivers/drivers.api.ts index e4246ce555..9ad8879115 100644 --- a/e2e/support/drivers/drivers.api.ts +++ b/e2e/support/drivers/drivers.api.ts @@ -129,10 +129,13 @@ export async function ensureDriver( * Wait until core holds a binary for the driver. * * `GET /drivers/:id/compiled` goes through core: 200 once the binary is on - * core's disk (core downloads it from the farm while answering), 404 while it is - * not, and 200 with `compilation_output` when the farm refused to build it. The - * driver row carries the same output, and is checked as well because core - * writes it there whether or not anyone asks. + * core's disk, 404 while it is not, and 200 with `compilation_output` when the + * farm refused to build it. core downloads the binary from the farm's S3 while + * answering, and a request that lands during that download waits for it, so the + * request gets the whole remaining budget rather than Playwright's 30 s default. + * The driver row carries the same output, and is checked as well because core + * writes it there whether or not anyone asks; `compilation_status=false` keeps + * that read from asking core too. */ export async function waitForDriverCompiled( api: APIRequestContext, @@ -146,7 +149,9 @@ export async function waitForDriverCompiled( let last = ''; let reported = 0; for (;;) { - const row = await getJson(api, `${ENGINE_API}/drivers/${driver_id}`); + const row = await getJson(api, `${ENGINE_API}/drivers/${driver_id}`, { + compilation_status: 'false', + }); if (row.compilation_output) { throw new Error( `${label} failed to build:\n${row.compilation_output}`, @@ -154,6 +159,7 @@ export async function waitForDriverCompiled( } const res = await api.get( `${ENGINE_API}/drivers/${driver_id}/compiled`, + { timeout: Math.max(deadline - Date.now(), 10_000) }, ); const body = await res.text(); if (res.ok()) { From 8b222dd4962bddbf6136e8d153b1fb3c3ae43e69 Mon Sep 17 00:00:00 2001 From: Alex Sorafumo Date: Thu, 8 Oct 2026 13:34:58 +1100 Subject: [PATCH 3/3] fix(e2e): make driver seeding safe to re-run on a reused stack - Move the drivers repository row to the configured uri and branch. - Replace a room's module from an earlier driver instead of adding a second one. - Retry /compiled on network errors, and ignore build output left by an earlier attempt. - Re-check the search-backed driver listing before creating a row. - Log a warning when drivers do not load, so a build farm outage does not fail every spec. - Share the listing helpers through e2e/support/api.ts and type the rows. Co-Authored-By: Claude Opus 5.5 (1M context) --- e2e/README.md | 3 +- e2e/support/api.ts | 24 +++ e2e/support/drivers/drivers.api.ts | 232 ++++++++++++++++++---------- e2e/support/drivers/drivers.seed.ts | 1 + e2e/support/seed.ts | 28 ++-- 5 files changed, 191 insertions(+), 97 deletions(-) diff --git a/e2e/README.md b/e2e/README.md index 8b8e40fa2c..888ac8b6d0 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -59,7 +59,8 @@ core runs real drivers from PlaceOS/drivers, pinned to a commit in from the PlaceOS build farm (build.placeos.run), which builds a commit it has not seen for the CPU architecture on first request and serves a download after that. `seed.ts` creates the driver rows and waits for core to hold the binaries, so a -spec never waits on the farm. To test a drivers branch against the suite, set +spec never waits on the farm. If the drivers do not load, the seed logs a warning +and continues: only the specs that need a room status fail. To test a drivers branch against the suite, set `E2E_DRIVERS_URI`, `E2E_DRIVERS_BRANCH` and `E2E_DRIVERS_COMMIT` before `up.sh`. Two things a cold start taught us that a long-lived stack hides: diff --git a/e2e/support/api.ts b/e2e/support/api.ts index 28666f8ff3..e41023b308 100644 --- a/e2e/support/api.ts +++ b/e2e/support/api.ts @@ -10,6 +10,30 @@ import { BACKEND_URL, RoleName, roleFor } from './env'; export const ENGINE_API = '/api/engine/v2'; export const STAFF_API = '/api/staff/v1'; +/** The rows of a listing, which is either a bare array or `{ results }`. */ +export function asList(body: unknown): T[] { + if (Array.isArray(body)) return body; + return (body as { results?: T[] } | null)?.results ?? []; +} + +/** Does this error body mean "the row is already there"? */ +export function alreadyExists(body: string): boolean { + return /already (exists|taken)|has already been taken|must be unique|should be unique|duplicate/i.test( + body, + ); +} + +/** GET a JSON body, or throw with the status and body. */ +export async function getJson( + api: APIRequestContext, + path: string, + params?: Record, +) { + const res = await api.get(path, { params }); + if (!res.ok()) throw new Error(`GET ${path} failed: HTTP ${res.status()} ${await res.text()}`); + return res.json(); +} + /** Read the raw bearer a worker's auth fixture wrote for a role. */ export function readToken(role: RoleName, workerIndex = 0): string { const { tokenPath } = roleFor(role, workerIndex); diff --git a/e2e/support/drivers/drivers.api.ts b/e2e/support/drivers/drivers.api.ts index 9ad8879115..20639c65fe 100644 --- a/e2e/support/drivers/drivers.api.ts +++ b/e2e/support/drivers/drivers.api.ts @@ -4,31 +4,24 @@ * Needs ADMIN: only an admin may create any of these. */ import { APIRequestContext } from '@playwright/test'; -import { ENGINE_API } from '../api'; +import { ENGINE_API, alreadyExists, asList, getJson } from '../api'; import { DRIVERS_COMMIT, DRIVERS_REPOSITORY, DriverSpec } from './drivers.env'; -function asList(body: unknown): any[] { - return Array.isArray(body) ? body : ((body as any)?.results ?? []); -} - -/** Does this error body mean the row is already there? */ -function alreadyExists(body: string): boolean { - return /already (exists|taken)|has already been taken|must be unique|should be unique|duplicate/i.test( - body, - ); +interface RepositoryRow { + id: string; + folder_name: string; + repo_type: string; + uri: string; + branch: string; } -async function getJson( - api: APIRequestContext, - path: string, - params?: Record, -) { - const res = await api.get(path, { params }); - if (!res.ok()) - throw new Error( - `GET ${path} failed: HTTP ${res.status()} ${await res.text()}`, - ); - return res.json(); +interface DriverRow { + id: string; + file_name: string; + module_name: string; + commit: string; + repository_id: string; + compilation_output?: string | null; } const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); @@ -39,13 +32,15 @@ const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); * `init` seeds its own `Drivers` row pointing at master; this one carries the * branch the suite is pinned to and is left alone by everything else. * `folder_name` is unique per repository type, so a second creator gets a - * uniqueness error rather than a second row. + * uniqueness error rather than a second row. A row left by an earlier pin or + * `E2E_DRIVERS_*` override is moved to the current uri and branch, because the + * build farm resolves a driver's commit on its repository's branch. */ export async function ensureDriversRepository( api: APIRequestContext, ): Promise<{ id: string; created: boolean }> { const find = async () => - asList( + asList( await getJson(api, `${ENGINE_API}/repositories`, { limit: '500' }), ).find( (r) => @@ -54,7 +49,10 @@ export async function ensureDriversRepository( ); const found = await find(); - if (found) return { id: found.id, created: false }; + if (found) { + await syncRepository(api, found); + return { id: found.id, created: false }; + } const res = await api.post(`${ENGINE_API}/repositories`, { data: { @@ -75,7 +73,10 @@ export async function ensureDriversRepository( // The listing is search-backed and can lag a create by a moment. for (let i = 0; i < 15; i++) { const raced = await find(); - if (raced) return { id: raced.id, created: false }; + if (raced) { + await syncRepository(api, raced); + return { id: raced.id, created: false }; + } await sleep(2000); } throw new Error( @@ -83,28 +84,56 @@ export async function ensureDriversRepository( ); } +/** Point an existing repository row at the configured uri and branch. */ +async function syncRepository( + api: APIRequestContext, + row: RepositoryRow, +): Promise { + const { uri, branch } = DRIVERS_REPOSITORY; + if (row.uri === uri && row.branch === branch) return; + const res = await api.patch(`${ENGINE_API}/repositories/${row.id}`, { + data: { uri, branch }, + }); + if (!res.ok()) { + throw new Error( + `update repository ${row.id} to ${uri}@${branch} failed: HTTP ` + + `${res.status()} ${await res.text()}`, + ); + } +} + /** * A driver row for `spec` at the pinned commit, which is what makes core fetch * (or have the farm build) its binary. * * Keyed on file, commit and repository: a row for another commit of the same - * file is a different driver, and nothing stops two rows for one file. + * file is a different driver, and nothing stops two rows for one file. The + * listing is search-backed, so when the repository already existed a miss is + * re-checked for a few seconds before creating: a seed re-run right after the + * first would otherwise add a second row. */ export async function ensureDriver( api: APIRequestContext, repository_id: string, spec: DriverSpec, + repositoryIsNew: boolean, ): Promise<{ id: string; created: boolean }> { - const existing = asList( - await getJson(api, `${ENGINE_API}/drivers`, { limit: '500' }), - ); - const found = existing.find( - (d) => - d.file_name === spec.file_name && - d.commit === DRIVERS_COMMIT && - d.repository_id === repository_id, - ); - if (found) return { id: found.id, created: false }; + const find = async () => + asList( + await getJson(api, `${ENGINE_API}/drivers`, { limit: '500' }), + ).find( + (d) => + d.file_name === spec.file_name && + d.commit === DRIVERS_COMMIT && + d.repository_id === repository_id, + ); + + const attempts = repositoryIsNew ? 1 : 5; + for (let i = 0; i < attempts; i++) { + if (i) await sleep(2000); + const found = await find(); + if (found) return { id: found.id, created: false }; + } const res = await api.post(`${ENGINE_API}/drivers`, { data: { @@ -133,9 +162,13 @@ export async function ensureDriver( * farm refused to build it. core downloads the binary from the farm's S3 while * answering, and a request that lands during that download waits for it, so the * request gets the whole remaining budget rather than Playwright's 30 s default. + * A request that fails outright (core still starting, a reset during the + * download) is retried like a 404. + * * The driver row carries the same output, and is checked as well because core * writes it there whether or not anyone asks; `compilation_status=false` keeps - * that read from asking core too. + * that read from asking core too. Output already on the row when the wait + * starts is from an earlier attempt, and only a change to it counts. */ export async function waitForDriverCompiled( api: APIRequestContext, @@ -144,61 +177,79 @@ export async function waitForDriverCompiled( timeoutMs: number, onProgress: (message: string) => void = () => {}, ): Promise { + const readRow = async (): Promise => + getJson(api, `${ENGINE_API}/drivers/${driver_id}`, { + compilation_status: 'false', + }); const deadline = Date.now() + timeoutMs; const started = Date.now(); + const stale = (await readRow()).compilation_output ?? null; let last = ''; let reported = 0; - for (;;) { - const row = await getJson(api, `${ENGINE_API}/drivers/${driver_id}`, { - compilation_status: 'false', - }); - if (row.compilation_output) { - throw new Error( - `${label} failed to build:\n${row.compilation_output}`, - ); + while (Date.now() <= deadline) { + const output = (await readRow()).compilation_output ?? null; + if (output && output !== stale) { + throw new Error(`${label} failed to build:\n${output}`); } - const res = await api.get( - `${ENGINE_API}/drivers/${driver_id}/compiled`, - { timeout: Math.max(deadline - Date.now(), 10_000) }, - ); - const body = await res.text(); - if (res.ok()) { - let parsed: any = null; - try { - parsed = body ? JSON.parse(body) : null; - } catch { - parsed = null; - } - if (parsed?.compilation_output) { - throw new Error( - `${label} failed to build:\n${parsed.compilation_output}`, - ); - } + const compiled = await api + .get(`${ENGINE_API}/drivers/${driver_id}/compiled`, { + timeout: Math.max(deadline - Date.now(), 10_000), + }) + .then(async (res) => ({ + ok: res.ok(), + status: res.status(), + body: await res.text(), + })) + .catch((error: unknown) => ({ + ok: false, + status: 0, + body: `${error}`, + })); + if (compiled.ok) { + const failure = compilationOutput(compiled.body); + if (failure) + throw new Error(`${label} failed to build:\n${failure}`); return; } - last = `HTTP ${res.status()} ${body.slice(0, 200)}`; + last = `${compiled.status ? `HTTP ${compiled.status} ` : ''}${compiled.body.slice(0, 200)}`; const elapsed = Math.round((Date.now() - started) / 1000); if (elapsed - reported >= 30) { reported = elapsed; onProgress(`${label}: not loaded after ${elapsed}s (${last})`); } - if (Date.now() > deadline) { - throw new Error( - `${label} was not loaded by core within ${Math.round(timeoutMs / 1000)}s ` + - `(last: ${last}). core fetches binaries from the build farm, which ` + - `compiles a commit it has not seen for this CPU architecture on first ` + - `request; that takes a few minutes and needs the farm reachable. ` + - `Check: docker compose -p placeos-e2e logs core`, - ); - } await sleep(5000); } + throw new Error( + `${label} was not loaded by core within ${Math.round(timeoutMs / 1000)}s ` + + `(last: ${last}). core fetches binaries from the build farm, which ` + + `compiles a commit it has not seen for this CPU architecture on first ` + + `request; that takes a few minutes and needs the farm reachable. ` + + `Check: docker compose -p placeos-e2e logs core`, + ); +} + +/** The `compilation_output` in a `/compiled` body, if it has one. */ +function compilationOutput(body: string): string | null { + try { + const parsed: unknown = body ? JSON.parse(body) : null; + if ( + parsed && + typeof parsed === 'object' && + 'compilation_output' in parsed + ) { + return `${parsed.compilation_output ?? ''}` || null; + } + } catch { + /* not JSON: a plain success body */ + } + return null; } export interface ModuleRow { id: string; driver_id: string; control_system_id?: string; + name: string; running: boolean; [k: string]: unknown; } @@ -208,7 +259,7 @@ export async function modulesInSystem( api: APIRequestContext, control_system_id: string, ): Promise { - return asList( + return asList( await getJson(api, `${ENGINE_API}/modules`, { control_system_id }), ); } @@ -219,15 +270,36 @@ export async function modulesInSystem( * Creating a logic module adds it to the system's module list on the backend * (`Module#add_logic_module`), so nothing has to patch the system. A module is * created stopped; `start` flips `running`, and core launches it on that change. + * + * A module with the same name from another driver is a leftover from an earlier + * pin, and is deleted first. Kept, it would stay `_1` and the new module + * would be `_2`, which nothing binds. Only call this on systems the suite + * owns. */ export async function ensureLogicModule( api: APIRequestContext, control_system_id: string, driver_id: string, ): Promise<{ id: string; created: boolean }> { - let mod = (await modulesInSystem(api, control_system_id)).find( - (m) => m.driver_id === driver_id, + const driver: DriverRow = await getJson( + api, + `${ENGINE_API}/drivers/${driver_id}`, + { compilation_status: 'false' }, + ); + const named = (await modulesInSystem(api, control_system_id)).filter( + (m) => m.name === driver.module_name, ); + for (const stale of named.filter((m) => m.driver_id !== driver_id)) { + const res = await api.delete(`${ENGINE_API}/modules/${stale.id}`); + if (!res.ok()) { + throw new Error( + `delete stale module ${stale.id} on ${control_system_id} failed: HTTP ` + + `${res.status()} ${await res.text()}`, + ); + } + } + + let mod = named.find((m) => m.driver_id === driver_id); let created = false; if (!mod) { const res = await api.post(`${ENGINE_API}/modules`, { @@ -239,18 +311,18 @@ export async function ensureLogicModule( `${res.status()} ${await res.text()}`, ); } - mod = await res.json(); + mod = (await res.json()) as ModuleRow; created = true; } - if (!mod!.running) { - const res = await api.post(`${ENGINE_API}/modules/${mod!.id}/start`); + if (!mod.running) { + const res = await api.post(`${ENGINE_API}/modules/${mod.id}/start`); if (!res.ok()) { throw new Error( - `start module ${mod!.id} failed: HTTP ${res.status()} ${await res.text()}`, + `start module ${mod.id} failed: HTTP ${res.status()} ${await res.text()}`, ); } } - return { id: mod!.id, created }; + return { id: mod.id, created }; } /** diff --git a/e2e/support/drivers/drivers.seed.ts b/e2e/support/drivers/drivers.seed.ts index 0d3379b6b6..a2105528db 100644 --- a/e2e/support/drivers/drivers.seed.ts +++ b/e2e/support/drivers/drivers.seed.ts @@ -43,6 +43,7 @@ export async function ensureDrivers( api, repository.id, ROOM_DRIVERS[key], + repository.created, ); } await Promise.all( diff --git a/e2e/support/seed.ts b/e2e/support/seed.ts index 2833212ed1..1050dc6ecf 100644 --- a/e2e/support/seed.ts +++ b/e2e/support/seed.ts @@ -25,7 +25,7 @@ import { staffEmail, } from './env'; import { mintToken, clientId, redirectUriFor } from './auth'; -import { ENGINE_API, STAFF_API } from './api'; +import { ENGINE_API, STAFF_API, alreadyExists, getJson as json } from './api'; import { describeDrivers, ensureDrivers } from './drivers/drivers.seed'; /** The app `init` guarantees exists — our way in before anything else is registered. */ @@ -41,19 +41,6 @@ export async function adminApi(): Promise { }); } -/** Does this error body mean "the row is already there"? */ -function alreadyExists(body: string): boolean { - return /already (exists|taken)|has already been taken|must be unique|should be unique|duplicate/i.test( - body, - ); -} - -async function json(api: APIRequestContext, path: string, params?: Record) { - const res = await api.get(path, { params }); - if (!res.ok()) throw new Error(`GET ${path} failed: HTTP ${res.status()} ${await res.text()}`); - return res.json(); -} - /** * The authority (domain) this stack serves. * @@ -286,8 +273,17 @@ export async function seed(): Promise { ` staff users ${users.created.length} created, ${users.present.length} present ` + `(${WORKERS} workers: ${staffEmail(0)} … ${staffEmail(WORKERS - 1)})`, ); - const drivers = await ensureDrivers(api, (message) => console.log(` drivers ${message}`)); - console.log(` drivers ${describeDrivers(drivers)}`); + // Only specs that bind a room's status need the drivers, so a build farm + // that is down or slow is reported here instead of failing every spec. + try { + const drivers = await ensureDrivers(api, (message) => console.log(` drivers ${message}`)); + console.log(` drivers ${describeDrivers(drivers)}`); + } catch (error) { + console.warn( + ` drivers NOT LOADED; specs that need a room status will fail.\n` + + `${error instanceof Error ? error.message : error}`, + ); + } } finally { await api.dispose(); }