diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9443aa1..9046caf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,10 +17,27 @@ permissions: contents: read jobs: + runtime-contracts: + name: Lua runtime contracts + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + with: + persist-credentials: false + - uses: ./.github/actions/setup-wippy + id: wippy + - name: Test real runtime contracts + env: + WIPPY: ${{ steps.wippy.outputs.path }} + RUNTIME_WORK_DIR: ${{ runner.temp }}/framework-runtime + run: make test-runtime test-runner: name: Test runner regressions runs-on: ubuntu-latest timeout-minutes: 10 + env: + WIPPY_FRAMEWORK_ROOT: ${{ github.workspace }} steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 with: @@ -44,6 +61,7 @@ jobs: run: | (cd src/test && wippy install && wippy lint --level error && wippy test) (cd test-harness/runner && wippy test) + make test-runner-go bash test-harness/runner-results/verify.sh bash test-harness/runner-hook-failure/verify.sh bash test-harness/runner-timeout/verify-timeouts.sh diff --git a/Makefile b/Makefile index 9387396..cf68e81 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,10 @@ # Wippy Framework # Modules that have test directories with wippy.lock TEST_MODULES = actor agent bootloader embeddings facade llm migration relay usage views +WIPPY ?= wippy +RUN_WIPPY = WIPPY="$(WIPPY)" bash scripts/run-wippy.sh -.PHONY: help check-manifests run-tests run-lint install +.PHONY: help check-manifests run-tests run-lint install test-runtime help: @echo "Wippy Framework" @@ -12,6 +14,18 @@ help: @echo " make run-lint Run lint for all modules" @echo " make check-manifests Validate package and test-app module types" @echo " make install Install dependencies for all test modules" + @echo " make test-runtime Test real process and control delivery" + +test-runtime: + WIPPY_TEST_REQUIRE_CASES=1 $(RUN_WIPPY) src/actor/test test test -- actor_runtime_test + WIPPY_TEST_REQUIRE_CASES=1 $(RUN_WIPPY) src/actor/test test test -- actor_runtime_resident_test + WIPPY_TEST_REQUIRE_CASES=1 $(RUN_WIPPY) src/agent/test test \ + -o wippy.llm:process_host:default=wippy.terminal:host \ + -o wippy.llm:env_storage:default=app:env_storage test -- tools_controls_runtime_test + WIPPY_TEST_REQUIRE_CASES=1 $(RUN_WIPPY) src/relay/test test \ + -o wippy.relay:application_host:default=app:processes \ + -o wippy.relay:user_security_scope:default=app:user test -- relay_runtime_test + WIPPY_TEST_REQUIRE_CASES=1 $(RUN_WIPPY) test-harness/runner test test -- aggregate_checks_test check-manifests: @python3 scripts/check_module_manifests.py @@ -20,8 +34,7 @@ run-tests: @failed=0; \ for mod in $(TEST_MODULES); do \ printf "%-14s " "$$mod"; \ - output=$$($(MAKE) -s -C src/$$mod/test test 2>&1); \ - if echo "$$output" | grep -q "PASSED"; then \ + if output=$$($(MAKE) -s -C src/$$mod/test test 2>&1); then \ echo "PASSED"; \ else \ echo "FAILED"; \ @@ -36,12 +49,12 @@ run-lint: @failed=0; \ for mod in $(TEST_MODULES); do \ printf "%-14s " "$$mod"; \ - output=$$($(MAKE) -s -C src/$$mod/test lint 2>&1); \ - if echo "$$output" | grep -q "errors"; then \ - echo "$$output" | grep -oP 'Checked.*'; \ - failed=1; \ + if output=$$($(MAKE) -s -C src/$$mod/test lint 2>&1); then \ + echo "PASSED"; \ else \ - echo "$$output" | grep -oP 'Checked.*'; \ + echo "FAILED"; \ + echo "$$output" | tail -5; \ + failed=1; \ fi; \ done; \ if [ $$failed -eq 1 ]; then echo ""; echo "Lint errors found"; exit 1; fi; \ diff --git a/README.md b/README.md index 60ed467..976216c 100644 --- a/README.md +++ b/README.md @@ -79,6 +79,15 @@ and configuration management — enabling developers to build sophisticated mult | **stream** | Data streaming interface with chunked reading support | | **cloudstorage** | Cloud storage provider interface for S3-like operations | +## Runtime regression tests + +`make test-runtime` exercises actor messages, relay delivery, tool controls, and aggregate +runner checks through Wippy using the local component and test-runner sources. Required +suites fail if they execute no cases. + +Set `WIPPY` to select the executable and `RUNTIME_WORK_DIR` to reuse an external dependency +directory. Test data stays outside the repository. + [module-actor]: https://github.com/wippyai/module-actor [module-agent]: https://github.com/wippyai/module-agent [module-bootloader]: https://github.com/wippyai/module-bootloader diff --git a/scripts/run-wippy.sh b/scripts/run-wippy.sh new file mode 100644 index 0000000..2417d0a --- /dev/null +++ b/scripts/run-wippy.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo=$(cd "$(dirname "$0")/.." && pwd -P) +source_root=$repo +app=$1 +shift +wippy=$(command -v "${WIPPY:-wippy}") +work_root=${RUNTIME_WORK_DIR:-$(mktemp -d "${TMPDIR:-/tmp}/wippy-framework.XXXXXX")} +mkdir -p "$work_root" +work_root=$(cd "$work_root" && pwd -P) +case "$work_root/" in "$repo/"*) echo 'Runtime files must be outside the repository' >&2; exit 1;; esac +if [ -z "${RUNTIME_WORK_DIR:-}" ]; then + trap 'rm -r -- "$work_root"' EXIT +fi + +app_root=$(cd "$source_root/$app" && pwd) +work="$work_root/${app//\//-}" +mkdir -p "$work" +source_path=$(awk '$1 == "src:" {print $2; exit}' "$app_root/wippy.lock") +source_path=$(cd "$app_root/${source_path:-.}" && pwd) +{ + printf 'directories:\n modules: %s\n src: %s\n' \ + "$(jq -Rn --arg value "$work/modules" '$value')" \ + "$(jq -Rn --arg value "$source_path" '$value')" + awk '/^modules:/ {active=1} /^replacements:/ {active=0} active {print}' "$app_root/wippy.lock" +} > "$work/wippy.lock" +jq -n --arg root "$source_root" --arg work "$work" '{ + version: "1.0", + registry: {dependency_vendor_dir: ($work + "/modules/vendor"), dependency_lock_path: ($work + "/wippy.lock")}, + workspace: {replacements: {"wippy/test": ($root + "/src/test")}} +}' > "$work/.wippy.yaml" + +while IFS=$'\t' read -r component replacement; do + replacement=$(cd "$app_root/$replacement" && pwd) + jq --arg component "$component" --arg path "$replacement" \ + '.workspace.replacements[$component] = $path' "$work/.wippy.yaml" > "$work/config.next" + mv "$work/config.next" "$work/.wippy.yaml" +done < <(awk '/^replacements:/ {active=1} active && $2 == "from:" {module=$3} active && $1 == "to:" {printf "%s\t%s\n", module, $2}' "$app_root/wippy.lock") + +if [ "$app" = src/facade/test ]; then + jq --arg path "$source_root/src/facade" '.workspace.replacements["wippy/facade"] = $path' \ + "$work/.wippy.yaml" > "$work/config.next" + mv "$work/config.next" "$work/.wippy.yaml" +fi + +export WIPPY_FRAMEWORK_ROOT="$repo" +export WIPPY_TEST_ARTIFACTS="$work" +cd "$work" +"$wippy" install +"$wippy" "$@" diff --git a/src/actor/src/actor.lua b/src/actor/src/actor.lua index 4d32ba6..b8cf23d 100644 --- a/src/actor/src/actor.lua +++ b/src/actor/src/actor.lua @@ -66,13 +66,7 @@ local function get_process(): any return actor._process end - return { - inbox = function() return process.inbox() end, - events = function() return process.events() end, - send = function(dest, topic, payload) return process.send(dest, topic, payload) end, - pid = function() return process.pid() end, - event = process.event - } + return process end function actor.new(initial_state: any, handlers: any): any @@ -108,15 +102,15 @@ function actor.new(initial_state: any, handlers: any): any internal_channel:case_receive() } + -- Update in place: go-lua detaches this upvalue from run_loop after a caught error. local function rebuild_select_cases() - select_cases = { - inbox:case_receive(), - events:case_receive(), - internal_channel:case_receive() - } - + local count = 3 for _, channel_info in pairs(registered_channels) do - table.insert(select_cases, channel_info.chan:case_receive()) + count = count + 1 + select_cases[count] = channel_info.chan:case_receive() + end + for index = #select_cases, count + 1, -1 do + select_cases[index] = nil end end @@ -277,7 +271,7 @@ function actor.new(initial_state: any, handlers: any): any -- Main actor loop while true do local result: SelectResult = channel.select(select_cases) - if not result.ok then + if not result.ok and not channel_to_id[result.channel] then break end diff --git a/src/actor/test/.wippy.yaml b/src/actor/test/.wippy.yaml new file mode 100644 index 0000000..977dbf4 --- /dev/null +++ b/src/actor/test/.wippy.yaml @@ -0,0 +1,4 @@ +version: "1.0" +workspace: + replacements: + wippy/test: ../../test diff --git a/src/actor/test/_index.yaml b/src/actor/test/_index.yaml index dea4565..76f017c 100644 --- a/src/actor/test/_index.yaml +++ b/src/actor/test/_index.yaml @@ -32,3 +32,54 @@ entries: - time - json method: run + + - name: runtime_processes + kind: process.host + lifecycle: + auto_start: true + + - name: actor_runtime_fixture + kind: process.lua + source: file://actor_runtime_fixture.lua + imports: + actor: wippy.actor:actor + modules: + - time + method: run + + - name: actor_runtime_test + kind: function.lua + meta: + type: test + suite: actor-runtime + timeout: 20s + source: file://actor_runtime_test.lua + imports: + actor: wippy.actor:actor + test: wippy.test:test + modules: + - system + - time + method: run + + - name: actor_resident_fixture + kind: process.lua + source: file://actor_runtime_fixture.lua + imports: + actor: wippy.actor:actor + modules: + - time + method: resident + + - name: actor_runtime_resident_test + kind: function.lua + meta: + type: test + suite: actor-runtime + timeout: 25s + source: file://actor_resident_test.lua + imports: + test: wippy.test:test + modules: + - time + method: run diff --git a/src/actor/test/actor_resident_test.lua b/src/actor/test/actor_resident_test.lua new file mode 100644 index 0000000..6918737 --- /dev/null +++ b/src/actor/test/actor_resident_test.lua @@ -0,0 +1,106 @@ +local test = require("test") +local time = require("time") + +local sequence = 0 + +local function with_residents(count: number, body: any): any + sequence = sequence + 1 + local topic = "actor.resident." .. sequence + local responses = process.listen(topic) + local events = process.events() + local active = {} + local pids = {} + local batch = 0 + local pending_deadline + local function dispatch() + batch = batch + 1 + for index, pid in ipairs(pids) do + assert(process.send(pid, "dispatch", { index = index, batch = batch })) + end + local seen = {} + local deadline = time.timer(10 * time.SECOND) + pending_deadline = deadline + for _ = 1, count do + local selected = channel.select({ responses:case_receive(), deadline:channel():case_receive() }) + assert(selected.channel == responses and selected.ok, "resident actor response deadline") + local value = selected.value + assert(active[value.pid] and not seen[value.pid], "resident actor response identity mismatch") + assert(pids[value.value.index] == value.pid and value.value.batch == batch and value.count == batch, + "resident actor dispatch mismatch") + seen[value.pid] = true + end + deadline:stop() + pending_deadline = nil + end + local ok: boolean, result: any = pcall(function() + for index = 1, count do + local pid, err = process.spawn_monitored("app:actor_resident_fixture", "app:runtime_processes", { + parent = process.pid(), topic = topic, + }) + assert(pid, tostring(err)) + active[pid] = true + pids[index] = pid + end + local ready = {} + local deadline = time.timer(10 * time.SECOND) + pending_deadline = deadline + for _ = 1, count do + local selected = channel.select({ responses:case_receive(), deadline:channel():case_receive() }) + assert(selected.channel == responses and selected.ok, "resident actor startup deadline") + local value = selected.value + assert(value.phase == "ready" and active[value.pid] and not ready[value.pid], + "resident actor startup identity mismatch") + ready[value.pid] = true + end + deadline:stop() + pending_deadline = nil + return body(dispatch) + end) + if pending_deadline then pending_deadline:stop() end + for pid in pairs(active) do process.cancel(pid, "2s") end + local deadline = time.timer(10 * time.SECOND) + while next(active) do + local selected = channel.select({ events:case_receive(), deadline:channel():case_receive() }) + if selected.channel ~= events then break end + local event = selected.value + if active[event.from] and event.kind == process.event.EXIT then + active[event.from] = nil + process.unmonitor(event.from) + if event.result.error then ok, result = false, tostring(event.result.error) end + end + end + deadline:stop() + local cleanup_expired = next(active) ~= nil + if cleanup_expired then + for pid in pairs(active) do process.terminate(pid) end + local forced = time.timer(3 * time.SECOND) + while next(active) do + local selected = channel.select({ events:case_receive(), forced:channel():case_receive() }) + if selected.channel ~= events then break end + local event = selected.value + if active[event.from] and event.kind == process.event.EXIT then + active[event.from] = nil + process.unmonitor(event.from) + end + end + forced:stop() + end + process.unlisten(responses) + assert(not next(active), tostring(result or "") .. "; resident actor termination deadline") + if not ok then error(result) end + assert(not cleanup_expired, "resident actor cleanup deadline") + return result +end + +local function define_tests() + test.describe("resident actor runtime", function() + test.it("starts, dispatches to, and cancels a real actor cohort", function() + with_residents(4, function(dispatch) + dispatch() + dispatch() + end) + end) + end) +end + +return { run = test.run_cases(define_tests) } diff --git a/src/actor/test/actor_runtime_fixture.lua b/src/actor/test/actor_runtime_fixture.lua new file mode 100644 index 0000000..3c6e5cf --- /dev/null +++ b/src/actor/test/actor_runtime_fixture.lua @@ -0,0 +1,131 @@ +local actor = require("actor") +local time = require("time") + +local function run(args) + local reply_topic = args.topic .. ".reply" + local proceed_topic = args.topic .. ".proceed" + local listener = process.listen(reply_topic) + local proceed = process.listen(proceed_topic) + local closed = channel.new(1) + local state = { count = 0, closures = 0 } + local function notify(value) + assert(process.send(args.parent, args.topic, value)) + end + local result = actor.new(state, { + __init = function(s) + if args.scenario == "closure" then + s.register_channel(closed, function(current, value, ok) + if not ok then + current.closures = current.closures + 1 + notify({ phase = "closed", value = value, ok = ok }) + end + end) + closed:close() + elseif args.scenario == "deadline" or args.scenario == "late" then + local value, err = s.wait(reply_topic, 5 * time.MILLISECOND) + notify({ phase = "expired", value = value, error = err }) + if args.scenario == "deadline" then + return actor.exit({ status = "deadline", error = err }) + end + local _, proceed_err = s.wait(proceed_topic, 2 * time.SECOND) + assert(not proceed_err, proceed_err) + local late, late_err = s.wait(reply_topic, 2 * time.SECOND) + return actor.exit({ status = "late", value = late, error = late_err }) + elseif args.scenario == "drain_exit" or args.scenario == "drain_next" then + local buffered = channel.new(4) + s.values = {} + s.register_channel(buffered, function(current, value, ok) + if ok then + current.values[#current.values + 1] = value + return + end + current.closures = current.closures + 1 + local outcome = { status = "drained", values = current.values, closures = current.closures } + if args.scenario == "drain_next" then return actor.next("next_result", outcome) end + return actor.exit(outcome) + end) + buffered:send(1) + buffered:send(false) + buffered:send("x") + buffered:close() + elseif args.scenario == "dispatch" then + notify({ phase = "ready" }) + elseif args.scenario == "init_next" then + return actor.next("next_result", { scenario = args.scenario }) + elseif args.scenario == "async_next" then + s.async(function() return actor.next("next_result", { scenario = args.scenario }) end) + elseif args.scenario == "deferred_async" then + coroutine.spawn(function() + assert(proceed:receive(), "deferred async proceed missing") + s.async(function() return actor.next("next_result", { scenario = args.scenario }) end) + end) + notify({ phase = "ready" }) + elseif args.scenario == "channel_next" then + s.register_channel(closed, function(_, value, ok) + assert(ok, "runtime channel closed before next") + return actor.next("next_result", value) + end) + closed:send({ scenario = args.scenario }) + elseif args.scenario == "event_next" then + notify({ phase = "ready" }) + elseif args.scenario == "cancel" then + notify({ phase = "waiting" }) + s.wait(reply_topic, 20 * time.MILLISECOND) + else + for index = 1, args.count or 1 do + notify({ phase = "waiting", index = index }) + local value, err = s.wait(reply_topic, 2 * time.SECOND) + if err then return actor.exit({ status = "failed", error = err }) end + s.count = s.count + 1 + s.last = value + end + return actor.exit({ status = "replied", count = s.count, value = s.last }) + end + end, + ping = function(s, payload, topic, from) + return actor.exit({ status = "inbox", closures = s.closures, + removed = not s.unregister_channel(closed), payload = payload, topic = topic, from = from }) + end, + dispatch = function(s, payload) + s.count = s.count + 1 + assert(process.send(args.parent, args.topic, { count = s.count, value = payload })) + end, + register_after_error = function(s) + assert(not pcall(error, "expected handler failure")) + s.register_channel(closed, function(_, value, ok) + return actor.exit({ status = "channel_after_error", value = value, ok = ok }) + end) + closed:send("after caught error") + end, + next_result = function(_, payload, topic, from) + return actor.exit({ status = "next", payload = payload, topic = topic, from = from }) + end, + __on_event = function(_, event) + if args.scenario == "event_next" and event.kind == process.event.CANCEL then + return actor.next("next_result", { scenario = args.scenario }) + end + end, + __on_cancel = function() + if args.scenario == "event_next" then return end + return actor.exit({ status = "canceled" }) + end, + }).run() + process.unlisten(listener) + process.unlisten(proceed) + return result +end + +local function resident(args) + return actor.new({ count = 0 }, { + __init = function() + assert(process.send(args.parent, args.topic, { phase = "ready", pid = process.pid() })) + end, + dispatch = function(state, value) + state.count = state.count + 1 + assert(process.send(args.parent, args.topic, { pid = process.pid(), count = state.count, value = value })) + end, + __on_cancel = function() return actor.exit({ status = "canceled" }) end, + }).run() +end + +return { run = run, resident = resident } diff --git a/src/actor/test/actor_runtime_test.lua b/src/actor/test/actor_runtime_test.lua new file mode 100644 index 0000000..6dbcabb --- /dev/null +++ b/src/actor/test/actor_runtime_test.lua @@ -0,0 +1,212 @@ +local test = require("test") +local time = require("time") +local actor = require("actor") +local system = require("system") + +local sequence = 0 + +local function receive(ch) + local timer = time.timer(3 * time.SECOND) + local result = channel.select({ ch:case_receive(), timer:channel():case_receive() }) + timer:stop() + assert(result.channel == ch and result.ok, "actor runtime response deadline") + return result.value +end + +local function with_fixture(scenario, count, body): any + sequence = sequence + 1 + local topic = "actor.runtime." .. sequence + local responses = process.listen(topic) + local events = process.events() + local pid, err = process.spawn_monitored("app:actor_runtime_fixture", "app:runtime_processes", { + parent = process.pid(), topic = topic, scenario = scenario, count = count, + }) + assert(pid, tostring(err)) + local exited = false + local function outcome(check_result) + local timer = time.timer(3 * time.SECOND) + while true do + local selected = channel.select({ events:case_receive(), timer:channel():case_receive() }) + if selected.channel == timer:channel() then + timer:stop() + error("actor runtime exit deadline") + end + local event = selected.value + if event.from == pid and event.kind == process.event.EXIT then + exited = true + timer:stop() + if check_result ~= false then + assert(not event.result.error, tostring(event.result.error)) + end + return event.result.value + end + end + end + local ok: boolean, result: any = pcall(body, pid, topic, responses, outcome) + if not ok and not exited then + process.terminate(pid) + local cleaned, cleanup_err = pcall(outcome, false) + if not cleaned then result = tostring(result) .. "; cleanup: " .. tostring(cleanup_err) end + end + process.unmonitor(pid) + process.unlisten(responses) + if not ok then error(result) end + return result +end + +local function wait_idle(pid) + local deadline = time.timer(3 * time.SECOND) + local cases = { deadline:channel():case_receive() } + while true do + local processes = assert(system.hosts.processes("app:runtime_processes")) + for _, child in ipairs(processes) do + if child.pid == pid and child.state == "idle" then + deadline:stop() + return + end + end + if channel.select(cases, true).channel == deadline:channel() then + deadline:stop() + error("actor idle state deadline") + end + coroutine.yield() + end +end + +local function define_tests() + test.describe("actor runtime", function() + test.it("initializes and completes repeated actors with the real process interface", function() + for index = 1, 8 do + local result = actor.new({ index = index }, { + __init = function(state) return actor.exit(state.index) end, + }).run() + test.eq(result, index) + end + end) + local next_sources = { init_next = "init", async_next = "async", + channel_next = "channel_handler", event_next = "event_handler" } + for _, scenario in ipairs({ "init_next", "async_next", "channel_next", "event_next" }) do + test.it("delivers internal next messages from " .. scenario, function() + with_fixture(scenario, 1, function(pid, _, responses, outcome) + if scenario == "event_next" then + test.eq(receive(responses).phase, "ready") + assert(process.cancel(pid, "2s")) + end + local result = outcome() + test.eq(result.status, "next") + test.eq(result.payload.scenario, scenario) + test.eq(result.topic, "next_result") + test.eq(result.from, next_sources[scenario]) + end) + end) + end + test.it("wakes an idle actor when an external coroutine invokes state.async", function() + with_fixture("deferred_async", 1, function(pid, topic, responses, outcome) + test.eq(receive(responses).phase, "ready") + wait_idle(pid) + assert(process.send(pid, topic .. ".proceed", true)) + local result = outcome() + test.eq(result.status, "next") + test.eq(result.payload.scenario, "deferred_async") + test.eq(result.from, "async") + end) + end) + test.it("receives a real topic reply before the wait deadline", function() + with_fixture("reply", 1, function(pid, topic, responses, outcome) + test.eq(receive(responses).phase, "waiting") + assert(process.send(pid, topic .. ".reply", { value = 42 })) + local result = outcome() + test.eq(result.status, "replied") + test.eq(result.value.value, 42) + test.eq(result.count, 1) + end) + end) + test.it("returns timeout when the real deadline expires", function() + with_fixture("deadline", 1, function(_, _, responses, outcome) + local expired = receive(responses) + test.eq(expired.phase, "expired") + test.is_nil(expired.value) + test.eq(expired.error, "timeout") + test.eq(outcome().status, "deadline") + end) + end) + test.it("preserves a late reply for the next wait on the same topic", function() + with_fixture("late", 1, function(pid, topic, responses, outcome) + test.eq(receive(responses).error, "timeout") + assert(process.send(pid, topic .. ".reply", { late = true })) + assert(process.send(pid, topic .. ".proceed", true)) + local result = outcome() + test.eq(result.status, "late") + test.eq(result.value.late, true) + test.is_nil(result.error) + end) + end) + test.it("reuses a topic listener across repeated waits without replaying replies", function() + with_fixture("reply", 3, function(pid, topic, responses, outcome) + for index = 1, 3 do + test.eq(receive(responses).index, index) + assert(process.send(pid, topic .. ".reply", { index = index })) + end + local result = outcome() + test.eq(result.count, 3) + test.eq(result.value.index, 3) + end) + end) + test.it("handles cancellation after a bounded outstanding wait", function() + with_fixture("cancel", 1, function(pid, _, responses, outcome) + test.eq(receive(responses).phase, "waiting") + assert(process.cancel(pid, "2s")) + test.eq(outcome().status, "canceled") + end) + end) + test.it("selects a channel registered after a handler catches an error", function() + with_fixture("dispatch", 0, function(pid, _, responses, outcome) + test.eq(receive(responses).phase, "ready") + assert(process.send(pid, "register_after_error", true)) + local result = outcome() + test.eq(result.status, "channel_after_error") + test.eq(result.value, "after caught error") + test.is_true(result.ok) + end) + end) + test.it("drains buffered channel values including false before one closure exit", function() + with_fixture("drain_exit", 0, function(_, _, _, outcome) + local result = outcome() + test.eq(result.status, "drained") + test.eq(result.closures, 1) + test.eq(#result.values, 3) + test.eq(result.values[1], 1) + test.eq(result.values[2], false) + test.eq(result.values[3], "x") + end) + end) + test.it("delivers an internal next returned by a closed channel callback", function() + with_fixture("drain_next", 0, function(_, _, _, outcome) + local result = outcome() + test.eq(result.status, "next") + test.eq(result.topic, "next_result") + test.eq(result.from, "channel_handler") + test.eq(result.payload.closures, 1) + test.eq(#result.payload.values, 3) + test.eq(result.payload.values[2], false) + end) + end) + test.it("unregisters a closed channel and continues handling inbox messages", function() + with_fixture("closure", 1, function(pid, _, responses, outcome) + local closed = receive(responses) + test.eq(closed.phase, "closed") + test.eq(closed.ok, false) + assert(process.send(pid, "ping", { value = "after closure" })) + local result = outcome() + test.eq(result.status, "inbox") + test.eq(result.closures, 1) + test.eq(result.removed, true) + test.eq(result.payload.value, "after closure") + test.eq(result.topic, "ping") + test.eq(result.from, (process.pid())) + end) + end) + end) +end + +return { run = test.run_cases(define_tests) } diff --git a/src/actor/test/actor_test.lua b/src/actor/test/actor_test.lua index 43284fb..1666e96 100644 --- a/src/actor/test/actor_test.lua +++ b/src/actor/test/actor_test.lua @@ -599,6 +599,69 @@ local function define_tests() end) end) + test.describe("select case maintenance", function() + local function run_with_select_counts(responses: {any}, init: (any) -> ()): any + local counts: any = { tables = {} } + local original_channel = _G.channel + local select_count = 0 + _G.channel = { + new = function() return mock_channel() end, + select = function(cases) + select_count = select_count + 1 + counts[select_count] = #cases + counts.tables[select_count] = cases + return responses[select_count] or { ok = false } + end + } + actor._process = mock_process(mock_channel(), mock_channel()) + local ok, err = pcall(function() + actor.new({}, { __init = function(state) init(state) end }).run() + end) + _G.channel = original_channel + if not ok then error(err) end + return counts + end + + test.it("adds and removes select cases as channels register and unregister", function() + local first, second = mock_channel(), mock_channel() + local counts = run_with_select_counts({}, function(state) + state.register_channel(first, function() end) + state.register_channel(second, function() end) + state.unregister_channel(first) + end) + test.eq(counts[1], 4) + end) + + test.it("removes a closed registered channel from the next select", function() + local chan = mock_channel() + local counts = run_with_select_counts({ + { ok = false, channel = chan }, + }, function(state) + state.register_channel(chan, function() end) + end) + test.eq(counts[1], 4) + test.eq(counts[2], 3) + test.is_true(rawequal(counts.tables[1], counts.tables[2])) + end) + + test.it("selects a channel registered after a caught error", function() + local chan = mock_channel() + local counts = run_with_select_counts({}, function(state) + pcall(error, "caught") + state.register_channel(chan, function() end) + end) + test.eq(counts[1], 4) + end) + + test.it("does not report an unknown channel as unregistered", function() + local result + run_with_select_counts({}, function(state) + result = state.unregister_channel(mock_channel()) + end) + test.is_false(result) + end) + end) + test.describe("async execution", function() test.it("state.async passes function to coroutine.spawn", function() local spawn_called = false diff --git a/src/agent/src/README.md b/src/agent/src/README.md index 5c307cf..6e3af97 100644 --- a/src/agent/src/README.md +++ b/src/agent/src/README.md @@ -219,10 +219,15 @@ local response, err = runner:step(builder, { stream_target = { reply_to = process.self(), topic = "stream" }, tool_call = "auto", disable_memory_recall = false, - previous_memory_ids = { "mem_1", "mem_2" } + previous_memory_ids = { "mem_1", "mem_2" }, + stop_check = function() return stop_requested end }) ``` +`stop_check` runs after prompt functions and memory recall, immediately before +provider generation. When it returns true, `step` returns `{ stopped = true }` +without a provider request. + ### Response Structure ```lua @@ -389,9 +394,11 @@ control handlers and apply proposals only after the round's outcomes settle. persistence. Returned `context` without `_control` is not a host configuration change. Legacy bindings do not acquire behavior controls implicitly. -Session and Dataflow persist proposals with the round and replay them after -recovery. Replay is at-least-once, not exactly-once external writes; providers -must make writes and lifecycle hooks idempotent using stable host refs. +Hosts explicitly collect these proposals from the lifecycle summary (`controls`) +and from `caller:get_wrapper_controls()`. A host that persists proposals with +the round and replays them after recovery +gives at-least-once delivery, not exactly-once external writes; providers must +make writes and lifecycle hooks idempotent using stable host refs. `memory.compact = true` requests checkpointing without a token threshold, but does not bypass an explicit disable or supply a missing provider. False does not cancel an earlier request. There are no automatic evaluator calls, new diff --git a/src/agent/src/agent.lua b/src/agent/src/agent.lua index cbe2d59..ac0d559 100644 --- a/src/agent/src/agent.lua +++ b/src/agent/src/agent.lua @@ -577,6 +577,10 @@ function agent:step(prompt_builder: any, runtime_options: any): (table?, string? --end + if runtime_options.stop_check and runtime_options.stop_check() then + return { stopped = true }, nil + end + local result, err = llm_instance.generate(final_messages, options) if err then return nil, err diff --git a/src/agent/src/tools/caller.lua b/src/agent/src/tools/caller.lua index 360122a..a3a3050 100644 --- a/src/agent/src/tools/caller.lua +++ b/src/agent/src/tools/caller.lua @@ -290,6 +290,14 @@ local function reset_wrapper_diagnostics(self: any) self.wrapper_controls = {} end +local function discard_stale_round(self: any) + if next(self.wrapper_observations) ~= nil then self.wrapper_observations = {} end + if next(self.wrapper_metadata) ~= nil then self.wrapper_metadata = {} end + if next(self.wrapper_errors) ~= nil then self.wrapper_errors = {} end + if next(self.wrapper_controls) ~= nil then self.wrapper_controls = {} end + if next(self.last_tool_calls) ~= nil then self.last_tool_calls = {} end +end + function tool_caller:get_wrapper_controls(): {table} return behavior_controls.prepare(self.wrapper_controls or {}) or {} end @@ -459,6 +467,7 @@ end function tool_caller:validate(tool_calls: {ToolCall}?): (any, string?) -- Check if there are any tool calls if not tool_calls or #tool_calls == 0 then + discard_stale_round(self) return {}, nil end diff --git a/src/agent/test/.wippy.yaml b/src/agent/test/.wippy.yaml new file mode 100644 index 0000000..977dbf4 --- /dev/null +++ b/src/agent/test/.wippy.yaml @@ -0,0 +1,4 @@ +version: "1.0" +workspace: + replacements: + wippy/test: ../../test diff --git a/src/agent/test/_index.yaml b/src/agent/test/_index.yaml index f05b138..b2a7da9 100644 --- a/src/agent/test/_index.yaml +++ b/src/agent/test/_index.yaml @@ -393,3 +393,36 @@ entries: - time - ctx method: delayed_echo + + - name: tools_controls_runtime_test + kind: function.lua + meta: + type: test + suite: agent-runtime + source: file://tools/controls_runtime_test.lua + imports: + test: wippy.test:test + tool_caller: wippy.agent.tools:caller + method: run_tests + + - name: control_fixture + kind: function.lua + meta: + type: tool + input_schema: '{"type":"object"}' + source: file://tools/control_fixture.lua + modules: [ctx] + method: execute + + - name: control_wrapper_apply + kind: function.lua + source: file://tools/control_fixture.lua + modules: [ctx] + method: apply + + - name: control_wrapper_binding + kind: contract.binding + contracts: + - contract: wippy.agent:tool_wrapper + methods: + apply: app:control_wrapper_apply diff --git a/src/agent/test/agent_integration_test.lua b/src/agent/test/agent_integration_test.lua index c6977b7..b9e6de9 100644 --- a/src/agent/test/agent_integration_test.lua +++ b/src/agent/test/agent_integration_test.lua @@ -10,11 +10,8 @@ local json = require("json") local env = require("env") local function define_integration_tests() - -- Skip all integration tests if not enabled - if not env.get("ENABLE_INTEGRATION_TESTS") then - print("WARNING: Integration tests skipped - set ENABLE_INTEGRATION_TESTS=true to run") - return - end + local enabled = env.get("ENABLE_INTEGRATION_TESTS") == "true" + local it = enabled and _G.it or _G.it_skip -- Test helpers for full chain testing local test_helpers = {} diff --git a/src/agent/test/agent_test.lua b/src/agent/test/agent_test.lua index 6aaa2c2..20b8fdb 100644 --- a/src/agent/test/agent_test.lua +++ b/src/agent/test/agent_test.lua @@ -614,6 +614,34 @@ local function define_tests() test.eq(seen.temperature, 0.7) end) + it("should skip provider generation when stop_check reports a committed stop", function() + local test_agent = agent.new({ + id = "stop-check-agent", + name = "Stop Check Agent", + description = "Checks stop before generation", + prompt = "You are stoppable." + }) + local calls = { generate = 0, checks = 0 } + agent._llm = { + generate = function() + calls.generate = calls.generate + 1 + return { result = "done", tokens = {} } + end + } + local result, err = test_agent:step(mock_prompt.new(), { + stop_check = function() calls.checks = calls.checks + 1; return true end + }) + test.is_nil(err) + test.is_true(result.stopped) + test.eq(calls.generate, 0) + test.eq(calls.checks, 1) + + result, err = test_agent:step(mock_prompt.new(), { stop_check = function() return false end }) + test.is_nil(err) + test.is_nil(result.stopped) + test.eq(calls.generate, 1) + end) + it("should carry an explicit zero temperature rather than treating it as unset", function() local spec = { id = "zero-temp-agent", diff --git a/src/agent/test/tools/caller_test.lua b/src/agent/test/tools/caller_test.lua index 73ca65b..e1094c1 100644 --- a/src/agent/test/tools/caller_test.lua +++ b/src/agent/test/tools/caller_test.lua @@ -756,6 +756,41 @@ local function define_tests() test.eq(count_table_elements(validated_tools), 0) end) + it("should discard the previous round state when validating no tool calls", function() + for _, empty in ipairs({ {}, false }) do + local caller = tool_caller.new() + caller.wrapper_observations = { { tool = "old" } } + caller.wrapper_metadata = { { tool = "old" } } + caller.wrapper_errors = { { error = "old" } } + caller.wrapper_controls = { { type = "stop" } } + caller.last_tool_calls = { { id = "call-old", name = "old" } } + + local validated_tools, err = caller:validate(empty or nil) + + test.is_nil(err) + test.eq(count_table_elements(validated_tools), 0) + test.eq(#caller:get_wrapper_observations(), 0) + test.eq(#caller:get_wrapper_metadata(), 0) + test.eq(#caller:get_wrapper_errors(), 0) + test.eq(#caller:get_wrapper_controls(), 0) + test.eq(#caller:get_last_tool_calls(), 0) + end + end) + + it("should keep empty round state tables when validating no tool calls", function() + local caller = tool_caller.new() + local observations, metadata = caller.wrapper_observations, caller.wrapper_metadata + local errors, controls, calls = caller.wrapper_errors, caller.wrapper_controls, caller.last_tool_calls + + caller:validate({}) + + test.is_true(rawequal(caller.wrapper_observations, observations)) + test.is_true(rawequal(caller.wrapper_metadata, metadata)) + test.is_true(rawequal(caller.wrapper_errors, errors)) + test.is_true(rawequal(caller.wrapper_controls, controls)) + test.is_true(rawequal(caller.last_tool_calls, calls)) + end) + it("should handle tool schema errors", function() local caller = tool_caller.new() diff --git a/src/agent/test/tools/control_fixture.lua b/src/agent/test/tools/control_fixture.lua new file mode 100644 index 0000000..28bf750 --- /dev/null +++ b/src/agent/test/tools/control_fixture.lua @@ -0,0 +1,47 @@ +local ctx = require("ctx") + +local function execute(args) + if args.raise then error("runtime fixture failure") end + local control = args.control + return { + success = args.fail ~= true, + error = args.fail and "explicit runtime fixture failure" or nil, + message = args.message, + call_id = ctx.get("call_id"), + scope = ctx.get("scope"), + _control = control, + wrapper_control = args.wrapper_control, + } +end + +local function apply(request) + if request.phase == "before_execute" then + if request.options.mode == "drop" then return {tool_calls = {}} end + local rewritten = {} + for _, call in ipairs(request.tool_calls) do + local args = {} + for key, value in pairs(call.arguments) do args[key] = value end + args.message = "wrapped " .. tostring(args.message) + rewritten[#rewritten + 1] = { + id = call.id, name = call.name, registry_id = call.registry_id, + arguments = args, context = call.context, + } + end + return {tool_calls = rewritten} + end + local proposal + for _, result in pairs(request.tool_results or {}) do + if result.result and result.result.wrapper_control then + proposal = result.result.wrapper_control + end + end + local call_ids = {} + for _, call in ipairs(request.tool_calls or {}) do call_ids[#call_ids + 1] = call.id end + return { + _control = proposal, + observations = {{level = "info", code = "runtime_outcome", content = request.outcome}}, + metadata = {phase = request.phase, call_ids = call_ids}, + } +end + +return {execute = execute, apply = apply} diff --git a/src/agent/test/tools/controls_runtime_test.lua b/src/agent/test/tools/controls_runtime_test.lua new file mode 100644 index 0000000..a47c71e --- /dev/null +++ b/src/agent/test/tools/controls_runtime_test.lua @@ -0,0 +1,176 @@ +local test = require("test") +local tool_caller = require("tool_caller") + +local function call(id, args) + return {id = id, name = "runtime_control", registry_id = "app:control_fixture", + arguments = args, context = {scope = "tool"}} +end + +local function count(value) + local total = 0 + for _ in pairs(value) do total = total + 1 end + return total +end + +local function wrapper(source, phases, options) + return {id = "runtime-wrapper", binding = "app:control_wrapper_binding", source = source, + phases = phases, options = options or {}, strict = true} +end + +local function define_tests() + describe("registered tool runtime controls", function() + for _, strategy in ipairs({tool_caller.STRATEGY.SEQUENTIAL, tool_caller.STRATEGY.PARALLEL}) do + it(strategy .. " preserves IDs, controls and successful peers across mixed failures", function() + local caller = tool_caller.new():set_strategy(strategy) + caller:set_wrapper_context({host = {kind = "session", session_id = "runtime"}}) + caller:set_tool_wrappers({wrapper(nil, {"after_execute"})}) + local validated, err = caller:validate({ + call("success-id", {message = "success", control = {config = {model = "next-model"}, + context = {session = {set = {scope = "next"}}}, memory = {compact = true}}}), + call("data-error-id", {fail = true, message = "failure", control = {memory = {compact = false}}}), + call("execution-error-id", {raise = true}), + call("json-error-id", "{invalid json"), + {id = "missing-id", name = "missing", registry_id = "app:missing", arguments = {}}, + }) + test.is_nil(err) + local results = caller:execute({scope = "session"}, validated) + test.eq(count(results), 5) + test.is_nil(results["success-id"].error) + local success = results["success-id"].result + test.eq(success.call_id, "success-id") + test.eq(success.scope, "session") + test.eq(success.message, "success") + test.eq(success._control.config.model, "next-model") + test.eq(success._control.context.session.set.scope, "next") + test.is_true(success._control.memory.compact) + test.eq(results["data-error-id"].error, "explicit runtime fixture failure") + test.is_false(results["data-error-id"].result._control.memory.compact) + test.eq(results["data-error-id"].result.call_id, "data-error-id") + test.not_nil(results["execution-error-id"].error) + test.is_nil(results["execution-error-id"].result) + test.contains(tostring(results["json-error-id"].error), "Failed to parse arguments") + test.not_nil(results["missing-id"].error) + for id, result in pairs(results) do test.eq(result.tool_call.call_id, id) end + test.eq(caller:get_wrapper_observations()[1].content.reason, "tool_execution_failed") + end) + + it(strategy .. " collects detached behavior proposals and resets each execution", function() + local caller = tool_caller.new():set_strategy(strategy) + caller:set_wrapper_context({host = {kind = "session", session_id = "runtime"}}) + caller:set_tool_wrappers({wrapper("behavior", {"before_execute", "after_execute"})}) + local validated, err = caller:validate({call("behavior-id", {message = "first", + wrapper_control = {config = {model = "stronger"}, memory = {compact = false}}})}) + test.is_nil(err) + local results = caller:execute({}, validated) + test.eq(results["behavior-id"].result.message, "wrapped first") + test.eq(results["behavior-id"].result.call_id, "behavior-id") + test.eq(#caller:get_wrapper_controls(), 1) + local controls = caller:get_wrapper_controls() + test.eq(controls[1].config.model, "stronger") + test.is_false(controls[1].memory.compact) + controls[1].config.model = "mutated" + test.eq(caller:get_wrapper_controls()[1].config.model, "stronger") + results["behavior-id"].result.wrapper_control.config.model = "changed-result" + test.eq(caller:get_wrapper_controls()[1].config.model, "stronger") + caller:set_tool_wrappers({}) + caller:execute({}, validated) + test.eq(#caller:get_wrapper_controls(), 0) + caller:set_tool_wrappers({wrapper("behavior", {"after_execute"})}) + validated, err = caller:validate({call("second-id", {message = "second"})}) + test.is_nil(err) + results = caller:execute({}, validated) + test.eq(results["second-id"].result.message, "second") + test.eq(#caller:get_wrapper_controls(), 0) + test.eq(#caller:get_wrapper_errors(), 0) + end) + + it(strategy .. " keeps different nested controls paired with two successful calls", function() + local caller = tool_caller.new():set_strategy(strategy) + caller:set_wrapper_context({host = {kind = "session", session_id = "runtime"}}) + local validated, err = caller:validate({ + call("left-id", {message = "left", control = {config = {model = "left-model"}, + context = {session = {set = {side = "left", nested = {depth = {value = 1}}}}}}}), + call("right-id", {message = "right", control = {config = {model = "right-model"}, + context = {session = {set = {side = "right", nested = {depth = {value = 2}}}}}}}), + }) + test.is_nil(err) + local results = caller:execute({}, validated) + test.eq(count(results), 2) + for id, side in pairs({["left-id"] = "left", ["right-id"] = "right"}) do + local entry = results[id] + test.is_nil(entry.error) + test.eq(entry.tool_call.call_id, id) + test.eq(entry.result.call_id, id) + test.eq(entry.result.message, side) + test.eq(entry.result._control.config.model, side .. "-model") + test.eq(entry.result._control.context.session.set.side, side) + end + test.eq(results["left-id"].result._control.context.session.set.nested.depth.value, 1) + test.eq(results["right-id"].result._control.context.session.set.nested.depth.value, 2) + test.is_false(results["left-id"].result._control == results["right-id"].result._control) + end) + + it(strategy .. " clears the previous round when a later round has no tool calls", function() + local caller = tool_caller.new():set_strategy(strategy) + caller:set_wrapper_context({host = {kind = "session", session_id = "runtime"}}) + caller:set_tool_wrappers({wrapper("behavior", {"after_execute"})}) + local validated, err = caller:validate({call("first-id", {message = "first", + wrapper_control = {memory = {compact = true}}})}) + test.is_nil(err) + caller:execute({}, validated) + test.eq(#caller:get_wrapper_controls(), 1) + test.eq(caller:get_wrapper_metadata()[1].metadata.call_ids[1], "first-id") + + validated, err = caller:validate({}) + test.is_nil(err) + test.eq(#caller:get_last_tool_calls(), 0) + test.eq(#caller:get_wrapper_observations(), 0) + test.eq(#caller:get_wrapper_metadata(), 0) + test.eq(#caller:get_wrapper_errors(), 0) + test.eq(#caller:get_wrapper_controls(), 0) + + local results = caller:execute({}, validated) + test.eq(count(results), 0) + local metadata = caller:get_wrapper_metadata() + test.eq(#metadata, 1) + test.eq(#metadata[1].metadata.call_ids, 0) + test.eq(#caller:get_wrapper_controls(), 0) + end) + + it(strategy .. " keeps legacy wrappers from proposing behavior controls", function() + local caller = tool_caller.new():set_strategy(strategy) + caller:set_wrapper_context({host = {kind = "session", session_id = "runtime"}}) + caller:set_tool_wrappers({wrapper(nil, {"after_execute"})}) + local validated, err = caller:validate({call("legacy-id", {wrapper_control = {memory = {compact = true}}})}) + test.is_nil(err) + local results = caller:execute({}, validated) + test.is_nil(results["legacy-id"].error) + test.eq(#caller:get_wrapper_controls(), 0) + test.eq(#caller:get_wrapper_observations(), 1) + end) + + it(strategy .. " rejects invalid behavior proposals without losing tool results", function() + local caller = tool_caller.new():set_strategy(strategy) + caller:set_wrapper_context({host = {kind = "session", session_id = "runtime"}}) + caller:set_tool_wrappers({wrapper("behavior", {"after_execute"})}) + local validated, err = caller:validate({call("invalid-id", {message = "completed", wrapper_control = {yield = {}}})}) + test.is_nil(err) + local results = caller:execute({}, validated) + test.eq(results["invalid-id"].result.message, "completed") + test.eq(#caller:get_wrapper_controls(), 0) + test.eq(#caller:get_wrapper_errors(), 1) + end) + end + + it("rejects a registered wrapper that drops provider call identities", function() + local caller = tool_caller.new() + caller:set_wrapper_context({host = {kind = "session", session_id = "runtime"}}) + caller:set_tool_wrappers({wrapper("behavior", {"before_execute"}, {mode = "drop"})}) + local validated, err = caller:validate({call("must-survive", {message = "first"})}) + test.is_nil(validated) + test.contains(err, "preserve every tool call ID") + end) + end) +end + +return {run_tests = test.run_cases(define_tests)} diff --git a/src/llm/test/bedrock/integration_test.lua b/src/llm/test/bedrock/integration_test.lua index c36681d..76a9df7 100644 --- a/src/llm/test/bedrock/integration_test.lua +++ b/src/llm/test/bedrock/integration_test.lua @@ -8,7 +8,7 @@ local json = require("json") local env = require("env") local function define_tests() - local RUN_INTEGRATION_TESTS = env.get("ENABLE_INTEGRATION_TESTS") + local RUN_INTEGRATION_TESTS = env.get("ENABLE_INTEGRATION_TESTS") == "true" local CLAUDE_MODEL = "us.anthropic.claude-haiku-4-5-20251001-v1:0" local CLAUDE_THINKING_MODEL = "us.anthropic.claude-sonnet-4-5-20250929-v1:0" local LLAMA_MODEL = "us.meta.llama4-maverick-17b-instruct-v1:0" diff --git a/src/llm/test/claude/integration_test.lua b/src/llm/test/claude/integration_test.lua index a78cbb3..934d850 100644 --- a/src/llm/test/claude/integration_test.lua +++ b/src/llm/test/claude/integration_test.lua @@ -8,7 +8,7 @@ local llm = require("llm") local function define_tests() -- Toggle to enable/disable real API integration tests - local RUN_INTEGRATION_TESTS = env.get("ENABLE_INTEGRATION_TESTS") + local RUN_INTEGRATION_TESTS = env.get("ENABLE_INTEGRATION_TESTS") == "true" describe("Claude Integration Tests", function() local actual_api_key = nil diff --git a/src/llm/test/e2e/integration_test.lua b/src/llm/test/e2e/integration_test.lua index 94b1f1d..a211444 100644 --- a/src/llm/test/e2e/integration_test.lua +++ b/src/llm/test/e2e/integration_test.lua @@ -3,28 +3,17 @@ local env = require("env") local bedrock_credentials = require("bedrock_credentials") local function define_tests() - local RUN_INTEGRATION_TESTS = env.get("ENABLE_INTEGRATION_TESTS") - local RUN_BEDROCK_TESTS = false + local enabled = env.get("ENABLE_INTEGRATION_TESTS") == "true" + local bedrock_enabled = false + if enabled then + local _, creds_err = bedrock_credentials.resolve() + bedrock_enabled = creds_err == nil + end + local it = enabled and _G.it or _G.it_skip + local bedrock_it = bedrock_enabled and _G.it or _G.it_skip describe("LLM Integration Tests", function() - before_all(function() - if not RUN_INTEGRATION_TESTS then - print("Integration tests disabled - set ENABLE_INTEGRATION_TESTS=true to enable") - return - end - local _, creds_err = bedrock_credentials.resolve() - if creds_err then - print("Bedrock integration tests disabled - " .. tostring(creds_err)) - else - RUN_BEDROCK_TESTS = true - end - end) - it("should call provider directly with provider_id and model", function() - if not RUN_INTEGRATION_TESTS then - print("Skipping direct provider test - not enabled") - return - end local messages = { { role = "user", @@ -51,10 +40,6 @@ local function define_tests() end) it("should handle streaming with provider_id", function() - if not RUN_INTEGRATION_TESTS then - print("Skipping streaming provider test - not enabled") - return - end local messages = { { @@ -84,10 +69,6 @@ local function define_tests() end) it("should handle tool calling with provider_id", function() - if not RUN_INTEGRATION_TESTS then - print("Skipping tool calling provider test - not enabled") - return - end local messages = { { @@ -132,8 +113,7 @@ local function define_tests() test.eq(result.tool_calls[1].name, "calculator") end) - it("should generate text via Bedrock provider using Converse API", function() - if not RUN_BEDROCK_TESTS then return end + bedrock_it("should generate text via Bedrock provider using Converse API", function() local messages = { { role = "user", content = { { type = "text", text = "Say 'Hello from Bedrock'" } } } @@ -156,8 +136,7 @@ local function define_tests() test.is_true(result.tokens.prompt_tokens > 0) end) - it("should handle Bedrock tool calling via Converse API", function() - if not RUN_BEDROCK_TESTS then return end + bedrock_it("should handle Bedrock tool calling via Converse API", function() local messages = { { role = "user", content = { { type = "text", text = "What is 15 * 23? Use the calculator tool." } } } @@ -195,8 +174,7 @@ local function define_tests() test.eq(result.tool_calls[1].name, "calculator") end) - it("should handle Bedrock structured output via Converse API", function() - if not RUN_BEDROCK_TESTS then return end + bedrock_it("should handle Bedrock structured output via Converse API", function() local messages = { { role = "user", content = { { type = "text", text = "Extract: Alice is 28 and lives in Paris" } } } @@ -230,8 +208,7 @@ local function define_tests() test.eq((result.result :: any).city, "Paris") end) - it("should handle Bedrock Titan embedding via InvokeModel", function() - if not RUN_BEDROCK_TESTS then return end + bedrock_it("should handle Bedrock Titan embedding via InvokeModel", function() local options = { provider_id = "wippy.llm.bedrock:provider", @@ -249,8 +226,7 @@ local function define_tests() test.not_nil(result.tokens) end) - it("should handle Bedrock Cohere embedding via InvokeModel", function() - if not RUN_BEDROCK_TESTS then return end + bedrock_it("should handle Bedrock Cohere embedding via InvokeModel", function() local options = { provider_id = "wippy.llm.bedrock:provider", @@ -268,8 +244,7 @@ local function define_tests() test.is_true(#result.result[2] > 0) end) - it("should handle Bedrock streaming via ConverseStream", function() - if not RUN_BEDROCK_TESTS then return end + bedrock_it("should handle Bedrock streaming via ConverseStream", function() local messages = { { role = "user", content = { { type = "text", text = "Count 1 2 3" } } } diff --git a/src/llm/test/google/integration_test.lua b/src/llm/test/google/integration_test.lua index af195e3..2616a64 100644 --- a/src/llm/test/google/integration_test.lua +++ b/src/llm/test/google/integration_test.lua @@ -5,7 +5,7 @@ local env = require("env") local function define_tests() -- Toggle to enable/disable real API integration tests - local RUN_INTEGRATION_TESTS = env.get("ENABLE_INTEGRATION_TESTS") + local RUN_INTEGRATION_TESTS = env.get("ENABLE_INTEGRATION_TESTS") == "true" describe("Google AI Integration Tests", function() local actual_gemini_api_key = nil diff --git a/src/llm/test/openai/integration_test.lua b/src/llm/test/openai/integration_test.lua index 932c316..eefd1d4 100644 --- a/src/llm/test/openai/integration_test.lua +++ b/src/llm/test/openai/integration_test.lua @@ -9,7 +9,7 @@ local test = require("test") local function define_tests() -- Toggle to enable/disable real API integration tests - local RUN_INTEGRATION_TESTS = env.get("ENABLE_INTEGRATION_TESTS") + local RUN_INTEGRATION_TESTS = env.get("ENABLE_INTEGRATION_TESTS") == "true" describe("OpenAI Integration Tests", function() local actual_api_key = nil diff --git a/src/llm/test/openai_compat/integration_test.lua b/src/llm/test/openai_compat/integration_test.lua index 17be6d5..89e07d5 100644 --- a/src/llm/test/openai_compat/integration_test.lua +++ b/src/llm/test/openai_compat/integration_test.lua @@ -9,7 +9,7 @@ local test = require("test") local function define_tests() -- Toggle to enable/disable real API integration tests - local RUN_INTEGRATION_TESTS = env.get("ENABLE_INTEGRATION_TESTS") + local RUN_INTEGRATION_TESTS = env.get("ENABLE_INTEGRATION_TESTS") == "true" describe("OpenAI-Compatible Integration Tests", function() local actual_api_key = nil diff --git a/src/llm/test/typesafe/integration_test.lua b/src/llm/test/typesafe/integration_test.lua index 16cae83..a414581 100644 --- a/src/llm/test/typesafe/integration_test.lua +++ b/src/llm/test/typesafe/integration_test.lua @@ -8,7 +8,7 @@ local function define_tests() -- Toggle to enable/disable real API integration tests -- A provider-specific opt-in exercises Jev without enabling unrelated -- integration suites that require other vendors' credentials. - local RUN_INTEGRATION_TESTS = env.get("ENABLE_INTEGRATION_TESTS") or env.get("TYPESAFE_INTEGRATION_TESTS") + local RUN_INTEGRATION_TESTS = env.get("ENABLE_INTEGRATION_TESTS") == "true" or env.get("TYPESAFE_INTEGRATION_TESTS") == "true" describe("TypeSafe Integration Tests", function() local actual_api_key = nil diff --git a/src/llm/test/util/compress_test.lua b/src/llm/test/util/compress_test.lua index 239d31e..b541fba 100644 --- a/src/llm/test/util/compress_test.lua +++ b/src/llm/test/util/compress_test.lua @@ -3,7 +3,7 @@ local env = require("env") local test = require("test") local function define_tests() - local RUN_INTEGRATION_TESTS = env.get("ENABLE_INTEGRATION_TESTS") + local RUN_INTEGRATION_TESTS = env.get("ENABLE_INTEGRATION_TESTS") == "true" describe("Compress Library", function() diff --git a/src/relay/src/user.lua b/src/relay/src/user.lua index 4a85ea4..ef10266 100644 --- a/src/relay/src/user.lua +++ b/src/relay/src/user.lua @@ -18,6 +18,12 @@ type PluginState = { status: string, } +type Route = { + prefix: string, + config: PluginConfig, + topic: string, +} + type UserState = { user_id: string, user_metadata: any, @@ -29,8 +35,12 @@ type UserState = { client_count: number, pg_scopes: {[string]: any}, pg_groups: {[string]: string}, + routes: {[any]: Route}, + route_count: number, } +local MAX_CACHED_ROUTES = 64 + type UserUpgradeState = { relay_user_upgrade: boolean, user_id: string, @@ -273,7 +283,10 @@ local function route_to_plugin(state: UserState, prefix: string, plugin_config: context = message_data.context } - process.send(pid, topic, payload_data) + local sent, send_err = process.send(pid, topic, payload_data) + if not sent then + return false, "Failed to send message to plugin: " .. tostring(send_err) + end return true end end @@ -284,21 +297,23 @@ end local function handle_client_join(state: UserState, payload_data: any) local client_pid = payload_data.client_pid :: string? if client_pid then - state.connected_clients[client_pid] = true - state.client_count = state.client_count + 1 - - logger:info("client connected", { - client_pid = client_pid, - client_count = state.client_count, - user_id = state.user_id - }) - - if state.client_count == 1 then - local session_plugin = state.active_plugins["session_"] - if session_plugin then - local pid = session_plugin.pid - if pid then - process.send(pid, "resume", {}) + if not state.connected_clients[client_pid] then + state.connected_clients[client_pid] = true + state.client_count = state.client_count + 1 + + logger:info("client connected", { + client_pid = client_pid, + client_count = state.client_count, + user_id = state.user_id + }) + + if state.client_count == 1 then + local session_plugin = state.active_plugins["session_"] + if session_plugin then + local pid = session_plugin.pid + if pid then + process.send(pid, "resume", {}) + end end end end @@ -342,8 +357,8 @@ local function handle_client_leave(state: UserState, payload_data: any) end local function handle_client_message(state: UserState, payload: any, from_pid: string) - local message_data, err = json.decode(string(payload:data())) - if not message_data then + local message_data, err = json.decode(payload:data()) + if err then process.send(from_pid, consts.CLIENT_TOPICS.ERROR, { error = consts.ERROR_CODES.INVALID_JSON, message = "Failed to decode JSON message" @@ -351,26 +366,42 @@ local function handle_client_message(state: UserState, payload: any, from_pid: s return end - local msg_type = message_data.type :: string? - if not msg_type then + if type(message_data) ~= "table" then process.send(from_pid, consts.CLIENT_TOPICS.ERROR, { error = consts.ERROR_CODES.UNKNOWN_COMMAND, - message = "Message type is required" + message = "Message must be a JSON object" }) return end - local plugin_prefix, plugin_config = find_plugin_for_command(state, msg_type) - if not plugin_prefix or not plugin_config then - process.send(from_pid, consts.CLIENT_TOPICS.ERROR, { - error = consts.ERROR_CODES.PLUGIN_NOT_FOUND, - message = "No plugin found for command: " .. msg_type - }) - return + local msg_type: any = message_data.type + local route = state.routes[msg_type] + if not route then + if type(msg_type) ~= "string" or msg_type == "" then + process.send(from_pid, consts.CLIENT_TOPICS.ERROR, { + error = consts.ERROR_CODES.UNKNOWN_COMMAND, + message = "Message type is required" + }) + return + end + + local plugin_prefix, plugin_config = find_plugin_for_command(state, msg_type) + if not plugin_prefix or not plugin_config then + process.send(from_pid, consts.CLIENT_TOPICS.ERROR, { + error = consts.ERROR_CODES.PLUGIN_NOT_FOUND, + message = "No plugin found for command: " .. msg_type + }) + return + end + + route = { prefix = plugin_prefix, config = plugin_config, topic = msg_type:sub(#plugin_prefix + 1) } + if state.route_count < MAX_CACHED_ROUTES then + state.routes[msg_type] = route + state.route_count = state.route_count + 1 + end end - local stripped_topic = msg_type:sub(#plugin_prefix + 1) - local success, route_err = route_to_plugin(state, plugin_prefix!, plugin_config!, stripped_topic, message_data, from_pid) + local success, route_err = route_to_plugin(state, route.prefix, route.config, route.topic, message_data, from_pid) if not success then process.send(from_pid, consts.CLIENT_TOPICS.ERROR, { error = consts.ERROR_CODES.PLUGIN_FAILED, @@ -449,7 +480,11 @@ local function run(args: any): any local resume_state = args :: UserUpgradeState active_plugins = resume_state.active_plugins connected_clients = resume_state.connected_clients - client_count = resume_state.client_count + -- Older duplicate JOINs could inflate the saved counter. Memberships + -- are authoritative; rebuilding the derived count keeps final LEAVE valid. + for _ in pairs(connected_clients) do + client_count = client_count + 1 + end upgrade_groups = resume_state.pg_groups end local user_id = string(args.user_id) @@ -466,7 +501,9 @@ local function run(args: any): any connected_clients = connected_clients, client_count = client_count, pg_scopes = {}, - pg_groups = {} + pg_groups = {}, + routes = {}, + route_count = 0, } local registry_name = consts.USER_HUB_REGISTRY_PREFIX .. state.user_id @@ -495,12 +532,10 @@ local function run(args: any): any local inbox = process.inbox() local events = process.events() + local select_cases = { inbox:case_receive(), events:case_receive() } while true do - local result = channel.select({ - inbox:case_receive(), - events:case_receive() - }) + local result = channel.select(select_cases) if not result.ok then break @@ -508,9 +543,9 @@ local function run(args: any): any if result.channel == inbox then local msg: any = result.value - local topic = string(msg:topic()) + local topic: string = msg:topic() local payload: any = msg:payload() - local from_pid = string(msg:from()) + local from_pid: string = msg:from() if topic == consts.WS_TOPICS.JOIN then handle_client_join(state, payload:data()) diff --git a/src/relay/test/.wippy.yaml b/src/relay/test/.wippy.yaml new file mode 100644 index 0000000..977dbf4 --- /dev/null +++ b/src/relay/test/.wippy.yaml @@ -0,0 +1,4 @@ +version: "1.0" +workspace: + replacements: + wippy/test: ../../test diff --git a/src/relay/test/_index.yaml b/src/relay/test/_index.yaml index 544773d..694d111 100644 --- a/src/relay/test/_index.yaml +++ b/src/relay/test/_index.yaml @@ -62,6 +62,7 @@ entries: modules: - time - security + - env method: run # app:central_test @@ -84,3 +85,43 @@ entries: modules: - time method: run + + - name: runtime_plugin_policy + kind: security.policy + groups: + - app:user + policy: + resources: '*' + actions: + - process.spawn + - process.spawn.linked + - process.spawn.monitored + - process.host + - process.cancel + effect: allow + + - name: relay_runtime_plugin + kind: process.lua + source: file://relay_runtime_plugin.lua + method: run + + - name: relay_runtime_test + kind: function.lua + meta: + type: test + suite: relay-runtime + timeout: 25s + source: file://relay_runtime_test.lua + imports: + test: wippy.test:test + consts: wippy.relay:consts + modules: + - time + - json + - security + method: run + + - name: relay_runtime_finished_plugin + kind: process.lua + source: file://relay_runtime_plugin.lua + method: finish diff --git a/src/relay/test/relay_runtime_plugin.lua b/src/relay/test/relay_runtime_plugin.lua new file mode 100644 index 0000000..75a079c --- /dev/null +++ b/src/relay/test/relay_runtime_plugin.lua @@ -0,0 +1,23 @@ +local function run(args) + local observer = args.config.runtime_observer + assert(process.send(observer, "relay.runtime.started", { pid = process.pid(), user_id = args.user_id })) + local inbox = process.inbox() + local events = process.events() + while true do + local result = channel.select({ inbox:case_receive(), events:case_receive() }) + if not result.ok then break end + if result.channel == events then + if result.value.kind == process.event.CANCEL then break end + else + local message = result.value + local payload = message:payload():data() + assert(process.send(args.user_hub_pid, "relay.runtime.response", { + topic = message:topic(), from = message:from(), plugin_pid = process.pid(), + user_id = args.user_id, user_metadata = args.user_metadata, payload = payload, + })) + end + end + return { status = "stopped" } +end + +return { run = run, finish = function() return { status = "finished" } end } diff --git a/src/relay/test/relay_runtime_test.lua b/src/relay/test/relay_runtime_test.lua new file mode 100644 index 0000000..6ef0d63 --- /dev/null +++ b/src/relay/test/relay_runtime_test.lua @@ -0,0 +1,348 @@ +local test = require("test") +local time = require("time") +local json = require("json") +local security = require("security") +local consts = require("consts") + +local sequence = 0 +local PIPELINE_WINDOW = 32 +local function pipeline_slot(index: any, first: number, last: number): number? + if type(index) ~= "number" or index < first or index > last or index % 1 ~= 0 then return nil end + return index - first + 1 +end + +local function receive(ch: any): any + local timer = time.timer(3 * time.SECOND) + local result = channel.select({ ch:case_receive(), timer:channel():case_receive() }) + timer:stop() + assert(result.channel == ch and result.ok, "relay runtime response deadline") + return result.value +end + +local function with_hub(plugin_id: string, body: any, active_pid: string?, session_pid: string?, saved_client_count: number?): any + sequence = sequence + 1 + local user_id = "relay-runtime-" .. process.pid() .. "-" .. sequence + local actor = security.new_actor(user_id, {}) + local scope, scope_err = security.named_scope("app:user") + assert(scope, tostring(scope_err)) + local config = consts.get_config() + config.runtime_observer = process.pid() + local welcome = process.listen(consts.CLIENT_TOPICS.WELCOME) + local response = process.listen("relay.runtime.response") + local errors = process.listen(consts.CLIENT_TOPICS.ERROR) + local started = process.listen("relay.runtime.started") + local events = process.events() + local args = { + user_id = user_id, user_metadata = { fixture = true }, + config = config, plugins = { + runtime_ = { prefix = "runtime_", process_id = plugin_id, + host = config.user_hub_host, auto_start = false }, + }, + } + if active_pid or session_pid then + args.relay_user_upgrade = true + args.active_plugins = {} + if active_pid then + args.active_plugins.runtime_ = { pid = active_pid, status = "running", restart_count = 0 } + end + if session_pid then + args.active_plugins.session_ = { pid = session_pid, status = "running", restart_count = 0 } + end + args.connected_clients = {} + args.client_count = 0 + if saved_client_count ~= nil then + args.connected_clients[process.pid()] = true + args.client_count = saved_client_count + end + args.pg_groups = {} + args = (json.encode(args)) + end + local pid, err = process.with_context({}):with_actor(actor):with_scope(scope):spawn_monitored( + consts.USER_HUB_PROCESS_ID, assert(config.user_hub_host), args) + assert(pid, tostring(err)) + local function join() + assert(process.send(pid, consts.WS_TOPICS.JOIN, { client_pid = process.pid() })) + return receive(welcome) + end + local function send(value) + local encoded, encode_err = json.encode(value) + assert(encoded, tostring(encode_err)) + assert(process.send(pid, consts.WS_TOPICS.MESSAGE, encoded)) + end + local children = {} + local function startup() + local plugin = receive(started) + assert(process.monitor(plugin.pid)) + children[plugin.pid] = true + return plugin + end + local ok: boolean, result: any = pcall(body, pid, user_id, join, send, response, errors, startup) + process.send(pid, consts.WS_TOPICS.CANCEL, {}) + local stopped = false + local timer = time.timer(3 * time.SECOND) + while not stopped or next(children) do + local selected = channel.select({ events:case_receive(), timer:channel():case_receive() }) + if selected.channel == timer:channel() then break end + local event = selected.value + if event.kind == process.event.EXIT and children[event.from] then + children[event.from] = nil + if event.result.error then ok, result = false, tostring(event.result.error) end + end + if event.from == pid and event.kind == process.event.EXIT then + stopped = true + if event.result.error then + ok, result = false, tostring(result or "") .. "; hub exited: " .. tostring(event.result.error) + end + end + end + timer:stop() + local cleanup_timed_out = not stopped or next(children) ~= nil + if cleanup_timed_out then + if not stopped then process.terminate(pid) end + for child in pairs(children) do process.terminate(child) end + local forced_deadline = time.timer(3 * time.SECOND) + while not stopped or next(children) do + local selected = channel.select({ events:case_receive(), forced_deadline:channel():case_receive() }) + if selected.channel == forced_deadline:channel() then break end + local event = selected.value + if event.kind == process.event.EXIT then + if event.from == pid then stopped = true end + children[event.from] = nil + end + end + forced_deadline:stop() + end + local leaked = not stopped or next(children) ~= nil + for _, ch in ipairs({ welcome, response, errors, started }) do process.unlisten(ch) end + if leaked then error(tostring(result or "") .. "; relay fixture termination deadline") end + if not ok then error(result) end + assert(not cleanup_timed_out and not leaked, "relay hub or plugin cleanup deadline") + return result +end + +local function define_tests() + test.describe("relay runtime", function() + test.it("starts plugins lazily and preserves command routing and payload fields", function() + with_hub("app:relay_runtime_plugin", function(pid, user_id, join, send, response, _, started) + local welcome = join() + test.eq(welcome.plugins[1].status, "not_started") + send({ type = "runtime_echo", data = { nested = { value = 42 } }, + request_id = "request-1", session_id = "session-1", start_token = "start-1", + attention_context_enabled = true, context = { label = "fixture" } }) + local boot = started() + local result = receive(response) + test.eq(result.plugin_pid, boot.pid) + test.eq(result.topic, "echo") + test.eq(result.from, pid) + test.eq(result.user_id, user_id) + test.eq(result.user_metadata.fixture, true) + test.eq(result.payload.conn_pid, (process.pid())) + test.eq(result.payload.type, "runtime_echo") + test.eq(result.payload.data.nested.value, 42) + test.eq(result.payload.request_id, "request-1") + test.eq(result.payload.session_id, "session-1") + test.eq(result.payload.start_token, "start-1") + test.eq(result.payload.attention_context_enabled, true) + test.eq(result.payload.context.label, "fixture") + end) + end) + test.it("delivers commands after disconnect and reconnect using the same plugin", function() + with_hub("app:relay_runtime_plugin", function(pid, _, join, send, response, _, started) + join() + send({ type = "runtime_echo", data = "before", request_id = "before" }) + local boot = started() + test.eq(receive(response).payload.data, "before") + assert(process.send(pid, consts.WS_TOPICS.LEAVE, { client_pid = process.pid() })) + local welcome = join() + test.eq(welcome.client_count, 1) + test.eq(welcome.plugins[1].status, "running") + send({ type = "runtime_echo", data = "after", request_id = "after", session_id = "reconnected" }) + local result = receive(response) + test.eq(result.plugin_pid, boot.pid) + test.eq(result.payload.data, "after") + test.eq(result.payload.request_id, "after") + test.eq(result.payload.session_id, "reconnected") + end) + end) + test.it("routes repeated commands and stops on a process cancellation event", function() + with_hub("app:relay_runtime_plugin", function(pid, _, join, send, response, _, started) + join() + local plugin_pid + for index = 1, 8 do + send({ type = "runtime_echo", request_id = tostring(index), data = index }) + if index == 1 then plugin_pid = started().pid end + local result = receive(response) + test.eq(result.plugin_pid, plugin_pid) + test.eq(result.payload.request_id, tostring(index)) + test.eq(result.payload.data, index) + end + assert(process.cancel(pid, "2s")) + end) + end) + test.it("preserves independent payloads across pipelined commands", function() + with_hub("app:relay_runtime_plugin", function(pid, _, join, send, response, _, started) + join() + for index = 1, 16 do + send({ type = "runtime_echo", request_id = tostring(index), session_id = "pipeline", + data = { index = index }, start_token = "token-" .. index, + attention_context_enabled = index % 2 == 0, context = { index = index } }) + end + local plugin = started() + local seen = {} + for _ = 1, 16 do + local value = receive(response) + local index = value.payload.data.index + test.is_true(index >= 1 and index <= 16 and not seen[index]) + seen[index] = true + test.eq(value.plugin_pid, plugin.pid) + test.eq(value.from, pid) + test.eq(value.topic, "echo") + test.eq(value.payload.conn_pid, (process.pid())) + test.eq(value.payload.type, "runtime_echo") + test.eq(value.payload.request_id, tostring(index)) + test.eq(value.payload.session_id, "pipeline") + test.eq(value.payload.start_token, "token-" .. index) + test.eq(value.payload.attention_context_enabled, index % 2 == 0) + test.eq(value.payload.context.index, index) + end + end) + end) + test.it("bounds pipeline identity slots by the window at high operation ids", function() + local first = 4294967297 + test.eq(pipeline_slot(first, first, first + PIPELINE_WINDOW - 1), 1) + test.eq(pipeline_slot(first + PIPELINE_WINDOW - 1, first, first + PIPELINE_WINDOW - 1), PIPELINE_WINDOW) + test.is_nil(pipeline_slot(first - 1, first, first + PIPELINE_WINDOW - 1)) + test.is_nil(pipeline_slot(first + PIPELINE_WINDOW, first, first + PIPELINE_WINDOW - 1)) + test.is_nil(pipeline_slot(first + 0.5, first, first + PIPELINE_WINDOW - 1)) + test.is_nil(pipeline_slot("1", 1, PIPELINE_WINDOW)) + end) + test.it("counts a repeated JOIN once so the last LEAVE shuts down the session plugin", function() + local resumed = process.listen("resume") + local shutdown = process.listen("shutdown") + local ok, err = pcall(with_hub, "app:relay_runtime_plugin", function(pid, _, join) + test.eq(join().client_count, 1) + receive(resumed) + test.eq(join().client_count, 1, "A repeated JOIN must keep one membership") + assert(process.send(pid, consts.WS_TOPICS.LEAVE, { client_pid = process.pid() })) + receive(shutdown) + test.eq(join().client_count, 1) + receive(resumed) + end, nil, process.pid()) + process.unlisten(resumed) + process.unlisten(shutdown) + if not ok then error(err) end + end) + for _, saved_count in ipairs({0, 1, 3}) do + test.it("restores the membership count before the final LEAVE, saved count " .. saved_count, function() + local shutdown = process.listen("shutdown") + local ok, err = pcall(with_hub, "app:relay_runtime_plugin", function(pid, _, join) + test.eq(join().client_count, 1, "Restored memberships are the source of the client count") + assert(process.send(pid, consts.WS_TOPICS.LEAVE, { client_pid = process.pid() })) + receive(shutdown) + end, nil, process.pid(), saved_count) + process.unlisten(shutdown) + if not ok then error(err) end + end) + end + test.it("reports a real plugin startup failure and keeps the hub responsive", function() + with_hub("app:missing_runtime_plugin", function(_, _, join, send, _, errors) + join() + send({ type = "runtime_echo", data = "first" }) + test.eq(receive(errors).error, consts.ERROR_CODES.PLUGIN_FAILED) + send({ type = "runtime_echo", data = "second" }) + local error_message = receive(errors) + test.eq(error_message.error, consts.ERROR_CODES.PLUGIN_FAILED) + test.is_true(error_message.message:find("failed permanently", 1, true) ~= nil) + end) + end) + test.it("reports delivery failure when a restored plugin pid has already exited", function() + local events = process.events() + local dead_pid, err = process.spawn_monitored("app:relay_runtime_finished_plugin", "app:processes") + assert(dead_pid, tostring(err)) + while true do + local event = receive(events) + if event.from == dead_pid and event.kind == process.event.EXIT then break end + end + with_hub("app:relay_runtime_plugin", function(_, _, join, send, _, errors) + test.eq(join().plugins[1].status, "running") + send({ type = "runtime_echo", request_id = "stale-plugin", data = "delivery" }) + test.eq(receive(errors).error, consts.ERROR_CODES.PLUGIN_FAILED) + end, dead_pid) + end) + test.it("rejects invalid JSON and missing or unknown commands without stopping", function() + with_hub("app:relay_runtime_plugin", function(pid, _, join, send, _, errors) + join() + assert(process.send(pid, consts.WS_TOPICS.MESSAGE, "{")) + test.eq(receive(errors).error, consts.ERROR_CODES.INVALID_JSON) + send({ data = "missing type" }) + test.eq(receive(errors).error, consts.ERROR_CODES.UNKNOWN_COMMAND) + send({ type = "missing_echo" }) + test.eq(receive(errors).error, consts.ERROR_CODES.PLUGIN_NOT_FOUND) + end) + end) + test.it("keeps routing to the same plugin after invalid input and a rejected plugin topic", function() + with_hub("app:relay_runtime_plugin", function(pid, _, join, send, response, errors, started) + join() + send({ type = "runtime_echo", request_id = "first", data = 1 }) + local plugin = started() + test.eq(receive(response).payload.request_id, "first") + assert(process.send(pid, consts.WS_TOPICS.MESSAGE, "{")) + test.eq(receive(errors).error, consts.ERROR_CODES.INVALID_JSON) + send({ type = "runtime_echo", request_id = "after-json", data = 2 }) + test.eq(receive(response).payload.request_id, "after-json") + send(42) + test.eq(receive(errors).error, consts.ERROR_CODES.UNKNOWN_COMMAND) + send({ type = "runtime_echo", request_id = "after-type", data = 3 }) + test.eq(receive(response).payload.request_id, "after-type") + send({ type = "runtime_@reserved", request_id = "reserved", data = 4 }) + local failure = receive(errors) + test.eq(failure.error, consts.ERROR_CODES.PLUGIN_FAILED) + test.contains(tostring(failure.message), "cannot send to @ topics") + send({ type = "runtime_echo", request_id = "after-reserved", data = 5 }) + local after = receive(response) + test.eq(after.payload.request_id, "after-reserved") + test.eq(after.plugin_pid, plugin.pid) + end) + end) + test.it("routes commands beyond the route cache bound and keeps rejecting invalid commands", function() + with_hub("app:relay_runtime_plugin", function(_, _, join, send, response, errors, started) + join() + for index = 1, 80 do + send({ type = "runtime_echo_" .. index, request_id = tostring(index) }) + end + started() + local seen = {} + for _ = 1, 80 do + local value = receive(response) + local request_id = value.payload.request_id + test.eq(value.topic, "echo_" .. request_id) + test.is_nil(seen[request_id]) + seen[request_id] = true + end + send({ type = "runtime_echo_1", request_id = "cached" }) + local cached = receive(response) + test.eq(cached.topic, "echo_1") + test.eq(cached.payload.request_id, "cached") + send({ type = 42 }) + test.eq(receive(errors).error, consts.ERROR_CODES.UNKNOWN_COMMAND) + send({ type = "missing_echo" }) + test.eq(receive(errors).error, consts.ERROR_CODES.PLUGIN_NOT_FOUND) + send({ type = "runtime_echo_80", request_id = "uncached" }) + test.eq(receive(response).topic, "echo_80") + end) + end) + test.it("rejects non-object JSON and non-string command types without stopping", function() + with_hub("app:relay_runtime_plugin", function(pid, _, join, send, _, errors) + join() + assert(process.send(pid, consts.WS_TOPICS.MESSAGE, "null")) + test.eq(receive(errors).error, consts.ERROR_CODES.UNKNOWN_COMMAND) + for _, value in ipairs({ 42, true, false, "scalar", {}, { type = 42 }, { type = true }, { type = "" } }) do + send(value) + test.eq(receive(errors).error, consts.ERROR_CODES.UNKNOWN_COMMAND) + end + end) + end) + end) +end + +return { run = test.run_cases(define_tests) } diff --git a/src/relay/test/user_test.lua b/src/relay/test/user_test.lua index d31fd4f..1b6b129 100644 --- a/src/relay/test/user_test.lua +++ b/src/relay/test/user_test.lua @@ -62,9 +62,13 @@ local function define_tests() client_count = 1, pg_scopes = { ["app:scope"] = { opaque = true } }, pg_groups = { ["workspace.1"] = "app:scope" }, + routes = { session_message = { prefix = "session_", config = { prefix = "session_", + process_id = "app:session", auto_start = true }, topic = "message" } }, + route_count = 1, }) test.eq(snapshot.relay_user_upgrade, true) + test.is_nil((snapshot :: any).routes) test.eq(snapshot.active_plugins.session_.pid, "app:plugin:1") test.eq(snapshot.connected_clients["app:client:1"], true) test.eq(snapshot.pg_groups["workspace.1"], "app:scope") diff --git a/src/test/README.md b/src/test/README.md index c892f8f..6c24e30 100644 --- a/src/test/README.md +++ b/src/test/README.md @@ -99,7 +99,12 @@ by test code are not automatically terminated and remain the test's responsibili Simple tests may return `true` or `nil` for success; `false`, an execution error, or an error-status result fails the run. BDD entries must report their plan, every declared case, and completion with matching pass/fail/skip counts. An aborted hook, -missing case, or inconsistent completion fails the run even if earlier cases passed. +missing case, replaced plan, or inconsistent completion fails the run even if earlier cases passed. +An error-status result also fails when a BDD entry reported otherwise successful cases. + +CI can set `WIPPY_TEST_REQUIRE_CASES=1` to reject empty discovery, filters with no matches, +entries without a BDD plan, and skipped-only plans. Without that opt-in, simple tests, +empty plans, and intentionally skipped suites retain their existing behavior. ### Deadlines and teardown diff --git a/src/test/_index.yaml b/src/test/_index.yaml index 17eb99b..8821095 100644 --- a/src/test/_index.yaml +++ b/src/test/_index.yaml @@ -14,6 +14,14 @@ entries: component: wippy/terminal version: "*" + - name: env_storage + kind: env.storage.os + + - name: require_cases + kind: env.variable + variable: WIPPY_TEST_REQUIRE_CASES + storage: wippy.test:env_storage + # wippy.test:terminal - name: terminal kind: library.lua @@ -114,6 +122,7 @@ entries: method: main modules: - io + - env - time - channel imports: @@ -132,5 +141,6 @@ entries: comment: Self-test for the test framework source: file://self_test.lua method: run + modules: [registry] imports: test: wippy.test:test diff --git a/src/test/runner.lua b/src/test/runner.lua index 2df1750..fcb0c6a 100644 --- a/src/test/runner.lua +++ b/src/test/runner.lua @@ -1,5 +1,6 @@ -- Test runner with real-time per-case TUI display local io = require("io") +local env = require("env") local registry = require("registry") local time = require("time") local channel = require("channel") @@ -60,6 +61,7 @@ end -- Main test runner logic local function run_tests(): number local args: {string}? = io.args() + local require_cases = env.get("WIPPY_TEST_REQUIRE_CASES") == "1" display.begin() @@ -72,7 +74,7 @@ local function run_tests(): number if not raw_entries or #raw_entries == 0 then display.info("No tests found") - return 0 + return require_cases and 1 or 0 end local entries: {discovery.TestEntry} = raw_entries @@ -82,7 +84,7 @@ local function run_tests(): number entries = discovery.filter_tests(entries, args) if #entries == 0 then display.info("No tests match filter: " .. table.concat(args, ", ")) - return 0 + return require_cases and 1 or 0 end display.info("Filter: " .. table.concat(args, ", ")) end @@ -156,11 +158,15 @@ local function run_tests(): number end if msg_type == "test:plan" then - local count = 0 - for _, planned_suite in ipairs(data.suites or {}) do - count = count + #(planned_suite.tests or {}) + if declared_cases[ref_id] ~= nil then + declared_cases[ref_id] = -1 + else + local count = 0 + for _, planned_suite in ipairs(data.suites or {}) do + count = count + #(planned_suite.tests or {}) + end + declared_cases[ref_id] = count end - declared_cases[ref_id] = count elseif msg_type == "test:case:pass" then local cs = case_stats[ref_id] @@ -291,6 +297,7 @@ local function run_tests(): number ("test process termination failed: " .. tostring(terminate_err)) or cancel_error or protocol_error local problem = verdict.check({ + require_cases = require_cases, entry_id = entry_id, declared = declared_cases[entry_id], observed = case_stats[entry_id] and ((case_stats[entry_id].passed or 0) + (case_stats[entry_id].failed or 0) + (case_stats[entry_id].skipped or 0)) or 0, @@ -314,14 +321,15 @@ local function run_tests(): number local value: any = control.value local result_err: any = control.result_error local problem = verdict.check({ + require_cases = require_cases, entry_id = entry_id, declared = declared_cases[entry_id], observed = case_count, cases = cs, completed = completion and completion.data or nil, result_error = result_err, - returned_false = value == false or (declared_cases[entry_id] == nil - and type(value) == "table" and value.status == "error"), + returned_false = value == false or (type(value) == "table" and value.status == "error" + and not (cs and cs.failed > 0)), }) if problem then diff --git a/src/test/runner_verdict.lua b/src/test/runner_verdict.lua index 9d476c5..e3b2ddb 100644 --- a/src/test/runner_verdict.lua +++ b/src/test/runner_verdict.lua @@ -1,6 +1,7 @@ -- Validate the protocol independently of the case events already rendered by the runner. type RunState = { entry_id: string?, + require_cases: boolean?, declared: number?, observed: number, cases: {passed: number, failed: number, skipped: number}?, @@ -51,6 +52,9 @@ local function check(state: RunState): RunnerError? return fail("test process did not report a plan") end if state.declared ~= nil then + if state.declared < 0 then + return fail("test process reported more than one plan") + end if state.observed ~= state.declared then return fail(tostring(state.declared) .. " declared cases, " .. tostring(state.observed) .. " executed") @@ -74,6 +78,10 @@ local function check(state: RunState): RunnerError? return fail("test completion counts disagree with declared cases") end end + if state.require_cases and (state.declared == nil or not state.completed + or state.completed.passed + state.completed.failed == 0) then + return fail("required test suite executed no cases") + end return nil end diff --git a/src/test/self_test.lua b/src/test/self_test.lua index 2a9ee83..9f721ce 100644 --- a/src/test/self_test.lua +++ b/src/test/self_test.lua @@ -1,7 +1,22 @@ -- Self-test for the test framework local test = require("test") +local registry = require("registry") local function define_tests() + test.describe("shared test resources", function() + test.it("does not publish benchmarking libraries or environment settings", function() + for _, name in ipairs({"benchmark", "benchmark_revision", "benchmark_runtime", + "benchmark_framework_revision", "benchmark_exact_allocations"}) do + local entry = registry.get("wippy.test:" .. name) + test.is_nil(entry and entry.id, "benchmarking stays outside the shared test module") + end + end) + test.it("does not register a benchmark filesystem for ordinary tests", function() + local output = registry.get("wippy.test:benchmark_output") + test.is_nil(output, "benchmark filesystems belong to the explicit benchmark harness") + end) + end) + -- Hook bookkeeping for the "suite hooks" suites below: a suite's before_all -- and after_all run once around the suite and all of its descendants. local hooks = { outer_before = 0, outer_after = 0, grandchild_ran = false } diff --git a/test-harness/go/go.mod b/test-harness/go/go.mod index 62cc21a..882d292 100644 --- a/test-harness/go/go.mod +++ b/test-harness/go/go.mod @@ -1,5 +1,5 @@ module github.com/wippyai/framework/test-harness -go 1.27.0 +go 1.26.0 require github.com/wippyai/go-lua v1.6.2 diff --git a/test-harness/go/integration_opt_in_test.go b/test-harness/go/integration_opt_in_test.go new file mode 100644 index 0000000..e5a5d33 --- /dev/null +++ b/test-harness/go/integration_opt_in_test.go @@ -0,0 +1,72 @@ +// SPDX-License-Identifier: MPL-2.0 + +package runner_test + +import ( + "os" + "regexp" + "testing" + + lua "github.com/wippyai/go-lua" +) + +// Exercise each provider suite's actual switch expression without executing +// the suite or allowing it to contact a provider. +func TestProviderIntegrationRequiresExplicitOptIn(t *testing.T) { + files := []string{ + "bedrock/integration_test.lua", "claude/integration_test.lua", + "google/integration_test.lua", "openai/integration_test.lua", + "openai_compat/integration_test.lua", "typesafe/integration_test.lua", + "util/compress_test.lua", + } + declaration := regexp.MustCompile(`(?m)^\s*local RUN_INTEGRATION_TESTS = (.+)$`) + scenarios := []struct { + name string + flags map[string]string + enabled bool + }{ + {name: "unset"}, + {name: "false", flags: map[string]string{"ENABLE_INTEGRATION_TESTS": "false"}}, + {name: "zero", flags: map[string]string{"ENABLE_INTEGRATION_TESTS": "0"}}, + {name: "other", flags: map[string]string{"ENABLE_INTEGRATION_TESTS": "yes"}}, + {name: "true", flags: map[string]string{"ENABLE_INTEGRATION_TESTS": "true"}, enabled: true}, + {name: "typesafe_false", flags: map[string]string{"TYPESAFE_INTEGRATION_TESTS": "false"}}, + {name: "typesafe_true", flags: map[string]string{"TYPESAFE_INTEGRATION_TESTS": "true"}}, + } + for _, file := range files { + t.Run(file, func(t *testing.T) { + source, err := os.ReadFile("../../src/llm/test/" + file) + if err != nil { + t.Fatal(err) + } + expression := declaration.FindSubmatch(source) + if len(expression) != 2 { + t.Fatal("integration switch declaration is missing") + } + for _, scenario := range scenarios { + t.Run(scenario.name, func(t *testing.T) { + l := lua.NewState() + defer l.Close() + env := l.NewTable() + env.RawSetString("get", l.NewFunction(func(l *lua.LState) int { + value, ok := scenario.flags[l.CheckString(1)] + if ok { + l.Push(lua.LString(value)) + } else { + l.Push(lua.LNil) + } + return 1 + })) + l.SetGlobal("env", env) + if err := l.DoString("return not not (" + string(expression[1]) + ")"); err != nil { + t.Fatal(err) + } + enabled := scenario.enabled || (file == "typesafe/integration_test.lua" && scenario.name == "typesafe_true") + if got := l.Get(-1); got != lua.LBool(enabled) { + t.Fatalf("integration switch returned %v (%s), want boolean %v", got, got.Type(), enabled) + } + }) + } + }) + } +} diff --git a/test-harness/go/runner_deadline_test.go b/test-harness/go/runner_deadline_test.go index f744ea5..2e7669c 100644 --- a/test-harness/go/runner_deadline_test.go +++ b/test-harness/go/runner_deadline_test.go @@ -55,7 +55,7 @@ if scenario ~= "unbounded" then entry.meta.suite_timeout="2m"; table.insert(entr if scenario == "expired_during_teardown" then entry.meta.timeout="1ms" end if scenario == "conflicting" then entries[2].meta.suite_timeout="3m" end local modules={ - io={args=function() return nil end}, registry={find=function() return entries end}, + io={args=function() return nil end}, env={get=function() return nil end}, registry={find=function() return entries end}, time={now=function() return stamp end,sleep=function() end,MILLISECOND=1,after=function(duration) observe_timeout(duration); local ch=new_channel(); ch.duration=duration; return ch end}, channel={new=new_channel,select=function(cases) if cases.default then diff --git a/test-harness/go/runner_teardown_test.go b/test-harness/go/runner_teardown_test.go index 58f9cf5..a9dcaf7 100644 --- a/test-harness/go/runner_teardown_test.go +++ b/test-harness/go/runner_teardown_test.go @@ -93,7 +93,7 @@ local function new_channel() return {receive=function() if current_id=="test:exp case_receive=function(self) return {channel=self,value=message("result")} end,close=function() end,send=function() end} end local entries={{id="test:expired",meta={timeout="1ms"}},{id="test:next",meta={}}} local modules={ - io={args=function() end},registry={find=function() return entries end}, + io={args=function() end},env={get=function() return nil end},registry={find=function() return entries end}, time={now=function() return stamp end,sleep=function() end,MILLISECOND=1,after=function(duration) local ch=new_channel();ch.duration=duration;return ch end}, channel={new=new_channel,select=function(cases) if current_id=="test:expired" then for _,case in ipairs(cases) do if case.channel.duration then return case end end end;return cases[1] end}, discovery={group_by_suite=function() return {suite=entries},{} end, sorted_keys=function() return {"suite"} end,short_name=function(id) return id end}, diff --git a/test-harness/runner-results/_index.yaml b/test-harness/runner-results/_index.yaml index 94359b9..c76a6c4 100644 --- a/test-harness/runner-results/_index.yaml +++ b/test-harness/runner-results/_index.yaml @@ -45,6 +45,16 @@ entries: meta: {type: test, suite: runner-results} source: file://results.lua method: conflicting_completion + - name: planned_error_status + kind: function.lua + meta: {type: test, suite: runner-results} + source: file://results.lua + method: planned_error_status + - name: replaced_plan + kind: function.lua + meta: {type: test, suite: runner-results} + source: file://results.lua + method: replaced_plan - name: empty_completion kind: function.lua meta: {type: test, suite: runner-results} @@ -64,3 +74,10 @@ entries: method: skipped_case imports: test: wippy.test:test + - name: empty_plan + kind: function.lua + meta: {type: test, suite: runner-results} + source: file://cases.lua + method: empty_plan + imports: + test: wippy.test:test diff --git a/test-harness/runner-results/cases.lua b/test-harness/runner-results/cases.lua index 8579694..01fbeb4 100644 --- a/test-harness/runner-results/cases.lua +++ b/test-harness/runner-results/cases.lua @@ -1,6 +1,7 @@ local test = require("test") return { + empty_plan = test.run_cases(function() end), failing_case = test.run_cases(function() test.describe("failed case", function() test.it("keeps its failure", function() diff --git a/test-harness/runner-results/results.lua b/test-harness/runner-results/results.lua index 2d95d31..aeea4b2 100644 --- a/test-harness/runner-results/results.lua +++ b/test-harness/runner-results/results.lua @@ -27,6 +27,17 @@ return { emit(args, "test:complete", {total = 1, passed = 0, failed = 1, skipped = 0}) return true end, + planned_error_status = function(args) + one_pass(args) + emit(args, "test:complete", {total = 1, passed = 1, failed = 0, skipped = 0}) + return {status = "error"} + end, + replaced_plan = function(args) + emit(args, "test:plan", {suites = {{tests = {"one", "two"}}}}) + one_pass(args) + emit(args, "test:complete", {total = 1, passed = 1, failed = 0, skipped = 0}) + return true + end, empty_completion = function(args) emit(args, "test:plan", {suites = {}}) emit(args, "test:complete", {}) diff --git a/test-harness/runner-results/verify.sh b/test-harness/runner-results/verify.sh index 324abad..b61e979 100644 --- a/test-harness/runner-results/verify.sh +++ b/test-harness/runner-results/verify.sh @@ -17,6 +17,8 @@ done <<'SCENARIOS' returns_false|test returned false raises_error|expected execution failure error_status|test returned false +planned_error_status|test returned false +replaced_plan|test process reported more than one plan completion_without_plan|test process did not report a plan missing_completion|test process did not report completion conflicting_completion|test completion counts disagree @@ -32,3 +34,20 @@ for scenario in returns_true returns_nil skipped_case; do fi done printf 'runner result and protocol regressions passed\n' + +for scenario in empty_plan skipped_case returns_nil no-such-entry; do + set +e + output=$(WIPPY_TEST_REQUIRE_CASES=1 "$wippy_bin" test --config .wippy.yaml test -- "app:$scenario" 2>&1) + status=$? + set -e + if [ "$status" -ne 1 ]; then + printf 'required runner accepted %s (exit %s):\n%s\n' "$scenario" "$status" "$output" + exit 1 + fi +done +if ! output=$("$wippy_bin" test --config .wippy.yaml test -- app:empty_plan 2>&1) || + ! grep -Fq 'PASSED' <<< "$output"; then + printf 'runner rejected a generic empty plan:\n%s\n' "$output" + exit 1 +fi +printf 'required case regressions passed\n' diff --git a/test-harness/runner/_index.yaml b/test-harness/runner/_index.yaml index ee88426..884ca9b 100644 --- a/test-harness/runner/_index.yaml +++ b/test-harness/runner/_index.yaml @@ -17,3 +17,30 @@ entries: imports: test: wippy.test:test verdict: wippy.test:runner_verdict + + - name: build_executor + kind: exec.native + + - name: aggregate_fixtures + kind: fs.directory + directory: /tmp/wippy-aggregate-checks + auto_init: true + + - name: env_storage + kind: env.storage.os + + - name: framework_root + kind: env.variable + variable: WIPPY_FRAMEWORK_ROOT + storage: app:env_storage + + - name: aggregate_checks_test + kind: function.lua + meta: + type: test + suite: build-checks + source: file://aggregate_checks_test.lua + method: run + modules: [exec, fs, env, time] + imports: + test: wippy.test:test diff --git a/test-harness/runner/aggregate_checks_test.lua b/test-harness/runner/aggregate_checks_test.lua new file mode 100644 index 0000000..ed7ce45 --- /dev/null +++ b/test-harness/runner/aggregate_checks_test.lua @@ -0,0 +1,84 @@ +local test = require("test") +local exec = require("exec") +local fs = require("fs") +local env = require("env") +local time = require("time") + +local function read_all(stream) + local chunks = {} + while true do + local data = stream:read() + if not data then break end + chunks[#chunks + 1] = data + end + stream:close() + return table.concat(chunks) +end + +local function run_check(target, message, exit_code) + local root = env.get("WIPPY_FRAMEWORK_ROOT") + test.not_nil(root, "WIPPY_FRAMEWORK_ROOT must select the real Framework checkout") + local files = fs.get("app:aggregate_fixtures") + local directory = "check-" .. tostring(time.now():unix_nano()) + for _, path in ipairs({directory, directory .. "/src", directory .. "/src/fixture", + directory .. "/src/fixture/test"}) do + local created, err = files:mkdir(path) + test.is_true(created, tostring(err)) + end + local module = directory .. "/src/fixture/test/Makefile" + local written, write_err = files:writefile(module, + "test lint:\n\t@printf '%s\\n' '" .. message .. "'; exit " .. tostring(exit_code) .. "\n") + test.is_true(written, tostring(write_err)) + local executor, executor_err = exec.get("app:build_executor") + test.not_nil(executor, tostring(executor_err)) + local command = "make --no-print-directory -f " .. string.format("%q", root .. "/Makefile") + .. " -C " .. string.format("%q", "/tmp/wippy-aggregate-checks/" .. directory) + .. " " .. target .. " TEST_MODULES=fixture" + local proc, create_err = executor:exec(command, {env = {MAKEFLAGS = ""}}) + test.not_nil(proc, tostring(create_err)) + local stdout, stdout_err = proc:stdout_stream() + local stderr, stderr_err = proc:stderr_stream() + test.not_nil(stdout, tostring(stdout_err)) + test.not_nil(stderr, tostring(stderr_err)) + local started, start_err = proc:start() + test.is_true(started, tostring(start_err)) + local output = read_all(stdout) + local errors = read_all(stderr) + local status, wait_err = proc:wait() + test.is_nil(wait_err) + executor:release() + files:remove(module) + files:remove(directory .. "/src/fixture/test") + files:remove(directory .. "/src/fixture") + files:remove(directory .. "/src") + files:remove(directory) + return status, output .. errors +end + +local function define_tests() + test.describe("aggregate build checks", function() + test.it("fails aggregate tests when a failed module prints PASSED", function() + local status, output = run_check("run-tests", "PASSED", 1) + test.is_true(status ~= 0) + test.contains(output, "FAILED") + end) + + test.it("passes aggregate tests without requiring a PASSED marker", function() + local status, output = run_check("run-tests", "completed", 0) + test.eq(status, 0, output) + end) + + test.it("fails aggregate lint without requiring an errors marker", function() + local status, output = run_check("run-lint", "Checked fixture", 1) + test.is_true(status ~= 0) + test.contains(output, "FAILED") + end) + + test.it("passes aggregate lint when successful output mentions errors", function() + local status, output = run_check("run-lint", "Checked fixture: 0 errors", 0) + test.eq(status, 0, output) + end) + end) +end + +return {run = test.run_cases(define_tests)} diff --git a/test-harness/runner/runner_verdict_test.lua b/test-harness/runner/runner_verdict_test.lua index fe1a867..80f8f9a 100644 --- a/test-harness/runner/runner_verdict_test.lua +++ b/test-harness/runner/runner_verdict_test.lua @@ -3,6 +3,32 @@ local verdict = require("verdict") local function define_tests() test.describe("runner result validation", function() + test.it("required suites reject a completed empty plan", function() + local err = verdict.check({require_cases = true, declared = 0, observed = 0, + cases = {passed = 0, failed = 0, skipped = 0}, + completed = {total = 0, passed = 0, failed = 0, skipped = 0}}) + test.not_nil(err) + test.contains(err.message, "required test suite executed no cases") + end) + + test.it("required suites reject skipped-only plans", function() + local err = verdict.check({require_cases = true, declared = 1, observed = 1, + cases = {passed = 0, failed = 0, skipped = 1}, + completed = {total = 1, passed = 0, failed = 0, skipped = 1}}) + test.not_nil(err) + test.contains(err.message, "required test suite executed no cases") + end) + + test.it("required suites accept an executed BDD case", function() + test.is_nil(verdict.check({require_cases = true, declared = 1, observed = 1, + cases = {passed = 1, failed = 0, skipped = 0}, + completed = {total = 1, passed = 1, failed = 0, skipped = 0}})) + end) + + test.it("required suites reject entries without a case plan", function() + test.not_nil(verdict.check({require_cases = true, observed = 0})) + end) + test.it("rejects completion without a plan", function() local err = verdict.check({observed = 0, completed = {total = 1, passed = 0, failed = 1, skipped = 0}}) @@ -61,6 +87,26 @@ local function define_tests() test.eq(err.message, "test process did not report completion") end) + test.it("rejects a replaced plan even when the counts agree", function() + local err = verdict.check({ declared = -1, observed = 1, + completed = { total = 1, passed = 1, failed = 0, skipped = 0 } }) + test.not_nil(err) + test.eq(err.message, "test process reported more than one plan") + end) + + test.it("required suites count failed cases as executed", function() + test.is_nil(verdict.check({require_cases = true, declared = 1, observed = 1, + cases = {passed = 0, failed = 1, skipped = 0}, + completed = {total = 1, passed = 0, failed = 1, skipped = 0}})) + end) + + test.it("reports a replaced plan before a missing required case", function() + local err = verdict.check({ require_cases = true, declared = -1, observed = 0, + completed = { total = 0, passed = 0, failed = 0, skipped = 0 } }) + test.not_nil(err) + test.eq(err.message, "test process reported more than one plan") + end) + test.it("accepts all declared cases with a successful completion", function() local err = verdict.check({ declared = 2, observed = 2, completed = { total = 2, passed = 2, failed = 0, skipped = 0 } })