From 3e8ea12d77b353e8c1ff9ab2d6808b16e18c901d Mon Sep 17 00:00:00 2001 From: Rodrigo Delduca Date: Wed, 7 Oct 2026 12:09:01 -0300 Subject: [PATCH 1/8] test: cover Lua runtime controls and message delivery --- .github/workflows/ci.yml | 30 +++ Makefile | 65 ++++- README.md | 15 ++ scripts/run-wippy.sh | 51 ++++ src/actor/src/actor.lua | 2 +- src/actor/test/.wippy.yaml | 4 + src/actor/test/_index.yaml | 62 +++++ src/actor/test/actor_runtime_fixture.lua | 67 +++++ src/actor/test/actor_runtime_test.lua | 168 ++++++++++++ src/agent/test/.wippy.yaml | 4 + src/agent/test/_index.yaml | 61 +++++ src/agent/test/agent_integration_test.lua | 7 +- src/agent/test/tools/control_fixture.lua | 45 ++++ src/agent/test/tools/controls_benchmark.lua | 47 ++++ .../test/tools/controls_runtime_test.lua | 123 +++++++++ src/llm/test/e2e/integration_test.lua | 53 +--- src/relay/src/user.lua | 17 +- src/relay/test/.wippy.yaml | 4 + src/relay/test/_index.yaml | 79 ++++++ src/relay/test/relay_runtime_plugin.lua | 23 ++ src/relay/test/relay_runtime_test.lua | 252 ++++++++++++++++++ src/test/_index.yaml | 46 ++++ src/test/benchmark.lua | 113 ++++++++ src/test/benchmark_test.lua | 143 ++++++++++ src/test/runner.lua | 8 +- src/test/runner_verdict.lua | 5 + test-harness/runner-results/_index.yaml | 7 + test-harness/runner-results/cases.lua | 1 + test-harness/runner-results/verify.sh | 17 ++ test-harness/runner/_index.yaml | 27 ++ test-harness/runner/aggregate_checks_test.lua | 84 ++++++ test-harness/runner/runner_verdict_test.lua | 26 ++ 32 files changed, 1598 insertions(+), 58 deletions(-) create mode 100644 scripts/run-wippy.sh create mode 100644 src/actor/test/.wippy.yaml create mode 100644 src/actor/test/actor_runtime_fixture.lua create mode 100644 src/actor/test/actor_runtime_test.lua create mode 100644 src/agent/test/.wippy.yaml create mode 100644 src/agent/test/tools/control_fixture.lua create mode 100644 src/agent/test/tools/controls_benchmark.lua create mode 100644 src/agent/test/tools/controls_runtime_test.lua create mode 100644 src/relay/test/.wippy.yaml create mode 100644 src/relay/test/relay_runtime_plugin.lua create mode 100644 src/relay/test/relay_runtime_test.lua create mode 100644 src/test/benchmark.lua create mode 100644 src/test/benchmark_test.lua create mode 100644 test-harness/runner/aggregate_checks_test.lua diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9443aa1a..8a63e814 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,10 +17,40 @@ permissions: contents: read jobs: + runtime-contracts: + name: Lua runtime contracts and benchmarks + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + with: + persist-credentials: false + - uses: ./.github/actions/setup-wippy + id: wippy + - name: Test real runtime contracts + env: + WIPPY: ${{ steps.wippy.outputs.path }} + RUNTIME_WORK_DIR: ${{ runner.temp }}/framework-runtime + run: make test-runtime + - name: Measure Lua operations + env: + WIPPY: ${{ steps.wippy.outputs.path }} + RUNTIME_WORK_DIR: ${{ runner.temp }}/framework-runtime + BENCH_OUTPUT_DIR: ${{ runner.temp }}/framework-benchmarks + run: make bench + - name: Save benchmark reports + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: framework-benchmarks + path: ${{ runner.temp }}/framework-benchmarks/*.json + if-no-files-found: error + test-runner: name: Test runner regressions runs-on: ubuntu-latest timeout-minutes: 10 + env: + WIPPY_FRAMEWORK_ROOT: ${{ github.workspace }} steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 with: diff --git a/Makefile b/Makefile index bc20e49f..84cd3fa1 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,15 @@ # Wippy Framework # Modules that have test directories with wippy.lock TEST_MODULES = actor agent bootloader embeddings facade llm migration relay usage views +WIPPY ?= wippy +BENCH_OUTPUT_DIR ?= /tmp/wippy-framework-benchmarks +BENCH_WARMUP ?= 5 +BENCH_SAMPLES ?= 30 +BENCH_SIZES ?= 1 8 32 +BENCH_REVISION ?= $(shell git rev-parse HEAD) +RUN_WIPPY = WIPPY="$(WIPPY)" bash scripts/run-wippy.sh -.PHONY: help check-manifests run-tests run-lint install +.PHONY: help check-manifests run-tests run-lint install test-runtime bench help: @echo "Wippy Framework" @@ -12,6 +19,49 @@ help: @echo " make run-lint Run lint for all modules" @echo " make check-manifests Validate package and test-app module types" @echo " make install Install dependencies for all test modules" + @echo " make test-runtime Test real process and control delivery" + @echo " make bench Write Lua operation benchmarks outside the repository" + +test-runtime: + WIPPY_TEST_REQUIRE_CASES=1 $(RUN_WIPPY) src/actor/test test test -- actor_runtime_test + WIPPY_TEST_REQUIRE_CASES=1 $(RUN_WIPPY) src/agent/test test \ + -o wippy.llm:process_host:default=wippy.terminal:host \ + -o wippy.llm:env_storage:default=app:env_storage test -- tools_controls_runtime_test + WIPPY_TEST_REQUIRE_CASES=1 $(RUN_WIPPY) src/relay/test test \ + -o wippy.relay:application_host:default=app:processes \ + -o wippy.relay:user_security_scope:default=app:user test -- relay_runtime_test + WIPPY_TEST_REQUIRE_CASES=1 $(RUN_WIPPY) test-harness/runner test test -- aggregate_checks_test + +bench: + @set -eu; \ + output="$(abspath $(BENCH_OUTPUT_DIR))"; \ + mkdir -p "$$output"; \ + output=$$(cd "$$output" && pwd -P); \ + repo=$$(pwd -P); \ + case "$$output/" in "$$repo/"*) echo 'Benchmark output must be outside the repository' >&2; exit 1;; esac; \ + export WIPPY_BENCH_WARMUP="$(BENCH_WARMUP)" WIPPY_BENCH_SAMPLES="$(BENCH_SAMPLES)"; \ + export WIPPY_TEST_REQUIRE_CASES=1; \ + export WIPPY_BENCH_REVISION="$(BENCH_REVISION)"; \ + WIPPY_BENCH_RUNTIME=$$("$(WIPPY)" version); export WIPPY_BENCH_RUNTIME; \ + for size in $(BENCH_SIZES); do \ + export WIPPY_BENCH_SIZE="$$size"; \ + for name in actor_roundtrip tools_sequential tools_parallel relay_roundtrip; do \ + rm -f "$$output/$$name-$$size.json"; \ + done; \ + $(RUN_WIPPY) src/actor/test test -c \ + -o "wippy.test:benchmark_output:directory=$$output" test -- actor_runtime_benchmark; \ + $(RUN_WIPPY) src/agent/test test -c \ + -o "wippy.test:benchmark_output:directory=$$output" \ + -o wippy.llm:process_host:default=wippy.terminal:host \ + -o wippy.llm:env_storage:default=app:env_storage test -- tools_controls_benchmark; \ + $(RUN_WIPPY) src/relay/test test -c \ + -o "wippy.test:benchmark_output:directory=$$output" \ + -o wippy.relay:application_host:default=app:processes \ + -o wippy.relay:user_security_scope:default=app:user test -- relay_runtime_benchmark; \ + for name in actor_roundtrip tools_sequential tools_parallel relay_roundtrip; do \ + test -s "$$output/$$name-$$size.json"; \ + done; \ + done check-manifests: @python3 scripts/check_module_manifests.py @@ -20,8 +70,7 @@ run-tests: @failed=0; \ for mod in $(TEST_MODULES); do \ printf "%-14s " "$$mod"; \ - output=$$($(MAKE) -s -C src/$$mod/test test 2>&1); \ - if echo "$$output" | grep -q "PASSED"; then \ + if output=$$($(MAKE) -s -C src/$$mod/test test 2>&1); then \ echo "PASSED"; \ else \ echo "FAILED"; \ @@ -36,12 +85,12 @@ run-lint: @failed=0; \ for mod in $(TEST_MODULES); do \ printf "%-14s " "$$mod"; \ - output=$$($(MAKE) -s -C src/$$mod/test lint 2>&1); \ - if echo "$$output" | grep -q "errors"; then \ - echo "$$output" | grep -oP 'Checked.*'; \ - failed=1; \ + if output=$$($(MAKE) -s -C src/$$mod/test lint 2>&1); then \ + echo "PASSED"; \ else \ - echo "$$output" | grep -oP 'Checked.*'; \ + echo "FAILED"; \ + echo "$$output" | tail -5; \ + failed=1; \ fi; \ done; \ if [ $$failed -eq 1 ]; then echo ""; echo "Lint errors found"; exit 1; fi; \ diff --git a/README.md b/README.md index 60ed4672..8d596672 100644 --- a/README.md +++ b/README.md @@ -79,6 +79,21 @@ and configuration management — enabling developers to build sophisticated mult | **stream** | Data streaming interface with chunked reading support | | **cloudstorage** | Cloud storage provider interface for S3-like operations | +## Runtime tests and benchmarks + +`make test-runtime` runs Lua tests for actor messages, relay delivery, and tool controls through Wippy. +It uses the local component and test-runner sources. Required suites fail if they execute no cases. + +`make bench` runs Lua workloads with five warmups and thirty samples at sizes 1, 8, and 32. +Reports contain measured samples, median and p95 latency, throughput, and source/runtime metadata. +Timing covers the operation inside Wippy, excluding CLI startup. Compare runs on the same system +without other test workloads; investigate repeatable slowdowns before delivery. + +Set `WIPPY` to select the executable, `RUNTIME_WORK_DIR` to reuse an external dependency directory, +and `BENCH_OUTPUT_DIR` to select the report directory. Reports default to `/tmp/wippy-framework-benchmarks`. +Use `BENCH_WARMUP`, `BENCH_SAMPLES`, and the space-separated `BENCH_SIZES` to change the workload. +Test data and benchmark reports must stay outside the repository. + [module-actor]: https://github.com/wippyai/module-actor [module-agent]: https://github.com/wippyai/module-agent [module-bootloader]: https://github.com/wippyai/module-bootloader diff --git a/scripts/run-wippy.sh b/scripts/run-wippy.sh new file mode 100644 index 00000000..d944d54d --- /dev/null +++ b/scripts/run-wippy.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo=$(cd "$(dirname "$0")/.." && pwd -P) +source_root=$(cd "${FRAMEWORK_SOURCE_ROOT:-$repo}" && pwd -P) +app=$1 +shift +wippy=$(command -v "${WIPPY:-wippy}") +work_root=${RUNTIME_WORK_DIR:-$(mktemp -d "${TMPDIR:-/tmp}/wippy-framework.XXXXXX")} +mkdir -p "$work_root" +work_root=$(cd "$work_root" && pwd -P) +case "$work_root/" in "$repo/"*) echo 'Runtime files must be outside the repository' >&2; exit 1;; esac +if [ -z "${RUNTIME_WORK_DIR:-}" ]; then + trap 'rm -r -- "$work_root"' EXIT +fi + +app_root=$(cd "$source_root/$app" && pwd) +work="$work_root/${app//\//-}" +mkdir -p "$work" +source_path=$(awk '$1 == "src:" {print $2; exit}' "$app_root/wippy.lock") +source_path=$(cd "$app_root/${source_path:-.}" && pwd) +{ + printf 'directories:\n modules: %s\n src: %s\n' \ + "$(jq -Rn --arg value "$work/modules" '$value')" \ + "$(jq -Rn --arg value "$source_path" '$value')" + awk '/^modules:/ {active=1} /^replacements:/ {active=0} active {print}' "$app_root/wippy.lock" +} > "$work/wippy.lock" +jq -n --arg root "$source_root" --arg work "$work" '{ + version: "1.0", + registry: {dependency_vendor_dir: ($work + "/modules/vendor"), dependency_lock_path: ($work + "/wippy.lock")}, + workspace: {replacements: {"wippy/test": ($root + "/src/test")}} +}' > "$work/.wippy.yaml" + +while IFS=$'\t' read -r component replacement; do + replacement=$(cd "$app_root/$replacement" && pwd) + jq --arg component "$component" --arg path "$replacement" \ + '.workspace.replacements[$component] = $path' "$work/.wippy.yaml" > "$work/config.next" + mv "$work/config.next" "$work/.wippy.yaml" +done < <(awk '/^replacements:/ {active=1} active && $2 == "from:" {module=$3} active && $1 == "to:" {printf "%s\t%s\n", module, $2}' "$app_root/wippy.lock") + +if [ "$app" = src/facade/test ]; then + jq --arg path "$source_root/src/facade" '.workspace.replacements["wippy/facade"] = $path' \ + "$work/.wippy.yaml" > "$work/config.next" + mv "$work/config.next" "$work/.wippy.yaml" +fi + +export WIPPY_FRAMEWORK_ROOT="$repo" +export WIPPY_TEST_ARTIFACTS="$work" +cd "$work" +"$wippy" install +"$wippy" "$@" diff --git a/src/actor/src/actor.lua b/src/actor/src/actor.lua index 4d32ba65..ad670028 100644 --- a/src/actor/src/actor.lua +++ b/src/actor/src/actor.lua @@ -277,7 +277,7 @@ function actor.new(initial_state: any, handlers: any): any -- Main actor loop while true do local result: SelectResult = channel.select(select_cases) - if not result.ok then + if not result.ok and not channel_to_id[result.channel] then break end diff --git a/src/actor/test/.wippy.yaml b/src/actor/test/.wippy.yaml new file mode 100644 index 00000000..977dbf41 --- /dev/null +++ b/src/actor/test/.wippy.yaml @@ -0,0 +1,4 @@ +version: "1.0" +workspace: + replacements: + wippy/test: ../../test diff --git a/src/actor/test/_index.yaml b/src/actor/test/_index.yaml index dea45655..fefcbd4c 100644 --- a/src/actor/test/_index.yaml +++ b/src/actor/test/_index.yaml @@ -32,3 +32,65 @@ entries: - time - json method: run + + - name: runtime_processes + kind: process.host + lifecycle: + auto_start: true + + - name: actor_runtime_fixture + kind: process.lua + source: file://actor_runtime_fixture.lua + imports: + actor: wippy.actor:actor + modules: + - time + method: run + + - name: actor_runtime_test + kind: function.lua + meta: + type: test + suite: actor-runtime + timeout: 20s + source: file://actor_runtime_test.lua + imports: + test: wippy.test:test + benchmark: wippy.test:benchmark + modules: + - time + - env + method: run + + - name: actor_runtime_benchmark + kind: function.lua + meta: + type: test + suite: actor-benchmark + timeout: 120s + source: file://actor_runtime_test.lua + imports: + test: wippy.test:test + benchmark: wippy.test:benchmark + modules: + - time + - env + method: benchmark + + - name: runtime_env_storage + kind: env.storage.os + + - name: runtime_bench_samples + kind: env.variable + variable: WIPPY_BENCH_SAMPLES + storage: app:runtime_env_storage + + - name: runtime_bench_warmup + kind: env.variable + variable: WIPPY_BENCH_WARMUP + storage: app:runtime_env_storage + + - name: runtime_bench_size + kind: env.variable + variable: WIPPY_BENCH_SIZE + storage: app:runtime_env_storage diff --git a/src/actor/test/actor_runtime_fixture.lua b/src/actor/test/actor_runtime_fixture.lua new file mode 100644 index 00000000..86a9a7a3 --- /dev/null +++ b/src/actor/test/actor_runtime_fixture.lua @@ -0,0 +1,67 @@ +local actor = require("actor") +local time = require("time") + +local function run(args) + local reply_topic = args.topic .. ".reply" + local proceed_topic = args.topic .. ".proceed" + local listener = process.listen(reply_topic) + local proceed = process.listen(proceed_topic) + local closed = channel.new(1) + local state = { count = 0, closures = 0 } + local function notify(value) + assert(process.send(args.parent, args.topic, value)) + end + local result = actor.new(state, { + __init = function(s) + if args.scenario == "closure" then + s.register_channel(closed, function(current, value, ok) + if not ok then + current.closures = current.closures + 1 + notify({ phase = "closed", value = value, ok = ok }) + end + end) + closed:close() + elseif args.scenario == "deadline" or args.scenario == "late" then + local value, err = s.wait(reply_topic, 5 * time.MILLISECOND) + notify({ phase = "expired", value = value, error = err }) + if args.scenario == "deadline" then + return actor.exit({ status = "deadline", error = err }) + end + local _, proceed_err = s.wait(proceed_topic, 2 * time.SECOND) + assert(not proceed_err, proceed_err) + local late, late_err = s.wait(reply_topic, 2 * time.SECOND) + return actor.exit({ status = "late", value = late, error = late_err }) + elseif args.scenario == "dispatch" then + notify({ phase = "ready" }) + elseif args.scenario == "cancel" then + notify({ phase = "waiting" }) + s.wait(reply_topic, 20 * time.MILLISECOND) + else + for index = 1, args.count or 1 do + notify({ phase = "waiting", index = index }) + local value, err = s.wait(reply_topic, 2 * time.SECOND) + if err then return actor.exit({ status = "failed", error = err }) end + s.count = s.count + 1 + s.last = value + end + return actor.exit({ status = "replied", count = s.count, value = s.last }) + end + end, + ping = function(s, payload, topic, from) + return actor.exit({ status = "inbox", closures = s.closures, + removed = not s.unregister_channel(closed), payload = payload, topic = topic, from = from }) + end, + dispatch = function(s, payload) + s.count = s.count + 1 + assert(process.send(args.parent, args.topic, { count = s.count, value = payload })) + end, + __on_cancel = function() + return actor.exit({ status = "canceled" }) + end, + }).run() + process.unlisten(listener) + process.unlisten(proceed) + return result +end + +return { run = run } diff --git a/src/actor/test/actor_runtime_test.lua b/src/actor/test/actor_runtime_test.lua new file mode 100644 index 00000000..80a04021 --- /dev/null +++ b/src/actor/test/actor_runtime_test.lua @@ -0,0 +1,168 @@ +local test = require("test") +local time = require("time") +local env = require("env") +local benchmark_report = require("benchmark") + +local sequence = 0 + +local function receive(ch) + local timer = time.timer(3 * time.SECOND) + local result = channel.select({ ch:case_receive(), timer:channel():case_receive() }) + timer:stop() + assert(result.channel == ch and result.ok, "actor runtime response deadline") + return result.value +end + +local function with_fixture(scenario, count, body): any + sequence = sequence + 1 + local topic = "actor.runtime." .. sequence + local responses = process.listen(topic) + local events = process.events() + local pid, err = process.spawn_monitored("app:actor_runtime_fixture", "app:runtime_processes", { + parent = process.pid(), topic = topic, scenario = scenario, count = count, + }) + assert(pid, tostring(err)) + local exited = false + local function outcome(check_result) + local timer = time.timer(3 * time.SECOND) + while true do + local selected = channel.select({ events:case_receive(), timer:channel():case_receive() }) + if selected.channel == timer:channel() then + timer:stop() + error("actor runtime exit deadline") + end + local event = selected.value + if event.from == pid and event.kind == process.event.EXIT then + exited = true + timer:stop() + if check_result ~= false then + assert(not event.result.error, tostring(event.result.error)) + end + return event.result.value + end + end + end + local ok: boolean, result: any = pcall(body, pid, topic, responses, outcome) + if not ok and not exited then + process.terminate(pid) + local cleaned, cleanup_err = pcall(outcome, false) + if not cleaned then result = tostring(result) .. "; cleanup: " .. tostring(cleanup_err) end + end + process.unmonitor(pid) + process.unlisten(responses) + if not ok then error(result) end + return result +end + +local function define_tests() + test.describe("actor runtime", function() + test.it("receives a real topic reply before the wait deadline", function() + with_fixture("reply", 1, function(pid, topic, responses, outcome) + test.eq(receive(responses).phase, "waiting") + assert(process.send(pid, topic .. ".reply", { value = 42 })) + local result = outcome() + test.eq(result.status, "replied") + test.eq(result.value.value, 42) + test.eq(result.count, 1) + end) + end) + test.it("returns timeout when the real deadline expires", function() + with_fixture("deadline", 1, function(_, _, responses, outcome) + local expired = receive(responses) + test.eq(expired.phase, "expired") + test.is_nil(expired.value) + test.eq(expired.error, "timeout") + test.eq(outcome().status, "deadline") + end) + end) + test.it("preserves a late reply for the next wait on the same topic", function() + with_fixture("late", 1, function(pid, topic, responses, outcome) + test.eq(receive(responses).error, "timeout") + assert(process.send(pid, topic .. ".reply", { late = true })) + assert(process.send(pid, topic .. ".proceed", true)) + local result = outcome() + test.eq(result.status, "late") + test.eq(result.value.late, true) + test.is_nil(result.error) + end) + end) + test.it("reuses a topic listener across repeated waits without replaying replies", function() + with_fixture("reply", 3, function(pid, topic, responses, outcome) + for index = 1, 3 do + test.eq(receive(responses).index, index) + assert(process.send(pid, topic .. ".reply", { index = index })) + end + local result = outcome() + test.eq(result.count, 3) + test.eq(result.value.index, 3) + end) + end) + test.it("handles cancellation after a bounded outstanding wait", function() + with_fixture("cancel", 1, function(pid, _, responses, outcome) + test.eq(receive(responses).phase, "waiting") + assert(process.cancel(pid, "2s")) + test.eq(outcome().status, "canceled") + end) + end) + test.it("unregisters a closed channel and continues handling inbox messages", function() + with_fixture("closure", 1, function(pid, _, responses, outcome) + local closed = receive(responses) + test.eq(closed.phase, "closed") + test.eq(closed.ok, false) + assert(process.send(pid, "ping", { value = "after closure" })) + local result = outcome() + test.eq(result.status, "inbox") + test.eq(result.closures, 1) + test.eq(result.removed, true) + test.eq(result.payload.value, "after closure") + test.eq(result.topic, "ping") + test.eq(result.from, (process.pid())) + end) + end) + end) +end + +local function benchmark(options) + local count = tonumber(options and options.iterations) or 100 + return with_fixture("dispatch", 0, function(pid, _, responses, outcome) + assert(receive(responses).phase == "ready") + local started = time.now() + for index = 1, count do + assert(process.send(pid, "dispatch", index)) + local value = receive(responses) + assert(value.count == index and value.value == index, "actor dispatch mismatch") + end + local duration_ms = time.now():sub(started):seconds() * 1000 + assert(process.cancel(pid, "2s")) + assert(outcome().status == "canceled") + return { operations = count, duration_ms = duration_ms } + end) +end + +local function benchmark_cases() + test.describe("actor_roundtrip benchmark", function() + local samples = tonumber((env.get("WIPPY_BENCH_SAMPLES"))) + if not samples then + test.it_skip("requires WIPPY_BENCH_SAMPLES", function() end) + return + end + test.it("measures verified runtime roundtrips", function() + local size = tonumber((env.get("WIPPY_BENCH_SIZE"))) or 1 + local warmup = tonumber((env.get("WIPPY_BENCH_WARMUP"))) or 5 + assert(samples > 0 and size > 0 and warmup >= 0, "invalid benchmark configuration") + for _ = 1, warmup do benchmark({ iterations = size }) end + local durations = {} + for index = 1, samples do + local result = benchmark({ iterations = size }) + assert(result.operations == size, "benchmark operation count mismatch") + durations[index] = result.duration_ms + end + local report, report_err = benchmark_report.report({ + name = "actor_roundtrip", size = size, operations_per_sample = size, samples_ms = durations, + }) + test.not_nil(report, tostring(report_err)) + end) + end) +end + +return { run = test.run_cases(define_tests), benchmark = test.run_cases(benchmark_cases), workload = benchmark } diff --git a/src/agent/test/.wippy.yaml b/src/agent/test/.wippy.yaml new file mode 100644 index 00000000..977dbf41 --- /dev/null +++ b/src/agent/test/.wippy.yaml @@ -0,0 +1,4 @@ +version: "1.0" +workspace: + replacements: + wippy/test: ../../test diff --git a/src/agent/test/_index.yaml b/src/agent/test/_index.yaml index f05b1383..90342c67 100644 --- a/src/agent/test/_index.yaml +++ b/src/agent/test/_index.yaml @@ -393,3 +393,64 @@ entries: - time - ctx method: delayed_echo + + - name: tools_controls_runtime_test + kind: function.lua + meta: + type: test + suite: agent-runtime + source: file://tools/controls_runtime_test.lua + imports: + test: wippy.test:test + tool_caller: wippy.agent.tools:caller + method: run_tests + + - name: control_fixture + kind: function.lua + meta: + type: tool + input_schema: '{"type":"object"}' + source: file://tools/control_fixture.lua + modules: [ctx] + method: execute + + - name: control_wrapper_apply + kind: function.lua + source: file://tools/control_fixture.lua + modules: [ctx] + method: apply + + - name: control_wrapper_binding + kind: contract.binding + contracts: + - contract: wippy.agent:tool_wrapper + methods: + apply: app:control_wrapper_apply + + - name: tools_controls_benchmark + kind: function.lua + meta: + type: test + suite: agent-benchmark + source: file://tools/controls_benchmark.lua + modules: [env, time, json] + imports: + benchmark: wippy.test:benchmark + test: wippy.test:test + tool_caller: wippy.agent.tools:caller + method: run_tests + + - name: bench_samples + kind: env.variable + variable: WIPPY_BENCH_SAMPLES + storage: app:env_storage + + - name: bench_warmup + kind: env.variable + variable: WIPPY_BENCH_WARMUP + storage: app:env_storage + + - name: bench_size + kind: env.variable + variable: WIPPY_BENCH_SIZE + storage: app:env_storage diff --git a/src/agent/test/agent_integration_test.lua b/src/agent/test/agent_integration_test.lua index c6977b7e..b9e6de95 100644 --- a/src/agent/test/agent_integration_test.lua +++ b/src/agent/test/agent_integration_test.lua @@ -10,11 +10,8 @@ local json = require("json") local env = require("env") local function define_integration_tests() - -- Skip all integration tests if not enabled - if not env.get("ENABLE_INTEGRATION_TESTS") then - print("WARNING: Integration tests skipped - set ENABLE_INTEGRATION_TESTS=true to run") - return - end + local enabled = env.get("ENABLE_INTEGRATION_TESTS") == "true" + local it = enabled and _G.it or _G.it_skip -- Test helpers for full chain testing local test_helpers = {} diff --git a/src/agent/test/tools/control_fixture.lua b/src/agent/test/tools/control_fixture.lua new file mode 100644 index 00000000..0ba42740 --- /dev/null +++ b/src/agent/test/tools/control_fixture.lua @@ -0,0 +1,45 @@ +local ctx = require("ctx") + +local function execute(args) + if args.raise then error("runtime fixture failure") end + local control = args.control + return { + success = args.fail ~= true, + error = args.fail and "explicit runtime fixture failure" or nil, + message = args.message, + call_id = ctx.get("call_id"), + scope = ctx.get("scope"), + _control = control, + wrapper_control = args.wrapper_control, + } +end + +local function apply(request) + if request.phase == "before_execute" then + if request.options.mode == "drop" then return {tool_calls = {}} end + local rewritten = {} + for _, call in ipairs(request.tool_calls) do + local args = {} + for key, value in pairs(call.arguments) do args[key] = value end + args.message = "wrapped " .. tostring(args.message) + rewritten[#rewritten + 1] = { + id = call.id, name = call.name, registry_id = call.registry_id, + arguments = args, context = call.context, + } + end + return {tool_calls = rewritten} + end + local proposal + for _, result in pairs(request.tool_results or {}) do + if result.result and result.result.wrapper_control then + proposal = result.result.wrapper_control + end + end + return { + _control = proposal, + observations = {{level = "info", code = "runtime_outcome", content = request.outcome}}, + metadata = {phase = request.phase}, + } +end + +return {execute = execute, apply = apply} diff --git a/src/agent/test/tools/controls_benchmark.lua b/src/agent/test/tools/controls_benchmark.lua new file mode 100644 index 00000000..ca80b781 --- /dev/null +++ b/src/agent/test/tools/controls_benchmark.lua @@ -0,0 +1,47 @@ +local test = require("test") +local tool_caller = require("tool_caller") +local time = require("time") +local env = require("env") +local benchmark = require("benchmark") + +local function define_tests() + local sample_count = tonumber((env.get("WIPPY_BENCH_SAMPLES"))) + local benchmark_it = sample_count and _G.it or _G.it_skip + describe("registered tool batch benchmarks", function() + for _, strategy in ipairs({tool_caller.STRATEGY.SEQUENTIAL, tool_caller.STRATEGY.PARALLEL}) do + benchmark_it("tools_" .. strategy, function() + local size = tonumber((env.get("WIPPY_BENCH_SIZE"))) or 1 + local warmup = tonumber((env.get("WIPPY_BENCH_WARMUP"))) or 5 + test.gt(sample_count, 0) + test.gt(size, 0) + local caller = tool_caller.new():set_strategy(strategy) + local calls = {} + for index = 1, size do + calls[index] = {id = "benchmark-" .. index, name = "runtime_control", + registry_id = "app:control_fixture", arguments = {message = "benchmark", + control = {context = {session = {set = {scope = "benchmark"}}}}}} + end + local samples = {} + for iteration = 1, warmup + sample_count do + local started = time.now() + local validated, err = caller:validate(calls) + test.is_nil(err) + local results = caller:execute({}, validated) + local elapsed = time.now():sub(started):seconds() * 1000 + for index = 1, size do + local result = results["benchmark-" .. index] + test.is_nil(result.error) + test.eq(result.result.call_id, "benchmark-" .. index) + test.eq(result.result._control.context.session.set.scope, "benchmark") + end + if iteration > warmup then samples[#samples + 1] = elapsed end + end + local report, report_err = benchmark.report({name = "tools_" .. strategy, + size = size, operations_per_sample = size, samples_ms = samples}) + test.not_nil(report, tostring(report_err)) + end) + end + end) +end + +return {run_tests = test.run_cases(define_tests)} diff --git a/src/agent/test/tools/controls_runtime_test.lua b/src/agent/test/tools/controls_runtime_test.lua new file mode 100644 index 00000000..37073056 --- /dev/null +++ b/src/agent/test/tools/controls_runtime_test.lua @@ -0,0 +1,123 @@ +local test = require("test") +local tool_caller = require("tool_caller") + +local function call(id, args) + return {id = id, name = "runtime_control", registry_id = "app:control_fixture", + arguments = args, context = {scope = "tool"}} +end + +local function count(value) + local total = 0 + for _ in pairs(value) do total = total + 1 end + return total +end + +local function wrapper(source, phases, options) + return {id = "runtime-wrapper", binding = "app:control_wrapper_binding", source = source, + phases = phases, options = options or {}, strict = true} +end + +local function define_tests() + describe("registered tool runtime controls", function() + for _, strategy in ipairs({tool_caller.STRATEGY.SEQUENTIAL, tool_caller.STRATEGY.PARALLEL}) do + it(strategy .. " preserves IDs, controls and successful peers across mixed failures", function() + local caller = tool_caller.new():set_strategy(strategy) + caller:set_wrapper_context({host = {kind = "session", session_id = "runtime"}}) + caller:set_tool_wrappers({wrapper(nil, {"after_execute"})}) + local validated, err = caller:validate({ + call("success-id", {message = "success", control = {config = {model = "next-model"}, + context = {session = {set = {scope = "next"}}}, memory = {compact = true}}}), + call("data-error-id", {fail = true, message = "failure", control = {memory = {compact = false}}}), + call("execution-error-id", {raise = true}), + call("json-error-id", "{invalid json"), + {id = "missing-id", name = "missing", registry_id = "app:missing", arguments = {}}, + }) + test.is_nil(err) + local results = caller:execute({scope = "session"}, validated) + test.eq(count(results), 5) + test.is_nil(results["success-id"].error) + local success = results["success-id"].result + test.eq(success.call_id, "success-id") + test.eq(success.scope, "session") + test.eq(success.message, "success") + test.eq(success._control.config.model, "next-model") + test.eq(success._control.context.session.set.scope, "next") + test.is_true(success._control.memory.compact) + test.eq(results["data-error-id"].error, "explicit runtime fixture failure") + test.is_false(results["data-error-id"].result._control.memory.compact) + test.eq(results["data-error-id"].result.call_id, "data-error-id") + test.not_nil(results["execution-error-id"].error) + test.is_nil(results["execution-error-id"].result) + test.contains(tostring(results["json-error-id"].error), "Failed to parse arguments") + test.not_nil(results["missing-id"].error) + for id, result in pairs(results) do test.eq(result.tool_call.call_id, id) end + test.eq(caller:get_wrapper_observations()[1].content.reason, "tool_execution_failed") + end) + + it(strategy .. " collects detached behavior proposals and resets each execution", function() + local caller = tool_caller.new():set_strategy(strategy) + caller:set_wrapper_context({host = {kind = "session", session_id = "runtime"}}) + caller:set_tool_wrappers({wrapper("behavior", {"before_execute", "after_execute"})}) + local validated, err = caller:validate({call("behavior-id", {message = "first", + wrapper_control = {config = {model = "stronger"}, memory = {compact = false}}})}) + test.is_nil(err) + local results = caller:execute({}, validated) + test.eq(results["behavior-id"].result.message, "wrapped first") + test.eq(results["behavior-id"].result.call_id, "behavior-id") + test.eq(#caller:get_wrapper_controls(), 1) + local controls = caller:get_wrapper_controls() + test.eq(controls[1].config.model, "stronger") + test.is_false(controls[1].memory.compact) + controls[1].config.model = "mutated" + test.eq(caller:get_wrapper_controls()[1].config.model, "stronger") + results["behavior-id"].result.wrapper_control.config.model = "changed-result" + test.eq(caller:get_wrapper_controls()[1].config.model, "stronger") + caller:set_tool_wrappers({}) + caller:execute({}, validated) + test.eq(#caller:get_wrapper_controls(), 0) + caller:set_tool_wrappers({wrapper("behavior", {"after_execute"})}) + validated, err = caller:validate({call("second-id", {message = "second"})}) + test.is_nil(err) + results = caller:execute({}, validated) + test.eq(results["second-id"].result.message, "second") + test.eq(#caller:get_wrapper_controls(), 0) + test.eq(#caller:get_wrapper_errors(), 0) + end) + + it(strategy .. " keeps legacy wrappers from proposing behavior controls", function() + local caller = tool_caller.new():set_strategy(strategy) + caller:set_wrapper_context({host = {kind = "session", session_id = "runtime"}}) + caller:set_tool_wrappers({wrapper(nil, {"after_execute"})}) + local validated, err = caller:validate({call("legacy-id", {wrapper_control = {memory = {compact = true}}})}) + test.is_nil(err) + local results = caller:execute({}, validated) + test.is_nil(results["legacy-id"].error) + test.eq(#caller:get_wrapper_controls(), 0) + test.eq(#caller:get_wrapper_observations(), 1) + end) + + it(strategy .. " rejects invalid behavior proposals without losing tool results", function() + local caller = tool_caller.new():set_strategy(strategy) + caller:set_wrapper_context({host = {kind = "session", session_id = "runtime"}}) + caller:set_tool_wrappers({wrapper("behavior", {"after_execute"})}) + local validated, err = caller:validate({call("invalid-id", {message = "completed", wrapper_control = {yield = {}}})}) + test.is_nil(err) + local results = caller:execute({}, validated) + test.eq(results["invalid-id"].result.message, "completed") + test.eq(#caller:get_wrapper_controls(), 0) + test.eq(#caller:get_wrapper_errors(), 1) + end) + end + + it("rejects a registered wrapper that drops provider call identities", function() + local caller = tool_caller.new() + caller:set_wrapper_context({host = {kind = "session", session_id = "runtime"}}) + caller:set_tool_wrappers({wrapper("behavior", {"before_execute"}, {mode = "drop"})}) + local validated, err = caller:validate({call("must-survive", {message = "first"})}) + test.is_nil(validated) + test.contains(err, "preserve every tool call ID") + end) + end) +end + +return {run_tests = test.run_cases(define_tests)} diff --git a/src/llm/test/e2e/integration_test.lua b/src/llm/test/e2e/integration_test.lua index 94b1f1da..a2114441 100644 --- a/src/llm/test/e2e/integration_test.lua +++ b/src/llm/test/e2e/integration_test.lua @@ -3,28 +3,17 @@ local env = require("env") local bedrock_credentials = require("bedrock_credentials") local function define_tests() - local RUN_INTEGRATION_TESTS = env.get("ENABLE_INTEGRATION_TESTS") - local RUN_BEDROCK_TESTS = false + local enabled = env.get("ENABLE_INTEGRATION_TESTS") == "true" + local bedrock_enabled = false + if enabled then + local _, creds_err = bedrock_credentials.resolve() + bedrock_enabled = creds_err == nil + end + local it = enabled and _G.it or _G.it_skip + local bedrock_it = bedrock_enabled and _G.it or _G.it_skip describe("LLM Integration Tests", function() - before_all(function() - if not RUN_INTEGRATION_TESTS then - print("Integration tests disabled - set ENABLE_INTEGRATION_TESTS=true to enable") - return - end - local _, creds_err = bedrock_credentials.resolve() - if creds_err then - print("Bedrock integration tests disabled - " .. tostring(creds_err)) - else - RUN_BEDROCK_TESTS = true - end - end) - it("should call provider directly with provider_id and model", function() - if not RUN_INTEGRATION_TESTS then - print("Skipping direct provider test - not enabled") - return - end local messages = { { role = "user", @@ -51,10 +40,6 @@ local function define_tests() end) it("should handle streaming with provider_id", function() - if not RUN_INTEGRATION_TESTS then - print("Skipping streaming provider test - not enabled") - return - end local messages = { { @@ -84,10 +69,6 @@ local function define_tests() end) it("should handle tool calling with provider_id", function() - if not RUN_INTEGRATION_TESTS then - print("Skipping tool calling provider test - not enabled") - return - end local messages = { { @@ -132,8 +113,7 @@ local function define_tests() test.eq(result.tool_calls[1].name, "calculator") end) - it("should generate text via Bedrock provider using Converse API", function() - if not RUN_BEDROCK_TESTS then return end + bedrock_it("should generate text via Bedrock provider using Converse API", function() local messages = { { role = "user", content = { { type = "text", text = "Say 'Hello from Bedrock'" } } } @@ -156,8 +136,7 @@ local function define_tests() test.is_true(result.tokens.prompt_tokens > 0) end) - it("should handle Bedrock tool calling via Converse API", function() - if not RUN_BEDROCK_TESTS then return end + bedrock_it("should handle Bedrock tool calling via Converse API", function() local messages = { { role = "user", content = { { type = "text", text = "What is 15 * 23? Use the calculator tool." } } } @@ -195,8 +174,7 @@ local function define_tests() test.eq(result.tool_calls[1].name, "calculator") end) - it("should handle Bedrock structured output via Converse API", function() - if not RUN_BEDROCK_TESTS then return end + bedrock_it("should handle Bedrock structured output via Converse API", function() local messages = { { role = "user", content = { { type = "text", text = "Extract: Alice is 28 and lives in Paris" } } } @@ -230,8 +208,7 @@ local function define_tests() test.eq((result.result :: any).city, "Paris") end) - it("should handle Bedrock Titan embedding via InvokeModel", function() - if not RUN_BEDROCK_TESTS then return end + bedrock_it("should handle Bedrock Titan embedding via InvokeModel", function() local options = { provider_id = "wippy.llm.bedrock:provider", @@ -249,8 +226,7 @@ local function define_tests() test.not_nil(result.tokens) end) - it("should handle Bedrock Cohere embedding via InvokeModel", function() - if not RUN_BEDROCK_TESTS then return end + bedrock_it("should handle Bedrock Cohere embedding via InvokeModel", function() local options = { provider_id = "wippy.llm.bedrock:provider", @@ -268,8 +244,7 @@ local function define_tests() test.is_true(#result.result[2] > 0) end) - it("should handle Bedrock streaming via ConverseStream", function() - if not RUN_BEDROCK_TESTS then return end + bedrock_it("should handle Bedrock streaming via ConverseStream", function() local messages = { { role = "user", content = { { type = "text", text = "Count 1 2 3" } } } diff --git a/src/relay/src/user.lua b/src/relay/src/user.lua index 4a85ea4a..209a67d1 100644 --- a/src/relay/src/user.lua +++ b/src/relay/src/user.lua @@ -273,7 +273,10 @@ local function route_to_plugin(state: UserState, prefix: string, plugin_config: context = message_data.context } - process.send(pid, topic, payload_data) + local sent, send_err = process.send(pid, topic, payload_data) + if not sent then + return false, "Failed to send message to plugin: " .. tostring(send_err) + end return true end end @@ -343,7 +346,7 @@ end local function handle_client_message(state: UserState, payload: any, from_pid: string) local message_data, err = json.decode(string(payload:data())) - if not message_data then + if err then process.send(from_pid, consts.CLIENT_TOPICS.ERROR, { error = consts.ERROR_CODES.INVALID_JSON, message = "Failed to decode JSON message" @@ -351,8 +354,16 @@ local function handle_client_message(state: UserState, payload: any, from_pid: s return end + if type(message_data) ~= "table" then + process.send(from_pid, consts.CLIENT_TOPICS.ERROR, { + error = consts.ERROR_CODES.UNKNOWN_COMMAND, + message = "Message must be a JSON object" + }) + return + end + local msg_type = message_data.type :: string? - if not msg_type then + if type(msg_type) ~= "string" or msg_type == "" then process.send(from_pid, consts.CLIENT_TOPICS.ERROR, { error = consts.ERROR_CODES.UNKNOWN_COMMAND, message = "Message type is required" diff --git a/src/relay/test/.wippy.yaml b/src/relay/test/.wippy.yaml new file mode 100644 index 00000000..977dbf41 --- /dev/null +++ b/src/relay/test/.wippy.yaml @@ -0,0 +1,4 @@ +version: "1.0" +workspace: + replacements: + wippy/test: ../../test diff --git a/src/relay/test/_index.yaml b/src/relay/test/_index.yaml index 544773dd..247357d0 100644 --- a/src/relay/test/_index.yaml +++ b/src/relay/test/_index.yaml @@ -62,6 +62,7 @@ entries: modules: - time - security + - env method: run # app:central_test @@ -84,3 +85,81 @@ entries: modules: - time method: run + + - name: runtime_plugin_policy + kind: security.policy + groups: + - app:user + policy: + resources: '*' + actions: + - process.spawn + - process.spawn.linked + - process.spawn.monitored + - process.host + - process.cancel + effect: allow + + - name: relay_runtime_plugin + kind: process.lua + source: file://relay_runtime_plugin.lua + method: run + + - name: relay_runtime_test + kind: function.lua + meta: + type: test + suite: relay-runtime + timeout: 25s + source: file://relay_runtime_test.lua + imports: + test: wippy.test:test + benchmark: wippy.test:benchmark + consts: wippy.relay:consts + modules: + - time + - json + - security + - env + method: run + + - name: relay_runtime_benchmark + kind: function.lua + meta: + type: test + suite: relay-benchmark + timeout: 120s + source: file://relay_runtime_test.lua + imports: + test: wippy.test:test + benchmark: wippy.test:benchmark + consts: wippy.relay:consts + modules: + - time + - json + - security + - env + method: benchmark + + - name: runtime_env_storage + kind: env.storage.os + + - name: runtime_bench_samples + kind: env.variable + variable: WIPPY_BENCH_SAMPLES + storage: app:runtime_env_storage + + - name: runtime_bench_warmup + kind: env.variable + variable: WIPPY_BENCH_WARMUP + storage: app:runtime_env_storage + + - name: runtime_bench_size + kind: env.variable + variable: WIPPY_BENCH_SIZE + storage: app:runtime_env_storage + + - name: relay_runtime_finished_plugin + kind: process.lua + source: file://relay_runtime_plugin.lua + method: finish diff --git a/src/relay/test/relay_runtime_plugin.lua b/src/relay/test/relay_runtime_plugin.lua new file mode 100644 index 00000000..75a079c0 --- /dev/null +++ b/src/relay/test/relay_runtime_plugin.lua @@ -0,0 +1,23 @@ +local function run(args) + local observer = args.config.runtime_observer + assert(process.send(observer, "relay.runtime.started", { pid = process.pid(), user_id = args.user_id })) + local inbox = process.inbox() + local events = process.events() + while true do + local result = channel.select({ inbox:case_receive(), events:case_receive() }) + if not result.ok then break end + if result.channel == events then + if result.value.kind == process.event.CANCEL then break end + else + local message = result.value + local payload = message:payload():data() + assert(process.send(args.user_hub_pid, "relay.runtime.response", { + topic = message:topic(), from = message:from(), plugin_pid = process.pid(), + user_id = args.user_id, user_metadata = args.user_metadata, payload = payload, + })) + end + end + return { status = "stopped" } +end + +return { run = run, finish = function() return { status = "finished" } end } diff --git a/src/relay/test/relay_runtime_test.lua b/src/relay/test/relay_runtime_test.lua new file mode 100644 index 00000000..a54653d8 --- /dev/null +++ b/src/relay/test/relay_runtime_test.lua @@ -0,0 +1,252 @@ +local test = require("test") +local time = require("time") +local env = require("env") +local benchmark_report = require("benchmark") +local json = require("json") +local security = require("security") +local consts = require("consts") + +local sequence = 0 +local function receive(ch: any): any + local timer = time.timer(3 * time.SECOND) + local result = channel.select({ ch:case_receive(), timer:channel():case_receive() }) + timer:stop() + assert(result.channel == ch and result.ok, "relay runtime response deadline") + return result.value +end + +local function with_hub(plugin_id: string, body: any, active_pid: string?): any + sequence = sequence + 1 + local user_id = "relay-runtime-" .. process.pid() .. "-" .. sequence + local actor = security.new_actor(user_id, {}) + local scope, scope_err = security.named_scope("app:user") + assert(scope, tostring(scope_err)) + local config = consts.get_config() + config.runtime_observer = process.pid() + local welcome = process.listen(consts.CLIENT_TOPICS.WELCOME) + local response = process.listen("relay.runtime.response") + local errors = process.listen(consts.CLIENT_TOPICS.ERROR) + local started = process.listen("relay.runtime.started") + local events = process.events() + local args = { + user_id = user_id, user_metadata = { fixture = true }, + config = config, plugins = { + runtime_ = { prefix = "runtime_", process_id = plugin_id, + host = config.user_hub_host, auto_start = false }, + }, + } + if active_pid then + args.relay_user_upgrade = true + args.active_plugins = { runtime_ = { pid = active_pid, status = "running", restart_count = 0 } } + args.connected_clients = {} + args.client_count = 0 + args.pg_groups = {} + args = (json.encode(args)) + end + local pid, err = process.with_context({}):with_actor(actor):with_scope(scope):spawn_monitored( + consts.USER_HUB_PROCESS_ID, assert(config.user_hub_host), args) + assert(pid, tostring(err)) + local function join() + assert(process.send(pid, consts.WS_TOPICS.JOIN, { client_pid = process.pid() })) + return receive(welcome) + end + local function send(value) + local encoded, encode_err = json.encode(value) + assert(encoded, tostring(encode_err)) + assert(process.send(pid, consts.WS_TOPICS.MESSAGE, encoded)) + end + local children = {} + local function startup() + local plugin = receive(started) + assert(process.monitor(plugin.pid)) + children[plugin.pid] = true + return plugin + end + local ok: boolean, result: any = pcall(body, pid, user_id, join, send, response, errors, startup) + process.send(pid, consts.WS_TOPICS.CANCEL, {}) + local stopped = false + local timer = time.timer(3 * time.SECOND) + while not stopped or next(children) do + local selected = channel.select({ events:case_receive(), timer:channel():case_receive() }) + if selected.channel == timer:channel() then break end + local event = selected.value + if event.kind == process.event.EXIT and children[event.from] then + children[event.from] = nil + if event.result.error then ok, result = false, tostring(event.result.error) end + end + if event.from == pid and event.kind == process.event.EXIT then + stopped = true + if event.result.error then + ok, result = false, tostring(result or "") .. "; hub exited: " .. tostring(event.result.error) + end + end + end + timer:stop() + local cleanup_timed_out = not stopped or next(children) ~= nil + if cleanup_timed_out then + if not stopped then process.terminate(pid) end + for child in pairs(children) do process.terminate(child) end + local forced_deadline = time.timer(3 * time.SECOND) + while not stopped or next(children) do + local selected = channel.select({ events:case_receive(), forced_deadline:channel():case_receive() }) + if selected.channel == forced_deadline:channel() then break end + local event = selected.value + if event.kind == process.event.EXIT then + if event.from == pid then stopped = true end + children[event.from] = nil + end + end + forced_deadline:stop() + end + local leaked = not stopped or next(children) ~= nil + for _, ch in ipairs({ welcome, response, errors, started }) do process.unlisten(ch) end + if leaked then error(tostring(result or "") .. "; relay fixture termination deadline") end + if not ok then error(result) end + assert(not cleanup_timed_out and not leaked, "relay hub or plugin cleanup deadline") + return result +end + +local function define_tests() + test.describe("relay runtime", function() + test.it("starts plugins lazily and preserves command routing and payload fields", function() + with_hub("app:relay_runtime_plugin", function(pid, user_id, join, send, response, _, started) + local welcome = join() + test.eq(welcome.plugins[1].status, "not_started") + send({ type = "runtime_echo", data = { nested = { value = 42 } }, + request_id = "request-1", session_id = "session-1", start_token = "start-1", + attention_context_enabled = true, context = { label = "fixture" } }) + local boot = started() + local result = receive(response) + test.eq(result.plugin_pid, boot.pid) + test.eq(result.topic, "echo") + test.eq(result.from, pid) + test.eq(result.user_id, user_id) + test.eq(result.user_metadata.fixture, true) + test.eq(result.payload.conn_pid, (process.pid())) + test.eq(result.payload.type, "runtime_echo") + test.eq(result.payload.data.nested.value, 42) + test.eq(result.payload.request_id, "request-1") + test.eq(result.payload.session_id, "session-1") + test.eq(result.payload.start_token, "start-1") + test.eq(result.payload.attention_context_enabled, true) + test.eq(result.payload.context.label, "fixture") + end) + end) + test.it("delivers commands after disconnect and reconnect using the same plugin", function() + with_hub("app:relay_runtime_plugin", function(pid, _, join, send, response, _, started) + join() + send({ type = "runtime_echo", data = "before", request_id = "before" }) + local boot = started() + test.eq(receive(response).payload.data, "before") + assert(process.send(pid, consts.WS_TOPICS.LEAVE, { client_pid = process.pid() })) + local welcome = join() + test.eq(welcome.client_count, 1) + test.eq(welcome.plugins[1].status, "running") + send({ type = "runtime_echo", data = "after", request_id = "after", session_id = "reconnected" }) + local result = receive(response) + test.eq(result.plugin_pid, boot.pid) + test.eq(result.payload.data, "after") + test.eq(result.payload.request_id, "after") + test.eq(result.payload.session_id, "reconnected") + end) + end) + test.it("reports a real plugin startup failure and keeps the hub responsive", function() + with_hub("app:missing_runtime_plugin", function(_, _, join, send, _, errors) + join() + send({ type = "runtime_echo", data = "first" }) + test.eq(receive(errors).error, consts.ERROR_CODES.PLUGIN_FAILED) + send({ type = "runtime_echo", data = "second" }) + local error_message = receive(errors) + test.eq(error_message.error, consts.ERROR_CODES.PLUGIN_FAILED) + test.is_true(error_message.message:find("failed permanently", 1, true) ~= nil) + end) + end) + test.it("reports delivery failure when a restored plugin pid has already exited", function() + local events = process.events() + local dead_pid, err = process.spawn_monitored("app:relay_runtime_finished_plugin", "app:processes") + assert(dead_pid, tostring(err)) + while true do + local event = receive(events) + if event.from == dead_pid and event.kind == process.event.EXIT then break end + end + with_hub("app:relay_runtime_plugin", function(_, _, join, send, _, errors) + test.eq(join().plugins[1].status, "running") + send({ type = "runtime_echo", request_id = "stale-plugin", data = "delivery" }) + test.eq(receive(errors).error, consts.ERROR_CODES.PLUGIN_FAILED) + end, dead_pid) + end) + test.it("rejects invalid JSON and missing or unknown commands without stopping", function() + with_hub("app:relay_runtime_plugin", function(pid, _, join, send, _, errors) + join() + assert(process.send(pid, consts.WS_TOPICS.MESSAGE, "{")) + test.eq(receive(errors).error, consts.ERROR_CODES.INVALID_JSON) + send({ data = "missing type" }) + test.eq(receive(errors).error, consts.ERROR_CODES.UNKNOWN_COMMAND) + send({ type = "missing_echo" }) + test.eq(receive(errors).error, consts.ERROR_CODES.PLUGIN_NOT_FOUND) + end) + end) + test.it("rejects non-object JSON and non-string command types without stopping", function() + with_hub("app:relay_runtime_plugin", function(pid, _, join, send, _, errors) + join() + assert(process.send(pid, consts.WS_TOPICS.MESSAGE, "null")) + test.eq(receive(errors).error, consts.ERROR_CODES.UNKNOWN_COMMAND) + for _, value in ipairs({ 42, true, false, "scalar", {}, { type = 42 }, { type = true }, { type = "" } }) do + send(value) + test.eq(receive(errors).error, consts.ERROR_CODES.UNKNOWN_COMMAND) + end + end) + end) + end) +end + +local function benchmark(options) + options = options or {} + local count = tonumber(options.iterations) or 100 + local samples = tonumber(options.samples) or 1 + local warmup = tonumber(options.warmup) or 0 + return with_hub("app:relay_runtime_plugin", function(_, _, join, send, response, _, started) + join() + send({ type = "runtime_echo", request_id = "startup", data = 0 }) + started() + assert(receive(response).payload.request_id == "startup") + local durations = {} + for iteration = 1, warmup + samples do + local began = time.now() + for index = 1, count do + local operation = (iteration - 1) * count + index + send({ type = "runtime_echo", request_id = tostring(operation), session_id = "benchmark", data = operation }) + local value = receive(response) + assert(value.payload.request_id == tostring(operation) and value.payload.data == operation, + "relay roundtrip mismatch") + end + local elapsed = time.now():sub(began):seconds() * 1000 + if iteration > warmup then durations[#durations + 1] = elapsed end + end + return { operations = count, duration_ms = durations[1], samples_ms = durations } + end) +end + +local function benchmark_cases() + test.describe("relay_roundtrip benchmark", function() + local samples = tonumber((env.get("WIPPY_BENCH_SAMPLES"))) + if not samples then + test.it_skip("requires WIPPY_BENCH_SAMPLES", function() end) + return + end + test.it("measures verified runtime roundtrips", function() + local size = tonumber((env.get("WIPPY_BENCH_SIZE"))) or 1 + local warmup = tonumber((env.get("WIPPY_BENCH_WARMUP"))) or 5 + assert(samples > 0 and size > 0 and warmup >= 0, "invalid benchmark configuration") + local result = benchmark({ iterations = size, samples = samples, warmup = warmup }) + test.eq(result.operations, size) + test.eq(#result.samples_ms, samples) + local report, report_err = benchmark_report.report({ + name = "relay_roundtrip", size = size, operations_per_sample = size, samples_ms = result.samples_ms, + }) + test.not_nil(report, tostring(report_err)) + end) + end) +end + +return { run = test.run_cases(define_tests), benchmark = test.run_cases(benchmark_cases), workload = benchmark } diff --git a/src/test/_index.yaml b/src/test/_index.yaml index a784b79a..c9e3f273 100644 --- a/src/test/_index.yaml +++ b/src/test/_index.yaml @@ -14,6 +14,14 @@ entries: component: wippy/terminal version: "*" + - name: env_storage + kind: env.storage.os + + - name: require_cases + kind: env.variable + variable: WIPPY_TEST_REQUIRE_CASES + storage: wippy.test:env_storage + # wippy.test:terminal - name: terminal kind: library.lua @@ -115,6 +123,7 @@ entries: method: main modules: - io + - env - time - channel imports: @@ -135,3 +144,40 @@ entries: method: run imports: test: wippy.test:test + + - name: benchmark_output + kind: fs.directory + directory: /tmp/wippy-benchmarks + auto_init: true + + - name: benchmark_revision + kind: env.variable + variable: WIPPY_BENCH_REVISION + storage: wippy.test:env_storage + + - name: benchmark_runtime + kind: env.variable + variable: WIPPY_BENCH_RUNTIME + storage: wippy.test:env_storage + + - name: benchmark_framework_revision + kind: env.variable + variable: WIPPY_BENCH_FRAMEWORK_REVISION + storage: wippy.test:env_storage + + - name: benchmark + kind: library.lua + source: file://benchmark.lua + modules: [fs, json, env, io] + + - name: benchmark_test + kind: function.lua + meta: + type: test + suite: test-framework + source: file://benchmark_test.lua + method: run + modules: [fs, json] + imports: + benchmark: wippy.test:benchmark + test: wippy.test:test diff --git a/src/test/benchmark.lua b/src/test/benchmark.lua new file mode 100644 index 00000000..9de2134f --- /dev/null +++ b/src/test/benchmark.lua @@ -0,0 +1,113 @@ +local fs = require("fs") +local json = require("json") +local env = require("env") +local io = require("io") + +local function finite(value: any): boolean + return type(value) == "number" and value == value and value < math.huge and value > -math.huge +end + +local function positive_integer(value: any): boolean + return finite(value) and value > 0 and value % 1 == 0 +end + +local function measurements(record: any): (any, string?) + if type(record) ~= "table" or type(record.name) ~= "string" + or not record.name:match("^[%w_-]+$") then + return nil, "benchmark name must contain only letters, digits, underscores or hyphens" + end + if not positive_integer(record.size) or not positive_integer(record.operations_per_sample) then + return nil, "benchmark size and operations_per_sample must be positive integers" + end + if type(record.samples_ms) ~= "table" or #record.samples_ms == 0 then + return nil, "benchmark samples_ms must be a nonempty dense list" + end + local ordered: {number} = {} + local duration = 0 + local count = 0 + for index, value in pairs(record.samples_ms) do + if not positive_integer(index) or index > #record.samples_ms or not finite(value) or value < 0 then + return nil, "benchmark samples must be finite nonnegative numbers in a dense list" + end + count = count + 1 + ordered[index] = value + duration = duration + value + end + if count ~= #record.samples_ms then + return nil, "benchmark samples_ms must be a nonempty dense list" + end + if not finite(duration) or duration <= 0 then + return nil, "benchmark total measured duration must be finite and positive" + end + table.sort(ordered) + local median = ordered[math.ceil(count / 2)] + if median == nil then return nil, "benchmark median sample is missing" end + if count % 2 == 0 then + local upper = ordered[count / 2 + 1] + if upper == nil then return nil, "benchmark median sample is missing" end + median = (median + upper) / 2 + end + local throughput = record.operations_per_sample / duration * count * 1000 + if not finite(throughput) then return nil, "benchmark throughput must be finite" end + return { + median_ms = median, + p95_ms = ordered[math.ceil(count * 0.95)], + throughput_ops_per_second = throughput, + sample_count = count, + } +end + +local function report(record) + local result, err = measurements(record) + if not result then return nil, err end + result.name = record.name + result.size = record.size + result.operations_per_sample = record.operations_per_sample + result.samples_ms = record.samples_ms + result.warmup_count = tonumber((env.get("WIPPY_BENCH_WARMUP"))) or 5 + result.metadata = { + revision = env.get("WIPPY_BENCH_REVISION") or "unknown", + runtime = env.get("WIPPY_BENCH_RUNTIME") or "unknown", + framework_revision = env.get("WIPPY_BENCH_FRAMEWORK_REVISION"), + } + local encoded, encode_err = json.encode(result) + if encode_err then return nil, tostring(encode_err) end + local output, output_err = fs.get("wippy.test:benchmark_output") + if not output then return nil, tostring(output_err) end + local filename = record.name .. "-" .. tostring(record.size) .. ".json" + local written, write_err = output:writefile(filename, encoded) + if not written then return nil, "failed to write benchmark report: " .. tostring(write_err) end + local summary = string.format("%s size=%d samples=%d median=%.3fms p95=%.3fms throughput=%.1f ops/s", + result.name, result.size, result.sample_count, result.median_ms, result.p95_ms, + result.throughput_ops_per_second) + if not io.print(summary) then print(summary) end + return result, filename +end + +local function compare(baseline, current) + local before, before_err = measurements(baseline) + if not before then return nil, before_err end + local after, after_err = measurements(current) + if not after then return nil, after_err end + if type(baseline.metadata) ~= "table" or type(current.metadata) ~= "table" + or type(baseline.metadata.runtime) ~= "string" or baseline.metadata.runtime == "" + or baseline.metadata.runtime ~= current.metadata.runtime then + return nil, "benchmark comparison requires matching runtime metadata" + end + if baseline.name ~= current.name or baseline.size ~= current.size + or baseline.operations_per_sample ~= current.operations_per_sample then + return nil, "benchmark comparison requires the same workload" + end + if before.median_ms <= 0 or before.p95_ms <= 0 then + return nil, "benchmark comparison requires positive baseline latencies" + end + return { + name = baseline.name, + size = baseline.size, + median_percent = (after.median_ms / before.median_ms - 1) * 100, + p95_percent = (after.p95_ms / before.p95_ms - 1) * 100, + throughput_percent = (after.throughput_ops_per_second / before.throughput_ops_per_second - 1) * 100, + } +end + +return {report = report, compare = compare} diff --git a/src/test/benchmark_test.lua b/src/test/benchmark_test.lua new file mode 100644 index 00000000..36b929b2 --- /dev/null +++ b/src/test/benchmark_test.lua @@ -0,0 +1,143 @@ +local test = require("test") +local benchmark = require("benchmark") +local fs = require("fs") +local json = require("json") + +local function define_tests() + test.describe("runtime benchmark reports", function() + test.it("does not report success when the artifact cannot be written", function() + local output = fs.get("wippy.test:benchmark_output") + output:remove("report-write-failure-1.json") + test.is_true((output:mkdir("report-write-failure-1.json"))) + local report, err = benchmark.report({name = "report-write-failure", size = 1, + operations_per_sample = 1, samples_ms = {2}}) + output:remove("report-write-failure-1.json") + test.is_nil(report) + test.not_nil(err) + end) + + test.it("compares equivalent workloads with independently derived percentage deltas", function() + local baseline = {name = "comparison", size = 2, operations_per_sample = 1, + samples_ms = {10, 20}, metadata = {runtime = "fixture"}} + local current = {name = "comparison", size = 2, operations_per_sample = 1, + samples_ms = {20, 40}, metadata = {runtime = "fixture"}} + local deltas, err = benchmark.compare(baseline, current) + test.not_nil(deltas, tostring(err)) + test.eq(deltas.median_percent, 100) + test.eq(deltas.p95_percent, 100) + test.eq(deltas.throughput_percent, -50) + end) + + test.it("rejects comparison across workloads or runtimes and missing records", function() + local baseline = {name = "comparison", size = 2, operations_per_sample = 1, + samples_ms = {10, 20}, metadata = {runtime = "fixture"}} + for _, current in ipairs({{}, + {name = "different", size = 2, operations_per_sample = 1, samples_ms = {1}, metadata = {runtime = "fixture"}}, + {name = "comparison", size = 3, operations_per_sample = 1, samples_ms = {1}, metadata = {runtime = "fixture"}}, + {name = "comparison", size = 2, operations_per_sample = 2, samples_ms = {1}, metadata = {runtime = "fixture"}}, + {name = "comparison", size = 2, operations_per_sample = 1, samples_ms = {1}, metadata = {runtime = "different"}}, + {name = "comparison", size = 2, operations_per_sample = 1, samples_ms = {1}}, + }) do + local deltas, err = benchmark.compare(baseline, current) + test.is_nil(deltas) + test.not_nil(err) + end + end) + + test.it("writes measured samples and independently derived aggregate metrics", function() + local samples = {40, 10, 30, 20} + local report, filename = benchmark.report({name = "report-roundtrip", size = 2, + operations_per_sample = 4, samples_ms = samples}) + test.not_nil(report, tostring(filename)) + test.eq(filename, "report-roundtrip-2.json") + local output = fs.get("wippy.test:benchmark_output") + local stored, err = json.decode((output:readfile(filename))) + test.is_nil(err) + test.eq(stored.name, "report-roundtrip") + test.eq(stored.size, 2) + test.eq(stored.operations_per_sample, 4) + test.eq(stored.sample_count, 4) + test.eq(stored.median_ms, 25) + test.eq(stored.p95_ms, 40) + test.eq(stored.throughput_ops_per_second, 160) + test.eq(stored.samples_ms[1], 40) + test.eq(stored.samples_ms[2], 10) + test.eq(samples[1], 40) + test.eq(samples[2], 10) + test.is_string(stored.metadata.revision) + test.is_string(stored.metadata.runtime) + end) + + test.it("retains zero samples when total measured time is positive", function() + local report, filename = benchmark.report({name = "report-zero-sample", size = 1, + operations_per_sample = 3, samples_ms = {0, 4}}) + test.not_nil(report, tostring(filename)) + test.eq(report.median_ms, 2) + test.eq(report.p95_ms, 4) + test.eq(report.throughput_ops_per_second, 1500) + end) + + test.it("selects the nearest rank p95 without replacing an odd median", function() + local report, filename = benchmark.report({name = "report-odd-sample", size = 1, + operations_per_sample = 1, samples_ms = {5, 1, 3}}) + test.not_nil(report, tostring(filename)) + test.eq(report.median_ms, 3) + test.eq(report.p95_ms, 5) + end) + + test.it("uses distinct files for benchmark names and sizes", function() + local first, first_file = benchmark.report({name = "report-distinct", size = 1, + operations_per_sample = 1, samples_ms = {2}}) + local second, second_file = benchmark.report({name = "report-distinct", size = 2, + operations_per_sample = 1, samples_ms = {8}}) + local third, third_file = benchmark.report({name = "report-other", size = 1, + operations_per_sample = 1, samples_ms = {6}}) + test.not_nil(first) + test.not_nil(second) + test.not_nil(third) + local output = fs.get("wippy.test:benchmark_output") + test.eq(json.decode((output:readfile(first_file))).median_ms, 2) + test.eq(json.decode((output:readfile(second_file))).median_ms, 8) + test.eq(json.decode((output:readfile(third_file))).median_ms, 6) + end) + + test.it("rejects missing measurements and malformed records before writing", function() + for _, record in ipairs({false, {}, + {name = "missing-size", operations_per_sample = 1, samples_ms = {1}}, + {name = "missing-ops", size = 1, samples_ms = {1}}, + {name = "missing-samples", size = 1, operations_per_sample = 1}, + {name = "empty-samples", size = 1, operations_per_sample = 1, samples_ms = {}}, + {name = "zero-duration", size = 1, operations_per_sample = 1, samples_ms = {0}}, + {name = "zero-ops", size = 1, operations_per_sample = 0, samples_ms = {1}}, + {name = "negative-size", size = -1, operations_per_sample = 1, samples_ms = {1}}, + {name = "fractional-size", size = 1.5, operations_per_sample = 1, samples_ms = {1}}, + {name = "../escaped", size = 1, operations_per_sample = 1, samples_ms = {1}}, + }) do + local report, err = benchmark.report(record) + test.is_nil(report) + test.not_nil(err) + end + end) + + test.it("rejects non-finite, negative, non-numeric and sparse samples", function() + local sparse = {1, 2, 3, 4, 5} + sparse[4] = nil + sparse[1000000000] = 6 + for _, samples in ipairs({{math.huge}, {-math.huge}, {0 / 0}, {-1}, {"1"}, + {[2] = 1}, sparse, {[1] = 1, extra = 2}, {1e308, 1e308}}) do + local report, err = benchmark.report({name = "report-invalid", size = 1, + operations_per_sample = 1, samples_ms = samples}) + test.is_nil(report) + test.not_nil(err) + end + local output = fs.get("wippy.test:benchmark_output") + local found = false + for entry in output:readdir("/") do + if entry.name == "report-invalid-1.json" then found = true end + end + test.is_false(found) + end) + end) +end + +return {run = test.run_cases(define_tests)} diff --git a/src/test/runner.lua b/src/test/runner.lua index 4d10f88c..88a03578 100644 --- a/src/test/runner.lua +++ b/src/test/runner.lua @@ -1,5 +1,6 @@ -- Test runner with real-time per-case TUI display local io = require("io") +local env = require("env") local registry = require("registry") local time = require("time") local channel = require("channel") @@ -59,6 +60,7 @@ end -- Main test runner logic local function run_tests(): number local args: {string}? = io.args() + local require_cases = env.get("WIPPY_TEST_REQUIRE_CASES") == "1" display.begin() @@ -71,7 +73,7 @@ local function run_tests(): number if not raw_entries or #raw_entries == 0 then display.info("No tests found") - return 0 + return require_cases and 1 or 0 end local entries: {discovery.TestEntry} = raw_entries @@ -81,7 +83,7 @@ local function run_tests(): number entries = discovery.filter_tests(entries, args) if #entries == 0 then display.info("No tests match filter: " .. table.concat(args, ", ")) - return 0 + return require_cases and 1 or 0 end display.info("Filter: " .. table.concat(args, ", ")) end @@ -265,6 +267,7 @@ local function run_tests(): number ("test process termination failed: " .. tostring(terminate_err)) or cancel_error or protocol_error local problem = verdict.check({ + require_cases = require_cases, entry_id = entry_id, declared = declared_cases[entry_id], observed = case_stats[entry_id] and ((case_stats[entry_id].passed or 0) + (case_stats[entry_id].failed or 0) + (case_stats[entry_id].skipped or 0)) or 0, @@ -287,6 +290,7 @@ local function run_tests(): number local value: any = control.value local result_err: any = control.result_error local problem = verdict.check({ + require_cases = require_cases, entry_id = entry_id, declared = declared_cases[entry_id], observed = case_count, diff --git a/src/test/runner_verdict.lua b/src/test/runner_verdict.lua index 9d476c52..30f3a033 100644 --- a/src/test/runner_verdict.lua +++ b/src/test/runner_verdict.lua @@ -1,6 +1,7 @@ -- Validate the protocol independently of the case events already rendered by the runner. type RunState = { entry_id: string?, + require_cases: boolean?, declared: number?, observed: number, cases: {passed: number, failed: number, skipped: number}?, @@ -74,6 +75,10 @@ local function check(state: RunState): RunnerError? return fail("test completion counts disagree with declared cases") end end + if state.require_cases and (state.declared == nil or not state.completed + or state.completed.passed + state.completed.failed == 0) then + return fail("required test suite executed no cases") + end return nil end diff --git a/test-harness/runner-results/_index.yaml b/test-harness/runner-results/_index.yaml index 94359b97..6679046b 100644 --- a/test-harness/runner-results/_index.yaml +++ b/test-harness/runner-results/_index.yaml @@ -64,3 +64,10 @@ entries: method: skipped_case imports: test: wippy.test:test + - name: empty_plan + kind: function.lua + meta: {type: test, suite: runner-results} + source: file://cases.lua + method: empty_plan + imports: + test: wippy.test:test diff --git a/test-harness/runner-results/cases.lua b/test-harness/runner-results/cases.lua index 85796940..01fbeb4f 100644 --- a/test-harness/runner-results/cases.lua +++ b/test-harness/runner-results/cases.lua @@ -1,6 +1,7 @@ local test = require("test") return { + empty_plan = test.run_cases(function() end), failing_case = test.run_cases(function() test.describe("failed case", function() test.it("keeps its failure", function() diff --git a/test-harness/runner-results/verify.sh b/test-harness/runner-results/verify.sh index 324abadb..200afbfc 100644 --- a/test-harness/runner-results/verify.sh +++ b/test-harness/runner-results/verify.sh @@ -32,3 +32,20 @@ for scenario in returns_true returns_nil skipped_case; do fi done printf 'runner result and protocol regressions passed\n' + +for scenario in empty_plan skipped_case returns_nil no-such-entry; do + set +e + output=$(WIPPY_TEST_REQUIRE_CASES=1 "$wippy_bin" test --config .wippy.yaml test -- "app:$scenario" 2>&1) + status=$? + set -e + if [ "$status" -ne 1 ]; then + printf 'required runner accepted %s (exit %s):\n%s\n' "$scenario" "$status" "$output" + exit 1 + fi +done +if ! output=$("$wippy_bin" test --config .wippy.yaml test -- app:empty_plan 2>&1) || + ! grep -Fq 'PASSED' <<< "$output"; then + printf 'runner rejected a generic empty plan:\n%s\n' "$output" + exit 1 +fi +printf 'required case regressions passed\n' diff --git a/test-harness/runner/_index.yaml b/test-harness/runner/_index.yaml index ee88426f..884ca9bd 100644 --- a/test-harness/runner/_index.yaml +++ b/test-harness/runner/_index.yaml @@ -17,3 +17,30 @@ entries: imports: test: wippy.test:test verdict: wippy.test:runner_verdict + + - name: build_executor + kind: exec.native + + - name: aggregate_fixtures + kind: fs.directory + directory: /tmp/wippy-aggregate-checks + auto_init: true + + - name: env_storage + kind: env.storage.os + + - name: framework_root + kind: env.variable + variable: WIPPY_FRAMEWORK_ROOT + storage: app:env_storage + + - name: aggregate_checks_test + kind: function.lua + meta: + type: test + suite: build-checks + source: file://aggregate_checks_test.lua + method: run + modules: [exec, fs, env, time] + imports: + test: wippy.test:test diff --git a/test-harness/runner/aggregate_checks_test.lua b/test-harness/runner/aggregate_checks_test.lua new file mode 100644 index 00000000..a81a7df7 --- /dev/null +++ b/test-harness/runner/aggregate_checks_test.lua @@ -0,0 +1,84 @@ +local test = require("test") +local exec = require("exec") +local fs = require("fs") +local env = require("env") +local time = require("time") + +local function read_all(stream) + local chunks = {} + while true do + local data = stream:read() + if not data then break end + chunks[#chunks + 1] = data + end + stream:close() + return table.concat(chunks) +end + +local function run_check(target, message, exit_code) + local root = env.get("WIPPY_FRAMEWORK_ROOT") + test.not_nil(root, "WIPPY_FRAMEWORK_ROOT must select the real Framework checkout") + local files = fs.get("app:aggregate_fixtures") + local directory = "check-" .. tostring(time.now():unix_nano()) + for _, path in ipairs({directory, directory .. "/src", directory .. "/src/fixture", + directory .. "/src/fixture/test"}) do + local created, err = files:mkdir(path) + test.is_true(created, tostring(err)) + end + local module = directory .. "/src/fixture/test/Makefile" + local written, write_err = files:writefile(module, + "test lint:\n\t@printf '%s\\n' '" .. message .. "'; exit " .. tostring(exit_code) .. "\n") + test.is_true(written, tostring(write_err)) + local executor, executor_err = exec.get("app:build_executor") + test.not_nil(executor, tostring(executor_err)) + local command = "make --no-print-directory -f " .. string.format("%q", root .. "/Makefile") + .. " -C " .. string.format("%q", "/tmp/wippy-aggregate-checks/" .. directory) + .. " " .. target .. " TEST_MODULES=fixture BENCH_REVISION=fixture" + local proc, create_err = executor:exec(command, {env = {MAKEFLAGS = ""}}) + test.not_nil(proc, tostring(create_err)) + local stdout, stdout_err = proc:stdout_stream() + local stderr, stderr_err = proc:stderr_stream() + test.not_nil(stdout, tostring(stdout_err)) + test.not_nil(stderr, tostring(stderr_err)) + local started, start_err = proc:start() + test.is_true(started, tostring(start_err)) + local output = read_all(stdout) + local errors = read_all(stderr) + local status, wait_err = proc:wait() + test.is_nil(wait_err) + executor:release() + files:remove(module) + files:remove(directory .. "/src/fixture/test") + files:remove(directory .. "/src/fixture") + files:remove(directory .. "/src") + files:remove(directory) + return status, output .. errors +end + +local function define_tests() + test.describe("aggregate build checks", function() + test.it("fails aggregate tests when a failed module prints PASSED", function() + local status, output = run_check("run-tests", "PASSED", 1) + test.is_true(status ~= 0) + test.contains(output, "FAILED") + end) + + test.it("passes aggregate tests without requiring a PASSED marker", function() + local status, output = run_check("run-tests", "completed", 0) + test.eq(status, 0, output) + end) + + test.it("fails aggregate lint without requiring an errors marker", function() + local status, output = run_check("run-lint", "Checked fixture", 1) + test.is_true(status ~= 0) + test.contains(output, "FAILED") + end) + + test.it("passes aggregate lint when successful output mentions errors", function() + local status, output = run_check("run-lint", "Checked fixture: 0 errors", 0) + test.eq(status, 0, output) + end) + end) +end + +return {run = test.run_cases(define_tests)} diff --git a/test-harness/runner/runner_verdict_test.lua b/test-harness/runner/runner_verdict_test.lua index fe1a867a..7dcd1b30 100644 --- a/test-harness/runner/runner_verdict_test.lua +++ b/test-harness/runner/runner_verdict_test.lua @@ -3,6 +3,32 @@ local verdict = require("verdict") local function define_tests() test.describe("runner result validation", function() + test.it("required suites reject a completed empty plan", function() + local err = verdict.check({require_cases = true, declared = 0, observed = 0, + cases = {passed = 0, failed = 0, skipped = 0}, + completed = {total = 0, passed = 0, failed = 0, skipped = 0}}) + test.not_nil(err) + test.contains(err.message, "required test suite executed no cases") + end) + + test.it("required suites reject skipped-only plans", function() + local err = verdict.check({require_cases = true, declared = 1, observed = 1, + cases = {passed = 0, failed = 0, skipped = 1}, + completed = {total = 1, passed = 0, failed = 0, skipped = 1}}) + test.not_nil(err) + test.contains(err.message, "required test suite executed no cases") + end) + + test.it("required suites accept an executed BDD case", function() + test.is_nil(verdict.check({require_cases = true, declared = 1, observed = 1, + cases = {passed = 1, failed = 0, skipped = 0}, + completed = {total = 1, passed = 1, failed = 0, skipped = 0}})) + end) + + test.it("required suites reject entries without a case plan", function() + test.not_nil(verdict.check({require_cases = true, observed = 0})) + end) + test.it("rejects completion without a plan", function() local err = verdict.check({observed = 0, completed = {total = 1, passed = 0, failed = 1, skipped = 0}}) From bf747462a365e111dad9291974d3db76a84bd845 Mon Sep 17 00:00:00 2001 From: Rodrigo Delduca Date: Wed, 7 Oct 2026 12:44:22 -0300 Subject: [PATCH 2/8] ci: keep performance measurements local --- .github/workflows/ci.yml | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8a63e814..12978b05 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,7 +18,7 @@ permissions: jobs: runtime-contracts: - name: Lua runtime contracts and benchmarks + name: Lua runtime contracts runs-on: ubuntu-latest timeout-minutes: 15 steps: @@ -32,19 +32,6 @@ jobs: WIPPY: ${{ steps.wippy.outputs.path }} RUNTIME_WORK_DIR: ${{ runner.temp }}/framework-runtime run: make test-runtime - - name: Measure Lua operations - env: - WIPPY: ${{ steps.wippy.outputs.path }} - RUNTIME_WORK_DIR: ${{ runner.temp }}/framework-runtime - BENCH_OUTPUT_DIR: ${{ runner.temp }}/framework-benchmarks - run: make bench - - name: Save benchmark reports - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 - with: - name: framework-benchmarks - path: ${{ runner.temp }}/framework-benchmarks/*.json - if-no-files-found: error - test-runner: name: Test runner regressions runs-on: ubuntu-latest From 6984d7d2778b8efa412d052ab402d3196a4671c4 Mon Sep 17 00:00:00 2001 From: Rodrigo Delduca Date: Wed, 7 Oct 2026 16:25:49 -0300 Subject: [PATCH 3/8] fix: close runtime correctness gaps in actor, relay, tools, and test runner - actor: update select cases in place, so channels registered after a handler catches an error are still selected - agent: honor runtime_options.stop_check before provider generation - tools: clear wrapper diagnostics and last tool calls on empty rounds - relay: count a repeated JOIN once, so the last LEAVE stops the session plugin - test runner: fail an error result hidden by a valid plan and reject a replaced plan - relay benchmark: bound pipeline identity slots by the window - README: state that wippy/session does not consume behavior proposals Adds runtime regression tests for each fix and edge-case coverage for buffered actor channels, relay recovery after rejected input, and paired tool controls. --- Makefile | 28 ++- README.md | 11 +- src/actor/src/actor.lua | 22 +- src/actor/test/_index.yaml | 65 ++++++ src/actor/test/actor_resident_test.lua | 146 +++++++++++++ src/actor/test/actor_runtime_fixture.lua | 66 +++++- src/actor/test/actor_runtime_test.lua | 193 +++++++++++++++-- src/agent/src/README.md | 15 +- src/agent/src/agent.lua | 4 + src/agent/src/tools/caller.lua | 9 + src/agent/test/_index.yaml | 5 + src/agent/test/agent_test.lua | 28 +++ src/agent/test/tools/control_fixture.lua | 4 +- src/agent/test/tools/controls_benchmark.lua | 31 ++- .../test/tools/controls_runtime_test.lua | 53 +++++ src/relay/src/user.lua | 42 ++-- src/relay/test/_index.yaml | 23 ++ src/relay/test/relay_runtime_test.lua | 179 +++++++++++++-- src/test/_index.yaml | 9 +- src/test/benchmark.lua | 203 +++++++++++++++++- src/test/benchmark_test.lua | 194 +++++++++++++++++ src/test/runner.lua | 16 +- src/test/runner_verdict.lua | 3 + test-harness/runner-results/_index.yaml | 10 + test-harness/runner-results/results.lua | 11 + test-harness/runner-results/verify.sh | 2 + test-harness/runner/runner_verdict_test.lua | 7 + 27 files changed, 1276 insertions(+), 103 deletions(-) create mode 100644 src/actor/test/actor_resident_test.lua diff --git a/Makefile b/Makefile index 84cd3fa1..165aadb9 100644 --- a/Makefile +++ b/Makefile @@ -3,10 +3,13 @@ TEST_MODULES = actor agent bootloader embeddings facade llm migration relay usage views WIPPY ?= wippy BENCH_OUTPUT_DIR ?= /tmp/wippy-framework-benchmarks -BENCH_WARMUP ?= 5 -BENCH_SAMPLES ?= 30 +BENCH_WARMUP ?= 100 +BENCH_SAMPLES ?= 100 BENCH_SIZES ?= 1 8 32 +BENCH_MEMORY_OPERATIONS ?= 10000 +BENCH_EXACT_ALLOCATIONS ?= 1 BENCH_REVISION ?= $(shell git rev-parse HEAD) +BENCH_PROFILER_FLAG = $(if $(filter 1,$(BENCH_EXACT_ALLOCATIONS)),--profiler) RUN_WIPPY = WIPPY="$(WIPPY)" bash scripts/run-wippy.sh .PHONY: help check-manifests run-tests run-lint install test-runtime bench @@ -24,6 +27,7 @@ help: test-runtime: WIPPY_TEST_REQUIRE_CASES=1 $(RUN_WIPPY) src/actor/test test test -- actor_runtime_test + WIPPY_TEST_REQUIRE_CASES=1 $(RUN_WIPPY) src/actor/test test test -- actor_runtime_resident_test WIPPY_TEST_REQUIRE_CASES=1 $(RUN_WIPPY) src/agent/test test \ -o wippy.llm:process_host:default=wippy.terminal:host \ -o wippy.llm:env_storage:default=app:env_storage test -- tools_controls_runtime_test @@ -40,25 +44,35 @@ bench: repo=$$(pwd -P); \ case "$$output/" in "$$repo/"*) echo 'Benchmark output must be outside the repository' >&2; exit 1;; esac; \ export WIPPY_BENCH_WARMUP="$(BENCH_WARMUP)" WIPPY_BENCH_SAMPLES="$(BENCH_SAMPLES)"; \ + export WIPPY_BENCH_MEMORY_OPERATIONS="$(BENCH_MEMORY_OPERATIONS)"; \ + export WIPPY_BENCH_EXACT_ALLOCATIONS="$(BENCH_EXACT_ALLOCATIONS)"; \ export WIPPY_TEST_REQUIRE_CASES=1; \ export WIPPY_BENCH_REVISION="$(BENCH_REVISION)"; \ WIPPY_BENCH_RUNTIME=$$("$(WIPPY)" version); export WIPPY_BENCH_RUNTIME; \ for size in $(BENCH_SIZES); do \ export WIPPY_BENCH_SIZE="$$size"; \ - for name in actor_roundtrip tools_sequential tools_parallel relay_roundtrip; do \ + for name in actor_roundtrip actor_lifecycle actor_resident tools_sequential tools_parallel relay_roundtrip relay_pipeline; do \ rm -f "$$output/$$name-$$size.json"; \ done; \ - $(RUN_WIPPY) src/actor/test test -c \ + $(RUN_WIPPY) src/actor/test test -c $(BENCH_PROFILER_FLAG) \ -o "wippy.test:benchmark_output:directory=$$output" test -- actor_runtime_benchmark; \ - $(RUN_WIPPY) src/agent/test test -c \ + $(RUN_WIPPY) src/actor/test test -c $(BENCH_PROFILER_FLAG) \ + -o "wippy.test:benchmark_output:directory=$$output" test -- actor_lifecycle_benchmark; \ + $(RUN_WIPPY) src/actor/test test -c $(BENCH_PROFILER_FLAG) \ + -o "wippy.test:benchmark_output:directory=$$output" test -- actor_resident_benchmark; \ + $(RUN_WIPPY) src/agent/test test -c $(BENCH_PROFILER_FLAG) \ -o "wippy.test:benchmark_output:directory=$$output" \ -o wippy.llm:process_host:default=wippy.terminal:host \ -o wippy.llm:env_storage:default=app:env_storage test -- tools_controls_benchmark; \ - $(RUN_WIPPY) src/relay/test test -c \ + $(RUN_WIPPY) src/relay/test test -c $(BENCH_PROFILER_FLAG) \ -o "wippy.test:benchmark_output:directory=$$output" \ -o wippy.relay:application_host:default=app:processes \ -o wippy.relay:user_security_scope:default=app:user test -- relay_runtime_benchmark; \ - for name in actor_roundtrip tools_sequential tools_parallel relay_roundtrip; do \ + $(RUN_WIPPY) src/relay/test test -c $(BENCH_PROFILER_FLAG) \ + -o "wippy.test:benchmark_output:directory=$$output" \ + -o wippy.relay:application_host:default=app:processes \ + -o wippy.relay:user_security_scope:default=app:user test -- relay_pipeline_benchmark; \ + for name in actor_roundtrip actor_lifecycle actor_resident tools_sequential tools_parallel relay_roundtrip relay_pipeline; do \ test -s "$$output/$$name-$$size.json"; \ done; \ done diff --git a/README.md b/README.md index 8d596672..d512d951 100644 --- a/README.md +++ b/README.md @@ -84,14 +84,17 @@ and configuration management — enabling developers to build sophisticated mult `make test-runtime` runs Lua tests for actor messages, relay delivery, and tool controls through Wippy. It uses the local component and test-runner sources. Required suites fail if they execute no cases. -`make bench` runs Lua workloads with five warmups and thirty samples at sizes 1, 8, and 32. -Reports contain measured samples, median and p95 latency, throughput, and source/runtime metadata. -Timing covers the operation inside Wippy, excluding CLI startup. Compare runs on the same system -without other test workloads; investigate repeatable slowdowns before delivery. +`make bench` runs local Lua workloads with 100 warmups and 100 samples at sizes 1, 8, and 32. +Reports include latency, throughput, heap use, exact allocation counts, and source/runtime metadata. +Timing covers the operation inside Wippy, excluding CLI startup. A separate memory pass uses +10,000 operations and the native profiler. Reports retain the cost of the measurement calls. +Compare runs on the same idle system, one workload at a time. CI runs correctness checks only. Set `WIPPY` to select the executable, `RUNTIME_WORK_DIR` to reuse an external dependency directory, and `BENCH_OUTPUT_DIR` to select the report directory. Reports default to `/tmp/wippy-framework-benchmarks`. Use `BENCH_WARMUP`, `BENCH_SAMPLES`, and the space-separated `BENCH_SIZES` to change the workload. +Use `BENCH_MEMORY_OPERATIONS` to set the separate memory workload size. The native profiler uses +the loopback interface on port 6060; keep that port free during local benchmarks. Test data and benchmark reports must stay outside the repository. [module-actor]: https://github.com/wippyai/module-actor diff --git a/src/actor/src/actor.lua b/src/actor/src/actor.lua index ad670028..b8cf23db 100644 --- a/src/actor/src/actor.lua +++ b/src/actor/src/actor.lua @@ -66,13 +66,7 @@ local function get_process(): any return actor._process end - return { - inbox = function() return process.inbox() end, - events = function() return process.events() end, - send = function(dest, topic, payload) return process.send(dest, topic, payload) end, - pid = function() return process.pid() end, - event = process.event - } + return process end function actor.new(initial_state: any, handlers: any): any @@ -108,15 +102,15 @@ function actor.new(initial_state: any, handlers: any): any internal_channel:case_receive() } + -- Update in place: go-lua detaches this upvalue from run_loop after a caught error. local function rebuild_select_cases() - select_cases = { - inbox:case_receive(), - events:case_receive(), - internal_channel:case_receive() - } - + local count = 3 for _, channel_info in pairs(registered_channels) do - table.insert(select_cases, channel_info.chan:case_receive()) + count = count + 1 + select_cases[count] = channel_info.chan:case_receive() + end + for index = #select_cases, count + 1, -1 do + select_cases[index] = nil end end diff --git a/src/actor/test/_index.yaml b/src/actor/test/_index.yaml index fefcbd4c..52a100b2 100644 --- a/src/actor/test/_index.yaml +++ b/src/actor/test/_index.yaml @@ -55,9 +55,11 @@ entries: timeout: 20s source: file://actor_runtime_test.lua imports: + actor: wippy.actor:actor test: wippy.test:test benchmark: wippy.test:benchmark modules: + - system - time - env method: run @@ -69,6 +71,64 @@ entries: suite: actor-benchmark timeout: 120s source: file://actor_runtime_test.lua + imports: + actor: wippy.actor:actor + test: wippy.test:test + benchmark: wippy.test:benchmark + modules: + - system + - time + - env + method: benchmark + + - name: actor_lifecycle_benchmark + kind: function.lua + meta: + type: test + suite: actor-benchmark + timeout: 120s + source: file://actor_runtime_test.lua + imports: + actor: wippy.actor:actor + test: wippy.test:test + benchmark: wippy.test:benchmark + modules: + - system + - time + - env + method: lifecycle + + - name: actor_resident_fixture + kind: process.lua + source: file://actor_runtime_fixture.lua + imports: + actor: wippy.actor:actor + modules: + - time + method: resident + + - name: actor_runtime_resident_test + kind: function.lua + meta: + type: test + suite: actor-runtime + timeout: 25s + source: file://actor_resident_test.lua + imports: + test: wippy.test:test + benchmark: wippy.test:benchmark + modules: + - time + - env + method: run + + - name: actor_resident_benchmark + kind: function.lua + meta: + type: test + suite: actor-benchmark + timeout: 120s + source: file://actor_resident_test.lua imports: test: wippy.test:test benchmark: wippy.test:benchmark @@ -94,3 +154,8 @@ entries: kind: env.variable variable: WIPPY_BENCH_SIZE storage: app:runtime_env_storage + + - name: runtime_bench_memory_operations + kind: env.variable + variable: WIPPY_BENCH_MEMORY_OPERATIONS + storage: app:runtime_env_storage diff --git a/src/actor/test/actor_resident_test.lua b/src/actor/test/actor_resident_test.lua new file mode 100644 index 00000000..a4746e54 --- /dev/null +++ b/src/actor/test/actor_resident_test.lua @@ -0,0 +1,146 @@ +local test = require("test") +local time = require("time") +local env = require("env") +local benchmark = require("benchmark") + +local sequence = 0 + +local function with_residents(count: number, body: any): any + sequence = sequence + 1 + local topic = "actor.resident." .. sequence + local responses = process.listen(topic) + local events = process.events() + local active = {} + local pids = {} + local batch = 0 + local pending_deadline + local function dispatch() + batch = batch + 1 + for index, pid in ipairs(pids) do + assert(process.send(pid, "dispatch", { index = index, batch = batch })) + end + local seen = {} + local deadline = time.timer(10 * time.SECOND) + pending_deadline = deadline + for _ = 1, count do + local selected = channel.select({ responses:case_receive(), deadline:channel():case_receive() }) + assert(selected.channel == responses and selected.ok, "resident actor response deadline") + local value = selected.value + assert(active[value.pid] and not seen[value.pid], "resident actor response identity mismatch") + assert(pids[value.value.index] == value.pid and value.value.batch == batch and value.count == batch, + "resident actor dispatch mismatch") + seen[value.pid] = true + end + deadline:stop() + pending_deadline = nil + end + local ok: boolean, result: any = pcall(function() + for index = 1, count do + local pid, err = process.spawn_monitored("app:actor_resident_fixture", "app:runtime_processes", { + parent = process.pid(), topic = topic, + }) + assert(pid, tostring(err)) + active[pid] = true + pids[index] = pid + end + local ready = {} + local deadline = time.timer(10 * time.SECOND) + pending_deadline = deadline + for _ = 1, count do + local selected = channel.select({ responses:case_receive(), deadline:channel():case_receive() }) + assert(selected.channel == responses and selected.ok, "resident actor startup deadline") + local value = selected.value + assert(value.phase == "ready" and active[value.pid] and not ready[value.pid], + "resident actor startup identity mismatch") + ready[value.pid] = true + end + deadline:stop() + pending_deadline = nil + return body(dispatch) + end) + if pending_deadline then pending_deadline:stop() end + for pid in pairs(active) do process.cancel(pid, "2s") end + local deadline = time.timer(10 * time.SECOND) + while next(active) do + local selected = channel.select({ events:case_receive(), deadline:channel():case_receive() }) + if selected.channel ~= events then break end + local event = selected.value + if active[event.from] and event.kind == process.event.EXIT then + active[event.from] = nil + process.unmonitor(event.from) + if event.result.error then ok, result = false, tostring(event.result.error) end + end + end + deadline:stop() + local cleanup_expired = next(active) ~= nil + if cleanup_expired then + for pid in pairs(active) do process.terminate(pid) end + local forced = time.timer(3 * time.SECOND) + while next(active) do + local selected = channel.select({ events:case_receive(), forced:channel():case_receive() }) + if selected.channel ~= events then break end + local event = selected.value + if active[event.from] and event.kind == process.event.EXIT then + active[event.from] = nil + process.unmonitor(event.from) + end + end + forced:stop() + end + process.unlisten(responses) + assert(not next(active), tostring(result or "") .. "; resident actor termination deadline") + if not ok then error(result) end + assert(not cleanup_expired, "resident actor cleanup deadline") + return result +end + +local function define_tests() + test.describe("resident actor runtime", function() + test.it("starts, dispatches to, and cancels a real actor cohort", function() + with_residents(4, function(dispatch) + dispatch() + dispatch() + end) + end) + end) +end + +local function benchmark_cases() + test.describe("actor_resident benchmark", function() + local samples = tonumber((env.get("WIPPY_BENCH_SAMPLES"))) + if not samples then + test.it_skip("requires WIPPY_BENCH_SAMPLES", function() end) + return + end + test.it("measures dispatch and live memory with actors resident", function() + local size = tonumber((env.get("WIPPY_BENCH_SIZE"))) or 256 + local warmup = tonumber((env.get("WIPPY_BENCH_WARMUP"))) or 5 + local memory_operations = tonumber((env.get("WIPPY_BENCH_MEMORY_OPERATIONS"))) + assert(size > 0 and samples > 0 and warmup >= 0, "invalid resident benchmark configuration") + local result = with_residents(size, function(dispatch) + local durations = {} + for iteration = 1, warmup + samples do + local began = time.now() + dispatch() + local elapsed = time.now():sub(began):seconds() * 1000 + if iteration > warmup then durations[#durations + 1] = elapsed end + end + local memory + if memory_operations then + local batches = math.ceil(memory_operations / size) + local memory_err + memory, memory_err = benchmark.measure_memory(function() + for _ = 1, batches do dispatch() end + end, batches * size) + assert(memory, tostring(memory_err)) + end + return { samples_ms = durations, memory = memory } + end) + local report, err = benchmark.report({ name = "actor_resident", size = size, + operations_per_sample = size, samples_ms = result.samples_ms, memory = result.memory }) + test.not_nil(report, tostring(err)) + end) + end) +end + +return { run = test.run_cases(define_tests), benchmark = test.run_cases(benchmark_cases) } diff --git a/src/actor/test/actor_runtime_fixture.lua b/src/actor/test/actor_runtime_fixture.lua index 86a9a7a3..3c6e5cf1 100644 --- a/src/actor/test/actor_runtime_fixture.lua +++ b/src/actor/test/actor_runtime_fixture.lua @@ -31,8 +31,43 @@ local function run(args) assert(not proceed_err, proceed_err) local late, late_err = s.wait(reply_topic, 2 * time.SECOND) return actor.exit({ status = "late", value = late, error = late_err }) + elseif args.scenario == "drain_exit" or args.scenario == "drain_next" then + local buffered = channel.new(4) + s.values = {} + s.register_channel(buffered, function(current, value, ok) + if ok then + current.values[#current.values + 1] = value + return + end + current.closures = current.closures + 1 + local outcome = { status = "drained", values = current.values, closures = current.closures } + if args.scenario == "drain_next" then return actor.next("next_result", outcome) end + return actor.exit(outcome) + end) + buffered:send(1) + buffered:send(false) + buffered:send("x") + buffered:close() elseif args.scenario == "dispatch" then notify({ phase = "ready" }) + elseif args.scenario == "init_next" then + return actor.next("next_result", { scenario = args.scenario }) + elseif args.scenario == "async_next" then + s.async(function() return actor.next("next_result", { scenario = args.scenario }) end) + elseif args.scenario == "deferred_async" then + coroutine.spawn(function() + assert(proceed:receive(), "deferred async proceed missing") + s.async(function() return actor.next("next_result", { scenario = args.scenario }) end) + end) + notify({ phase = "ready" }) + elseif args.scenario == "channel_next" then + s.register_channel(closed, function(_, value, ok) + assert(ok, "runtime channel closed before next") + return actor.next("next_result", value) + end) + closed:send({ scenario = args.scenario }) + elseif args.scenario == "event_next" then + notify({ phase = "ready" }) elseif args.scenario == "cancel" then notify({ phase = "waiting" }) s.wait(reply_topic, 20 * time.MILLISECOND) @@ -55,7 +90,23 @@ local function run(args) s.count = s.count + 1 assert(process.send(args.parent, args.topic, { count = s.count, value = payload })) end, + register_after_error = function(s) + assert(not pcall(error, "expected handler failure")) + s.register_channel(closed, function(_, value, ok) + return actor.exit({ status = "channel_after_error", value = value, ok = ok }) + end) + closed:send("after caught error") + end, + next_result = function(_, payload, topic, from) + return actor.exit({ status = "next", payload = payload, topic = topic, from = from }) + end, + __on_event = function(_, event) + if args.scenario == "event_next" and event.kind == process.event.CANCEL then + return actor.next("next_result", { scenario = args.scenario }) + end + end, __on_cancel = function() + if args.scenario == "event_next" then return end return actor.exit({ status = "canceled" }) end, }).run() @@ -64,4 +115,17 @@ local function run(args) return result end -return { run = run } +local function resident(args) + return actor.new({ count = 0 }, { + __init = function() + assert(process.send(args.parent, args.topic, { phase = "ready", pid = process.pid() })) + end, + dispatch = function(state, value) + state.count = state.count + 1 + assert(process.send(args.parent, args.topic, { pid = process.pid(), count = state.count, value = value })) + end, + __on_cancel = function() return actor.exit({ status = "canceled" }) end, + }).run() +end + +return { run = run, resident = resident } diff --git a/src/actor/test/actor_runtime_test.lua b/src/actor/test/actor_runtime_test.lua index 80a04021..b461d612 100644 --- a/src/actor/test/actor_runtime_test.lua +++ b/src/actor/test/actor_runtime_test.lua @@ -2,6 +2,8 @@ local test = require("test") local time = require("time") local env = require("env") local benchmark_report = require("benchmark") +local actor = require("actor") +local system = require("system") local sequence = 0 @@ -54,8 +56,63 @@ local function with_fixture(scenario, count, body): any return result end +local function wait_idle(pid) + local deadline = time.timer(3 * time.SECOND) + local cases = { deadline:channel():case_receive() } + while true do + local processes = assert(system.hosts.processes("app:runtime_processes")) + for _, child in ipairs(processes) do + if child.pid == pid and child.state == "idle" then + deadline:stop() + return + end + end + if channel.select(cases, true).channel == deadline:channel() then + deadline:stop() + error("actor idle state deadline") + end + coroutine.yield() + end +end + local function define_tests() test.describe("actor runtime", function() + test.it("initializes and completes repeated actors with the real process interface", function() + for index = 1, 8 do + local result = actor.new({ index = index }, { + __init = function(state) return actor.exit(state.index) end, + }).run() + test.eq(result, index) + end + end) + local next_sources = { init_next = "init", async_next = "async", + channel_next = "channel_handler", event_next = "event_handler" } + for _, scenario in ipairs({ "init_next", "async_next", "channel_next", "event_next" }) do + test.it("delivers internal next messages from " .. scenario, function() + with_fixture(scenario, 1, function(pid, _, responses, outcome) + if scenario == "event_next" then + test.eq(receive(responses).phase, "ready") + assert(process.cancel(pid, "2s")) + end + local result = outcome() + test.eq(result.status, "next") + test.eq(result.payload.scenario, scenario) + test.eq(result.topic, "next_result") + test.eq(result.from, next_sources[scenario]) + end) + end) + end + test.it("wakes an idle actor when an external coroutine invokes state.async", function() + with_fixture("deferred_async", 1, function(pid, topic, responses, outcome) + test.eq(receive(responses).phase, "ready") + wait_idle(pid) + assert(process.send(pid, topic .. ".proceed", true)) + local result = outcome() + test.eq(result.status, "next") + test.eq(result.payload.scenario, "deferred_async") + test.eq(result.from, "async") + end) + end) test.it("receives a real topic reply before the wait deadline", function() with_fixture("reply", 1, function(pid, topic, responses, outcome) test.eq(receive(responses).phase, "waiting") @@ -104,6 +161,38 @@ local function define_tests() test.eq(outcome().status, "canceled") end) end) + test.it("selects a channel registered after a handler catches an error", function() + with_fixture("dispatch", 0, function(pid, _, responses, outcome) + test.eq(receive(responses).phase, "ready") + assert(process.send(pid, "register_after_error", true)) + local result = outcome() + test.eq(result.status, "channel_after_error") + test.eq(result.value, "after caught error") + test.is_true(result.ok) + end) + end) + test.it("drains buffered channel values including false before one closure exit", function() + with_fixture("drain_exit", 0, function(_, _, _, outcome) + local result = outcome() + test.eq(result.status, "drained") + test.eq(result.closures, 1) + test.eq(#result.values, 3) + test.eq(result.values[1], 1) + test.eq(result.values[2], false) + test.eq(result.values[3], "x") + end) + end) + test.it("delivers an internal next returned by a closed channel callback", function() + with_fixture("drain_next", 0, function(_, _, _, outcome) + local result = outcome() + test.eq(result.status, "next") + test.eq(result.topic, "next_result") + test.eq(result.from, "channel_handler") + test.eq(result.payload.closures, 1) + test.eq(#result.payload.values, 3) + test.eq(result.payload.values[2], false) + end) + end) test.it("unregisters a closed channel and continues handling inbox messages", function() with_fixture("closure", 1, function(pid, _, responses, outcome) local closed = receive(responses) @@ -123,22 +212,69 @@ local function define_tests() end local function benchmark(options) - local count = tonumber(options and options.iterations) or 100 + options = options or {} + local count = tonumber(options.iterations) or 100 + local samples = tonumber(options.samples) or 1 + local warmup = tonumber(options.warmup) or 0 return with_fixture("dispatch", 0, function(pid, _, responses, outcome) assert(receive(responses).phase == "ready") - local started = time.now() - for index = 1, count do - assert(process.send(pid, "dispatch", index)) - local value = receive(responses) - assert(value.count == index and value.value == index, "actor dispatch mismatch") + local operation = 0 + local function dispatch(iterations) + for _ = 1, iterations do + operation = operation + 1 + assert(process.send(pid, "dispatch", operation)) + local value = receive(responses) + assert(value.count == operation and value.value == operation, "actor dispatch mismatch") + end + end + local durations = {} + for iteration = 1, warmup + samples do + local started = time.now() + dispatch(count) + local duration_ms = time.now():sub(started):seconds() * 1000 + if iteration > warmup then durations[#durations + 1] = duration_ms end + end + local memory + if options.memory_operations then + local memory_err + memory, memory_err = benchmark_report.measure_memory(function() dispatch(options.memory_operations) end, + options.memory_operations) + assert(memory, tostring(memory_err)) end - local duration_ms = time.now():sub(started):seconds() * 1000 assert(process.cancel(pid, "2s")) assert(outcome().status == "canceled") - return { operations = count, duration_ms = duration_ms } + return { operations = count, duration_ms = durations[1], samples_ms = durations, memory = memory } end) end +local function lifecycle(options) + options = options or {} + local count = tonumber(options.iterations) or 100 + local samples = tonumber(options.samples) or 1 + local warmup = tonumber(options.warmup) or 0 + local handlers = { __init = function(state) return actor.exit(state.index) end } + local function complete(iterations) + for index = 1, iterations do + assert(actor.new({ index = index }, handlers).run() == index, "actor lifecycle result mismatch") + end + end + local durations = {} + for iteration = 1, warmup + samples do + local began = time.now() + complete(count) + local elapsed = time.now():sub(began):seconds() * 1000 + if iteration > warmup then durations[#durations + 1] = elapsed end + end + local memory + if options.memory_operations then + local memory_err + memory, memory_err = benchmark_report.measure_memory(function() complete(options.memory_operations) end, + options.memory_operations) + assert(memory, tostring(memory_err)) + end + return { operations = count, samples_ms = durations, memory = memory } +end + local function benchmark_cases() test.describe("actor_roundtrip benchmark", function() local samples = tonumber((env.get("WIPPY_BENCH_SAMPLES"))) @@ -150,19 +286,42 @@ local function benchmark_cases() local size = tonumber((env.get("WIPPY_BENCH_SIZE"))) or 1 local warmup = tonumber((env.get("WIPPY_BENCH_WARMUP"))) or 5 assert(samples > 0 and size > 0 and warmup >= 0, "invalid benchmark configuration") - for _ = 1, warmup do benchmark({ iterations = size }) end - local durations = {} - for index = 1, samples do - local result = benchmark({ iterations = size }) - assert(result.operations == size, "benchmark operation count mismatch") - durations[index] = result.duration_ms - end + local result = benchmark({ iterations = size, samples = samples, warmup = warmup, + memory_operations = tonumber((env.get("WIPPY_BENCH_MEMORY_OPERATIONS"))) }) + test.eq(result.operations, size) + test.eq(#result.samples_ms, samples) + local report, report_err = benchmark_report.report({ + name = "actor_roundtrip", size = size, operations_per_sample = size, samples_ms = result.samples_ms, + memory = result.memory, + }) + test.not_nil(report, tostring(report_err)) + end) + end) +end + +local function lifecycle_cases() + test.describe("actor_lifecycle benchmark", function() + local samples = tonumber((env.get("WIPPY_BENCH_SAMPLES"))) + if not samples then + test.it_skip("requires WIPPY_BENCH_SAMPLES", function() end) + return + end + test.it("measures verified actor initialization and completion", function() + local size = tonumber((env.get("WIPPY_BENCH_SIZE"))) or 1 + local warmup = tonumber((env.get("WIPPY_BENCH_WARMUP"))) or 5 + assert(samples > 0 and size > 0 and warmup >= 0, "invalid benchmark configuration") + local result = lifecycle({ iterations = size, samples = samples, warmup = warmup, + memory_operations = tonumber((env.get("WIPPY_BENCH_MEMORY_OPERATIONS"))) }) + test.eq(result.operations, size) + test.eq(#result.samples_ms, samples) local report, report_err = benchmark_report.report({ - name = "actor_roundtrip", size = size, operations_per_sample = size, samples_ms = durations, + name = "actor_lifecycle", size = size, operations_per_sample = size, samples_ms = result.samples_ms, + memory = result.memory, }) test.not_nil(report, tostring(report_err)) end) end) end -return { run = test.run_cases(define_tests), benchmark = test.run_cases(benchmark_cases), workload = benchmark } +return { run = test.run_cases(define_tests), benchmark = test.run_cases(benchmark_cases), workload = benchmark, + lifecycle = test.run_cases(lifecycle_cases) } diff --git a/src/agent/src/README.md b/src/agent/src/README.md index 5c307cf8..9f897964 100644 --- a/src/agent/src/README.md +++ b/src/agent/src/README.md @@ -219,10 +219,15 @@ local response, err = runner:step(builder, { stream_target = { reply_to = process.self(), topic = "stream" }, tool_call = "auto", disable_memory_recall = false, - previous_memory_ids = { "mem_1", "mem_2" } + previous_memory_ids = { "mem_1", "mem_2" }, + stop_check = function() return stop_requested end }) ``` +`stop_check` runs after prompt functions and memory recall, immediately before +provider generation. When it returns true, `step` returns `{ stopped = true }` +without a provider request. + ### Response Structure ```lua @@ -389,9 +394,11 @@ control handlers and apply proposals only after the round's outcomes settle. persistence. Returned `context` without `_control` is not a host configuration change. Legacy bindings do not acquire behavior controls implicitly. -Session and Dataflow persist proposals with the round and replay them after -recovery. Replay is at-least-once, not exactly-once external writes; providers -must make writes and lifecycle hooks idempotent using stable host refs. +Hosts collect these proposals from the lifecycle summary (`controls`) and from +`caller:get_wrapper_controls()`. `wippy/session` does not consume them yet. A +host that persists proposals with the round and replays them after recovery +gives at-least-once delivery, not exactly-once external writes; providers must +make writes and lifecycle hooks idempotent using stable host refs. `memory.compact = true` requests checkpointing without a token threshold, but does not bypass an explicit disable or supply a missing provider. False does not cancel an earlier request. There are no automatic evaluator calls, new diff --git a/src/agent/src/agent.lua b/src/agent/src/agent.lua index cbe2d599..ac0d5594 100644 --- a/src/agent/src/agent.lua +++ b/src/agent/src/agent.lua @@ -577,6 +577,10 @@ function agent:step(prompt_builder: any, runtime_options: any): (table?, string? --end + if runtime_options.stop_check and runtime_options.stop_check() then + return { stopped = true }, nil + end + local result, err = llm_instance.generate(final_messages, options) if err then return nil, err diff --git a/src/agent/src/tools/caller.lua b/src/agent/src/tools/caller.lua index 360122a4..a3a30506 100644 --- a/src/agent/src/tools/caller.lua +++ b/src/agent/src/tools/caller.lua @@ -290,6 +290,14 @@ local function reset_wrapper_diagnostics(self: any) self.wrapper_controls = {} end +local function discard_stale_round(self: any) + if next(self.wrapper_observations) ~= nil then self.wrapper_observations = {} end + if next(self.wrapper_metadata) ~= nil then self.wrapper_metadata = {} end + if next(self.wrapper_errors) ~= nil then self.wrapper_errors = {} end + if next(self.wrapper_controls) ~= nil then self.wrapper_controls = {} end + if next(self.last_tool_calls) ~= nil then self.last_tool_calls = {} end +end + function tool_caller:get_wrapper_controls(): {table} return behavior_controls.prepare(self.wrapper_controls or {}) or {} end @@ -459,6 +467,7 @@ end function tool_caller:validate(tool_calls: {ToolCall}?): (any, string?) -- Check if there are any tool calls if not tool_calls or #tool_calls == 0 then + discard_stale_round(self) return {}, nil end diff --git a/src/agent/test/_index.yaml b/src/agent/test/_index.yaml index 90342c67..71e9addb 100644 --- a/src/agent/test/_index.yaml +++ b/src/agent/test/_index.yaml @@ -454,3 +454,8 @@ entries: kind: env.variable variable: WIPPY_BENCH_SIZE storage: app:env_storage + + - name: bench_memory_operations + kind: env.variable + variable: WIPPY_BENCH_MEMORY_OPERATIONS + storage: app:env_storage diff --git a/src/agent/test/agent_test.lua b/src/agent/test/agent_test.lua index 6aaa2c29..20b8fdb0 100644 --- a/src/agent/test/agent_test.lua +++ b/src/agent/test/agent_test.lua @@ -614,6 +614,34 @@ local function define_tests() test.eq(seen.temperature, 0.7) end) + it("should skip provider generation when stop_check reports a committed stop", function() + local test_agent = agent.new({ + id = "stop-check-agent", + name = "Stop Check Agent", + description = "Checks stop before generation", + prompt = "You are stoppable." + }) + local calls = { generate = 0, checks = 0 } + agent._llm = { + generate = function() + calls.generate = calls.generate + 1 + return { result = "done", tokens = {} } + end + } + local result, err = test_agent:step(mock_prompt.new(), { + stop_check = function() calls.checks = calls.checks + 1; return true end + }) + test.is_nil(err) + test.is_true(result.stopped) + test.eq(calls.generate, 0) + test.eq(calls.checks, 1) + + result, err = test_agent:step(mock_prompt.new(), { stop_check = function() return false end }) + test.is_nil(err) + test.is_nil(result.stopped) + test.eq(calls.generate, 1) + end) + it("should carry an explicit zero temperature rather than treating it as unset", function() local spec = { id = "zero-temp-agent", diff --git a/src/agent/test/tools/control_fixture.lua b/src/agent/test/tools/control_fixture.lua index 0ba42740..28bf7501 100644 --- a/src/agent/test/tools/control_fixture.lua +++ b/src/agent/test/tools/control_fixture.lua @@ -35,10 +35,12 @@ local function apply(request) proposal = result.result.wrapper_control end end + local call_ids = {} + for _, call in ipairs(request.tool_calls or {}) do call_ids[#call_ids + 1] = call.id end return { _control = proposal, observations = {{level = "info", code = "runtime_outcome", content = request.outcome}}, - metadata = {phase = request.phase}, + metadata = {phase = request.phase, call_ids = call_ids}, } end diff --git a/src/agent/test/tools/controls_benchmark.lua b/src/agent/test/tools/controls_benchmark.lua index ca80b781..c98ed109 100644 --- a/src/agent/test/tools/controls_benchmark.lua +++ b/src/agent/test/tools/controls_benchmark.lua @@ -22,22 +22,39 @@ local function define_tests() control = {context = {session = {set = {scope = "benchmark"}}}}}} end local samples = {} - for iteration = 1, warmup + sample_count do - local started = time.now() - local validated, err = caller:validate(calls) - test.is_nil(err) - local results = caller:execute({}, validated) - local elapsed = time.now():sub(started):seconds() * 1000 + local function verify(results) for index = 1, size do local result = results["benchmark-" .. index] test.is_nil(result.error) test.eq(result.result.call_id, "benchmark-" .. index) test.eq(result.result._control.context.session.set.scope, "benchmark") end + end + local function execute() + local validated, err = caller:validate(calls) + test.is_nil(err) + return caller:execute({}, validated) + end + for iteration = 1, warmup + sample_count do + local started = time.now() + local results = execute() + local elapsed = time.now():sub(started):seconds() * 1000 + verify(results) if iteration > warmup then samples[#samples + 1] = elapsed end end + local memory + local memory_operations = tonumber((env.get("WIPPY_BENCH_MEMORY_OPERATIONS"))) + if memory_operations then + test.gt(memory_operations, 0) + local rounds = math.ceil(memory_operations / size) + local memory_err + memory, memory_err = benchmark.measure_memory(function() + for _ = 1, rounds do verify(execute()) end + end, rounds * size) + test.not_nil(memory, tostring(memory_err)) + end local report, report_err = benchmark.report({name = "tools_" .. strategy, - size = size, operations_per_sample = size, samples_ms = samples}) + size = size, operations_per_sample = size, samples_ms = samples, memory = memory}) test.not_nil(report, tostring(report_err)) end) end diff --git a/src/agent/test/tools/controls_runtime_test.lua b/src/agent/test/tools/controls_runtime_test.lua index 37073056..a47c71e8 100644 --- a/src/agent/test/tools/controls_runtime_test.lua +++ b/src/agent/test/tools/controls_runtime_test.lua @@ -84,6 +84,59 @@ local function define_tests() test.eq(#caller:get_wrapper_errors(), 0) end) + it(strategy .. " keeps different nested controls paired with two successful calls", function() + local caller = tool_caller.new():set_strategy(strategy) + caller:set_wrapper_context({host = {kind = "session", session_id = "runtime"}}) + local validated, err = caller:validate({ + call("left-id", {message = "left", control = {config = {model = "left-model"}, + context = {session = {set = {side = "left", nested = {depth = {value = 1}}}}}}}), + call("right-id", {message = "right", control = {config = {model = "right-model"}, + context = {session = {set = {side = "right", nested = {depth = {value = 2}}}}}}}), + }) + test.is_nil(err) + local results = caller:execute({}, validated) + test.eq(count(results), 2) + for id, side in pairs({["left-id"] = "left", ["right-id"] = "right"}) do + local entry = results[id] + test.is_nil(entry.error) + test.eq(entry.tool_call.call_id, id) + test.eq(entry.result.call_id, id) + test.eq(entry.result.message, side) + test.eq(entry.result._control.config.model, side .. "-model") + test.eq(entry.result._control.context.session.set.side, side) + end + test.eq(results["left-id"].result._control.context.session.set.nested.depth.value, 1) + test.eq(results["right-id"].result._control.context.session.set.nested.depth.value, 2) + test.is_false(results["left-id"].result._control == results["right-id"].result._control) + end) + + it(strategy .. " clears the previous round when a later round has no tool calls", function() + local caller = tool_caller.new():set_strategy(strategy) + caller:set_wrapper_context({host = {kind = "session", session_id = "runtime"}}) + caller:set_tool_wrappers({wrapper("behavior", {"after_execute"})}) + local validated, err = caller:validate({call("first-id", {message = "first", + wrapper_control = {memory = {compact = true}}})}) + test.is_nil(err) + caller:execute({}, validated) + test.eq(#caller:get_wrapper_controls(), 1) + test.eq(caller:get_wrapper_metadata()[1].metadata.call_ids[1], "first-id") + + validated, err = caller:validate({}) + test.is_nil(err) + test.eq(#caller:get_last_tool_calls(), 0) + test.eq(#caller:get_wrapper_observations(), 0) + test.eq(#caller:get_wrapper_metadata(), 0) + test.eq(#caller:get_wrapper_errors(), 0) + test.eq(#caller:get_wrapper_controls(), 0) + + local results = caller:execute({}, validated) + test.eq(count(results), 0) + local metadata = caller:get_wrapper_metadata() + test.eq(#metadata, 1) + test.eq(#metadata[1].metadata.call_ids, 0) + test.eq(#caller:get_wrapper_controls(), 0) + end) + it(strategy .. " keeps legacy wrappers from proposing behavior controls", function() local caller = tool_caller.new():set_strategy(strategy) caller:set_wrapper_context({host = {kind = "session", session_id = "runtime"}}) diff --git a/src/relay/src/user.lua b/src/relay/src/user.lua index 209a67d1..28e6e79f 100644 --- a/src/relay/src/user.lua +++ b/src/relay/src/user.lua @@ -287,21 +287,23 @@ end local function handle_client_join(state: UserState, payload_data: any) local client_pid = payload_data.client_pid :: string? if client_pid then - state.connected_clients[client_pid] = true - state.client_count = state.client_count + 1 - - logger:info("client connected", { - client_pid = client_pid, - client_count = state.client_count, - user_id = state.user_id - }) - - if state.client_count == 1 then - local session_plugin = state.active_plugins["session_"] - if session_plugin then - local pid = session_plugin.pid - if pid then - process.send(pid, "resume", {}) + if not state.connected_clients[client_pid] then + state.connected_clients[client_pid] = true + state.client_count = state.client_count + 1 + + logger:info("client connected", { + client_pid = client_pid, + client_count = state.client_count, + user_id = state.user_id + }) + + if state.client_count == 1 then + local session_plugin = state.active_plugins["session_"] + if session_plugin then + local pid = session_plugin.pid + if pid then + process.send(pid, "resume", {}) + end end end end @@ -506,12 +508,10 @@ local function run(args: any): any local inbox = process.inbox() local events = process.events() + local select_cases = { inbox:case_receive(), events:case_receive() } while true do - local result = channel.select({ - inbox:case_receive(), - events:case_receive() - }) + local result = channel.select(select_cases) if not result.ok then break @@ -519,9 +519,9 @@ local function run(args: any): any if result.channel == inbox then local msg: any = result.value - local topic = string(msg:topic()) + local topic: string = msg:topic() local payload: any = msg:payload() - local from_pid = string(msg:from()) + local from_pid: string = msg:from() if topic == consts.WS_TOPICS.JOIN then handle_client_join(state, payload:data()) diff --git a/src/relay/test/_index.yaml b/src/relay/test/_index.yaml index 247357d0..afc136ea 100644 --- a/src/relay/test/_index.yaml +++ b/src/relay/test/_index.yaml @@ -141,6 +141,24 @@ entries: - env method: benchmark + - name: relay_pipeline_benchmark + kind: function.lua + meta: + type: test + suite: relay-benchmark + timeout: 120s + source: file://relay_runtime_test.lua + imports: + test: wippy.test:test + benchmark: wippy.test:benchmark + consts: wippy.relay:consts + modules: + - time + - json + - security + - env + method: pipeline + - name: runtime_env_storage kind: env.storage.os @@ -159,6 +177,11 @@ entries: variable: WIPPY_BENCH_SIZE storage: app:runtime_env_storage + - name: runtime_bench_memory_operations + kind: env.variable + variable: WIPPY_BENCH_MEMORY_OPERATIONS + storage: app:runtime_env_storage + - name: relay_runtime_finished_plugin kind: process.lua source: file://relay_runtime_plugin.lua diff --git a/src/relay/test/relay_runtime_test.lua b/src/relay/test/relay_runtime_test.lua index a54653d8..ddacdbc5 100644 --- a/src/relay/test/relay_runtime_test.lua +++ b/src/relay/test/relay_runtime_test.lua @@ -7,6 +7,12 @@ local security = require("security") local consts = require("consts") local sequence = 0 +local PIPELINE_WINDOW = 32 +local function pipeline_slot(index: any, first: number, last: number): number? + if type(index) ~= "number" or index < first or index > last or index % 1 ~= 0 then return nil end + return index - first + 1 +end + local function receive(ch: any): any local timer = time.timer(3 * time.SECOND) local result = channel.select({ ch:case_receive(), timer:channel():case_receive() }) @@ -15,7 +21,7 @@ local function receive(ch: any): any return result.value end -local function with_hub(plugin_id: string, body: any, active_pid: string?): any +local function with_hub(plugin_id: string, body: any, active_pid: string?, session_pid: string?): any sequence = sequence + 1 local user_id = "relay-runtime-" .. process.pid() .. "-" .. sequence local actor = security.new_actor(user_id, {}) @@ -35,9 +41,15 @@ local function with_hub(plugin_id: string, body: any, active_pid: string?): any host = config.user_hub_host, auto_start = false }, }, } - if active_pid then + if active_pid or session_pid then args.relay_user_upgrade = true - args.active_plugins = { runtime_ = { pid = active_pid, status = "running", restart_count = 0 } } + args.active_plugins = {} + if active_pid then + args.active_plugins.runtime_ = { pid = active_pid, status = "running", restart_count = 0 } + end + if session_pid then + args.active_plugins.session_ = { pid = session_pid, status = "running", restart_count = 0 } + end args.connected_clients = {} args.client_count = 0 args.pg_groups = {} @@ -150,6 +162,74 @@ local function define_tests() test.eq(result.payload.session_id, "reconnected") end) end) + test.it("routes repeated commands and stops on a process cancellation event", function() + with_hub("app:relay_runtime_plugin", function(pid, _, join, send, response, _, started) + join() + local plugin_pid + for index = 1, 8 do + send({ type = "runtime_echo", request_id = tostring(index), data = index }) + if index == 1 then plugin_pid = started().pid end + local result = receive(response) + test.eq(result.plugin_pid, plugin_pid) + test.eq(result.payload.request_id, tostring(index)) + test.eq(result.payload.data, index) + end + assert(process.cancel(pid, "2s")) + end) + end) + test.it("preserves independent payloads across pipelined commands", function() + with_hub("app:relay_runtime_plugin", function(pid, _, join, send, response, _, started) + join() + for index = 1, 16 do + send({ type = "runtime_echo", request_id = tostring(index), session_id = "pipeline", + data = { index = index }, start_token = "token-" .. index, + attention_context_enabled = index % 2 == 0, context = { index = index } }) + end + local plugin = started() + local seen = {} + for _ = 1, 16 do + local value = receive(response) + local index = value.payload.data.index + test.is_true(index >= 1 and index <= 16 and not seen[index]) + seen[index] = true + test.eq(value.plugin_pid, plugin.pid) + test.eq(value.from, pid) + test.eq(value.topic, "echo") + test.eq(value.payload.conn_pid, (process.pid())) + test.eq(value.payload.type, "runtime_echo") + test.eq(value.payload.request_id, tostring(index)) + test.eq(value.payload.session_id, "pipeline") + test.eq(value.payload.start_token, "token-" .. index) + test.eq(value.payload.attention_context_enabled, index % 2 == 0) + test.eq(value.payload.context.index, index) + end + end) + end) + test.it("bounds pipeline identity slots by the window at high operation ids", function() + local first = 4294967297 + test.eq(pipeline_slot(first, first, first + PIPELINE_WINDOW - 1), 1) + test.eq(pipeline_slot(first + PIPELINE_WINDOW - 1, first, first + PIPELINE_WINDOW - 1), PIPELINE_WINDOW) + test.is_nil(pipeline_slot(first - 1, first, first + PIPELINE_WINDOW - 1)) + test.is_nil(pipeline_slot(first + PIPELINE_WINDOW, first, first + PIPELINE_WINDOW - 1)) + test.is_nil(pipeline_slot(first + 0.5, first, first + PIPELINE_WINDOW - 1)) + test.is_nil(pipeline_slot("1", 1, PIPELINE_WINDOW)) + end) + test.it("counts a repeated JOIN once so the last LEAVE shuts down the session plugin", function() + local resumed = process.listen("resume") + local shutdown = process.listen("shutdown") + local ok, err = pcall(with_hub, "app:relay_runtime_plugin", function(pid, _, join) + test.eq(join().client_count, 1) + receive(resumed) + test.eq(join().client_count, 1, "A repeated JOIN must keep one membership") + assert(process.send(pid, consts.WS_TOPICS.LEAVE, { client_pid = process.pid() })) + receive(shutdown) + test.eq(join().client_count, 1) + receive(resumed) + end, nil, process.pid()) + process.unlisten(resumed) + process.unlisten(shutdown) + if not ok then error(err) end + end) test.it("reports a real plugin startup failure and keeps the hub responsive", function() with_hub("app:missing_runtime_plugin", function(_, _, join, send, _, errors) join() @@ -186,6 +266,30 @@ local function define_tests() test.eq(receive(errors).error, consts.ERROR_CODES.PLUGIN_NOT_FOUND) end) end) + test.it("keeps routing to the same plugin after invalid input and a rejected plugin topic", function() + with_hub("app:relay_runtime_plugin", function(pid, _, join, send, response, errors, started) + join() + send({ type = "runtime_echo", request_id = "first", data = 1 }) + local plugin = started() + test.eq(receive(response).payload.request_id, "first") + assert(process.send(pid, consts.WS_TOPICS.MESSAGE, "{")) + test.eq(receive(errors).error, consts.ERROR_CODES.INVALID_JSON) + send({ type = "runtime_echo", request_id = "after-json", data = 2 }) + test.eq(receive(response).payload.request_id, "after-json") + send(42) + test.eq(receive(errors).error, consts.ERROR_CODES.UNKNOWN_COMMAND) + send({ type = "runtime_echo", request_id = "after-type", data = 3 }) + test.eq(receive(response).payload.request_id, "after-type") + send({ type = "runtime_@reserved", request_id = "reserved", data = 4 }) + local failure = receive(errors) + test.eq(failure.error, consts.ERROR_CODES.PLUGIN_FAILED) + test.contains(tostring(failure.message), "cannot send to @ topics") + send({ type = "runtime_echo", request_id = "after-reserved", data = 5 }) + local after = receive(response) + test.eq(after.payload.request_id, "after-reserved") + test.eq(after.plugin_pid, plugin.pid) + end) + end) test.it("rejects non-object JSON and non-string command types without stopping", function() with_hub("app:relay_runtime_plugin", function(pid, _, join, send, _, errors) join() @@ -205,30 +309,71 @@ local function benchmark(options) local count = tonumber(options.iterations) or 100 local samples = tonumber(options.samples) or 1 local warmup = tonumber(options.warmup) or 0 - return with_hub("app:relay_runtime_plugin", function(_, _, join, send, response, _, started) + return with_hub("app:relay_runtime_plugin", function(pid, _, join, send, response, _, started) join() send({ type = "runtime_echo", request_id = "startup", data = 0 }) started() assert(receive(response).payload.request_id == "startup") - local durations = {} - for iteration = 1, warmup + samples do - local began = time.now() - for index = 1, count do - local operation = (iteration - 1) * count + index + local operation = 0 + local client_pid = process.pid() + local function roundtrip(iterations) + if options.pipelined then + local remaining = iterations + while remaining > 0 do + local chunk = math.min(count, PIPELINE_WINDOW, remaining) + local first = operation + 1 + for _ = 1, chunk do + operation = operation + 1 + send({ type = "runtime_echo", request_id = tostring(operation), session_id = "benchmark", + data = { index = operation }, start_token = "benchmark-token", + attention_context_enabled = false, context = { index = operation } }) + end + local seen = {} + for _ = 1, chunk do + local value = receive(response) + local index = value.payload.data.index + local slot = pipeline_slot(index, first, operation) + assert(slot and not seen[slot], "relay pipeline identity mismatch") + seen[slot] = true + assert(value.from == pid and value.topic == "echo" and value.payload.type == "runtime_echo" + and value.payload.conn_pid == client_pid and value.payload.request_id == tostring(index) + and value.payload.session_id == "benchmark" and value.payload.start_token == "benchmark-token" + and value.payload.attention_context_enabled == false and value.payload.context.index == index, + "relay pipeline payload mismatch") + end + remaining = remaining - chunk + end + return + end + for _ = 1, iterations do + operation = operation + 1 send({ type = "runtime_echo", request_id = tostring(operation), session_id = "benchmark", data = operation }) local value = receive(response) assert(value.payload.request_id == tostring(operation) and value.payload.data == operation, "relay roundtrip mismatch") end + end + local durations = {} + for iteration = 1, warmup + samples do + local began = time.now() + roundtrip(count) local elapsed = time.now():sub(began):seconds() * 1000 if iteration > warmup then durations[#durations + 1] = elapsed end end - return { operations = count, duration_ms = durations[1], samples_ms = durations } + local memory + if options.memory_operations then + local memory_err + memory, memory_err = benchmark_report.measure_memory(function() roundtrip(options.memory_operations) end, + options.memory_operations) + assert(memory, tostring(memory_err)) + end + return { operations = count, duration_ms = durations[1], samples_ms = durations, memory = memory } end) end -local function benchmark_cases() - test.describe("relay_roundtrip benchmark", function() +local function define_benchmark(pipelined) + local name = pipelined and "relay_pipeline" or "relay_roundtrip" + test.describe(name .. " benchmark", function() local samples = tonumber((env.get("WIPPY_BENCH_SAMPLES"))) if not samples then test.it_skip("requires WIPPY_BENCH_SAMPLES", function() end) @@ -238,15 +383,19 @@ local function benchmark_cases() local size = tonumber((env.get("WIPPY_BENCH_SIZE"))) or 1 local warmup = tonumber((env.get("WIPPY_BENCH_WARMUP"))) or 5 assert(samples > 0 and size > 0 and warmup >= 0, "invalid benchmark configuration") - local result = benchmark({ iterations = size, samples = samples, warmup = warmup }) + local result = benchmark({ iterations = size, samples = samples, warmup = warmup, + memory_operations = tonumber((env.get("WIPPY_BENCH_MEMORY_OPERATIONS"))), pipelined = pipelined }) test.eq(result.operations, size) test.eq(#result.samples_ms, samples) local report, report_err = benchmark_report.report({ - name = "relay_roundtrip", size = size, operations_per_sample = size, samples_ms = result.samples_ms, + name = name, size = size, operations_per_sample = size, samples_ms = result.samples_ms, + memory = result.memory, }) test.not_nil(report, tostring(report_err)) end) end) end -return { run = test.run_cases(define_tests), benchmark = test.run_cases(benchmark_cases), workload = benchmark } +return { run = test.run_cases(define_tests), + benchmark = test.run_cases(function() define_benchmark(false) end), + pipeline = test.run_cases(function() define_benchmark(true) end), workload = benchmark } diff --git a/src/test/_index.yaml b/src/test/_index.yaml index c9e3f273..7782331b 100644 --- a/src/test/_index.yaml +++ b/src/test/_index.yaml @@ -165,10 +165,15 @@ entries: variable: WIPPY_BENCH_FRAMEWORK_REVISION storage: wippy.test:env_storage + - name: benchmark_exact_allocations + kind: env.variable + variable: WIPPY_BENCH_EXACT_ALLOCATIONS + storage: wippy.test:env_storage + - name: benchmark kind: library.lua source: file://benchmark.lua - modules: [fs, json, env, io] + modules: [fs, json, env, io, system, http_client] - name: benchmark_test kind: function.lua @@ -177,7 +182,7 @@ entries: suite: test-framework source: file://benchmark_test.lua method: run - modules: [fs, json] + modules: [fs, json, system, env] imports: benchmark: wippy.test:benchmark test: wippy.test:test diff --git a/src/test/benchmark.lua b/src/test/benchmark.lua index 9de2134f..3aa163d1 100644 --- a/src/test/benchmark.lua +++ b/src/test/benchmark.lua @@ -2,6 +2,8 @@ local fs = require("fs") local json = require("json") local env = require("env") local io = require("io") +local system = require("system") +local http_client = require("http_client") local function finite(value: any): boolean return type(value) == "number" and value == value and value < math.huge and value > -math.huge @@ -11,6 +13,156 @@ local function positive_integer(value: any): boolean return finite(value) and value > 0 and value % 1 == 0 end +local function profiler_stats(): (any, string?) + local before = assert(system.memory.stats()) + local response, err = http_client.get("http://127.0.0.1:6060/debug/pprof/heap?debug=1", {timeout = 5}) + if not response then return nil, tostring(err) end + local after = assert(system.memory.stats()) + if response.status_code ~= 200 or type(response.body) ~= "string" then + return nil, "native profiler returned an invalid response" + end + local trailer = response.body:match("# runtime%.MemStats\n(.*)") + if not trailer then return nil, "native profiler memory counters are missing" end + local result = {} + for _, field in ipairs({"TotalAlloc", "Mallocs", "Frees", "HeapAlloc", "HeapObjects", "NumGC"}) do + local value = tonumber(trailer:match("# " .. field .. " = (%d+)\n")) + if not finite(value) or value < 0 then return nil, "invalid profiler counter: " .. field end + result[field] = value + end + if result.NumGC ~= before.num_gc or result.NumGC ~= after.num_gc + or result.TotalAlloc < before.total_alloc or result.TotalAlloc > after.total_alloc + or result.HeapObjects < before.heap_objects or result.HeapObjects > after.heap_objects + or result.Mallocs - result.Frees ~= result.HeapObjects then + return nil, "native profiler belongs to a different runtime or changed during measurement" + end + return result +end + +local function measure_memory(workload: any, operations: any, exact: boolean?): (any, string?) + if type(workload) ~= "function" or not positive_integer(operations) then + return nil, "memory measurement requires bounded work and a positive operation count" + end + if exact == nil then exact = env.get("WIPPY_BENCH_EXACT_ALLOCATIONS") == "1" end + local previous_limit, limit_err = system.memory.set_limit(-1) + if previous_limit == nil then return nil, tostring(limit_err) end + if previous_limit >= 2 ^ 63 then previous_limit = -1 end + local initial, initial_err = system.memory.stats() + if not initial or initial.next_gc > 2 ^ 53 then + local restored, restore_err = system.memory.set_limit(previous_limit) + if restored == nil then return nil, "failed to restore memory limit: " .. tostring(restore_err) end + return nil, tostring(initial_err or "memory measurement requires supported enabled garbage collection") + end + local previous_gc, gc_err = system.gc.set_percent(-1) + if previous_gc == nil then + local restored, restore_err = system.memory.set_limit(previous_limit) + if restored == nil then return nil, "failed to restore memory limit: " .. tostring(restore_err) end + return nil, tostring(gc_err) + end + local measured, result = pcall(function() + assert(system.gc.collect()) + assert(system.gc.collect()) + if exact then + local witness = assert(profiler_stats()) + assert(system.gc.collect()) + assert(system.gc.collect()) + local confirmed = assert(profiler_stats()) + assert(confirmed.NumGC == witness.NumGC + 2, "native profiler belongs to a different runtime") + end + local control_before = assert(system.memory.stats()) + local control_after = assert(system.memory.stats()) + local instrumentation_bytes = control_after.total_alloc - control_before.total_alloc + local instrumentation_objects = control_after.heap_objects - control_before.heap_objects + local instrumentation_allocations + if exact then + local profile_before = assert(profiler_stats()) + local profile_after = assert(profiler_stats()) + instrumentation_allocations = profile_after.Mallocs - profile_before.Mallocs + instrumentation_bytes = profile_after.TotalAlloc - profile_before.TotalAlloc + end + control_before = nil + control_after = nil + assert(system.gc.collect()) + assert(system.gc.collect()) + local live_heap_before = assert(system.memory.allocated()) + local profile_before + if exact then profile_before = assert(profiler_stats()) end + local before = assert(system.memory.stats()) + workload() + local after = assert(system.memory.stats()) + local profile_after + if exact then profile_after = assert(profiler_stats()) end + assert(after.num_gc == before.num_gc, "GC ran during allocation measurement") + if exact then + assert(profile_after.NumGC == profile_before.NumGC, "GC ran during exact allocation measurement") + end + local memory: any = { + method = exact and "pprof_memstats_mallocs_delta" or "gc_disabled_heap_objects_delta", operations = operations, + allocated_bytes = exact and (profile_after.TotalAlloc - profile_before.TotalAlloc) + or (after.total_alloc - before.total_alloc), + heap_objects = after.heap_objects - before.heap_objects, + heap_growth_bytes = after.heap_alloc - before.heap_alloc, + live_heap_before_bytes = live_heap_before, + gc_before = before.num_gc, gc_after = after.num_gc, + instrumentation_allocated_bytes = instrumentation_bytes, + instrumentation_heap_objects = instrumentation_objects, + allocations = exact and (profile_after.Mallocs - profile_before.Mallocs) or nil, + instrumentation_allocations = instrumentation_allocations, + } + if exact then memory.allocations_per_op = memory.allocations / operations end + memory.allocated_bytes_per_op = memory.allocated_bytes / operations + memory.heap_objects_per_op = memory.heap_objects / operations + memory.heap_growth_bytes_per_op = memory.heap_growth_bytes / operations + before = nil + after = nil + profile_before = nil + profile_after = nil + assert(system.gc.collect()) + assert(system.gc.collect()) + memory.live_heap_after_bytes = assert(system.memory.allocated()) + memory.retained_heap_bytes = memory.live_heap_after_bytes - memory.live_heap_before_bytes + return memory + end) + local restored_gc, restore_gc_err = system.gc.set_percent(previous_gc) + local restored_limit, restore_limit_err = system.memory.set_limit(previous_limit) + if restored_gc == nil or restored_limit == nil then + return nil, "failed to restore runtime settings: " .. tostring(restore_gc_err or restore_limit_err) + end + if not measured then return nil, tostring(result) end + return result +end + +local function memory_measurements(memory: any): (any, string?) + if type(memory) ~= "table" or (memory.method ~= "gc_disabled_heap_objects_delta" + and memory.method ~= "pprof_memstats_mallocs_delta") + or not positive_integer(memory.operations) or memory.gc_before ~= memory.gc_after then + return nil, "memory measurement requires bounded operations without garbage collection" + end + local result = {} + for _, field in ipairs({"allocated_bytes", "heap_objects", "heap_growth_bytes", + "live_heap_before_bytes", "live_heap_after_bytes", "gc_before", "gc_after", + "instrumentation_allocated_bytes", "instrumentation_heap_objects"}) do + local value = memory[field] + if not finite(value) or value < 0 then return nil, "invalid memory counter: " .. field end + result[field] = value + end + result.method = memory.method + result.operations = memory.operations + result.allocated_bytes_per_op = memory.allocated_bytes / memory.operations + result.heap_objects_per_op = memory.heap_objects / memory.operations + result.heap_growth_bytes_per_op = memory.heap_growth_bytes / memory.operations + result.retained_heap_bytes = memory.live_heap_after_bytes - memory.live_heap_before_bytes + if memory.method == "pprof_memstats_mallocs_delta" then + if not finite(memory.allocations) or memory.allocations < 0 + or not finite(memory.instrumentation_allocations) or memory.instrumentation_allocations < 0 then + return nil, "invalid exact allocation counter" + end + result.allocations = memory.allocations + result.instrumentation_allocations = memory.instrumentation_allocations + result.allocations_per_op = memory.allocations / memory.operations + end + return result +end + local function measurements(record: any): (any, string?) if type(record) ~= "table" or type(record.name) ~= "string" or not record.name:match("^[%w_-]+$") then @@ -49,12 +201,18 @@ local function measurements(record: any): (any, string?) end local throughput = record.operations_per_sample / duration * count * 1000 if not finite(throughput) then return nil, "benchmark throughput must be finite" end - return { + local result = { median_ms = median, p95_ms = ordered[math.ceil(count * 0.95)], throughput_ops_per_second = throughput, sample_count = count, } + if record.memory ~= nil then + local memory, memory_err = memory_measurements(record.memory) + if not memory then return nil, memory_err end + result.memory = memory + end + return result end local function report(record) @@ -84,7 +242,7 @@ local function report(record) return result, filename end -local function compare(baseline, current) +local function compare(baseline: any, current: any, require_improvement: boolean?): (any, string?) local before, before_err = measurements(baseline) if not before then return nil, before_err end local after, after_err = measurements(current) @@ -98,16 +256,53 @@ local function compare(baseline, current) or baseline.operations_per_sample ~= current.operations_per_sample then return nil, "benchmark comparison requires the same workload" end + if require_improvement and (baseline.warmup_count ~= current.warmup_count + or baseline.sample_count ~= current.sample_count or before.sample_count ~= after.sample_count + or (baseline.sample_count ~= nil and baseline.sample_count ~= before.sample_count) + or (current.sample_count ~= nil and current.sample_count ~= after.sample_count)) then + return nil, "benchmark comparison requires the same measurement plan" + end if before.median_ms <= 0 or before.p95_ms <= 0 then return nil, "benchmark comparison requires positive baseline latencies" end - return { + local result = { name = baseline.name, size = baseline.size, median_percent = (after.median_ms / before.median_ms - 1) * 100, p95_percent = (after.p95_ms / before.p95_ms - 1) * 100, throughput_percent = (after.throughput_ops_per_second / before.throughput_ops_per_second - 1) * 100, } + if (before.memory == nil) ~= (after.memory == nil) then + return nil, "benchmark comparison requires matching memory measurements" + end + if before.memory then + if before.memory.method ~= after.memory.method then return nil, "benchmark comparison requires the same memory method" end + if before.memory.operations ~= after.memory.operations then + return nil, "benchmark comparison requires the same memory operation count" + end + for _, field in ipairs({"allocated_bytes_per_op", "heap_objects_per_op", "heap_growth_bytes_per_op"}) do + local previous = before.memory[field] + if previous <= 0 then return nil, "benchmark comparison requires positive baseline memory counters" end + result[field .. "_percent"] = (after.memory[field] / previous - 1) * 100 + end + result.live_heap_after_bytes_delta = after.memory.live_heap_after_bytes - before.memory.live_heap_after_bytes + result.retained_heap_bytes_delta = after.memory.retained_heap_bytes - before.memory.retained_heap_bytes + if before.memory.allocations_per_op then + if before.memory.allocations_per_op <= 0 then return nil, "benchmark comparison requires positive baseline allocations" end + result.allocations_per_op_percent = (after.memory.allocations_per_op / before.memory.allocations_per_op - 1) * 100 + end + end + if require_improvement then + local failures = {} + for _, field in ipairs({"median_percent", "p95_percent", "allocated_bytes_per_op_percent", + "allocations_per_op_percent", "heap_growth_bytes_per_op_percent", "live_heap_after_bytes_delta"}) do + if result[field] == nil or result[field] >= 0 then failures[#failures + 1] = field end + end + if #failures > 0 then + return nil, "benchmark did not improve required metrics: " .. table.concat(failures, ", ") + end + end + return result end -return {report = report, compare = compare} +return {report = report, compare = compare, measure_memory = measure_memory} diff --git a/src/test/benchmark_test.lua b/src/test/benchmark_test.lua index 36b929b2..f0f4a34a 100644 --- a/src/test/benchmark_test.lua +++ b/src/test/benchmark_test.lua @@ -2,9 +2,203 @@ local test = require("test") local benchmark = require("benchmark") local fs = require("fs") local json = require("json") +local system = require("system") +local env = require("env") local function define_tests() test.describe("runtime benchmark reports", function() + test.it("measures bounded allocation work and restores runtime settings", function() + local gc_percent = system.gc.get_percent() + local memory_limit = system.memory.get_limit() + local retained = {} + local memory, err = benchmark.measure_memory(function() + for index = 1, 100 do retained[index] = {index, string.rep("x", 128)} end + end, 100, false) + test.not_nil(memory, tostring(err)) + test.eq(memory.operations, 100) + test.eq(memory.method, "gc_disabled_heap_objects_delta") + test.eq(memory.gc_before, memory.gc_after) + test.gt(memory.allocated_bytes, 0) + test.gt(memory.heap_objects, 0) + test.eq(memory.allocated_bytes_per_op, memory.allocated_bytes / 100) + test.eq(memory.heap_objects_per_op, memory.heap_objects / 100) + test.gt(memory.live_heap_before_bytes, 0) + test.gt(memory.live_heap_after_bytes, 0) + test.eq(system.gc.get_percent(), gc_percent) + test.eq(system.memory.get_limit(), memory_limit) + test.eq(retained[100][1], 100) + end) + + test.it("restores runtime settings after allocation work throws", function() + local gc_percent = system.gc.get_percent() + local memory_limit = system.memory.get_limit() + local memory, err = benchmark.measure_memory(function() error("allocation fixture failed") end, 10, false) + test.is_nil(memory) + test.is_true(tostring(err):find("allocation fixture failed", 1, true) ~= nil) + test.eq(system.gc.get_percent(), gc_percent) + test.eq(system.memory.get_limit(), memory_limit) + end) + + test.it("rejects already-disabled garbage collection without enabling it", function() + local previous_gc = assert(system.gc.set_percent(-1)) + local previous_limit = assert(system.memory.get_limit()) + local invoked = false + local memory, err = benchmark.measure_memory(function() invoked = true end, 10, false) + local observed_limit = system.memory.get_limit() + local saved_limit = assert(system.memory.set_limit(-1)) + local next_gc = assert(system.memory.stats()).next_gc + system.memory.set_limit(saved_limit) + system.gc.set_percent(previous_gc) + test.is_nil(memory) + test.not_nil(err) + test.is_false(invoked) + test.eq(observed_limit, previous_limit) + test.gt(next_gc, 2 ^ 53) + end) + + test.it("preserves an initially unbounded memory limit", function() + local previous_limit = assert(system.memory.set_limit(-1)) + local memory, err = benchmark.measure_memory(function() end, 10, false) + local observed_limit = system.memory.get_limit() + system.memory.set_limit(previous_limit) + test.not_nil(memory, tostring(err)) + test.eq(observed_limit, 2 ^ 63) + end) + + local exact_it = env.get("WIPPY_BENCH_EXACT_ALLOCATIONS") == "1" and test.it or test.it_skip + exact_it("reads exact cumulative allocations from the native profiler", function() + local retained = {} + local memory, err = benchmark.measure_memory(function() + for index = 1, 1000 do retained[index] = {index, string.rep("y", 128)} end + end, 1000, true) + test.not_nil(memory, tostring(err)) + test.eq(memory.method, "pprof_memstats_mallocs_delta") + test.gt(memory.allocations, 0) + test.eq(memory.allocations_per_op, memory.allocations / 1000) + test.gt(memory.instrumentation_allocations, 0) + test.eq(memory.gc_before, memory.gc_after) + test.eq(retained[1000][1], 1000) + end) + + local foreign_it = env.get("WIPPY_BENCH_EXACT_ALLOCATIONS") == "foreign" and test.it or test.it_skip + foreign_it("rejects a profiler served by a different runtime before invoking work", function() + local invoked = false + local previous_gc = system.gc.get_percent() + local previous_limit = system.memory.get_limit() + local memory, err = benchmark.measure_memory(function() invoked = true end, 10, true) + test.is_nil(memory) + test.is_false(invoked) + test.is_true(tostring(err):find("different runtime", 1, true) ~= nil) + test.eq(system.gc.get_percent(), previous_gc) + test.eq(system.memory.get_limit(), previous_limit) + end) + + test.it("rejects unbounded allocation measurement inputs before invoking work", function() + local invoked = false + for _, operations in ipairs({0, -1, 1.5, math.huge}) do + local memory, err = benchmark.measure_memory(function() invoked = true end, operations) + test.is_nil(memory) + test.not_nil(err) + end + test.is_false(invoked) + end) + + test.it("persists allocation counters and derives normalized memory comparisons", function() + local memory = {method = "gc_disabled_heap_objects_delta", operations = 10, + allocated_bytes = 1000, heap_objects = 20, heap_growth_bytes = 1000, + live_heap_before_bytes = 20000, live_heap_after_bytes = 21000, + retained_heap_bytes = 1000, gc_before = 2, gc_after = 2, + instrumentation_allocated_bytes = 100, instrumentation_heap_objects = 2} + local baseline, filename = benchmark.report({name = "memory-report", size = 1, + operations_per_sample = 1, samples_ms = {2}, memory = memory}) + test.not_nil(baseline, tostring(filename)) + local stored = json.decode((fs.get("wippy.test:benchmark_output"):readfile(filename))) + test.not_nil(stored.memory) + test.eq(stored.memory.allocated_bytes_per_op, 100) + test.eq(stored.memory.heap_objects_per_op, 2) + test.eq(stored.memory.heap_growth_bytes_per_op, 100) + memory.allocated_bytes = 500 + memory.heap_objects = 10 + memory.heap_growth_bytes = 500 + memory.live_heap_after_bytes = 20500 + memory.retained_heap_bytes = 500 + local current = benchmark.report({name = "memory-report", size = 1, + operations_per_sample = 1, samples_ms = {1}, memory = memory}) + local deltas, err = benchmark.compare(stored, current) + test.not_nil(deltas, tostring(err)) + test.eq(deltas.allocated_bytes_per_op_percent, -50) + test.eq(deltas.heap_objects_per_op_percent, -50) + test.eq(deltas.heap_growth_bytes_per_op_percent, -50) + test.eq(deltas.live_heap_after_bytes_delta, -500) + end) + + test.it("rejects allocation records invalidated by garbage collection", function() + local report, err = benchmark.report({name = "memory-gc", size = 1, + operations_per_sample = 1, samples_ms = {2}, memory = { + method = "gc_disabled_heap_objects_delta", operations = 10, + allocated_bytes = 1000, heap_objects = 20, heap_growth_bytes = 1000, + live_heap_before_bytes = 20000, live_heap_after_bytes = 21000, + retained_heap_bytes = 1000, gc_before = 2, gc_after = 3, + instrumentation_allocated_bytes = 100, instrumentation_heap_objects = 2}}) + test.is_nil(report) + test.not_nil(err) + end) + + test.it("requires faster latencies and lower exact allocations and heap footprint", function() + local function record(latency, bytes, allocations, growth, live) + return {name = "strict-improvement", size = 1, operations_per_sample = 1, + samples_ms = {latency}, metadata = {runtime = "fixture"}, memory = { + method = "pprof_memstats_mallocs_delta", operations = 10, + allocated_bytes = bytes, allocations = allocations, heap_objects = allocations, + heap_growth_bytes = growth, live_heap_before_bytes = 20000, + live_heap_after_bytes = live, gc_before = 2, gc_after = 2, + instrumentation_allocated_bytes = 100, instrumentation_heap_objects = 2, + instrumentation_allocations = 3}} + end + local baseline = record(10, 1000, 20, 1000, 21000) + for _, current in ipairs({record(10, 500, 10, 500, 20500), + record(11, 500, 10, 500, 20500), record(5, 1000, 10, 500, 20500), + record(5, 500, 20, 500, 20500), record(5, 500, 10, 1000, 20500), + record(5, 500, 10, 500, 21000)}) do + local deltas, err = benchmark.compare(baseline, current, true) + test.is_nil(deltas) + test.not_nil(err) + end + local deltas, err = benchmark.compare(baseline, record(5, 500, 10, 500, 20500), true) + test.not_nil(deltas, tostring(err)) + end) + + test.it("rejects comparisons with different allocation operation counts", function() + local function record(operations) + return {name = "matched-operations", size = 1, operations_per_sample = 1, + samples_ms = {10}, metadata = {runtime = "fixture"}, memory = { + method = "gc_disabled_heap_objects_delta", operations = operations, + allocated_bytes = 1000, heap_objects = 20, heap_growth_bytes = 1000, + live_heap_before_bytes = 20000, live_heap_after_bytes = 21000, + gc_before = 2, gc_after = 2, instrumentation_allocated_bytes = 100, + instrumentation_heap_objects = 2}} + end + local deltas, err = benchmark.compare(record(10), record(100)) + test.is_nil(deltas) + test.not_nil(err) + end) + + test.it("rejects strict comparison with mismatched warmup or sample plans", function() + local baseline = {name = "matched-plan", size = 1, operations_per_sample = 1, + samples_ms = {10, 20}, warmup_count = 5, sample_count = 2, + metadata = {runtime = "fixture"}} + for _, current in ipairs({ + {name = "matched-plan", size = 1, operations_per_sample = 1, + samples_ms = {5, 10}, warmup_count = 100, sample_count = 2, metadata = {runtime = "fixture"}}, + {name = "matched-plan", size = 1, operations_per_sample = 1, + samples_ms = {5}, warmup_count = 5, sample_count = 1, metadata = {runtime = "fixture"}}, + }) do + local deltas, err = benchmark.compare(baseline, current, true) + test.is_nil(deltas) + test.is_true(tostring(err):find("measurement plan", 1, true) ~= nil) + end + end) + test.it("does not report success when the artifact cannot be written", function() local output = fs.get("wippy.test:benchmark_output") output:remove("report-write-failure-1.json") diff --git a/src/test/runner.lua b/src/test/runner.lua index 88a03578..94dc9262 100644 --- a/src/test/runner.lua +++ b/src/test/runner.lua @@ -157,11 +157,15 @@ local function run_tests(): number end if msg_type == "test:plan" then - local count = 0 - for _, planned_suite in ipairs(data.suites or {}) do - count = count + #(planned_suite.tests or {}) + if declared_cases[ref_id] ~= nil then + declared_cases[ref_id] = -1 + else + local count = 0 + for _, planned_suite in ipairs(data.suites or {}) do + count = count + #(planned_suite.tests or {}) + end + declared_cases[ref_id] = count end - declared_cases[ref_id] = count elseif msg_type == "test:case:pass" then local cs = case_stats[ref_id] @@ -297,8 +301,8 @@ local function run_tests(): number cases = cs, completed = completion and completion.data or nil, result_error = result_err, - returned_false = value == false or (declared_cases[entry_id] == nil - and type(value) == "table" and value.status == "error"), + returned_false = value == false or (type(value) == "table" and value.status == "error" + and not (cs and cs.failed > 0)), }) if problem then diff --git a/src/test/runner_verdict.lua b/src/test/runner_verdict.lua index 30f3a033..e3b2ddb2 100644 --- a/src/test/runner_verdict.lua +++ b/src/test/runner_verdict.lua @@ -52,6 +52,9 @@ local function check(state: RunState): RunnerError? return fail("test process did not report a plan") end if state.declared ~= nil then + if state.declared < 0 then + return fail("test process reported more than one plan") + end if state.observed ~= state.declared then return fail(tostring(state.declared) .. " declared cases, " .. tostring(state.observed) .. " executed") diff --git a/test-harness/runner-results/_index.yaml b/test-harness/runner-results/_index.yaml index 6679046b..c76a6c42 100644 --- a/test-harness/runner-results/_index.yaml +++ b/test-harness/runner-results/_index.yaml @@ -45,6 +45,16 @@ entries: meta: {type: test, suite: runner-results} source: file://results.lua method: conflicting_completion + - name: planned_error_status + kind: function.lua + meta: {type: test, suite: runner-results} + source: file://results.lua + method: planned_error_status + - name: replaced_plan + kind: function.lua + meta: {type: test, suite: runner-results} + source: file://results.lua + method: replaced_plan - name: empty_completion kind: function.lua meta: {type: test, suite: runner-results} diff --git a/test-harness/runner-results/results.lua b/test-harness/runner-results/results.lua index 2d95d312..aeea4b25 100644 --- a/test-harness/runner-results/results.lua +++ b/test-harness/runner-results/results.lua @@ -27,6 +27,17 @@ return { emit(args, "test:complete", {total = 1, passed = 0, failed = 1, skipped = 0}) return true end, + planned_error_status = function(args) + one_pass(args) + emit(args, "test:complete", {total = 1, passed = 1, failed = 0, skipped = 0}) + return {status = "error"} + end, + replaced_plan = function(args) + emit(args, "test:plan", {suites = {{tests = {"one", "two"}}}}) + one_pass(args) + emit(args, "test:complete", {total = 1, passed = 1, failed = 0, skipped = 0}) + return true + end, empty_completion = function(args) emit(args, "test:plan", {suites = {}}) emit(args, "test:complete", {}) diff --git a/test-harness/runner-results/verify.sh b/test-harness/runner-results/verify.sh index 200afbfc..b61e9792 100644 --- a/test-harness/runner-results/verify.sh +++ b/test-harness/runner-results/verify.sh @@ -17,6 +17,8 @@ done <<'SCENARIOS' returns_false|test returned false raises_error|expected execution failure error_status|test returned false +planned_error_status|test returned false +replaced_plan|test process reported more than one plan completion_without_plan|test process did not report a plan missing_completion|test process did not report completion conflicting_completion|test completion counts disagree diff --git a/test-harness/runner/runner_verdict_test.lua b/test-harness/runner/runner_verdict_test.lua index 7dcd1b30..29177cfc 100644 --- a/test-harness/runner/runner_verdict_test.lua +++ b/test-harness/runner/runner_verdict_test.lua @@ -87,6 +87,13 @@ local function define_tests() test.eq(err.message, "test process did not report completion") end) + test.it("rejects a replaced plan even when the counts agree", function() + local err = verdict.check({ declared = -1, observed = 1, + completed = { total = 1, passed = 1, failed = 0, skipped = 0 } }) + test.not_nil(err) + test.eq(err.message, "test process reported more than one plan") + end) + test.it("accepts all declared cases with a successful completion", function() local err = verdict.check({ declared = 2, observed = 2, completed = { total = 2, passed = 2, failed = 0, skipped = 0 } }) From 5f4d1e8a95a58a1d7ce3cd1202696a62042a2767 Mon Sep 17 00:00:00 2001 From: Rodrigo Delduca Date: Wed, 7 Oct 2026 16:53:39 -0300 Subject: [PATCH 4/8] perf(relay): cache validated command routes in the user hub Validated commands skip the type checks, the plugin prefix scan, and the topic substring on later messages. The cache holds at most 64 routes and is not part of the upgrade state. JSON payloads are decoded without a redundant string conversion. Relay roundtrip at size 256 against the original baseline, five matched rounds: median -5.0% to -6.8%, p95 -4.0% to -6.4%, allocations/op -7.7% to -8.1%. --- src/relay/src/user.lua | 58 ++++++++++++++++++--------- src/relay/test/relay_runtime_test.lua | 27 +++++++++++++ src/relay/test/user_test.lua | 4 ++ 3 files changed, 70 insertions(+), 19 deletions(-) diff --git a/src/relay/src/user.lua b/src/relay/src/user.lua index 28e6e79f..90e8bbcc 100644 --- a/src/relay/src/user.lua +++ b/src/relay/src/user.lua @@ -18,6 +18,12 @@ type PluginState = { status: string, } +type Route = { + prefix: string, + config: PluginConfig, + topic: string, +} + type UserState = { user_id: string, user_metadata: any, @@ -29,8 +35,12 @@ type UserState = { client_count: number, pg_scopes: {[string]: any}, pg_groups: {[string]: string}, + routes: {[any]: Route}, + route_count: number, } +local MAX_CACHED_ROUTES = 64 + type UserUpgradeState = { relay_user_upgrade: boolean, user_id: string, @@ -347,7 +357,7 @@ local function handle_client_leave(state: UserState, payload_data: any) end local function handle_client_message(state: UserState, payload: any, from_pid: string) - local message_data, err = json.decode(string(payload:data())) + local message_data, err = json.decode(payload:data()) if err then process.send(from_pid, consts.CLIENT_TOPICS.ERROR, { error = consts.ERROR_CODES.INVALID_JSON, @@ -364,26 +374,34 @@ local function handle_client_message(state: UserState, payload: any, from_pid: s return end - local msg_type = message_data.type :: string? - if type(msg_type) ~= "string" or msg_type == "" then - process.send(from_pid, consts.CLIENT_TOPICS.ERROR, { - error = consts.ERROR_CODES.UNKNOWN_COMMAND, - message = "Message type is required" - }) - return - end + local msg_type: any = message_data.type + local route = state.routes[msg_type] + if not route then + if type(msg_type) ~= "string" or msg_type == "" then + process.send(from_pid, consts.CLIENT_TOPICS.ERROR, { + error = consts.ERROR_CODES.UNKNOWN_COMMAND, + message = "Message type is required" + }) + return + end - local plugin_prefix, plugin_config = find_plugin_for_command(state, msg_type) - if not plugin_prefix or not plugin_config then - process.send(from_pid, consts.CLIENT_TOPICS.ERROR, { - error = consts.ERROR_CODES.PLUGIN_NOT_FOUND, - message = "No plugin found for command: " .. msg_type - }) - return + local plugin_prefix, plugin_config = find_plugin_for_command(state, msg_type) + if not plugin_prefix or not plugin_config then + process.send(from_pid, consts.CLIENT_TOPICS.ERROR, { + error = consts.ERROR_CODES.PLUGIN_NOT_FOUND, + message = "No plugin found for command: " .. msg_type + }) + return + end + + route = { prefix = plugin_prefix, config = plugin_config, topic = msg_type:sub(#plugin_prefix + 1) } + if state.route_count < MAX_CACHED_ROUTES then + state.routes[msg_type] = route + state.route_count = state.route_count + 1 + end end - local stripped_topic = msg_type:sub(#plugin_prefix + 1) - local success, route_err = route_to_plugin(state, plugin_prefix!, plugin_config!, stripped_topic, message_data, from_pid) + local success, route_err = route_to_plugin(state, route.prefix, route.config, route.topic, message_data, from_pid) if not success then process.send(from_pid, consts.CLIENT_TOPICS.ERROR, { error = consts.ERROR_CODES.PLUGIN_FAILED, @@ -479,7 +497,9 @@ local function run(args: any): any connected_clients = connected_clients, client_count = client_count, pg_scopes = {}, - pg_groups = {} + pg_groups = {}, + routes = {}, + route_count = 0, } local registry_name = consts.USER_HUB_REGISTRY_PREFIX .. state.user_id diff --git a/src/relay/test/relay_runtime_test.lua b/src/relay/test/relay_runtime_test.lua index ddacdbc5..1a9ae595 100644 --- a/src/relay/test/relay_runtime_test.lua +++ b/src/relay/test/relay_runtime_test.lua @@ -290,6 +290,33 @@ local function define_tests() test.eq(after.plugin_pid, plugin.pid) end) end) + test.it("routes commands beyond the route cache bound and keeps rejecting invalid commands", function() + with_hub("app:relay_runtime_plugin", function(_, _, join, send, response, errors, started) + join() + for index = 1, 80 do + send({ type = "runtime_echo_" .. index, request_id = tostring(index) }) + end + started() + local seen = {} + for _ = 1, 80 do + local value = receive(response) + local request_id = value.payload.request_id + test.eq(value.topic, "echo_" .. request_id) + test.is_nil(seen[request_id]) + seen[request_id] = true + end + send({ type = "runtime_echo_1", request_id = "cached" }) + local cached = receive(response) + test.eq(cached.topic, "echo_1") + test.eq(cached.payload.request_id, "cached") + send({ type = 42 }) + test.eq(receive(errors).error, consts.ERROR_CODES.UNKNOWN_COMMAND) + send({ type = "missing_echo" }) + test.eq(receive(errors).error, consts.ERROR_CODES.PLUGIN_NOT_FOUND) + send({ type = "runtime_echo_80", request_id = "uncached" }) + test.eq(receive(response).topic, "echo_80") + end) + end) test.it("rejects non-object JSON and non-string command types without stopping", function() with_hub("app:relay_runtime_plugin", function(pid, _, join, send, _, errors) join() diff --git a/src/relay/test/user_test.lua b/src/relay/test/user_test.lua index d31fd4f1..1b6b129e 100644 --- a/src/relay/test/user_test.lua +++ b/src/relay/test/user_test.lua @@ -62,9 +62,13 @@ local function define_tests() client_count = 1, pg_scopes = { ["app:scope"] = { opaque = true } }, pg_groups = { ["workspace.1"] = "app:scope" }, + routes = { session_message = { prefix = "session_", config = { prefix = "session_", + process_id = "app:session", auto_start = true }, topic = "message" } }, + route_count = 1, }) test.eq(snapshot.relay_user_upgrade, true) + test.is_nil((snapshot :: any).routes) test.eq(snapshot.active_plugins.session_.pid, "app:plugin:1") test.eq(snapshot.connected_clients["app:client:1"], true) test.eq(snapshot.pg_groups["workspace.1"], "app:scope") From 013377f89fdc06d0d5ad1d2e59bf7158bf19a9d0 Mon Sep 17 00:00:00 2001 From: Rodrigo Delduca Date: Wed, 7 Oct 2026 18:45:45 -0300 Subject: [PATCH 5/8] test: add unit tests for select cases, stale tool rounds and plan verdicts --- src/actor/test/actor_test.lua | 63 +++++++++++++++++++++ src/agent/test/tools/caller_test.lua | 35 ++++++++++++ test-harness/runner/runner_verdict_test.lua | 13 +++++ 3 files changed, 111 insertions(+) diff --git a/src/actor/test/actor_test.lua b/src/actor/test/actor_test.lua index 43284fbd..1666e96b 100644 --- a/src/actor/test/actor_test.lua +++ b/src/actor/test/actor_test.lua @@ -599,6 +599,69 @@ local function define_tests() end) end) + test.describe("select case maintenance", function() + local function run_with_select_counts(responses: {any}, init: (any) -> ()): any + local counts: any = { tables = {} } + local original_channel = _G.channel + local select_count = 0 + _G.channel = { + new = function() return mock_channel() end, + select = function(cases) + select_count = select_count + 1 + counts[select_count] = #cases + counts.tables[select_count] = cases + return responses[select_count] or { ok = false } + end + } + actor._process = mock_process(mock_channel(), mock_channel()) + local ok, err = pcall(function() + actor.new({}, { __init = function(state) init(state) end }).run() + end) + _G.channel = original_channel + if not ok then error(err) end + return counts + end + + test.it("adds and removes select cases as channels register and unregister", function() + local first, second = mock_channel(), mock_channel() + local counts = run_with_select_counts({}, function(state) + state.register_channel(first, function() end) + state.register_channel(second, function() end) + state.unregister_channel(first) + end) + test.eq(counts[1], 4) + end) + + test.it("removes a closed registered channel from the next select", function() + local chan = mock_channel() + local counts = run_with_select_counts({ + { ok = false, channel = chan }, + }, function(state) + state.register_channel(chan, function() end) + end) + test.eq(counts[1], 4) + test.eq(counts[2], 3) + test.is_true(rawequal(counts.tables[1], counts.tables[2])) + end) + + test.it("selects a channel registered after a caught error", function() + local chan = mock_channel() + local counts = run_with_select_counts({}, function(state) + pcall(error, "caught") + state.register_channel(chan, function() end) + end) + test.eq(counts[1], 4) + end) + + test.it("does not report an unknown channel as unregistered", function() + local result + run_with_select_counts({}, function(state) + result = state.unregister_channel(mock_channel()) + end) + test.is_false(result) + end) + end) + test.describe("async execution", function() test.it("state.async passes function to coroutine.spawn", function() local spawn_called = false diff --git a/src/agent/test/tools/caller_test.lua b/src/agent/test/tools/caller_test.lua index 73ca65b5..e1094c1f 100644 --- a/src/agent/test/tools/caller_test.lua +++ b/src/agent/test/tools/caller_test.lua @@ -756,6 +756,41 @@ local function define_tests() test.eq(count_table_elements(validated_tools), 0) end) + it("should discard the previous round state when validating no tool calls", function() + for _, empty in ipairs({ {}, false }) do + local caller = tool_caller.new() + caller.wrapper_observations = { { tool = "old" } } + caller.wrapper_metadata = { { tool = "old" } } + caller.wrapper_errors = { { error = "old" } } + caller.wrapper_controls = { { type = "stop" } } + caller.last_tool_calls = { { id = "call-old", name = "old" } } + + local validated_tools, err = caller:validate(empty or nil) + + test.is_nil(err) + test.eq(count_table_elements(validated_tools), 0) + test.eq(#caller:get_wrapper_observations(), 0) + test.eq(#caller:get_wrapper_metadata(), 0) + test.eq(#caller:get_wrapper_errors(), 0) + test.eq(#caller:get_wrapper_controls(), 0) + test.eq(#caller:get_last_tool_calls(), 0) + end + end) + + it("should keep empty round state tables when validating no tool calls", function() + local caller = tool_caller.new() + local observations, metadata = caller.wrapper_observations, caller.wrapper_metadata + local errors, controls, calls = caller.wrapper_errors, caller.wrapper_controls, caller.last_tool_calls + + caller:validate({}) + + test.is_true(rawequal(caller.wrapper_observations, observations)) + test.is_true(rawequal(caller.wrapper_metadata, metadata)) + test.is_true(rawequal(caller.wrapper_errors, errors)) + test.is_true(rawequal(caller.wrapper_controls, controls)) + test.is_true(rawequal(caller.last_tool_calls, calls)) + end) + it("should handle tool schema errors", function() local caller = tool_caller.new() diff --git a/test-harness/runner/runner_verdict_test.lua b/test-harness/runner/runner_verdict_test.lua index 29177cfc..80f8f9ab 100644 --- a/test-harness/runner/runner_verdict_test.lua +++ b/test-harness/runner/runner_verdict_test.lua @@ -94,6 +94,19 @@ local function define_tests() test.eq(err.message, "test process reported more than one plan") end) + test.it("required suites count failed cases as executed", function() + test.is_nil(verdict.check({require_cases = true, declared = 1, observed = 1, + cases = {passed = 0, failed = 1, skipped = 0}, + completed = {total = 1, passed = 0, failed = 1, skipped = 0}})) + end) + + test.it("reports a replaced plan before a missing required case", function() + local err = verdict.check({ require_cases = true, declared = -1, observed = 0, + completed = { total = 0, passed = 0, failed = 0, skipped = 0 } }) + test.not_nil(err) + test.eq(err.message, "test process reported more than one plan") + end) + test.it("accepts all declared cases with a successful completion", function() local err = verdict.check({ declared = 2, observed = 2, completed = { total = 2, passed = 2, failed = 0, skipped = 0 } }) From ab8c45a995149b3542f72b706625b85f42c7fc3b Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Sun, 11 Oct 2026 15:08:23 -0400 Subject: [PATCH 6/8] fix(test): isolate benchmark filesystem from shared module --- .github/workflows/ci.yml | 2 ++ Makefile | 19 +++++++++++------- README.md | 4 ++++ scripts/run-wippy.sh | 12 +++++++++++ src/test/_index.yaml | 18 +---------------- src/test/self_test.lua | 8 ++++++++ test-harness/benchmark/_index.yaml | 20 +++++++++++++++++++ .../benchmark}/benchmark_test.lua | 1 + test-harness/benchmark/output/_index.yaml | 8 ++++++++ test-harness/benchmark/wippy.lock | 10 ++++++++++ 10 files changed, 78 insertions(+), 24 deletions(-) create mode 100644 test-harness/benchmark/_index.yaml rename {src/test => test-harness/benchmark}/benchmark_test.lua (99%) create mode 100644 test-harness/benchmark/output/_index.yaml create mode 100644 test-harness/benchmark/wippy.lock diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 12978b05..0b17f106 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -61,6 +61,8 @@ jobs: run: | (cd src/test && wippy install && wippy lint --level error && wippy test) (cd test-harness/runner && wippy test) + bash scripts/run-wippy.sh test-harness/benchmark lint --level error + make test-benchmarks bash test-harness/runner-results/verify.sh bash test-harness/runner-hook-failure/verify.sh bash test-harness/runner-timeout/verify-timeouts.sh diff --git a/Makefile b/Makefile index 165aadb9..c500d0bb 100644 --- a/Makefile +++ b/Makefile @@ -11,8 +11,9 @@ BENCH_EXACT_ALLOCATIONS ?= 1 BENCH_REVISION ?= $(shell git rev-parse HEAD) BENCH_PROFILER_FLAG = $(if $(filter 1,$(BENCH_EXACT_ALLOCATIONS)),--profiler) RUN_WIPPY = WIPPY="$(WIPPY)" bash scripts/run-wippy.sh +RUN_BENCHMARK = WIPPY="$(WIPPY)" bash scripts/run-wippy.sh --benchmark -.PHONY: help check-manifests run-tests run-lint install test-runtime bench +.PHONY: help check-manifests run-tests run-lint install test-runtime test-benchmarks bench help: @echo "Wippy Framework" @@ -23,6 +24,7 @@ help: @echo " make check-manifests Validate package and test-app module types" @echo " make install Install dependencies for all test modules" @echo " make test-runtime Test real process and control delivery" + @echo " make test-benchmarks Test benchmark reports in their isolated harness" @echo " make bench Write Lua operation benchmarks outside the repository" test-runtime: @@ -36,6 +38,9 @@ test-runtime: -o wippy.relay:user_security_scope:default=app:user test -- relay_runtime_test WIPPY_TEST_REQUIRE_CASES=1 $(RUN_WIPPY) test-harness/runner test test -- aggregate_checks_test +test-benchmarks: + WIPPY_TEST_REQUIRE_CASES=1 $(RUN_WIPPY) test-harness/benchmark test test + bench: @set -eu; \ output="$(abspath $(BENCH_OUTPUT_DIR))"; \ @@ -54,21 +59,21 @@ bench: for name in actor_roundtrip actor_lifecycle actor_resident tools_sequential tools_parallel relay_roundtrip relay_pipeline; do \ rm -f "$$output/$$name-$$size.json"; \ done; \ - $(RUN_WIPPY) src/actor/test test -c $(BENCH_PROFILER_FLAG) \ + $(RUN_BENCHMARK) src/actor/test test -c $(BENCH_PROFILER_FLAG) \ -o "wippy.test:benchmark_output:directory=$$output" test -- actor_runtime_benchmark; \ - $(RUN_WIPPY) src/actor/test test -c $(BENCH_PROFILER_FLAG) \ + $(RUN_BENCHMARK) src/actor/test test -c $(BENCH_PROFILER_FLAG) \ -o "wippy.test:benchmark_output:directory=$$output" test -- actor_lifecycle_benchmark; \ - $(RUN_WIPPY) src/actor/test test -c $(BENCH_PROFILER_FLAG) \ + $(RUN_BENCHMARK) src/actor/test test -c $(BENCH_PROFILER_FLAG) \ -o "wippy.test:benchmark_output:directory=$$output" test -- actor_resident_benchmark; \ - $(RUN_WIPPY) src/agent/test test -c $(BENCH_PROFILER_FLAG) \ + $(RUN_BENCHMARK) src/agent/test test -c $(BENCH_PROFILER_FLAG) \ -o "wippy.test:benchmark_output:directory=$$output" \ -o wippy.llm:process_host:default=wippy.terminal:host \ -o wippy.llm:env_storage:default=app:env_storage test -- tools_controls_benchmark; \ - $(RUN_WIPPY) src/relay/test test -c $(BENCH_PROFILER_FLAG) \ + $(RUN_BENCHMARK) src/relay/test test -c $(BENCH_PROFILER_FLAG) \ -o "wippy.test:benchmark_output:directory=$$output" \ -o wippy.relay:application_host:default=app:processes \ -o wippy.relay:user_security_scope:default=app:user test -- relay_runtime_benchmark; \ - $(RUN_WIPPY) src/relay/test test -c $(BENCH_PROFILER_FLAG) \ + $(RUN_BENCHMARK) src/relay/test test -c $(BENCH_PROFILER_FLAG) \ -o "wippy.test:benchmark_output:directory=$$output" \ -o wippy.relay:application_host:default=app:processes \ -o wippy.relay:user_security_scope:default=app:user test -- relay_pipeline_benchmark; \ diff --git a/README.md b/README.md index d512d951..97070295 100644 --- a/README.md +++ b/README.md @@ -83,6 +83,8 @@ and configuration management — enabling developers to build sophisticated mult `make test-runtime` runs Lua tests for actor messages, relay delivery, and tool controls through Wippy. It uses the local component and test-runner sources. Required suites fail if they execute no cases. +Ordinary tests do not register a benchmark filesystem. `make test-benchmarks` runs report and +GC-setting checks in a separate harness; its filesystem opens an existing directory without creating one. `make bench` runs local Lua workloads with 100 warmups and 100 samples at sizes 1, 8, and 32. Reports include latency, throughput, heap use, exact allocation counts, and source/runtime metadata. @@ -96,6 +98,8 @@ Use `BENCH_WARMUP`, `BENCH_SAMPLES`, and the space-separated `BENCH_SIZES` to ch Use `BENCH_MEMORY_OPERATIONS` to set the separate memory workload size. The native profiler uses the loopback interface on port 6060; keep that port free during local benchmarks. Test data and benchmark reports must stay outside the repository. +Only `make bench` adds the benchmark output resource to its isolated workload sources and explicitly +creates the requested report directory. The shared `wippy/test` module never creates that directory. [module-actor]: https://github.com/wippyai/module-actor [module-agent]: https://github.com/wippyai/module-agent diff --git a/scripts/run-wippy.sh b/scripts/run-wippy.sh index d944d54d..020d080f 100644 --- a/scripts/run-wippy.sh +++ b/scripts/run-wippy.sh @@ -3,6 +3,11 @@ set -euo pipefail repo=$(cd "$(dirname "$0")/.." && pwd -P) source_root=$(cd "${FRAMEWORK_SOURCE_ROOT:-$repo}" && pwd -P) +benchmark=false +if [ "${1:-}" = --benchmark ]; then + benchmark=true + shift +fi app=$1 shift wippy=$(command -v "${WIPPY:-wippy}") @@ -19,6 +24,13 @@ work="$work_root/${app//\//-}" mkdir -p "$work" source_path=$(awk '$1 == "src:" {print $2; exit}' "$app_root/wippy.lock") source_path=$(cd "$app_root/${source_path:-.}" && pwd) +if "$benchmark"; then + benchmark_source=$(mktemp -d "$work/benchmark-source.XXXXXX") + cp -R "$source_path/." "$benchmark_source/" + mkdir -p "$benchmark_source/benchmark-output" + cp "$source_root/test-harness/benchmark/output/_index.yaml" "$benchmark_source/benchmark-output/_index.yaml" + source_path=$benchmark_source +fi { printf 'directories:\n modules: %s\n src: %s\n' \ "$(jq -Rn --arg value "$work/modules" '$value')" \ diff --git a/src/test/_index.yaml b/src/test/_index.yaml index 7782331b..eb35ae55 100644 --- a/src/test/_index.yaml +++ b/src/test/_index.yaml @@ -142,14 +142,10 @@ entries: comment: Self-test for the test framework source: file://self_test.lua method: run + modules: [registry] imports: test: wippy.test:test - - name: benchmark_output - kind: fs.directory - directory: /tmp/wippy-benchmarks - auto_init: true - - name: benchmark_revision kind: env.variable variable: WIPPY_BENCH_REVISION @@ -174,15 +170,3 @@ entries: kind: library.lua source: file://benchmark.lua modules: [fs, json, env, io, system, http_client] - - - name: benchmark_test - kind: function.lua - meta: - type: test - suite: test-framework - source: file://benchmark_test.lua - method: run - modules: [fs, json, system, env] - imports: - benchmark: wippy.test:benchmark - test: wippy.test:test diff --git a/src/test/self_test.lua b/src/test/self_test.lua index 2a9ee838..d9b837a4 100644 --- a/src/test/self_test.lua +++ b/src/test/self_test.lua @@ -1,7 +1,15 @@ -- Self-test for the test framework local test = require("test") +local registry = require("registry") local function define_tests() + test.describe("shared test resources", function() + test.it("does not register a benchmark filesystem for ordinary tests", function() + local output = registry.get("wippy.test:benchmark_output") + test.is_nil(output, "benchmark filesystems belong to the explicit benchmark harness") + end) + end) + -- Hook bookkeeping for the "suite hooks" suites below: a suite's before_all -- and after_all run once around the suite and all of its descendants. local hooks = { outer_before = 0, outer_after = 0, grandchild_ran = false } diff --git a/test-harness/benchmark/_index.yaml b/test-harness/benchmark/_index.yaml new file mode 100644 index 00000000..cd290c73 --- /dev/null +++ b/test-harness/benchmark/_index.yaml @@ -0,0 +1,20 @@ +version: "1.0" +namespace: app + +entries: + - name: test_dep + kind: ns.dependency + component: wippy/test + version: "*" + + - name: benchmark_test + kind: function.lua + meta: + type: test + suite: benchmark-reports + source: file://benchmark_test.lua + method: run + modules: [fs, json, system, env] + imports: + benchmark: wippy.test:benchmark + test: wippy.test:test diff --git a/src/test/benchmark_test.lua b/test-harness/benchmark/benchmark_test.lua similarity index 99% rename from src/test/benchmark_test.lua rename to test-harness/benchmark/benchmark_test.lua index f0f4a34a..f1d03ce7 100644 --- a/src/test/benchmark_test.lua +++ b/test-harness/benchmark/benchmark_test.lua @@ -1,3 +1,4 @@ +-- Filesystem and process-wide GC checks run only in this explicit harness. local test = require("test") local benchmark = require("benchmark") local fs = require("fs") diff --git a/test-harness/benchmark/output/_index.yaml b/test-harness/benchmark/output/_index.yaml new file mode 100644 index 00000000..e603b44a --- /dev/null +++ b/test-harness/benchmark/output/_index.yaml @@ -0,0 +1,8 @@ +version: "1.0" +namespace: wippy.test + +entries: + - name: benchmark_output + kind: fs.directory + directory: . + auto_init: false diff --git a/test-harness/benchmark/wippy.lock b/test-harness/benchmark/wippy.lock new file mode 100644 index 00000000..b15c5ddc --- /dev/null +++ b/test-harness/benchmark/wippy.lock @@ -0,0 +1,10 @@ +directories: + modules: .wippy + src: . +modules: + - name: wippy/terminal + version: 0.4.2 + hash: 34b99aac2eba9c5016fd02c783117fb2b0381ce14696c7b199265458b99a3271 + - name: wippy/test + version: 0.4.18 + hash: 048d88d2748ec6901fba703ed1317c27f7027650735f653a51d38c07741e1b99 From 6a712456e65433530eab8e95b3ab40098593b0ef Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Mon, 12 Oct 2026 13:58:26 -0400 Subject: [PATCH 7/8] docs(agent): describe proposal delivery as a host contract --- src/agent/src/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/agent/src/README.md b/src/agent/src/README.md index 9f897964..6e3af977 100644 --- a/src/agent/src/README.md +++ b/src/agent/src/README.md @@ -394,9 +394,9 @@ control handlers and apply proposals only after the round's outcomes settle. persistence. Returned `context` without `_control` is not a host configuration change. Legacy bindings do not acquire behavior controls implicitly. -Hosts collect these proposals from the lifecycle summary (`controls`) and from -`caller:get_wrapper_controls()`. `wippy/session` does not consume them yet. A -host that persists proposals with the round and replays them after recovery +Hosts explicitly collect these proposals from the lifecycle summary (`controls`) +and from `caller:get_wrapper_controls()`. A host that persists proposals with +the round and replays them after recovery gives at-least-once delivery, not exactly-once external writes; providers must make writes and lifecycle hooks idempotent using stable host refs. `memory.compact = true` requests checkpointing without a token threshold, but From fb43c36ea1b75e017c0fd44573363beb40dace01 Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Mon, 12 Oct 2026 14:12:18 -0400 Subject: [PATCH 8/8] fix(relay): derive restored client count from memberships MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cover valid, inflated and undercounted restored counters without changing upgrade payloads. Align the verification harness with CI’s Go 1.26 baseline; the same race and vet checks pass on Go 1.26.5. --- src/relay/src/user.lua | 6 +++++- src/relay/test/relay_runtime_test.lua | 18 +++++++++++++++++- test-harness/go/go.mod | 2 +- 3 files changed, 23 insertions(+), 3 deletions(-) diff --git a/src/relay/src/user.lua b/src/relay/src/user.lua index 90e8bbcc..ef10266e 100644 --- a/src/relay/src/user.lua +++ b/src/relay/src/user.lua @@ -480,7 +480,11 @@ local function run(args: any): any local resume_state = args :: UserUpgradeState active_plugins = resume_state.active_plugins connected_clients = resume_state.connected_clients - client_count = resume_state.client_count + -- Older duplicate JOINs could inflate the saved counter. Memberships + -- are authoritative; rebuilding the derived count keeps final LEAVE valid. + for _ in pairs(connected_clients) do + client_count = client_count + 1 + end upgrade_groups = resume_state.pg_groups end local user_id = string(args.user_id) diff --git a/src/relay/test/relay_runtime_test.lua b/src/relay/test/relay_runtime_test.lua index fccdeccf..6ef0d631 100644 --- a/src/relay/test/relay_runtime_test.lua +++ b/src/relay/test/relay_runtime_test.lua @@ -19,7 +19,7 @@ local function receive(ch: any): any return result.value end -local function with_hub(plugin_id: string, body: any, active_pid: string?, session_pid: string?): any +local function with_hub(plugin_id: string, body: any, active_pid: string?, session_pid: string?, saved_client_count: number?): any sequence = sequence + 1 local user_id = "relay-runtime-" .. process.pid() .. "-" .. sequence local actor = security.new_actor(user_id, {}) @@ -50,6 +50,10 @@ local function with_hub(plugin_id: string, body: any, active_pid: string?, sessi end args.connected_clients = {} args.client_count = 0 + if saved_client_count ~= nil then + args.connected_clients[process.pid()] = true + args.client_count = saved_client_count + end args.pg_groups = {} args = (json.encode(args)) end @@ -228,6 +232,18 @@ local function define_tests() process.unlisten(shutdown) if not ok then error(err) end end) + for _, saved_count in ipairs({0, 1, 3}) do + test.it("restores the membership count before the final LEAVE, saved count " .. saved_count, function() + local shutdown = process.listen("shutdown") + local ok, err = pcall(with_hub, "app:relay_runtime_plugin", function(pid, _, join) + test.eq(join().client_count, 1, "Restored memberships are the source of the client count") + assert(process.send(pid, consts.WS_TOPICS.LEAVE, { client_pid = process.pid() })) + receive(shutdown) + end, nil, process.pid(), saved_count) + process.unlisten(shutdown) + if not ok then error(err) end + end) + end test.it("reports a real plugin startup failure and keeps the hub responsive", function() with_hub("app:missing_runtime_plugin", function(_, _, join, send, _, errors) join() diff --git a/test-harness/go/go.mod b/test-harness/go/go.mod index 62cc21a0..882d2926 100644 --- a/test-harness/go/go.mod +++ b/test-harness/go/go.mod @@ -1,5 +1,5 @@ module github.com/wippyai/framework/test-harness -go 1.27.0 +go 1.26.0 require github.com/wippyai/go-lua v1.6.2