From 3faf97f6314ee9f1663a27641e3d56640318ed90 Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Wed, 7 Oct 2026 18:20:27 -0400 Subject: [PATCH 1/2] feat(llm): support OpenAI Decisions evaluation --- src/llm/src/README.md | 78 +++++- src/llm/src/openai/_index.yaml | 32 ++- src/llm/src/openai/client.lua | 8 + src/llm/src/openai/decisions_mapper.lua | 260 ++++++++++++++++++ src/llm/src/openai/evaluate.lua | 39 +++ src/llm/test/_index.yaml | 47 ++++ src/llm/test/llm_test.lua | 64 +++++ src/llm/test/openai/client_test.lua | 17 ++ .../openai/decisions_integration_test.lua | 71 +++++ src/llm/test/openai/decisions_mapper_test.lua | 257 +++++++++++++++++ src/llm/test/openai/evaluate_test.lua | 137 +++++++++ 11 files changed, 1007 insertions(+), 3 deletions(-) create mode 100644 src/llm/src/openai/decisions_mapper.lua create mode 100644 src/llm/src/openai/evaluate.lua create mode 100644 src/llm/test/openai/decisions_integration_test.lua create mode 100644 src/llm/test/openai/decisions_mapper_test.lua create mode 100644 src/llm/test/openai/evaluate_test.lua diff --git a/src/llm/src/README.md b/src/llm/src/README.md index 34328196..151662dd 100644 --- a/src/llm/src/README.md +++ b/src/llm/src/README.md @@ -332,6 +332,80 @@ entries: The module ships the `wippy.llm.typesafe:provider` entry bound to its driver and credential variables. Driver env vars: `TYPESAFE_API_KEY`, `TYPESAFE_BASE_URL` (default `https://api.typesafe.ai/v1`), `TYPESAFE_TIMEOUT`. +### OpenAI Decisions + +The native OpenAI provider also implements `evaluate` through +[`POST /v1/decisions`](https://developers.openai.com/api/docs/guides/decisions). +It uses the same questions and readings as the TypeSafe evaluator. A direct +call uses your OpenAI credentials and skips model discovery: + +```lua +local result, err = llm.evaluate("Export fails in Safari but works in Chrome.", { + severity = { + type = "score", + instructions = "How severe is this issue?", + domain = { + "Cosmetic: appearance only; no lost functionality", + "Workaround available: a task fails, but another way works", + "Fully blocked: a task fails with no workaround" + } + } +}, { + model = "gpt-6-luna", + provider_id = "wippy.llm.openai:provider" +}) +if err then + return nil, err +end +local severity = result.result.severity +-- severity.score is the expected 1-based level index. +-- severity.level is the highest-probability 1-based level index. +-- severity.probabilities preserves the distribution over all three levels. +``` + +For model discovery, register a model with `class: [evaluate]`, +`capabilities: [evaluate]`, and a provider mapping to +`wippy.llm.openai:provider` with `provider_model: gpt-6-luna`, following the +registration example above. Credentials and transport settings are shared with +the native OpenAI provider: `OPENAI_API_KEY`, `OPENAI_BASE_URL`, +`OPENAI_ORGANIZATION`, and `OPENAI_TIMEOUT`. The base URL defaults to +`https://api.openai.com/v1`; evaluation sends a nonstreaming `/decisions` request. +Per-call options support `timeout`, `retry`, and an explicitly supplied +`safety_identifier` of at most 128 bytes. The actor ID used by Wippy's usage +tracking is not automatically sent as a safety identifier. Generation options +such as temperature, tools, and streaming are rejected for evaluation. + +The adapter sends a string state as text and serializes a table state as JSON +text. Table instructions are also serialized as JSON text. Slot keys stay local; +the request uses generated question names and maps answers back to your keys. +Predicate `domain.yes` and `domain.no` descriptions are included in the question +instructions. Choice domains support between 2 and 255 string options. Score +domains retain their declaration order: OpenAI's zero-based expected score is +shifted by one to match Wippy's existing reading contract. Confidence remains a +provider-specific statistic; choose thresholds using labeled examples from your +own application. + +Every declared question must receive a valid answer. A refusal fails the whole +evaluation with a `content_filtered` provider error; malformed or incomplete +answers fail with a `model_error`. The adapter never returns partial readings as +a successful result. Independent questions can share one request; a question +that depends on another answer needs a later call. + +This adapter supports the existing text/JSON evaluation contract. Images inside +a JSON table are treated as text; image evaluation needs a separate typed input +extension. The Decisions API is currently in public beta and the official guide +lists `gpt-6-luna` as its supported model. OpenAI bills this endpoint for input +tokens only; reported token usage is preserved for accounting. Availability, +pricing, and limits should be checked in the official guide before rollout. +Configure usage-tracker pricing for the Decisions endpoint when registering an +evaluation model; generation prices for the same model name can differ. The raw +provider usage remains in `result.metadata.usage`, while normalized token +categories keep uncached input, cache reads, and cache writes disjoint. + +The optional live test runs only with `ENABLE_DECISIONS_INTEGRATION_TESTS=true` +and valid OpenAI credentials. The regular suite uses fixtures and mocked HTTP; +it does not require a paid API call. + ## Streaming ```lua @@ -537,7 +611,7 @@ entries: - `wippy.llm.claude` - Anthropic Claude (direct API) - `wippy.llm.bedrock` - AWS Bedrock (Converse API for text generation, InvokeModel for embeddings) -- `wippy.llm.openai` - OpenAI native via the Responses API (`/v1/responses`) — GPT-5.x, o-series, encrypted reasoning persistence, `xhigh`/`minimal` reasoning effort. Use this for `api.openai.com`. +- `wippy.llm.openai` - OpenAI native via the Responses API (`/v1/responses`) — GPT-5.x, o-series, encrypted reasoning persistence, `xhigh`/`minimal` reasoning effort; also supports typed evaluation via `/v1/decisions`. Use this for `api.openai.com`. - `wippy.llm.openai_compat` - OpenAI-compatible Chat Completions (`/v1/chat/completions`) — Ollama, vLLM, llama.cpp, LM Studio, OpenRouter, Together, Groq, Fireworks, DeepInfra, Mistral, DeepSeek, etc. Use this for any non-OpenAI backend that exposes a `/chat/completions` endpoint. - `wippy.llm.google.vertex` - Google Vertex AI - `wippy.llm.google.generative_ai` - Google Generative AI (Gemini) @@ -593,7 +667,7 @@ In ECS/EKS pods, AWS credentials are resolved automatically from the container m - `wippy.llm.claude` - Claude provider (direct API) - `wippy.llm.bedrock` - AWS Bedrock provider -- `wippy.llm.openai` - OpenAI native (Responses API) +- `wippy.llm.openai` - OpenAI native (Responses, embeddings, and Decisions evaluation) - `wippy.llm.openai_compat` - OpenAI-compatible (Chat Completions) for Ollama / vLLM / OpenRouter / Together / Groq / etc. - `wippy.llm.google` - Google providers (Vertex AI, Generative AI) - `wippy.llm.typesafe` - TypeSafe Jev evaluation provider diff --git a/src/llm/src/openai/_index.yaml b/src/llm/src/openai/_index.yaml index 8ea7a898..32b27c99 100644 --- a/src/llm/src/openai/_index.yaml +++ b/src/llm/src/openai/_index.yaml @@ -58,6 +58,9 @@ entries: - contract: wippy.llm:structured_output methods: structured_output: wippy.llm.openai:structured_output + - contract: wippy.llm:evaluator + methods: + evaluate: wippy.llm.openai:evaluate - contract: wippy.llm:provider methods: status: wippy.llm.openai:status @@ -98,6 +101,33 @@ entries: pool: max_size: 50 + # wippy.llm.openai:evaluate + - name: evaluate + kind: function.lua + meta: + comment: Typed predicate, choice and score evaluation via the OpenAI Decisions API + source: file://evaluate.lua + imports: + openai_client: wippy.llm.openai:client + openai_mapper: wippy.llm.openai:mapper + decisions_mapper: wippy.llm.openai:decisions_mapper + output: wippy.llm:output + method: handler + pool: + max_size: 50 + + # wippy.llm.openai:decisions_mapper + - name: decisions_mapper + kind: library.lua + meta: + comment: Mapper between the evaluator contract and OpenAI Decisions questions, readings and usage + source: file://decisions_mapper.lua + modules: + - json + imports: + evaluation: wippy.llm:evaluation + output: wippy.llm:output + # wippy.llm.openai:generate - name: generate kind: function.lua @@ -141,7 +171,7 @@ entries: name: openai type: llm.provider title: OpenAI - comment: OpenAI provider via /v1/responses for GPT/o-series models, plus embeddings + comment: OpenAI provider via /v1/responses for GPT/o-series models, embeddings, and typed evaluation via /v1/decisions tags: - llm - provider diff --git a/src/llm/src/openai/client.lua b/src/llm/src/openai/client.lua index 58286e8b..19abe5be 100644 --- a/src/llm/src/openai/client.lua +++ b/src/llm/src/openai/client.lua @@ -182,6 +182,14 @@ function openai_client.request(endpoint_path, payload, options) } end + if type(parsed) ~= "table" then + return nil, { + status_code = response.status_code, + message = "Invalid OpenAI response: expected a JSON object", + metadata = extract_response_metadata(response) + } + end + parsed.metadata = extract_response_metadata(response) return parsed end diff --git a/src/llm/src/openai/decisions_mapper.lua b/src/llm/src/openai/decisions_mapper.lua new file mode 100644 index 00000000..cd67a1c8 --- /dev/null +++ b/src/llm/src/openai/decisions_mapper.lua @@ -0,0 +1,260 @@ +local json = require("json") +local evaluation = require("evaluation") +local output = require("output") + +local decisions_mapper = {} + +local function sorted_keys(value): {string} + local keys: {string} = {} + for key in pairs(value) do keys[#keys + 1] = key end + table.sort(keys) + return keys +end + +local function dense_array(value): number? + if type(value) ~= "table" then return nil end + local count = 0 + for key in pairs(value) do + if type(key) ~= "number" or key < 1 or key % 1 ~= 0 then return nil end + count = count + 1 + end + for index = 1, count do + if value[index] == nil then return nil end + end + return count +end + +local function as_text(value): (string?, string?) + if type(value) == "string" then return value, nil end + local encoded, err = json.encode(value) + if err then return nil, "Failed to encode JSON input: " .. tostring(err) end + return encoded, nil +end + +-- Only safety_identifier is an inference option. These other options belong +-- to the facade's actor and usage tracking, and never reach the provider. +local facade_options = { user = true, timestamp = true, metadata = true } +local contract_fields = { + state = true, questions = true, model = true, options = true, + timeout = true, retry = true, _provider_id = true +} + +function decisions_mapper.map_request(args: any): (any?, string?) + if type(args) ~= "table" then return nil, "Evaluation arguments must be a table" end + local input_err = evaluation.validate(args.state, args.questions, args.model) + if input_err then return nil, input_err end + for key in pairs(args) do + if not contract_fields[key] then + return nil, "OpenAI Decisions does not support argument '" .. tostring(key) .. "'" + end + end + if args.options ~= nil and type(args.options) ~= "table" then + return nil, "OpenAI Decisions options must be a table" + end + local options = args.options or {} + for key in pairs(options) do + if key ~= "safety_identifier" and not facade_options[key] then + return nil, "OpenAI Decisions does not support option '" .. tostring(key) .. "'" + end + end + if options.safety_identifier ~= nil then + if type(options.safety_identifier) ~= "string" or #options.safety_identifier > 128 then + return nil, "OpenAI Decisions safety_identifier must be a string of at most 128 bytes" + end + end + + local input, text_err = as_text(args.state) + if text_err then return nil, text_err end + local questions: {any} = {} + for index, key in ipairs(sorted_keys(args.questions)) do + local slot = args.questions[key] + local instructions, instructions_err = as_text(slot.instructions) + if instructions_err then return nil, instructions_err end + -- Caller keys are identifiers, not evidence or instructions for the model. + local question: any = { type = slot.type, name = "q" .. tostring(index), instructions = instructions } + if slot.type == "choice" then + local choices: {any} = {} + if #slot.domain > 0 then + for _, value in ipairs(slot.domain) do choices[#choices + 1] = { value = value } end + else + for _, value in ipairs(sorted_keys(slot.domain)) do + choices[#choices + 1] = { value = value, description = slot.domain[value] } + end + end + if #choices > 255 then return nil, "OpenAI Decisions choice domain supports at most 255 options in slot: " .. key end + question.choices = choices + elseif slot.type == "score" then + local levels: {any} = {} + for level, description in ipairs(slot.domain) do + levels[level] = { label = "level_" .. tostring(level - 1), description = description } + end + question.levels = levels + elseif slot.domain and next(slot.domain) ~= nil then + -- Decisions predicates have no separate domain field. + local criteria = "\n\nOutcome criteria:" + if slot.domain.yes ~= nil then criteria = criteria .. "\nTrue: " .. slot.domain.yes end + if slot.domain.no ~= nil then criteria = criteria .. "\nFalse: " .. slot.domain.no end + question.instructions = tostring(instructions) .. criteria + end + questions[index] = question + end + return { model = args.model, input = input, questions = questions, safety_identifier = options.safety_identifier } +end + +local function probability(value): boolean + return type(value) == "number" and value == value and value >= 0 and value <= 1 +end + +local function normalized(total: number, count: number): boolean + return math.abs(total - 1) <= math.max(0.002, count * 0.00005 + 0.0001) +end + +local function read_distribution(key: string, slot: any, answer: any): (any?, string?) + if not probability(answer.confidence) then return nil, "slot '" .. key .. "' answer carries an invalid confidence" end + local count = dense_array(answer.probabilities) + if not count then return nil, "slot '" .. key .. "' answer carries no probability array" end + local probabilities: any = {} + local expected_count = 0 + if slot.type == "choice" then + if #slot.domain > 0 then + for _, value in ipairs(slot.domain) do probabilities[value] = false; expected_count = expected_count + 1 end + else + for value in pairs(slot.domain) do probabilities[value] = false; expected_count = expected_count + 1 end + end + else + expected_count = #slot.domain + for index = 1, expected_count do probabilities[index] = false end + end + if count ~= expected_count then return nil, "slot '" .. key .. "' probability array does not cover its domain" end + local total = 0 + local expected = 0 + local highest = -1 + for _, entry in ipairs(answer.probabilities) do + if type(entry) ~= "table" or not probability(entry.probability) then + return nil, "slot '" .. key .. "' answer carries an invalid probability" + end + local value = entry.value + local target: any = value + if slot.type == "choice" then + if type(value) ~= "string" then return nil, "slot '" .. key .. "' probability has an invalid choice value" end + else + if type(value) ~= "number" or value % 1 ~= 0 or value < 0 or value >= expected_count then + return nil, "slot '" .. key .. "' probability has an invalid score level" + end + if entry.label ~= "level_" .. tostring(value) then return nil, "slot '" .. key .. "' probability has an unexpected level label" end + target = value + 1 + expected = expected + value * entry.probability + end + if probabilities[target] ~= false then return nil, "slot '" .. key .. "' probabilities contain an unknown or duplicate value" end + probabilities[target] = entry.probability + total = total + entry.probability + if entry.probability > highest then highest = entry.probability end + end + if not normalized(total, count) then return nil, "slot '" .. key .. "' probabilities must sum to 1" end + if slot.type == "choice" then + if type(answer.choice) ~= "string" or type(probabilities[answer.choice]) ~= "number" then + return nil, "slot '" .. key .. "' selected choice is outside its domain" + end + if probabilities[answer.choice] < highest then return nil, "slot '" .. key .. "' selected choice is not a highest-probability option" end + return { type = "choice", choice = answer.choice, probabilities = probabilities, confidence = answer.confidence } + end + if type(answer.score) ~= "number" or answer.score ~= answer.score or answer.score < 0 or answer.score > count - 1 then + return nil, "slot '" .. key .. "' score is outside its level range" + end + if math.abs(answer.score - expected) > math.max(0.005, count * count * 0.00005) then + return nil, "slot '" .. key .. "' score disagrees with its level probabilities" + end + local level = 1 + for index = 2, count do + if probabilities[index] > probabilities[level] then level = index end + end + return { type = "score", score = answer.score + 1, level = level, probabilities = probabilities, confidence = answer.confidence } +end + +function decisions_mapper.map_response(body: any, questions: any): (any?, string?, string?) + if type(body) ~= "table" or not dense_array(body.answers) then return nil, "OpenAI Decisions response carries no answer array" end + local slots: {[string]: any} = {} + local keys = sorted_keys(questions) + for index, key in ipairs(keys) do slots["q" .. tostring(index)] = { key = key, slot = questions[key] } end + if #body.answers ~= #keys then return nil, "OpenAI Decisions response does not answer every declared slot" end + local seen: {[string]: boolean} = {} + for _, answer in ipairs(body.answers) do + if type(answer) ~= "table" or type(answer.name) ~= "string" or not slots[answer.name] then + return nil, "OpenAI Decisions response contains an unnamed or undeclared answer" + end + if seen[answer.name] then return nil, "OpenAI Decisions response contains a duplicate answer" end + seen[answer.name] = true + end + -- Any refusal fails the complete evaluation, even when other slots succeeded. + for _, answer in ipairs(body.answers) do + if answer.type == "refusal" then + return nil, "OpenAI Decisions refused to answer slot '" .. slots[answer.name].key .. "'", output.ERROR_TYPE.CONTENT_FILTER + end + end + local readings: {[string]: any} = {} + for _, answer in ipairs(body.answers) do + local entry = slots[answer.name] + local key = entry.key :: string + local slot = entry.slot + if answer.type ~= slot.type then return nil, "slot '" .. key .. "' answer has an unexpected type" end + if slot.type == "predicate" then + if not probability(answer.probability) then return nil, "slot '" .. key .. "' answer carries an invalid probability" end + readings[key] = { type = "predicate", probability = answer.probability } + else + local reading, err = read_distribution(key, slot, answer) + if not reading then return nil, err end + readings[key] = reading + end + end + return readings +end + +local function token_count(value): boolean + return type(value) == "number" and value == value and value >= 0 and value < math.huge and value % 1 == 0 +end + +function decisions_mapper.map_tokens(usage: any): (any?, string?) + if usage == nil then return nil, nil end + if type(usage) ~= "table" then return nil, "OpenAI Decisions response carries invalid usage" end + for _, key in ipairs({ "input_tokens", "output_tokens", "total_tokens" }) do + if usage[key] ~= nil and not token_count(usage[key]) then return nil, "OpenAI Decisions usage has invalid " .. key end + end + local tokens: any = { prompt_tokens = usage.input_tokens, completion_tokens = usage.output_tokens, total_tokens = usage.total_tokens } + if usage.input_tokens ~= nil and usage.output_tokens ~= nil and usage.total_tokens ~= nil + and usage.total_tokens ~= (usage.input_tokens :: number) + (usage.output_tokens :: number) then + return nil, "OpenAI Decisions total_tokens disagrees with input and output tokens" + end + if tokens.total_tokens == nil and usage.input_tokens ~= nil and usage.output_tokens ~= nil then + tokens.total_tokens = (usage.input_tokens :: number) + (usage.output_tokens :: number) + end + if usage.input_tokens_details ~= nil then + if type(usage.input_tokens_details) ~= "table" then return nil, "OpenAI Decisions usage has invalid input token details" end + local details = usage.input_tokens_details + for _, key in ipairs({ "cached_tokens", "cache_write_tokens" }) do + if details[key] ~= nil and not token_count(details[key]) then return nil, "OpenAI Decisions usage has invalid " .. key end + end + tokens.cache_read_tokens = details.cached_tokens + tokens.cache_write_tokens = details.cache_write_tokens + tokens.cache_read_input_tokens = details.cached_tokens + tokens.cache_creation_input_tokens = details.cache_write_tokens + if usage.input_tokens ~= nil then + local cached = ((details.cached_tokens or 0) :: number) + ((details.cache_write_tokens or 0) :: number) + if cached > usage.input_tokens then return nil, "OpenAI Decisions cached token counts exceed input tokens" end + -- Canonical token categories are disjoint, so context and tracking + -- can sum them without counting cached input twice. + tokens.prompt_tokens = (usage.input_tokens :: number) - cached + end + end + if usage.output_tokens_details ~= nil then + if type(usage.output_tokens_details) ~= "table" then return nil, "OpenAI Decisions usage has invalid output token details" end + local thinking = usage.output_tokens_details.reasoning_tokens + if thinking ~= nil and not token_count(thinking) then return nil, "OpenAI Decisions usage has invalid reasoning_tokens" end + if thinking ~= nil and usage.output_tokens ~= nil and thinking > usage.output_tokens then + return nil, "OpenAI Decisions reasoning tokens exceed output tokens" + end + tokens.thinking_tokens = thinking + end + return tokens +end + +return decisions_mapper diff --git a/src/llm/src/openai/evaluate.lua b/src/llm/src/openai/evaluate.lua new file mode 100644 index 00000000..0edb8ea4 --- /dev/null +++ b/src/llm/src/openai/evaluate.lua @@ -0,0 +1,39 @@ +local openai_client = require("openai_client") +local openai_mapper = require("openai_mapper") +local decisions_mapper = require("decisions_mapper") +local output = require("output") + +type ClassifyError = (http_err: any?) -> (string, string, table?) + +local evaluate_handler = { _client = openai_client, _mapper = decisions_mapper } + +function evaluate_handler.handler(contract_args) + if type(contract_args) ~= "table" then contract_args = {} end + local err = output.errors.evaluate(contract_args):classifier(openai_mapper.classify_error :: ClassifyError) + local payload, map_err = evaluate_handler._mapper.map_request(contract_args) + if not payload then return nil, err:kind(output.ERROR_TYPE.INVALID_REQUEST):message(tostring(map_err)):build() end + + local response, request_err = evaluate_handler._client.request("/decisions", payload, { + timeout = contract_args.timeout, retry = contract_args.retry + }) + if request_err then return nil, err:from(request_err):build() end + + local readings, response_err, response_kind = evaluate_handler._mapper.map_response(response, contract_args.questions) + local metadata: any = {} + if type(response) == "table" then + if type(response.metadata) == "table" then + for key, value in pairs(response.metadata) do metadata[key] = value end + end + metadata.model = response.model + metadata.usage = response.usage + end + if not readings then + return nil, err:kind(response_kind or output.ERROR_TYPE.MODEL_ERROR):message(tostring(response_err)):details(metadata):build() + end + local tokens, usage_err = evaluate_handler._mapper.map_tokens(response.usage) + if usage_err then return nil, err:kind(output.ERROR_TYPE.MODEL_ERROR):message(usage_err):details(metadata):build() end + + return { success = true, result = { readings = readings }, tokens = tokens, metadata = metadata } +end + +return evaluate_handler diff --git a/src/llm/test/_index.yaml b/src/llm/test/_index.yaml index fa9898df..ea530dfc 100644 --- a/src/llm/test/_index.yaml +++ b/src/llm/test/_index.yaml @@ -123,6 +123,7 @@ entries: - security imports: llm: wippy.llm:llm + decisions_handler: wippy.llm.openai:evaluate test: wippy.test:test method: run_tests @@ -939,6 +940,52 @@ entries: test: wippy.test:test method: run_tests + - name: openai_decisions_mapper_test + kind: function.lua + meta: + name: OpenAI Decisions Mapper Unit Tests + type: test + group: LLM / Drivers / OpenAI + tags: [unit-test, mapper, evaluate] + source: file://openai/decisions_mapper_test.lua + modules: + - json + imports: + decisions_mapper: wippy.llm.openai:decisions_mapper + output: wippy.llm:output + test: wippy.test:test + method: run_tests + + - name: openai_evaluate_test + kind: function.lua + meta: + name: OpenAI Decisions Evaluation Handler Unit Tests + type: test + group: LLM / Drivers / OpenAI + tags: [unit-test, evaluate, contract] + source: file://openai/evaluate_test.lua + modules: + - json + imports: + evaluate_handler: wippy.llm.openai:evaluate + test: wippy.test:test + method: run_tests + + - name: openai_decisions_integration_test + kind: function.lua + meta: + name: OpenAI Decisions Contract Integration Tests + type: test + group: LLM / Drivers / OpenAI + tags: [integration-test, evaluate, contract] + source: file://openai/decisions_integration_test.lua + modules: + - env + - contract + imports: + test: wippy.test:test + method: run_tests + # app:openai_compat_client_test - name: openai_compat_client_test kind: function.lua diff --git a/src/llm/test/llm_test.lua b/src/llm/test/llm_test.lua index 70952f23..86a871a3 100644 --- a/src/llm/test/llm_test.lua +++ b/src/llm/test/llm_test.lua @@ -1,4 +1,5 @@ local llm = require("llm") +local decisions_handler = require("decisions_handler") local json = require("json") local security = require("security") @@ -1766,6 +1767,69 @@ local function define_tests() test.is_nil(args.options.timeout) test.is_nil(args.options.retry) end) + + it("should evaluate Decisions through the direct native provider and normalize input usage", function() + local original_client = decisions_handler._client + local seen_payload, seen_transport + decisions_handler._client = { request = function(path, payload, transport_options) + test.eq(path, "/decisions") + seen_payload, seen_transport = payload, transport_options + return { model = "gpt-6-luna", answers = { { name = "q1", type = "predicate", probability = 0.9 } }, + usage = { input_tokens = 100, output_tokens = 0, total_tokens = 100, + input_tokens_details = { cached_tokens = 20, cache_write_tokens = 10 } } } + end } + mock_providers.open = function(provider_id, context) + test.eq(provider_id, "wippy.llm.openai:provider") + return { evaluate = function(self, args) return decisions_handler.handler(args) end } + end + local ok, result, err = pcall(llm.evaluate, { text = "broken" }, { + damaged = { type = "predicate", instructions = "Is it broken?" } + }, { model = "gpt-6-luna", provider_id = "wippy.llm.openai:provider", safety_identifier = "opaque-hash", + timeout = 7, retry = { attempts = 2 }, metadata = { purpose = "usage" } }) + decisions_handler._client = original_client + test.is_true(ok) + test.is_nil(err) + assert(result) + test.eq(result.result.damaged.probability, 0.9) + test.eq(result.tokens.context_tokens, 100) + test.eq(result.tokens.prompt_tokens, 70) + test.eq(result.tokens.cache_read_tokens, 20) + test.eq(result.tokens.cache_write_tokens, 10) + test.eq(result.tokens.completion_tokens, 0) + test.eq(result.metadata.usage.input_tokens, 100) + test.eq(seen_payload.safety_identifier, "opaque-hash") + test.is_nil(seen_payload.user) + test.eq(seen_transport.timeout, 7) + test.eq(seen_transport.retry.attempts, 2) + test.eq(mock_usage_tracker.last_model_id, "gpt-6-luna") + end) + + it("should route a model card declaring evaluate to native OpenAI Decisions", function() + local seen_model, seen_options + mock_models.get_by_name = function(name) + return { id = "app.models:luna-decisions", name = "luna-decisions", title = "Luna Decisions", + capabilities = { "evaluate" }, classes = { "evaluate" }, priority = 100, + providers = { { id = "wippy.llm.openai:provider", provider_model = "gpt-6-luna", + options = { safety_identifier = "provider-default" } } } } + end + mock_providers.open = function(provider_id, context) + test.eq(provider_id, "wippy.llm.openai:provider") + test.eq(context.safety_identifier, "provider-default") + return { evaluate = function(self, args) + seen_model, seen_options = args.model, args.options + return { success = true, result = { readings = { damaged = { type = "predicate", probability = 0.9 } } }, + tokens = { prompt_tokens = 42, completion_tokens = 0, total_tokens = 42 } } + end } + end + local result, err = llm.evaluate("broken", { damaged = { type = "predicate", instructions = "Is it broken?" } }, + { model = "luna-decisions", safety_identifier = "caller-override" }) + test.is_nil(err) + assert(result) + test.eq(seen_model, "gpt-6-luna") + test.eq(seen_options.safety_identifier, "caller-override") + test.eq(result.result.damaged.probability, 0.9) + test.eq(mock_usage_tracker.last_model_id, "luna-decisions") + end) end) describe("Caller Options Ownership", function() diff --git a/src/llm/test/openai/client_test.lua b/src/llm/test/openai/client_test.lua index c0e52020..133513d0 100644 --- a/src/llm/test/openai/client_test.lua +++ b/src/llm/test/openai/client_test.lua @@ -12,6 +12,23 @@ local function define_tests() openai_client._http_client = nil end) + it("returns an error for scalar or null JSON before attaching metadata", function() + openai_client._ctx = { all = function() return { api_key = "test-key" } end } + openai_client._env = { get = function() return nil end } + for _, body in ipairs({ "null", "false", "42", '"unexpected"' }) do + openai_client._http_client = { + post = function() + return { status_code = 200, body = body, headers = { ["X-Request-Id"] = "malformed-response" } } + end + } + local response, err = openai_client.request("/decisions", {}) + test.is_nil(response) + test.not_nil(err) + test.contains(err.message, "expected a JSON object") + test.eq(err.metadata.request_id, "malformed-response") + end + end) + describe("HTTP Method Support", function() it("should default to POST method", function() openai_client._ctx = { diff --git a/src/llm/test/openai/decisions_integration_test.lua b/src/llm/test/openai/decisions_integration_test.lua new file mode 100644 index 00000000..ab0fad4f --- /dev/null +++ b/src/llm/test/openai/decisions_integration_test.lua @@ -0,0 +1,71 @@ +local contract = require("contract") +local env = require("env") +local test = require("test") + +local function define_tests() + describe("OpenAI Decisions Contract Integration", function() + it("binds evaluation on native OpenAI and rejects unsupported options before HTTP", function() + local evaluator, get_err = contract.get("wippy.llm:evaluator") + test.is_nil(get_err) + assert(evaluator) + local driver, open_err = evaluator:with_context({ api_key = "test-key" }):open("wippy.llm.openai:driver") + test.is_nil(open_err) + assert(driver) + local response, err = driver:evaluate({ model = "gpt-6-luna", state = "text", + questions = { truth = { type = "predicate", instructions = "True?" } }, options = { temperature = 0.5 } }) + test.is_nil(response) + test.not_nil(err) + test.contains(err:message(), "temperature") + local compat, compat_err = evaluator:open("wippy.llm.openai_compat:driver") + test.is_nil(compat) + test.not_nil(compat_err) + end) + + it("evaluates predicates, choices and scores through the native OpenAI binding", function() + -- This dedicated opt-in does not follow ENABLE_INTEGRATION_TESTS: + -- adding Decisions must not cause existing live suites to spend more. + if env.get("ENABLE_DECISIONS_INTEGRATION_TESTS") ~= "true" then + print("Skipping live Decisions test - set ENABLE_DECISIONS_INTEGRATION_TESTS=true to enable") + return + end + local key = env.get("OPENAI_API_KEY") + test.not_nil(key, "OPENAI_API_KEY is required for the opted-in Decisions test") + local evaluator, get_err = contract.get("wippy.llm:evaluator") + test.is_nil(get_err) + assert(evaluator) + local driver, open_err = evaluator:with_context({ api_key = key }):open("wippy.llm.openai:driver") + test.is_nil(open_err) + assert(driver) + local response, err = driver:evaluate({ + model = "gpt-6-luna", + state = "The delivered phone has a broken screen and cannot be used.", + questions = { + damaged = { type = "predicate", instructions = "Does the customer report physical damage?" }, + department = { type = "choice", instructions = "Which department should handle this complaint?", + domain = { billing = "payment errors", support = "damaged or broken products" } }, + severity = { type = "score", instructions = "Rate the problem's severity.", + domain = { "cosmetic only", "usable with a workaround", "unusable" } } + } + }) + test.is_nil(err) + assert(response) + test.is_true(response.success) + test.eq(response.result.readings.damaged.type, "predicate") + test.is_true(response.result.readings.damaged.probability >= 0 and response.result.readings.damaged.probability <= 1) + test.eq(response.result.readings.department.type, "choice") + test.not_nil(response.result.readings.department.probabilities.billing) + test.not_nil(response.result.readings.department.probabilities.support) + test.eq(response.result.readings.severity.type, "score") + test.is_true(response.result.readings.severity.score >= 1 and response.result.readings.severity.score <= 3) + test.eq(#response.result.readings.severity.probabilities, 3) + test.is_true(response.metadata.usage.input_tokens > 0) + test.eq(response.tokens.prompt_tokens + (response.tokens.cache_read_tokens or 0) + (response.tokens.cache_write_tokens or 0), + response.metadata.usage.input_tokens) + test.eq(response.tokens.completion_tokens, 0) + test.eq(response.tokens.total_tokens, response.metadata.usage.input_tokens + response.metadata.usage.output_tokens) + test.not_nil(response.metadata.model) + end) + end) +end + +return require("test").run_cases(define_tests) diff --git a/src/llm/test/openai/decisions_mapper_test.lua b/src/llm/test/openai/decisions_mapper_test.lua new file mode 100644 index 00000000..1eed81b5 --- /dev/null +++ b/src/llm/test/openai/decisions_mapper_test.lua @@ -0,0 +1,257 @@ +local mapper = require("decisions_mapper") +local json = require("json") +local output = require("output") +local test = require("test") + +local function questions() + return { + a_truth = { type = "predicate", instructions = "Is it broken?", domain = { yes = "damaged", no = "intact" } }, + b_route = { type = "choice", instructions = { task = "Route it" }, domain = { billing = "payments", technical = "defects" } }, + c_severity = { type = "score", instructions = "Rate severity", domain = { "low", "medium", "high" } } + } +end + +local function response() + return { answers = { + { type = "predicate", name = "q1", probability = 0.9 }, + { type = "choice", name = "q2", choice = "technical", confidence = 0.8, probabilities = { + { value = "technical", probability = 0.9 }, { value = "billing", probability = 0.1 } + } }, + { type = "score", name = "q3", score = 1.1, confidence = 0.55, probabilities = { + { value = 2, label = "level_2", probability = 0.2 }, + { value = 0, label = "level_0", probability = 0.1 }, + { value = 1, label = "level_1", probability = 0.7 } + } } + } } +end + +local function define_tests() + describe("OpenAI Decisions Mapper", function() + it("maps all question types and JSON text without exposing caller keys", function() + local payload, err = mapper.map_request({ model = "gpt-6-luna", state = { ticket = "broken" }, questions = questions() }) + test.is_nil(err) + assert(payload) + test.eq(json.decode(payload.input).ticket, "broken") + test.eq(#payload.questions, 3) + test.eq(payload.questions[1].name, "q1") + test.eq(payload.questions[1].type, "predicate") + test.eq(payload.questions[1].instructions, "Is it broken?\n\nOutcome criteria:\nTrue: damaged\nFalse: intact") + test.eq(json.decode(payload.questions[2].instructions).task, "Route it") + test.eq(payload.questions[2].choices[1].value, "billing") + test.eq(payload.questions[2].choices[1].description, "payments") + test.eq(payload.questions[3].levels[1].label, "level_0") + test.eq(payload.questions[3].levels[1].description, "low") + local encoded = json.encode(payload) + test.is_nil((encoded:find("a_truth", 1, true))) + test.is_nil((encoded:find("b_route", 1, true))) + test.is_nil((encoded:find("c_severity", 1, true))) + end) + + it("preserves ordered array choices, repeated score descriptions and plain predicate instructions", function() + local payload, err = mapper.map_request({ model = "future-model", state = "text", questions = { + a = { type = "choice", instructions = "Choose", domain = { "second", "first" } }, + b = { type = "predicate", instructions = "True?", domain = {} }, + c = { type = "score", instructions = "Rate", domain = { "same", "same" } } + } }) + test.is_nil(err) + assert(payload) + test.eq(payload.input, "text") + test.eq(payload.questions[1].choices[1].value, "second") + test.is_nil(payload.questions[1].choices[1].description) + test.eq(payload.questions[2].instructions, "True?") + test.eq(payload.questions[3].levels[2].label, "level_1") + test.eq(payload.questions[3].levels[2].description, "same") + end) + + it("accepts explicit safety identifiers and excludes facade tracking options", function() + local payload, err = mapper.map_request({ model = "gpt-6-luna", state = "text", questions = questions(), + options = { safety_identifier = "opaque-hash", user = "actor-id", metadata = { tag = "usage" }, timestamp = 42 } }) + test.is_nil(err) + assert(payload) + test.eq(payload.safety_identifier, "opaque-hash") + test.is_nil(payload.user) + test.is_nil(payload.metadata) + payload, err = mapper.map_request({ model = "gpt-6-luna", state = "text", questions = questions(), options = { user = "actor-id" } }) + test.is_nil(err) + assert(payload) + test.is_nil(payload.safety_identifier) + end) + + it("rejects invalid input and unsupported inference options", function() + local invalid = { + { model = "gpt-6-luna", state = "text", questions = {} }, + { model = "gpt-6-luna", state = 42, questions = questions() }, + { model = "gpt-6-luna", state = "text", questions = questions(), options = "invalid" }, + { model = "gpt-6-luna", state = "text", questions = questions(), options = { temperature = 0 } }, + { model = "gpt-6-luna", state = "text", questions = questions(), stream = false }, + { model = "gpt-6-luna", state = "text", questions = questions(), tools = {} }, + { model = "gpt-6-luna", state = "text", questions = questions(), options = { safety_identifier = 42 } }, + { model = "gpt-6-luna", state = "text", questions = questions(), options = { safety_identifier = string.rep("x", 129) } } + } + for _, args in ipairs(invalid) do + local payload, err = mapper.map_request(args) + test.is_nil(payload) + test.not_nil(err) + end + local domain = {} + for index = 1, 256 do domain[index] = "option_" .. tostring(index) end + local payload, err = mapper.map_request({ model = "gpt-6-luna", state = "text", questions = { + route = { type = "choice", instructions = "Choose", domain = domain } + } }) + test.is_nil(payload) + test.contains(err, "255") + end) + + it("matches out of order answers and probability values to the original slots", function() + local body = response() + body.answers[1], body.answers[3] = body.answers[3], body.answers[1] + local readings, err = mapper.map_response(body, questions()) + test.is_nil(err) + assert(readings) + test.eq(readings.a_truth.probability, 0.9) + test.eq(readings.b_route.choice, "technical") + test.eq(readings.b_route.probabilities.billing, 0.1) + test.eq(readings.b_route.confidence, 0.8) + test.eq(readings.c_severity.score, 2.1) + test.eq(readings.c_severity.level, 2) + test.eq(readings.c_severity.probabilities[1], 0.1) + test.eq(readings.c_severity.probabilities[3], 0.2) + test.eq(readings.c_severity.confidence, 0.55) + end) + + it("converts endpoint score extremes and selects the first modal level on ties", function() + local slot = { score = { type = "score", instructions = "Rate", domain = { "low", "high" } } } + for _, value in ipairs({ 0, 1 }) do + local reading, err = mapper.map_response({ answers = { { name = "q1", type = "score", score = value, confidence = 1, + probabilities = { { value = 0, label = "level_0", probability = 1 - value }, { value = 1, label = "level_1", probability = value } } + } } }, slot) + test.is_nil(err) + assert(reading) + test.eq(reading.score.score, value + 1) + test.eq(reading.score.level, value + 1) + end + local body = response() + body.answers[3].score = 0.5 + body.answers[3].probabilities[1].probability = 0 + body.answers[3].probabilities[2].probability = 0.5 + body.answers[3].probabilities[3].probability = 0.5 + local readings, err = mapper.map_response(body, questions()) + test.is_nil(err) + assert(readings) + test.eq(readings.c_severity.level, 1) + end) + + it("fails the whole evaluation when any known slot is refused", function() + local body = response() + body.answers[2] = { name = "q2", type = "refusal" } + local readings, err, kind = mapper.map_response(body, questions()) + test.is_nil(readings) + test.contains(err, "b_route") + test.eq(kind, output.ERROR_TYPE.CONTENT_FILTER) + end) + + it("rejects missing, duplicate, unknown and unnamed answers and malformed arrays", function() + local mutations = { + function(body) body.answers = nil end, + function(body) body.answers[3] = nil end, + function(body) body.answers[2].name = "q1" end, + function(body) body.answers[2].name = "unknown" end, + function(body) body.answers[2].name = nil end, + function(body) body.answers.extra = {} end, + function(body) body.answers[2] = "invalid" end, + function(body) body.answers[1].type = "choice" end + } + for _, mutate in ipairs(mutations) do + local body = response() + mutate(body) + local readings, err = mapper.map_response(body, questions()) + test.is_nil(readings) + test.not_nil(err) + end + for _, body in ipairs({ false, 42, "invalid" }) do + local readings, err = mapper.map_response(body, questions()) + test.is_nil(readings) + test.not_nil(err) + end + end) + + it("rejects malformed and inconsistent probabilities, confidence and scores", function() + local mutations = { + function(body) body.answers[1].probability = 0 / 0 end, + function(body) body.answers[1].probability = math.huge end, + function(body) body.answers[2].confidence = nil end, + function(body) body.answers[2].confidence = 1.1 end, + function(body) body.answers[2].choice = "billing" end, + function(body) body.answers[2].choice = true end, + function(body) body.answers[2].probabilities[2].value = "technical" end, + function(body) body.answers[2].probabilities[2].value = "outside" end, + function(body) body.answers[2].probabilities[2].value = false end, + function(body) body.answers[2].probabilities[2].probability = -0.1 end, + function(body) body.answers[2].probabilities[2].probability = 0.5 end, + function(body) body.answers[2].probabilities.extra = {} end, + function(body) body.answers[2].probabilities[1] = nil end, + function(body) body.answers[3].score = 0 / 0 end, + function(body) body.answers[3].score = 3 end, + function(body) body.answers[3].score = 0.5 end, + function(body) body.answers[3].probabilities[1].value = 0.5 end, + function(body) body.answers[3].probabilities[1].value = 4 end, + function(body) body.answers[3].probabilities[1].label = "wrong" end, + function(body) body.answers[3].probabilities[1].value = 0; body.answers[3].probabilities[1].label = "level_0" end + } + for _, mutate in ipairs(mutations) do + local body = response() + mutate(body) + local readings, err = mapper.map_response(body, questions()) + test.is_nil(readings) + test.not_nil(err) + end + end) + + it("preserves input only usage without inventing absent details", function() + local tokens, err = mapper.map_tokens({ input_tokens = 42, output_tokens = 0, total_tokens = 42 }) + test.is_nil(err) + assert(tokens) + test.eq(tokens.prompt_tokens, 42) + test.eq(tokens.completion_tokens, 0) + test.eq(tokens.total_tokens, 42) + test.is_nil(tokens.thinking_tokens) + test.is_nil(tokens.cache_read_tokens) + test.is_nil(tokens.cache_write_tokens) + test.is_nil(mapper.map_tokens(nil)) + tokens, err = mapper.map_tokens({ input_tokens = 42 }) + test.is_nil(err) + assert(tokens) + test.is_nil(tokens.completion_tokens) + test.is_nil(tokens.total_tokens) + end) + + it("keeps reported cache and reasoning categories disjoint for usage tracking", function() + local tokens, err = mapper.map_tokens({ input_tokens = 100, output_tokens = 5, total_tokens = 105, + input_tokens_details = { cached_tokens = 20, cache_write_tokens = 10 }, output_tokens_details = { reasoning_tokens = 2 } }) + test.is_nil(err) + assert(tokens) + test.eq(tokens.prompt_tokens, 70) + test.eq(tokens.cache_read_tokens, 20) + test.eq(tokens.cache_write_tokens, 10) + test.eq(tokens.cache_read_input_tokens, 20) + test.eq(tokens.cache_creation_input_tokens, 10) + test.eq(tokens.thinking_tokens, 2) + test.eq(tokens.total_tokens, 105) + end) + + it("rejects malformed usage instead of silently replacing it with zero", function() + for _, usage in ipairs({ false, { input_tokens = "42" }, { output_tokens = -1 }, { total_tokens = math.huge }, + { input_tokens_details = false }, { input_tokens_details = { cached_tokens = 0.5 } }, + { input_tokens = 1, input_tokens_details = { cached_tokens = 2 } }, + { output_tokens_details = { reasoning_tokens = -1 } }, + { input_tokens = 42, output_tokens = 0, total_tokens = 41 }, + { output_tokens = 0, output_tokens_details = { reasoning_tokens = 1 } } }) do + local tokens, err = mapper.map_tokens(usage) + test.is_nil(tokens) + test.not_nil(err) + end + end) + end) +end + +return require("test").run_cases(define_tests) diff --git a/src/llm/test/openai/evaluate_test.lua b/src/llm/test/openai/evaluate_test.lua new file mode 100644 index 00000000..4b31b6f9 --- /dev/null +++ b/src/llm/test/openai/evaluate_test.lua @@ -0,0 +1,137 @@ +local handler = require("evaluate_handler") +local json = require("json") +local test = require("test") + +local function args() + return { model = "gpt-6-luna", state = "broken screen", _provider_id = "wippy.llm.openai:provider", questions = { + damaged = { type = "predicate", instructions = "Is it broken?" } + } } +end + +local function body() + return { model = "gpt-6-luna-2026-10-01", answers = { { type = "predicate", name = "q1", probability = 0.95 } }, + usage = { input_tokens = 42, output_tokens = 0, total_tokens = 42 }, metadata = { request_id = "req_decision" } } +end + +local function define_tests() + describe("OpenAI Decisions Evaluation Handler", function() + local original_client = handler._client + after_each(function() handler._client = original_client end) + + it("dispatches Decisions with transport options and returns readings, reported model and usage", function() + local seen_path, seen_payload, seen_options + local response_body: any = body() + handler._client = { request = function(path, payload, options) + seen_path, seen_payload, seen_options = path, payload, options + return response_body + end } + local request = args() + request.timeout = 7 + request.retry = { attempts = 2 } + local response, err = handler.handler(request) + test.is_nil(err) + assert(response) + test.eq(seen_path, "/decisions") + test.eq(seen_payload.input, "broken screen") + test.eq(seen_options.timeout, 7) + test.eq(seen_options.retry.attempts, 2) + test.is_true(response.success) + test.eq(response.result.readings.damaged.probability, 0.95) + test.eq(response.tokens.prompt_tokens, 42) + test.eq(response.tokens.completion_tokens, 0) + test.eq(response.metadata.model, "gpt-6-luna-2026-10-01") + test.eq(response.metadata.request_id, "req_decision") + test.eq(response.metadata.usage.input_tokens, 42) + test.is_nil(response_body.metadata.model) + end) + + it("validates before sending any HTTP request", function() + handler._client = { request = function() error("must not call HTTP") end } + local request = args() + request.options = { temperature = 0.5 } + local response, err = handler.handler(request) + test.is_nil(response) + test.eq(err:kind(), errors.INVALID) + test.is_false(err:retryable()) + test.contains(err:message(), "temperature") + response, err = handler.handler(nil) + test.is_nil(response) + test.eq(err:kind(), errors.INVALID) + end) + + it("returns a nonretryable error for refusals without partial readings", function() + local response_body = body() + response_body.answers[1] = { type = "refusal", name = "q1" } + handler._client = { request = function() return response_body end } + local response, err = handler.handler(args()) + test.is_nil(response) + test.eq(err:kind(), errors.INVALID) + test.is_false(err:retryable()) + test.contains(err:message(), "refused") + test.eq(err:details().request_id, "req_decision") + test.eq(err:details().operation, "evaluate") + end) + + it("returns model errors for malformed answers and usage", function() + local response_body = body() + response_body.answers[1].probability = "invalid" + handler._client = { request = function() return response_body end } + local response, err = handler.handler(args()) + test.is_nil(response) + test.eq(err:kind(), errors.NOT_FOUND) + test.eq(err:details().request_id, "req_decision") + response_body = body() + response_body.usage = { input_tokens = "invalid" } + response, err = handler.handler(args()) + test.is_nil(response) + test.eq(err:kind(), errors.NOT_FOUND) + test.contains(err:message(), "usage") + end) + + it("reuses OpenAI error classification for rate limits, auth and transient failures", function() + for _, item in ipairs({ { status = 401, kind = errors.PERMISSION_DENIED, retryable = false }, + { status = 429, kind = errors.RATE_LIMITED, retryable = true }, + { status = 503, kind = errors.UNAVAILABLE, retryable = true } }) do + handler._client = { request = function() + return nil, { status_code = item.status, message = "provider error", metadata = { request_id = "req_error" } } + end } + local response, err = handler.handler(args()) + test.is_nil(response) + test.eq(err:kind(), item.kind) + test.eq(err:retryable(), item.retryable) + test.eq(err:details().request_id, "req_error") + test.eq(err:details().provider, "wippy.llm.openai:provider") + end + end) + + it("uses the existing authenticated HTTP client and retries transient errors", function() + local client = original_client + local saved_ctx, saved_env, saved_http = client._ctx, client._env, client._http_client + local calls = 0 + client._ctx = { all = function() return { api_key = "test-key", base_url = "https://example.test/v1" } end } + client._env = { get = function() return nil end } + client._http_client = { post = function(url, options) + calls = calls + 1 + test.eq(url, "https://example.test/v1/decisions") + test.eq(options.headers.Authorization, "Bearer test-key") + test.eq(options.timeout, 7) + test.eq(json.decode(options.body).questions[1].type, "predicate") + if calls == 1 then return { status_code = 503, body = json.encode({ error = { message = "unavailable" } }), headers = {} } end + return { status_code = 200, body = json.encode(body()), headers = { ["X-Request-Id"] = "req_http" } } + end } + handler._client = client + local request = args() + request.timeout = 7 + request.retry = { attempts = 2, backoff_ms = 0 } + local ok, response, err = pcall(handler.handler, request) + client._ctx, client._env, client._http_client = saved_ctx, saved_env, saved_http + test.is_true(ok) + test.is_nil(err) + assert(response) + test.eq(calls, 2) + test.eq(response.metadata.request_id, "req_http") + end) + end) +end + +return require("test").run_cases(define_tests) From 9aaef4099e1073c294c6eb82a4ccc16f991093e6 Mon Sep 17 00:00:00 2001 From: Wolfy-J Date: Wed, 7 Oct 2026 18:23:58 -0400 Subject: [PATCH 2/2] test(llm): check lazy evaluator binding methods --- src/llm/test/openai/decisions_integration_test.lua | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/llm/test/openai/decisions_integration_test.lua b/src/llm/test/openai/decisions_integration_test.lua index ab0fad4f..448b2b14 100644 --- a/src/llm/test/openai/decisions_integration_test.lua +++ b/src/llm/test/openai/decisions_integration_test.lua @@ -17,8 +17,11 @@ local function define_tests() test.not_nil(err) test.contains(err:message(), "temperature") local compat, compat_err = evaluator:open("wippy.llm.openai_compat:driver") - test.is_nil(compat) - test.not_nil(compat_err) + test.is_nil(compat_err) + assert(compat) + -- Opening a binding is lazy. Methods come from the contracts the + -- binding actually implements, so unsupported evaluation is absent. + test.is_nil(compat.evaluate) end) it("evaluates predicates, choices and scores through the native OpenAI binding", function()