From 6807ac13ffa606084284c5fb0f6dab4f1df5b291 Mon Sep 17 00:00:00 2001 From: PEDRO LOBATO CARCAMO Date: Tue, 15 Sep 2026 21:32:16 -0600 Subject: [PATCH 1/9] fix(upload): name the egress on every parse failure Qwen's WAF challenges POST /api/v2/files/parse per source IP, not per account. The 529s on qwen-next on 2026-09-16 were one WARP egress (104.28.222.16) challenged with all 236 accounts behind it; the logs said WAF_CAPTCHA and nothing about which proxy, so the branch was suspected before the IP was. describeEgress(account) returns the proxy an account leaves through (credentials masked) or 'direct'. Parse failures carry it in the message (`via socks5://host:port`) and as error.egress; the rate-limiter and breaker warnings name it too. Groundwork for keying both by egress once more than one egress exists (.scratch/egress-waf). Co-Authored-By: Claude Fable 5.1 --- src/utils/proxy-helper.js | 12 +++++++ src/utils/upload.js | 27 ++++++++------ tests/proxy-egress-describe.test.js | 55 +++++++++++++++++++++++++++++ 3 files changed, 84 insertions(+), 10 deletions(-) create mode 100644 tests/proxy-egress-describe.test.js diff --git a/src/utils/proxy-helper.js b/src/utils/proxy-helper.js index 23f59d6e..8974c476 100644 --- a/src/utils/proxy-helper.js +++ b/src/utils/proxy-helper.js @@ -155,6 +155,17 @@ const resolveProxyUrl = (account) => { return config.proxyUrl || null } +/** + * Egress identity for logs: the proxy URL an account's requests leave through, + * credentials masked, or 'direct' when none applies. Qwen's WAF judges by egress + * IP, not by account, so parse failures name this instead of the account. + */ +const describeEgress = (account) => { + const url = resolveProxyUrl(account) + if (!url) return 'direct' + return url.replace(/\/\/[^/@]*@/, '//***@') +} + /** * Evict oldest entry from agent cache when over limit. * Map iteration order is insertion order, so first key is oldest. @@ -292,6 +303,7 @@ const fetchWithProxy = (url, fetchOptions = {}, account) => { module.exports = { resolveProxyUrl, + describeEgress, getProxyAgent, invalidateProxyAgent, getChatBaseUrl, diff --git a/src/utils/upload.js b/src/utils/upload.js index 15827ce1..352f9de4 100644 --- a/src/utils/upload.js +++ b/src/utils/upload.js @@ -3,7 +3,7 @@ const OSS = require('ali-oss') const mimetypes = require('mime-types') const { logger } = require('./logger') const { generateUUID } = require('./tools.js') -const { getProxyAgent, getChatBaseUrl, applyProxyToAxiosConfig } = require('./proxy-helper') +const { getProxyAgent, getChatBaseUrl, applyProxyToAxiosConfig, describeEgress } = require('./proxy-helper') const { buildRequestHeaders } = require('./header-profile') const config = require('../config/index.js') @@ -391,17 +391,22 @@ const parseServiceFailureCode = (response) => { return null } -const throwIfParseServiceFailed = (response, fileId) => { +// `egress` names the proxy (or 'direct') the parse left through. The WAF +// challenge is per egress IP, so this is the field that tells one burnt proxy +// apart from a Qwen-side outage. +const throwIfParseServiceFailed = (response, fileId, egress = 'unknown') => { const code = parseServiceFailureCode(response) if (code === null) return - const error = new Error(`Qwen 文档解析服务失败: ${code} (${fileId})`) + const error = new Error(`Qwen 文档解析服务失败: ${code} (${fileId}, via ${egress})`) error.code = code === WAF_CAPTCHA_CODE ? 'qwen_parse_waf_challenge' : 'qwen_parse_unavailable' error.parseCode = code + error.egress = egress throw error } const parseUploadedTextFile = async (fileId, authToken, account, options = {}) => { if (!fileId || !authToken) throw new Error('解析文档缺少 fileId 或认证 Token') + const egress = describeEgress(account) const baseUrl = getChatBaseUrl() const requestConfig = applyProxyToAxiosConfig({ @@ -410,7 +415,7 @@ const parseUploadedTextFile = async (fileId, authToken, account, options = {}) = }, account) const parseResponse = await axios.post(`${baseUrl}/api/v2/files/parse`, { file_id: fileId }, requestConfig) - throwIfParseServiceFailed(parseResponse, fileId) + throwIfParseServiceFailed(parseResponse, fileId, egress) const maxAttempts = Math.max(1, Number(options.maxAttempts) || 30) const intervalMs = Math.max(50, Number(options.intervalMs) || 500) @@ -421,7 +426,7 @@ const parseUploadedTextFile = async (fileId, authToken, account, options = {}) = { file_id_list: [fileId] }, requestConfig ) - throwIfParseServiceFailed(response, fileId) + throwIfParseServiceFailed(response, fileId, egress) const payload = unwrapApiData(response) const records = Array.isArray(payload) ? payload : (payload?.list || payload?.items || []) const record = records.find(item => item?.file_id === fileId) || records[0] @@ -519,7 +524,7 @@ const noteParseOutcome = (error) => { if (cooldownSeconds > 0 && parseBreaker.strikes >= PARSE_BREAKER_STRIKES) { parseBreaker.openUntil = nowMs() + cooldownSeconds * 1000 error.retryAfterSeconds = cooldownSeconds - logger.warn(`Agent 上下文解析被 WAF 连续拦截 ${parseBreaker.strikes} 次,${cooldownSeconds}s 内不再上传`, 'UPLOAD') + logger.warn(`Agent 上下文解析被 WAF 连续拦截 ${parseBreaker.strikes} 次,${cooldownSeconds}s 内不再上传 (egress ${error.egress || 'unknown'})`, 'UPLOAD') } } @@ -550,7 +555,7 @@ const parseWindow = [] const resetParseRateLimiter = () => { parseWindow.length = 0 } -const takeParseSlot = () => { +const takeParseSlot = (account) => { const max = Math.max(0, parseInt(config.agentParseMaxPerWindow, 10) || 0) if (max <= 0) return const windowSeconds = Math.max(1, parseInt(config.agentParseWindowSeconds, 10) || 120) @@ -563,10 +568,12 @@ const takeParseSlot = () => { } const untilFree = Math.ceil((parseWindow[0] + windowMs - now) / 1000) const retryAfter = Math.min(PARSE_RATE_RETRY_MAX_SECONDS, Math.max(PARSE_RATE_RETRY_MIN_SECONDS, untilFree)) - logger.warn(`Agent 上下文解析已达速率上限 (${max}/${windowSeconds}s),${retryAfter}s 后重试`, 'UPLOAD') - const error = new Error(`Qwen 文档解析服务失败: ${PARSE_RATE_LIMITED_CODE} (${max}/${windowSeconds}s reached, upload skipped)`) + const egress = describeEgress(account) + logger.warn(`Agent 上下文解析已达速率上限 (${max}/${windowSeconds}s),${retryAfter}s 后重试 (egress ${egress})`, 'UPLOAD') + const error = new Error(`Qwen 文档解析服务失败: ${PARSE_RATE_LIMITED_CODE} (${max}/${windowSeconds}s reached, upload skipped, via ${egress})`) error.code = 'qwen_parse_rate_limited' error.parseCode = PARSE_RATE_LIMITED_CODE + error.egress = egress error.retryAfterSeconds = retryAfter error.breakerOpen = false throw error @@ -576,7 +583,7 @@ const uploadAgentContextFile = async (text, authToken, account, options = {}) => const content = Buffer.from(String(text || ''), 'utf8') if (content.length === 0) throw new Error('Agent 上下文为空') assertParseBreakerClosed() - takeParseSlot() + takeParseSlot(account) const filename = options.filename || `QWEN2API_AGENT_CONTEXT_${Date.now()}.txt` const uploaded = await uploadFileToQwenOss(content, filename, authToken, account) try { diff --git a/tests/proxy-egress-describe.test.js b/tests/proxy-egress-describe.test.js new file mode 100644 index 00000000..57678043 --- /dev/null +++ b/tests/proxy-egress-describe.test.js @@ -0,0 +1,55 @@ +const test = require('node:test') +const assert = require('node:assert/strict') + +const config = require('../src/config/index.js') +const { describeEgress } = require('../src/utils/proxy-helper') + +// The WAF challenge is per egress IP. Every parse failure names its egress so a +// burnt proxy can be told apart from a Qwen-side outage in one log line. + +const withGlobalProxy = (value, fn) => { + const saved = config.proxyUrl + config.proxyUrl = value + try { + return fn() + } finally { + config.proxyUrl = saved + } +} + +test('describeEgress: "direct" when neither the account nor PROXY_URL sets a proxy', () => { + withGlobalProxy(null, () => { + assert.equal(describeEgress({ email: 'a@example.com' }), 'direct') + assert.equal(describeEgress(null), 'direct') + assert.equal(describeEgress(undefined), 'direct') + }) +}) + +test('describeEgress: the account proxy wins over PROXY_URL and is trimmed', () => { + withGlobalProxy('http://global.example:8080', () => { + assert.equal( + describeEgress({ proxy: ' socks5://lohari-warp-qwen:9091 ' }), + 'socks5://lohari-warp-qwen:9091' + ) + }) +}) + +test('describeEgress: falls back to PROXY_URL when the account has no proxy', () => { + withGlobalProxy('socks5://127.0.0.1:1080', () => { + assert.equal(describeEgress({ proxy: '' }), 'socks5://127.0.0.1:1080') + assert.equal(describeEgress({}), 'socks5://127.0.0.1:1080') + }) +}) + +test('describeEgress: proxy credentials never reach the log line', () => { + withGlobalProxy(null, () => { + assert.equal( + describeEgress({ proxy: 'http://user:s3cr3t@proxy.example:3128' }), + 'http://***@proxy.example:3128' + ) + assert.equal( + describeEgress({ proxy: 'socks5://u:p@10.0.0.2:1080' }), + 'socks5://***@10.0.0.2:1080' + ) + }) +}) From cb28e1282b27eabbeca559f5c0295966e250cd58 Mon Sep 17 00:00:00 2001 From: PEDRO LOBATO CARCAMO Date: Tue, 15 Sep 2026 21:46:19 -0600 Subject: [PATCH 2/9] parse(egress): drop proxy credentials from egress labels; name egress on breaker-open too Review findings on 3ec9b9b: - describeEgress now returns protocol//host:port via new URL(); userinfo is dropped, not masked. The old regex (/\/\/[^@]*@/) cut at the FIRST "@", so "http://user:p@ss@proxy:3128" leaked "ss@proxy", and "http://proxy?token=a@b" lost its host. The regex survives only as the fallback when URL() throws, now anchored to the authority (/\/\/[^/?#]*@/). - assertParseBreakerClosed(account) names the egress like every other parse failure; both callers already had the account in hand. Without it, a burnt proxy was indistinguishable from a Qwen outage on the one log line that matters while the breaker is open. - Tests: "@" in password/query/path, unparseable URL still masked, breaker-open error carries egress and no credentials. expected-counts 1151 -> 1154. Co-Authored-By: Claude Fable 5.1 --- src/utils/proxy-helper.js | 15 +++++--- src/utils/upload.js | 8 +++-- tests/proxy-egress-describe.test.js | 56 +++++++++++++++++++++++++++-- 3 files changed, 69 insertions(+), 10 deletions(-) diff --git a/src/utils/proxy-helper.js b/src/utils/proxy-helper.js index 8974c476..01869987 100644 --- a/src/utils/proxy-helper.js +++ b/src/utils/proxy-helper.js @@ -156,14 +156,21 @@ const resolveProxyUrl = (account) => { } /** - * Egress identity for logs: the proxy URL an account's requests leave through, - * credentials masked, or 'direct' when none applies. Qwen's WAF judges by egress - * IP, not by account, so parse failures name this instead of the account. + * Egress identity for logs: `protocol//host:port` of the proxy an account's + * requests leave through, or 'direct' when none applies. Qwen's WAF judges by + * egress IP, not by account, so parse failures name this instead of the account. + * Credentials never make it out: the WHATWG parser drops userinfo, and the regex + * fallback (unparseable URL) masks up to the last `@` of the authority. */ const describeEgress = (account) => { const url = resolveProxyUrl(account) if (!url) return 'direct' - return url.replace(/\/\/[^/@]*@/, '//***@') + try { + const parsed = new URL(url) + return `${parsed.protocol}//${parsed.host}` + } catch { + return url.replace(/\/\/[^/?#]*@/, '//***@') + } } /** diff --git a/src/utils/upload.js b/src/utils/upload.js index 352f9de4..eb8b3d45 100644 --- a/src/utils/upload.js +++ b/src/utils/upload.js @@ -528,12 +528,14 @@ const noteParseOutcome = (error) => { } } -const assertParseBreakerClosed = () => { +const assertParseBreakerClosed = (account) => { const remaining = parseBreakerRemainingSeconds() if (remaining <= 0) return - const error = new Error(`Qwen 文档解析服务失败: ${WAF_CAPTCHA_CODE} (breaker open, ${remaining}s left, upload skipped)`) + const egress = describeEgress(account) + const error = new Error(`Qwen 文档解析服务失败: ${WAF_CAPTCHA_CODE} (breaker open, ${remaining}s left, upload skipped, via ${egress})`) error.code = 'qwen_parse_waf_challenge' error.parseCode = WAF_CAPTCHA_CODE + error.egress = egress error.retryAfterSeconds = remaining error.breakerOpen = true throw error @@ -582,7 +584,7 @@ const takeParseSlot = (account) => { const uploadAgentContextFile = async (text, authToken, account, options = {}) => { const content = Buffer.from(String(text || ''), 'utf8') if (content.length === 0) throw new Error('Agent 上下文为空') - assertParseBreakerClosed() + assertParseBreakerClosed(account) takeParseSlot(account) const filename = options.filename || `QWEN2API_AGENT_CONTEXT_${Date.now()}.txt` const uploaded = await uploadFileToQwenOss(content, filename, authToken, account) diff --git a/tests/proxy-egress-describe.test.js b/tests/proxy-egress-describe.test.js index 57678043..0333a50e 100644 --- a/tests/proxy-egress-describe.test.js +++ b/tests/proxy-egress-describe.test.js @@ -3,6 +3,7 @@ const assert = require('node:assert/strict') const config = require('../src/config/index.js') const { describeEgress } = require('../src/utils/proxy-helper') +const { assertParseBreakerClosed, noteParseOutcome, resetParseBreaker } = require('../src/utils/upload') // The WAF challenge is per egress IP. Every parse failure names its egress so a // burnt proxy can be told apart from a Qwen-side outage in one log line. @@ -41,15 +42,64 @@ test('describeEgress: falls back to PROXY_URL when the account has no proxy', () }) }) -test('describeEgress: proxy credentials never reach the log line', () => { +test('describeEgress: credentials are dropped, only protocol//host:port remains', () => { withGlobalProxy(null, () => { assert.equal( describeEgress({ proxy: 'http://user:s3cr3t@proxy.example:3128' }), - 'http://***@proxy.example:3128' + 'http://proxy.example:3128' ) assert.equal( describeEgress({ proxy: 'socks5://u:p@10.0.0.2:1080' }), - 'socks5://***@10.0.0.2:1080' + 'socks5://10.0.0.2:1080' ) }) }) + +test('describeEgress: an "@" in the password, query or path never leaks or eats the host', () => { + withGlobalProxy(null, () => { + // password containing "@": userinfo ends at the LAST "@" of the authority + const leaky = describeEgress({ proxy: 'http://user:p@ss@proxy.example:3128' }) + assert.equal(leaky, 'http://proxy.example:3128') + assert.doesNotMatch(leaky, /ss/) + // "@" in the query with no path segment: no credentials, host must survive + assert.equal( + describeEgress({ proxy: 'http://proxy.example:3128?token=abc@def' }), + 'http://proxy.example:3128' + ) + // "@" in the path + assert.equal( + describeEgress({ proxy: 'socks5://proxy.example:1080/a@b/c' }), + 'socks5://proxy.example:1080' + ) + }) +}) + +test('describeEgress: an unparseable URL still gets its userinfo masked', () => { + withGlobalProxy(null, () => { + assert.throws(() => new URL('http://u:p@ss@[::1'), 'precondition: this shape must not parse') + assert.equal(describeEgress({ proxy: 'http://u:p@ss@[::1' }), 'http://***@[::1') + assert.equal(describeEgress({ proxy: 'http://[::1' }), 'http://[::1') + }) +}) + +test('breaker-open rejection names the egress like every other parse failure', () => { + const savedBreaker = config.agentParseBreakerSeconds + config.agentParseBreakerSeconds = 90 + resetParseBreaker() + try { + const waf = () => Object.assign(new Error('waf'), { code: 'qwen_parse_waf_challenge', parseCode: 'WAF_CAPTCHA' }) + for (let i = 0; i < 5; i++) noteParseOutcome(waf()) + assert.throws( + () => assertParseBreakerClosed({ proxy: 'socks5://u:p@lohari-warp-qwen:9091' }), + (error) => + error.breakerOpen === true && + error.parseCode === 'WAF_CAPTCHA' && + error.egress === 'socks5://lohari-warp-qwen:9091' && + /upload skipped, via socks5:\/\/lohari-warp-qwen:9091\)$/.test(error.message) && + !error.message.includes('u:p@') + ) + } finally { + resetParseBreaker() + config.agentParseBreakerSeconds = savedBreaker + } +}) From b2ebd25df26b3a694ef965391f97bcfb2c3e9611 Mon Sep 17 00:00:00 2001 From: PEDRO LOBATO CARCAMO Date: Tue, 15 Sep 2026 22:16:56 -0600 Subject: [PATCH 3/9] fix(proxy): stop bridging undici bodies through web streams under Bun Under Bun, Readable.toWeb/fromWeb drops undici's body error on a mid-body close: the SSE consumer hangs and the rejection escapes as an unhandled rejection, which the runtime turns into a process exit. qwen-next died 6x in 2.5 h this way (2026-09-16, UND_ERR_SOCKET "other side closed" at 5.7 KB into the stream, always via the socks5 hop). The proxy transport now talks to undici directly: stream responses get undici's own Node Readable, everything else is buffered in the transport, and the fetch wrapper returns a buffered Response. Transport failures are mapped onto the codes request.js already retries (UND_ERR_SOCKET -> ECONNRESET, connect/headers/body timeouts -> ETIMEDOUT/ECONNABORTED) and axios' timeout becomes headersTimeout/bodyTimeout, so the HTTP-proxy path no longer cuts slow first bytes at the 10 s pool default. Repro: tools/dev-probes/repro-transport.js under bun -> 3/3 caught, 0 escapes (before: 2 escaped rejections + hang per cut). Co-Authored-By: Claude Fable 5.1 --- src/utils/proxy-helper.js | 79 +++++++---- tests/expected-counts.json | 2 +- tests/proxy-transport-no-bridge.test.js | 171 ++++++++++++++++++++++++ 3 files changed, 227 insertions(+), 25 deletions(-) create mode 100644 tests/proxy-transport-no-bridge.test.js diff --git a/src/utils/proxy-helper.js b/src/utils/proxy-helper.js index 01869987..f6bbbd1d 100644 --- a/src/utils/proxy-helper.js +++ b/src/utils/proxy-helper.js @@ -1,7 +1,7 @@ const { once } = require('node:events'); const { STATUS_CODES } = require('node:http'); const { isIP } = require('node:net'); -const { PassThrough, Readable } = require('node:stream'); +const { PassThrough } = require('node:stream'); const { checkServerIdentity } = require('node:tls'); const axios = require('axios'); const { HttpsProxyAgent } = require('https-proxy-agent'); @@ -19,6 +19,14 @@ const MAX_AGENT_CACHE_SIZE = 50 const PROXY_CONNECT_TIMEOUT_MS = 10_000; const proxyUrls = new WeakMap(); const proxyTransports = new WeakMap(); +// undici transport failures mapped onto the codes request.js already retries / cools down on. +const UNDICI_ERROR_CODES = { + UND_ERR_SOCKET: 'ECONNRESET', + UND_ERR_CONNECT_TIMEOUT: 'ETIMEDOUT', + UND_ERR_HEADERS_TIMEOUT: 'ECONNABORTED', + UND_ERR_BODY_TIMEOUT: 'ECONNABORTED', + UND_ERR_ABORTED: 'ERR_CANCELED' +}; /** * Reuse the account's SOCKS/CONNECT implementation with an HTTP client that honors sockets. @@ -66,6 +74,10 @@ const getProxyTransport = (proxyAgent) => { } }); const requestDispatcher = dispatcher.compose(interceptors.redirect({ maxRedirections: 20 }), interceptors.decompress()); + // Never hand a Readable.toWeb/fromWeb-bridged body to a consumer: under Bun the bridge drops + // undici's body error on a mid-body close, so the consumer hangs and the rejection escapes as a + // process crash (qwen-next died 6x in 2.5 h on 2026-09-16; tools/dev-probes/repro-bridge.js). + // Streams are returned as undici's own Node Readable; everything else is buffered right here. const fetch = async (url, options) => { const request = new globalThis.Request(url, options); // Keep buffered upstream payloads replayable across 307/308 redirects. @@ -80,34 +92,52 @@ const getProxyTransport = (proxyAgent) => { }); const noBody = request.method === 'HEAD' || [204, 205, 304].includes(response.statusCode); if (noBody) await response.body.dump(); - // Bun stalls on Undici fetch's WebStream bridge; its native Response streams correctly. - try { - return new globalThis.Response(noBody ? null : Readable.toWeb(response.body), { - status: response.statusCode, - statusText: STATUS_CODES[response.statusCode] || '', - headers: response.headers - }); - } catch (error) { - response.body.destroy(); - throw error; - } + const payload = noBody ? null : Buffer.from(await response.body.arrayBuffer()); + return new globalThis.Response(payload, { + status: response.statusCode, + statusText: STATUS_CODES[response.statusCode] || '', + headers: response.headers + }); + }; + const readResponseData = (response, responseType) => { + if (responseType === 'stream') return response.body; + if (responseType === 'arraybuffer') return response.body.arrayBuffer().then((buffer) => Buffer.from(buffer)); + // json/text/undefined: axios' transformResponse parses the text, exactly as with the Node adapter. + return response.body.text(); }; const adapter = async (requestConfig) => { - const adaptedConfig = { - ...requestConfig, - env: { ...requestConfig.env, fetch, Request: globalThis.Request, Response: globalThis.Response } - }; + let response; try { - const response = await axios.getAdapter('fetch', adaptedConfig)(adaptedConfig); - // Existing SSE consumers rely on Node streams for both success and error bodies. - if (requestConfig.responseType === 'stream') response.data = Readable.fromWeb(response.data); - return response; + response = await proxyRequest(axios.getUri(requestConfig), { + dispatcher: requestDispatcher, + method: String(requestConfig.method || 'get').toUpperCase(), + headers: axios.AxiosHeaders.from(requestConfig.headers).toJSON(), + body: requestConfig.data, + signal: requestConfig.signal, + headersTimeout: requestConfig.timeout || undefined, + bodyTimeout: requestConfig.timeout || undefined, + maxRedirections: 20 + }); } catch (error) { - if (requestConfig.responseType === 'stream' && error.response?.data?.getReader) { - error.response.data = Readable.fromWeb(error.response.data); - } - throw error; + throw axios.AxiosError.from(error, UNDICI_ERROR_CODES[error.code] || error.code || axios.AxiosError.ERR_NETWORK, requestConfig); } + const axiosResponse = { + data: await readResponseData(response, requestConfig.responseType), + status: response.statusCode, + statusText: STATUS_CODES[response.statusCode] || '', + headers: axios.AxiosHeaders.from(response.headers), + config: requestConfig, + request: null + }; + const { validateStatus } = requestConfig; + if (!validateStatus || validateStatus(axiosResponse.status)) return axiosResponse; + throw new axios.AxiosError( + `Request failed with status code ${axiosResponse.status}`, + [axios.AxiosError.ERR_BAD_REQUEST, axios.AxiosError.ERR_BAD_RESPONSE][Math.floor(axiosResponse.status / 100) - 4], + requestConfig, + null, + axiosResponse + ); }; transport = { dispatcher, fetch, adapter }; proxyTransports.set(proxyAgent, transport); @@ -316,6 +346,7 @@ module.exports = { getChatBaseUrl, getCliBaseUrl, applyProxyToAxiosConfig, + getProxyTransport, fetchWithProxy, isValidProxyUrl } diff --git a/tests/expected-counts.json b/tests/expected-counts.json index b070841b..5a0633ee 100644 --- a/tests/expected-counts.json +++ b/tests/expected-counts.json @@ -1,5 +1,5 @@ { - "tests": 1146, + "tests": 1159, "suites": 134, "note": "Authoritative count. Verify with the per-file sum in AGENTS.md (\"The test gate\"). The -a on that grep is load-bearing: tool-prompt.test.js emits bytes that make grep call the stream binary, and without -a its whole summary line — 133 tests — is silently dropped from the sum.", "updated": "2026-09-16" diff --git a/tests/proxy-transport-no-bridge.test.js b/tests/proxy-transport-no-bridge.test.js new file mode 100644 index 00000000..ee9cb0bb --- /dev/null +++ b/tests/proxy-transport-no-bridge.test.js @@ -0,0 +1,171 @@ +const test = require('node:test') +const assert = require('node:assert/strict') +const http = require('node:http') +const net = require('node:net') +const axios = require('axios') + +// The Bun/undici proxy transport must never bridge a body through Readable.toWeb/fromWeb: under +// Bun the bridge drops undici's body error on a mid-body close, the consumer hangs and the +// rejection escapes as a process crash (qwen-next, 2026-09-16). The transport itself is +// runtime-agnostic, so this drives it under Node through a real SOCKS5 hop against an upstream +// that cuts the socket mid-body. + +process.env.DATA_SAVE_MODE = 'none' +process.env.ACCOUNTS = '' +const { getProxyAgent, getProxyTransport } = require('../src/utils/proxy-helper') + +const SSE_FRAME = `data: ${'x'.repeat(120)}\n\n` +const listen = (server) => new Promise((resolve) => server.listen(0, '127.0.0.1', () => resolve(server.address().port))) + +// Minimal SOCKS5: no auth, CONNECT, IPv4 or domain targets; propagates the upstream close. +const startSocks5 = () => net.createServer((client) => { + client.on('error', () => {}) + client.once('data', (greeting) => { + if (greeting[0] !== 5) return client.destroy() + client.write(Buffer.from([5, 0])) + client.once('data', (request) => { + if (request[0] !== 5 || request[1] !== 1) return client.destroy() + let host, offset + if (request[3] === 1) { + host = request.subarray(4, 8).join('.') + offset = 8 + } else if (request[3] === 3) { + const length = request[4] + host = request.subarray(5, 5 + length).toString() + offset = 5 + length + } else { + return client.destroy() + } + const upstream = net.connect(request.readUInt16BE(offset), host, () => { + client.write(Buffer.from([5, 0, 0, 1, 0, 0, 0, 0, 0, 0])) + client.pipe(upstream).pipe(client) + }) + upstream.on('error', () => client.destroy()) + upstream.on('close', () => client.destroy()) + }) + }) +}) + +const startUpstream = () => http.createServer((req, res) => { + const chunks = [] + req.on('data', (chunk) => chunks.push(chunk)) + req.on('end', () => { + const [route, query = ''] = req.url.split('?') + if (route === '/sse-cut') { + res.writeHead(200, { 'content-type': 'text/event-stream' }) + for (let sent = 0; sent < 5708; sent += SSE_FRAME.length) res.write(SSE_FRAME) + setTimeout(() => res.socket.destroy(), 30) + } else if (route === '/sse-401') { + res.writeHead(401, { 'content-type': 'application/json' }) + res.end(JSON.stringify({ error: 'unauthorized' })) + } else if (route === '/json') { + res.writeHead(200, { 'content-type': 'application/json', 'x-probe': '1' }) + res.end(JSON.stringify({ ok: true, method: req.method, query, body: Buffer.concat(chunks).toString(), auth: req.headers.authorization || '' })) + } else if (route === '/json-500') { + res.writeHead(500, { 'content-type': 'application/json' }) + res.end(JSON.stringify({ error: 'boom' })) + } else if (route === '/bytes') { + res.writeHead(200, { 'content-type': 'application/octet-stream' }) + res.end(Buffer.from([1, 2, 3, 250])) + } else if (route === '/reset') { + req.socket.destroy() + } else if (route === '/slow-headers') { + setTimeout(() => { res.writeHead(200); res.end('late') }, 1500) + } else { + res.writeHead(404) + res.end() + } + }) +}) + +let socks, upstream, base, transport +const escaped = [] +const onEscape = (error) => escaped.push(error) + +test.before(async () => { + process.on('unhandledRejection', onEscape) + process.on('uncaughtException', onEscape) + socks = startSocks5() + upstream = startUpstream() + const socksPort = await listen(socks) + base = `http://127.0.0.1:${await listen(upstream)}` + transport = getProxyTransport(getProxyAgent({ email: 'transport@example.com', proxy: `socks5://127.0.0.1:${socksPort}` })) +}) + +test.after(() => { + process.off('unhandledRejection', onEscape) + process.off('uncaughtException', onEscape) + socks.close() + upstream.close() +}) + +const request = (config) => axios.request({ adapter: transport.adapter, proxy: false, ...config }) +const settle = () => new Promise((resolve) => setTimeout(resolve, 150)) +const rejection = (promise) => promise.then(() => assert.fail('expected a rejection'), (error) => error) +const readAll = async (stream) => { + let text = '' + for await (const chunk of stream) text += chunk + return text +} + +test('stream: a mid-body close rejects the consumer and escapes nowhere', async () => { + const response = await request({ url: `${base}/sse-cut`, method: 'post', data: '{}', responseType: 'stream', timeout: 5000 }) + assert.equal(response.status, 200) + assert.equal(typeof response.data.on, 'function', 'SSE consumers expect a Node stream') + let received = 0 + await assert.rejects( + (async () => { for await (const chunk of response.data) received += chunk.length })(), + (error) => error.code === 'UND_ERR_SOCKET' + ) + assert.ok(received > 0, 'the cut happens after bytes were delivered') + await settle() + assert.deepEqual(escaped, []) +}) + +test('stream: a non-2xx still rejects with the body as a Node stream', async () => { + const error = await rejection(request({ url: `${base}/sse-401`, responseType: 'stream' })) + assert.equal(error.response?.status, 401) + assert.equal(typeof error.response.data.on, 'function') + assert.equal(await readAll(error.response.data), '{"error":"unauthorized"}') +}) + +test('json: method, query, body and headers reach the upstream; axios parses the text', async () => { + const response = await request({ + url: `${base}/json`, + method: 'post', + params: { chat_id: 'abc' }, + data: { hello: 'world' }, + headers: { Authorization: 'Bearer test' } + }) + assert.equal(response.status, 200) + assert.equal(response.headers['x-probe'], '1') + assert.deepEqual(response.data, { ok: true, method: 'POST', query: 'chat_id=abc', body: '{"hello":"world"}', auth: 'Bearer test' }) + const error = await rejection(request({ url: `${base}/json-500` })) + assert.equal(error.response?.status, 500) + assert.deepEqual(error.response.data, { error: 'boom' }) +}) + +test('arraybuffer: returns a Buffer like the Node adapter', async () => { + const response = await request({ url: `${base}/bytes`, responseType: 'arraybuffer' }) + assert.ok(Buffer.isBuffer(response.data)) + assert.deepEqual([...response.data], [1, 2, 3, 250]) +}) + +test('request-phase transport errors map onto the codes request.js retries', async () => { + const reset = await rejection(request({ url: `${base}/reset`, responseType: 'stream' })) + assert.equal(reset.isAxiosError, true) + assert.equal(reset.code, 'ECONNRESET') + assert.equal(reset.cause?.code, 'UND_ERR_SOCKET') + const timeout = await rejection(request({ url: `${base}/slow-headers`, timeout: 200 })) + assert.equal(timeout.code, 'ECONNABORTED') + assert.equal(timeout.cause?.code, 'UND_ERR_HEADERS_TIMEOUT') +}) + +test('fetch: the Response is buffered, so a mid-body close rejects instead of hanging', async () => { + const ok = await transport.fetch(`${base}/json`) + assert.equal(ok.status, 200) + assert.equal((await ok.json()).ok, true) + await assert.rejects(transport.fetch(`${base}/sse-cut`, { method: 'POST', body: '{}' }), (error) => error.code === 'UND_ERR_SOCKET') + await settle() + assert.deepEqual(escaped, []) +}) From afd1a7934670bad0829c7bdccd4e4683e61bf1dd Mon Sep 17 00:00:00 2001 From: PEDRO LOBATO CARCAMO Date: Wed, 16 Sep 2026 07:16:09 -0600 Subject: [PATCH 4/9] proxy: accept socks5h:// so DNS resolves at the proxy, not on the host MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit socks5:// makes socks-proxy-agent resolve the target hostname locally and hand the proxy a bare IP, so the DNS query leaves through the host resolver while the TCP goes through the proxy — two egresses for one request. On qwen-next 76/80 upstream connections reached sing-box as bare IPs. socks5h:// (curl semantics) delegates resolution to the proxy: the agent reads the scheme itself and sets shouldLookup=false. Accept it in the backend validator, the dashboard regex it mirrors, the accounts API error text and the three locales. Co-Authored-By: Claude Fable 5.1 --- public/src/locales/en.json | 2 +- public/src/locales/ru.json | 2 +- public/src/locales/zh.json | 2 +- public/src/views/dashboard.vue | 2 +- src/routes/accounts.js | 2 +- src/utils/proxy-helper.js | 11 ++++++-- tests/proxy-socks5h.test.js | 46 ++++++++++++++++++++++++++++++++++ 7 files changed, 60 insertions(+), 7 deletions(-) create mode 100644 tests/proxy-socks5h.test.js diff --git a/public/src/locales/en.json b/public/src/locales/en.json index c3df5599..a7eadbda 100644 --- a/public/src/locales/en.json +++ b/public/src/locales/en.json @@ -118,7 +118,7 @@ "forceRefreshAllConfirm": "Are you sure you want to force refresh the tokens of all accounts? \nThis will refresh all accounts, regardless of whether they are about to expire, and may take a long time.", "forceRefreshAllFailed": "Forced refresh failed:", "progressRetry": "Progress query failed, retrying...", - "proxyInvalid": "Format error: must start with http://, https:// or socks5://", + "proxyInvalid": "Format error: must start with http://, https://, socks5:// or socks5h://", "proxyUpdateFailed": "Agent update failed:", "proxyUpdateSuccess": "Agent updated successfully", "refreshAllComplete": "Batch refresh completed, {n} accounts were successfully refreshed", diff --git a/public/src/locales/ru.json b/public/src/locales/ru.json index ec2899f0..ec302a9e 100644 --- a/public/src/locales/ru.json +++ b/public/src/locales/ru.json @@ -120,7 +120,7 @@ "forceRefreshAllFailed": "Ошибка принудительного обновления: ", "proxyUpdateSuccess": "Proxy обновлён", "proxyUpdateFailed": "Ошибка обновления proxy: ", - "proxyInvalid": "Неверный формат: ожидается http://, https:// или socks5://" + "proxyInvalid": "Неверный формат: ожидается http://, https://, socks5:// или socks5h://" }, "settings": { "title": "Настройки", diff --git a/public/src/locales/zh.json b/public/src/locales/zh.json index 8f968c5d..d917f7a0 100644 --- a/public/src/locales/zh.json +++ b/public/src/locales/zh.json @@ -120,7 +120,7 @@ "forceRefreshAllFailed": "强制刷新失败: ", "proxyUpdateSuccess": "代理更新成功", "proxyUpdateFailed": "代理更新失败: ", - "proxyInvalid": "格式错误:必须以 http://、https:// 或 socks5:// 开头" + "proxyInvalid": "格式错误:必须以 http://、https://、socks5:// 或 socks5h:// 开头" }, "settings": { "title": "系统设置", diff --git a/public/src/views/dashboard.vue b/public/src/views/dashboard.vue index 96bb1c02..9d08eaa2 100644 --- a/public/src/views/dashboard.vue +++ b/public/src/views/dashboard.vue @@ -576,7 +576,7 @@ const editProxy = ref({ email: '', proxy: '' }) const isSavingProxy = ref(false) // 与后端 src/utils/proxy-helper.js#PROXY_URL_REGEX 保持一致 -const PROXY_URL_REGEX = /^(https?|socks5):\/\/[^\s]+$/i +const PROXY_URL_REGEX = /^(https?|socks5h?):\/\/[^\s]+$/i const isValidProxy = (value) => { if (!value) return true const trimmed = String(value).trim() diff --git a/src/routes/accounts.js b/src/routes/accounts.js index 19ce0c93..3d2ab03e 100644 --- a/src/routes/accounts.js +++ b/src/routes/accounts.js @@ -11,7 +11,7 @@ const { isValidProxyUrl } = require('../utils/proxy-helper') const { DEFAULT_CLI_QUOTA_LIMIT, getAccountCliState } = require('../utils/cli-support') // 仅在 proxy 字段存在时触发;空字符串/null 一律视为"清除代理",无需校验 -const PROXY_FORMAT_ERROR = '代理 URL 格式无效,应以 http://、https:// 或 socks5:// 开头' +const PROXY_FORMAT_ERROR = '代理 URL 格式无效,应以 http://、https://、socks5:// 或 socks5h:// 开头' const batchAccountTasks = new Map() const BATCH_TASK_RETENTION_MS = 1000 * 60 * 30 diff --git a/src/utils/proxy-helper.js b/src/utils/proxy-helper.js index f6bbbd1d..5e83602d 100644 --- a/src/utils/proxy-helper.js +++ b/src/utils/proxy-helper.js @@ -155,8 +155,13 @@ const destroyProxyAgent = (agent) => { agent.destroy(); }; -// Accept http/https/socks5 protocols; regex intentionally loose to catch common typos only -const PROXY_URL_REGEX = /^(https?|socks5):\/\/[^\s]+$/i +// Accept http/https/socks5/socks5h; regex intentionally loose to catch common typos only. +// socks5:// resolves the target hostname LOCALLY and hands the proxy an IP (socks-proxy-agent +// sets `lookup = true`), so the DNS query leaves through the host's resolver while the TCP +// goes through the proxy. socks5h:// delegates resolution to the proxy (curl semantics): +// DNS and TCP share one egress. Measured on qwen-next 2026-09-16: 76/80 upstream connections +// reached sing-box as bare IPs under socks5://. +const PROXY_URL_REGEX = /^(https?|socks5h?):\/\/[^\s]+$/i /** * Validate proxy URL format. @@ -243,6 +248,8 @@ const getOrCreateAgent = (url, account) => { const proxyUrl = new URL(url); switch (proxyUrl.protocol) { case 'socks5:': + case 'socks5h:': + // The agent reads the scheme itself: socks5h → shouldLookup=false. agent = new SocksProxyAgent(proxyUrl); break; case 'http:': diff --git a/tests/proxy-socks5h.test.js b/tests/proxy-socks5h.test.js new file mode 100644 index 00000000..771f846c --- /dev/null +++ b/tests/proxy-socks5h.test.js @@ -0,0 +1,46 @@ +const test = require('node:test') +const assert = require('node:assert/strict') + +const { isValidProxyUrl, getProxyAgent, describeEgress, invalidateProxyAgent } = require('../src/utils/proxy-helper') + +// socks5:// resolves the target hostname locally and hands the proxy an IP; socks5h:// +// lets the proxy resolve (curl semantics). On qwen-next (2026-09-16) 76 of 80 upstream +// connections reached sing-box as bare IPs: DNS for Qwen's domains was leaving through +// Hetzner's resolver while the TCP went through WARP. The `h` variant closes that gap, +// and the dashboard/route validators must let it through or the field cannot be saved. + +test('socks5h:// is a valid account proxy URL, alongside socks5/http/https', () => { + assert.equal(isValidProxyUrl('socks5h://lohari-warp-qwen:9091'), true) + assert.equal(isValidProxyUrl('socks5://lohari-warp-qwen:9091'), true) + assert.equal(isValidProxyUrl('SOCKS5H://x:1'), true) + assert.equal(isValidProxyUrl('http://x:1'), true) + assert.equal(isValidProxyUrl('socks4://x:1'), false) + assert.equal(isValidProxyUrl('socks5hh://x:1'), false) + assert.equal(isValidProxyUrl('socks5h:/x:1'), false) +}) + +test('socks5h:// builds a SOCKS agent that leaves DNS to the proxy; socks5:// resolves locally', () => { + const remoteUrl = 'socks5h://127.0.0.1:1080' + const localUrl = 'socks5://127.0.0.1:1080' + const remote = getProxyAgent({ email: 'socks5h@example.com', proxy: remoteUrl }) + const local = getProxyAgent({ email: 'socks5@example.com', proxy: localUrl }) + try { + assert.equal(remote.shouldLookup, false, 'socks5h must hand the hostname to the proxy') + assert.equal(local.shouldLookup, true, 'socks5 keeps the historical local lookup') + assert.equal(remote.proxy.type, 5) + } finally { + invalidateProxyAgent(remoteUrl) + invalidateProxyAgent(localUrl) + } +}) + +test('describeEgress keeps the scheme so a log line tells socks5h apart from socks5, without credentials', () => { + assert.equal( + describeEgress({ email: 'a@example.com', proxy: 'socks5h://user:secret@lohari-warp-qwen:9091' }), + 'socks5h://lohari-warp-qwen:9091' + ) + assert.equal( + describeEgress({ email: 'a@example.com', proxy: 'socks5://lohari-warp-qwen:9091' }), + 'socks5://lohari-warp-qwen:9091' + ) +}) From 64f859389b231d949b8bf5d58fe9c9778c7c93fd Mon Sep 17 00:00:00 2001 From: PEDRO LOBATO CARCAMO Date: Wed, 16 Sep 2026 07:16:09 -0600 Subject: [PATCH 5/9] anthropic: fail over to another account when the stream dies before the first block MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Live on qwen-next (2026-09-16, 8 h): 3 of 70 /v1/messages streams ended in undici `UND_ERR_SOCKET: other side closed` with 63–90 KiB already read, and Claude Code showed "Connection lost mid-response". Until now that surfaced as a 500 with nothing the client could do. - upstream-error.js#isTransportInterruption: socket closes and timeouts as Node, undici and streams emit them. Excludes client cancels (AbortError / ERR_CANCELED) and anything carrying an HTTP response. - sse.js#consumeSSEStream counts bytes, and stamps upstreamBytesRead / upstreamEventCount on whatever error escapes — the controller uses it to decide whether a replay is safe, the [EGRESS] log to tell early from late cuts. - anthropic.js#handleAnthropicStream: one failover per request, only while no content block has been emitted (replaying after visible text would duplicate it), for transport cuts and for a RateLimited first frame. The burned account is paused (recordFailedAccount) before the pool is asked again with excludeEmails, so the retry never lands on it. - request.js#sendChatRequest forwards excludeEmails to the rotator. - The outer 429 handler no longer marks an account the failover already marked (error.accountFailureRecorded) — same guard chat.js already had. Without it, a single-account deployment logged the quota exhaustion twice. Tests: anthropic-midstream-failover (9), sse-interrupt-metrics, and the upstream-quota-429 harness now honours excludeEmails like the real rotator. Gate blessed at 1175 tests / 136 suites (was 1159/134). Co-Authored-By: Claude Fable 5.1 --- src/controllers/anthropic.js | 98 +++++++++- src/utils/request.js | 5 +- src/utils/sse.js | 27 ++- src/utils/upstream-error.js | 24 +++ tests/anthropic-midstream-failover.test.js | 199 +++++++++++++++++++++ tests/expected-counts.json | 4 +- tests/sse-interrupt-metrics.test.js | 67 +++++++ tests/upstream-quota-429.test.js | 15 +- 8 files changed, 422 insertions(+), 17 deletions(-) create mode 100644 tests/anthropic-midstream-failover.test.js create mode 100644 tests/sse-interrupt-metrics.test.js diff --git a/src/controllers/anthropic.js b/src/controllers/anthropic.js index d1f6344c..4f9a5d9b 100644 --- a/src/controllers/anthropic.js +++ b/src/controllers/anthropic.js @@ -57,10 +57,14 @@ const { logger } = require('../utils/logger'); const { assertNoUpstreamFailure, describeUpstreamFailure, + isRateLimitError, + isTransportInterruption, noteRateLimitedAccount, RATE_LIMIT_ANTHROPIC_TYPE, UpstreamResponseError } = require('../utils/upstream-error.js'); +const { describeEgress } = require('../utils/proxy-helper.js'); +const { recordFailedAccount } = require('../utils/agent-account-failover.js'); const { analyzeAnthropicCompatibility, buildAnthropicCompatibilityHeaders @@ -1209,6 +1213,44 @@ const runWithAnthropicPing = async (res, work, intervalMs) => { } }; +// Reintento por cuenta cuando el upstream se cae ANTES de que el cliente haya visto un solo +// bloque de contenido. Dos causas, una regla: +// - transporte: el socket se cerro a mitad del SSE (`UND_ERR_SOCKET: other side closed`, +// 3 de 70 peticiones en 8 h el 2026-09-16 en qwen-next; sing-box sin un solo error, asi +// que no se sabe si corto WARP o Qwen); +// - cuota: el primer frame util es `RateLimited` — la cuenta se pausa (recordFailedAccount) +// y otra sirve el turno en vez de devolverle al cliente un 429 a medio stream. +// El guard es "cero content_block emitidos": con uno ya en el cable, reenviar duplicaria +// texto en la pantalla del cliente, asi que ahi el error sigue saliendo como hasta ahora y +// el cliente (Claude Code) reintenta el. Una sola vuelta por peticion: la segunda caida +// consecutiva es senal, no ruido. +const MID_STREAM_FAILOVER_MAX_RETRIES = 1; + +const classifyMidStreamFailure = (error) => { + if (isRateLimitError(error)) return 'quota'; + if (isTransportInterruption(error)) return 'transport'; + return null; +}; + +/** + * Una linea por interrupcion, se reintente o no. Es la medida que faltaba: sin tasa de + * cierres por egress no hay forma de saber si WARP los empeora respecto a salir directo. + * `bytes`/`frames` los anota consumeSSEStream en el error; `proxy` nunca lleva credenciales + * (describeEgress). + */ +const logEgressInterruption = (kind, error, account, { emittedBlocks, action }) => { + logger.warn( + `mid_stream_${kind} code=${error?.code || 'unknown'}` + + ` bytes=${Number(error?.upstreamBytesRead) || 0}` + + ` frames=${Number(error?.upstreamEventCount) || 0}` + + ` emitted_blocks=${emittedBlocks}` + + ` account=${account?.email || 'none'}` + + ` proxy=${describeEgress(account)}` + + ` action=${action}`, + 'EGRESS' + ); +}; + /** * 处理流式 Anthropic 响应 * @param {object} res - Express 响应 @@ -1706,6 +1748,7 @@ const handleAnthropicStream = async (res, ctx, upstream) => { let attemptsMade = 0; let retriedAfterVisibleText = false; let protocolRecoveryRetried = false; + let failoverRetries = 0; for (;;) { attemptsMade += 1; @@ -1719,8 +1762,47 @@ const handleAnthropicStream = async (res, ctx, upstream) => { upstreamCompleted = result.completed; upstreamEventCount = result.eventCount; } catch (e) { - logger.error('Anthropic 流式心跳包装失败', 'ANTHROPIC', '', e); - throw e; + const kind = classifyMidStreamFailure(e); + const emittedBlocks = blockIndex + 1; + const canFailover = kind !== null + && emittedBlocks === 0 + && failoverRetries < MID_STREAM_FAILOVER_MAX_RETRIES + && !res.writableEnded && !res.destroyed; + if (kind) { + logEgressInterruption(kind, e, ctx.currentAccount, { + emittedBlocks, + action: canFailover ? 'failover' : 'deliver_error' + }); + } + if (!canFailover) { + logger.error('Anthropic 流式心跳包装失败', 'ANTHROPIC', '', e); + throw e; + } + + failoverRetries += 1; + const failedEmail = ctx.currentAccount?.email || null; + // Cuota: pausa la cuenta antes de sortear otra. Transporte: solo anota el email; un + // cierre a mitad de stream no es culpa de la cuenta y no debe acercarla al cooldown. + recordFailedAccount(e, ctx.currentAccount); + let retryResp = null; + try { + await runWithAnthropicPing(res, async () => { + retryResp = await sendRequest(requestBody, { + ...upstreamOptions, + excludeEmails: failedEmail ? [failedEmail] : [] + }); + }); + } catch (retryError) { + logger.error('Anthropic 流式 failover 重试失败', 'ANTHROPIC', '', retryError); + throw retryError.publicMessage ? retryError : e; + } + if (!retryResp?.status || !retryResp.response) throw e; + currentUpstream = retryResp.response; + // Stats y un eventual 429 posterior se atribuyen a quien sirvio de verdad. + if (retryResp.currentAccount) ctx.currentAccount = retryResp.currentAccount; + // El failover no gasta cupo de correccion de protocolo: el modelo aun no ha hablado. + attemptsMade -= 1; + continue; } // 本轮收尾。解析器的尾巴属于这一轮,必须在判定之前放出来。文本通道截断之后例外: @@ -2638,6 +2720,9 @@ const handleAnthropicMessages = async (req, res) => { // Tambien fuera: el catch decide si olvidar un prefijo de historial reutilizado. let upstreamResp = null; let contextPrefixKey = null; + // Y el contexto del handler: un failover a mitad de stream cambia ctx.currentAccount por + // la cuenta que sirvio de verdad; el catch tiene que marcar ESA, no la del sorteo inicial. + let ctx = null; try { const compatibility = analyzeAnthropicCompatibility(req.body || {}); const compatibilityHeaders = buildAnthropicCompatibilityHeaders(compatibility); @@ -2677,7 +2762,7 @@ const handleAnthropicMessages = async (req, res) => { } const message_id = `msg_${generateUUID().replace(/-/g, '').slice(0, 24)}`; - const ctx = { + ctx = { message_id, model, hasTools, @@ -2706,7 +2791,12 @@ const handleAnthropicMessages = async (req, res) => { : (failure.overloaded ? 'overloaded_error' : 'api_error'); // La otra mitad: sin esto el cliente deja de reintentar pero el servidor sigue // devolviendo la misma cuenta agotada al sorteo, y la quema en cada vuelta. - noteRateLimitedAccount(error, currentAccount); + // Si el failover a mitad de stream ya paso la cuenta a cooldown (recordFailedAccount) + // y luego no hubo otra cuenta a la que saltar, el error que llega aqui es el mismo: + // no se marca dos veces. + if (!error?.accountFailureRecorded) { + noteRateLimitedAccount(error, ctx?.currentAccount || currentAccount); + } // Un prefijo de historial reutilizado pudo ser la causa (file_id que Qwen ya no // reconoce): se olvida y el reintento del cliente hornea uno nuevo. Un 529 por // ContextExternalizationError nunca llega aqui con contextPrefixReused. diff --git a/src/utils/request.js b/src/utils/request.js index ac69d75b..6ff08c87 100644 --- a/src/utils/request.js +++ b/src/utils/request.js @@ -649,9 +649,12 @@ const externalizeOversizedAgentContext = async ( */ const sendChatRequest = async (body, options = {}) => { // 获取可用的账户(包含 proxy 等完整字段) + // excludeEmails:本次 HTTP 请求里已经烧掉的账户(流中途 failover)——轮换器跳过它们, + // 即使它们对其他请求仍然可用。 + const excludeEmails = Array.isArray(options.excludeEmails) ? options.excludeEmails : [] const currentAccount = options.currentAccount?.token ? options.currentAccount - : accountManager.getAccount() + : accountManager.getAccount(excludeEmails) const currentToken = currentAccount ? currentAccount.token : null if (!currentToken) { diff --git a/src/utils/sse.js b/src/utils/sse.js index fcf3955c..4b2ef67e 100644 --- a/src/utils/sse.js +++ b/src/utils/sse.js @@ -154,6 +154,7 @@ const consumeSSEStream = async (stream, onFrame, options = {}) => { const decoder = new SSEDecoder() let sawDone = false let eventCount = 0 + let bytesRead = 0 let stopped = false const consumeFrames = async (frames) => { @@ -168,16 +169,28 @@ const consumeSSEStream = async (stream, onFrame, options = {}) => { } } - for await (const chunk of stream) { - await consumeFrames(decoder.push(chunk)) - if (stopped) { - if (typeof stream.on === 'function') stream.on('error', () => {}) - break + try { + for await (const chunk of stream) { + bytesRead += typeof chunk === 'string' ? Buffer.byteLength(chunk) : chunk.length + await consumeFrames(decoder.push(chunk)) + if (stopped) { + if (typeof stream.on === 'function') stream.on('error', () => {}) + break + } + } + if (!stopped) await consumeFrames(decoder.end()) + } catch (error) { + // 中途失败带上"上游走到了哪":控制器据此判断重试是否安全(已转发多少), + // egress 日志据此区分早断(几 KB)与晚断(几十 KB)。onFrame 抛出的业务错误 + // (RateLimited 帧)同样经过这里,同样标注。 + if (error && typeof error === 'object') { + error.upstreamBytesRead = bytesRead + error.upstreamEventCount = eventCount } + throw error } - if (!stopped) await consumeFrames(decoder.end()) - return { sawDone, eventCount, completed: true, stopped } + return { sawDone, eventCount, bytesRead, completed: true, stopped } } /** diff --git a/src/utils/upstream-error.js b/src/utils/upstream-error.js index 29c4eec6..553e38a4 100644 --- a/src/utils/upstream-error.js +++ b/src/utils/upstream-error.js @@ -46,6 +46,29 @@ const isRateLimitError = (error) => { return RATE_LIMIT_MESSAGE_RE.test(String(error.publicMessage || error.message || '')); }; +/** + * ¿El transporte se cayo antes o a mitad de la respuesta? Cierres de socket y timeouts tal + * y como los emiten Node (`ECONNRESET`), undici (`UND_ERR_SOCKET: other side closed` — + * 3 de 70 peticiones en 8 h el 2026-09-16 en qwen-next, con 63–90 KiB ya escritos) y los + * streams (`ERR_STREAM_PREMATURE_CLOSE`). NO incluye la cancelacion del propio cliente + * (`ERR_CANCELED` / AbortError): si quien corto fue el cliente no hay a quien reintentarle. + * Tampoco un error con respuesta HTTP: eso lo decidio el upstream, no la red. + */ +const TRANSPORT_INTERRUPTION_CODES = new Set([ + 'ECONNRESET', 'ECONNABORTED', 'ETIMEDOUT', 'EPIPE', + 'UND_ERR_SOCKET', 'UND_ERR_BODY_TIMEOUT', 'UND_ERR_HEADERS_TIMEOUT', + 'ERR_STREAM_PREMATURE_CLOSE' +]); +const TRANSPORT_INTERRUPTION_MESSAGE_RE = /other side closed|socket hang up|premature close/i; +const isTransportInterruption = (error) => { + if (!error || typeof error !== 'object') return false; + if (error.response) return false; + if (error.name === 'AbortError' || error.code === 'ERR_CANCELED') return false; + const code = String(error.code || error.cause?.code || ''); + if (TRANSPORT_INTERRUPTION_CODES.has(code)) return true; + return TRANSPORT_INTERRUPTION_MESSAGE_RE.test(String(error.message || error.cause?.message || '')); +}; + /** * El adjunto de contexto largo (upload + parse en Qwen) fallo en una peticion que NO * puede compactarse (lleva tools). Es una averia temporal del upstream —el servicio de @@ -225,6 +248,7 @@ module.exports = { assertNoUpstreamFailure, isRateLimitError, isWafChallengeError, + isTransportInterruption, rateLimitRetryAfterSeconds, describeUpstreamFailure, noteRateLimitedAccount, diff --git a/tests/anthropic-midstream-failover.test.js b/tests/anthropic-midstream-failover.test.js new file mode 100644 index 00000000..96048b1c --- /dev/null +++ b/tests/anthropic-midstream-failover.test.js @@ -0,0 +1,199 @@ +// Failover por cuenta cuando el upstream se cae ANTES del primer bloque de contenido. +// +// Medido en qwen-next el 2026-09-16: 3 de 70 peticiones terminaron en +// `UND_ERR_SOCKET: other side closed` a mitad del SSE y 1 en `RateLimited` en el primer +// frame. Las cuatro salieron al cliente como error a medio stream aunque todavia no habia +// visto nada. Con cero content_block emitidos reenviar es seguro (no duplica texto), asi +// que el handler sortea otra cuenta y sigue; con uno ya en el cable, el error sale como +// siempre y el cliente reintenta el. +// Misma cabecera que tests/agent-account-failover.test.js: sin API_KEY el arranque corta el +// proceso antes de que el runner reporte, y sin DATA_SAVE_MODE=none account.js intentaria +// un login real al importarse. +process.env.API_KEY = 'midstream-failover-test-key' +process.env.DATA_SAVE_MODE = 'none' +process.env.ACCOUNTS = '' +process.env.ENABLE_CLI = 'false' +process.env.ENABLE_FILE_LOG = 'false' +process.env.PROXY_URL = '' + +const { describe, it, after } = require('node:test') +const assert = require('node:assert/strict') +const { Readable } = require('node:stream') + +const accountManager = require('../src/utils/account') +const { handleAnthropicStream } = require('../src/controllers/anthropic.js') +const { isTransportInterruption } = require('../src/utils/upstream-error.js') + +after(() => { accountManager.destroy() }) + +const createMockResponse = () => ({ + output: '', + headers: {}, + writableEnded: false, + destroyed: false, + set(headers) { Object.assign(this.headers, headers); return this }, + status() { return this }, + write(chunk) { this.output += String(chunk); return true }, + end(chunk = '') { this.output += String(chunk); this.writableEnded = true } +}) + +const frame = (content) => + `data: ${JSON.stringify({ choices: [{ delta: { phase: 'answer', content }, finish_reason: null }] })}\n\n` +const DONE = 'data: {"choices":[{"delta":{},"finish_reason":"stop"}]}\n\ndata: [DONE]\n\n' +// El paquete real de cuota (tests/upstream-quota-429.test.js lo fija). +const QUOTA = `data: ${JSON.stringify({ + success: false, + data: { code: 'RateLimited', details: "You've reached the upper limit for today's usage." } +})}\n\n` + +const socketClose = () => Object.assign(new Error('other side closed'), { code: 'UND_ERR_SOCKET' }) +const streamOf = (chunks, failure = null) => Readable.from((async function* () { + for (const chunk of chunks) yield chunk + if (failure) throw failure +})()) + +const emittedText = (output) => + [...output.matchAll(/"type":"text_delta","text":("(?:[^"\\]|\\.)*")/g)].map(m => JSON.parse(m[1])).join('') +const errorEvents = (output) => output.match(/^event: error$/gm) || [] + +const FIRST = { email: 'first@example.com', proxy: 'socks5h://lohari-warp-qwen:9091' } +const SECOND = { email: 'second@example.com', proxy: 'socks5h://lohari-warp-qwen:9091' } + +const recordingSendRequest = (respond) => { + const calls = [] + const fn = async (body, options) => { + calls.push({ body, options }) + return respond(calls.length) + } + fn.calls = calls + return fn +} +const recovered = () => ({ status: true, response: streamOf([frame('Recovered.'), DONE]), currentAccount: { ...SECOND } }) + +const ctxFor = (sendRequest, overrides = {}) => ({ + message_id: 'msg_failover', + model: 'qwen-test', + hasTools: false, + toolChoice: 'auto', + requestBody: { messages: [{ role: 'user', content: 'hola' }] }, + currentAccount: { ...FIRST }, + upstreamOptions: { allowContextCompaction: true, contextPrefixKey: 'session-k1' }, + sendRequest, + ...overrides +}) + +describe('mid-stream failover before the first content block', () => { + it('a socket close with nothing emitted is served by another account, transparently', async () => { + const sendRequest = recordingSendRequest(() => recovered()) + const res = createMockResponse() + const ctx = ctxFor(sendRequest) + + await handleAnthropicStream(res, ctx, streamOf([], socketClose())) + + assert.equal(emittedText(res.output), 'Recovered.') + assert.equal(errorEvents(res.output).length, 0, 'the client never sees the failure') + assert.match(res.output, /event: message_stop/) + assert.equal(sendRequest.calls.length, 1) + const [{ body, options }] = sendRequest.calls + assert.deepEqual(body, ctx.requestBody, 'the same internal body is replayed, no hint appended') + assert.deepEqual(options.excludeEmails, ['first@example.com'], 'the account that failed is skipped') + assert.equal(options.contextPrefixKey, 'session-k1', 'the uploaded history prefix is reused, not re-parsed') + assert.equal(options.allowContextCompaction, true) + assert.equal(ctx.currentAccount.email, 'second@example.com', 'stats and later 429s go to who actually served') + }) + + it('a RateLimited first frame rotates to another account instead of a 429 mid-stream', async () => { + const sendRequest = recordingSendRequest(() => recovered()) + const res = createMockResponse() + const ctx = ctxFor(sendRequest) + + await handleAnthropicStream(res, ctx, streamOf([QUOTA])) + + assert.equal(emittedText(res.output), 'Recovered.') + assert.equal(errorEvents(res.output).length, 0) + assert.equal(sendRequest.calls.length, 1) + assert.deepEqual(sendRequest.calls[0].options.excludeEmails, ['first@example.com']) + assert.equal(ctx.currentAccount.email, 'second@example.com') + }) + + it('a socket close AFTER a content block is delivered as an error: replaying would duplicate text', async () => { + const sendRequest = recordingSendRequest(() => recovered()) + const res = createMockResponse() + + await assert.rejects( + handleAnthropicStream(res, ctxFor(sendRequest), streamOf([frame('Half an ans')], socketClose())), + (err) => err.code === 'UND_ERR_SOCKET' + ) + assert.equal(sendRequest.calls.length, 0, 'no failover once the client has seen a block') + assert.equal(emittedText(res.output), 'Half an ans', 'what was already sent stays sent') + }) + + it('only one failover per request: a second consecutive close propagates', async () => { + const sendRequest = recordingSendRequest(() => ({ + status: true, response: streamOf([], socketClose()), currentAccount: { ...SECOND } + })) + const res = createMockResponse() + + await assert.rejects( + handleAnthropicStream(res, ctxFor(sendRequest), streamOf([], socketClose())), + (err) => err.code === 'UND_ERR_SOCKET' + ) + assert.equal(sendRequest.calls.length, 1) + }) + + it('a failed replay surfaces the ORIGINAL failure when the retry carries no public message', async () => { + const sendRequest = recordingSendRequest(() => ({ status: false, message: 'no account' })) + const res = createMockResponse() + + await assert.rejects( + handleAnthropicStream(res, ctxFor(sendRequest), streamOf([], socketClose())), + (err) => err.code === 'UND_ERR_SOCKET' + ) + assert.equal(sendRequest.calls.length, 1) + }) + + it('no failover when the client is already gone', async () => { + const sendRequest = recordingSendRequest(() => recovered()) + const res = createMockResponse() + res.destroyed = true + + await assert.rejects( + handleAnthropicStream(res, ctxFor(sendRequest), streamOf([], socketClose())), + (err) => err.code === 'UND_ERR_SOCKET' + ) + assert.equal(sendRequest.calls.length, 0) + }) + + it('an unrelated error (a bug, not the transport) is not retried', async () => { + const sendRequest = recordingSendRequest(() => recovered()) + const res = createMockResponse() + const bug = new TypeError('cannot read properties of undefined') + + await assert.rejects( + handleAnthropicStream(res, ctxFor(sendRequest), streamOf([], bug)), + (err) => err === bug + ) + assert.equal(sendRequest.calls.length, 0) + }) +}) + +describe('isTransportInterruption', () => { + it('recognises socket closes and transport timeouts from Node, undici and streams', () => { + for (const code of ['ECONNRESET', 'ECONNABORTED', 'ETIMEDOUT', 'EPIPE', 'UND_ERR_SOCKET', + 'UND_ERR_BODY_TIMEOUT', 'UND_ERR_HEADERS_TIMEOUT', 'ERR_STREAM_PREMATURE_CLOSE']) { + assert.equal(isTransportInterruption(Object.assign(new Error('x'), { code })), true, code) + } + assert.equal(isTransportInterruption(new Error('other side closed')), true, 'undici message without code') + assert.equal(isTransportInterruption(new Error('socket hang up')), true) + assert.equal(isTransportInterruption(Object.assign(new TypeError('fetch failed'), { cause: { code: 'ECONNRESET' } })), true, 'fetch wraps the code in cause') + }) + + it('never treats the client cancelling, an HTTP response, or a business error as a transport failure', () => { + assert.equal(isTransportInterruption(Object.assign(new Error('canceled'), { code: 'ERR_CANCELED' })), false) + assert.equal(isTransportInterruption(Object.assign(new Error('aborted'), { name: 'AbortError' })), false) + assert.equal(isTransportInterruption(Object.assign(new Error('other side closed'), { response: { status: 502 } })), false) + assert.equal(isTransportInterruption(Object.assign(new Error('quota'), { code: 'RateLimited' })), false) + assert.equal(isTransportInterruption(null), false) + assert.equal(isTransportInterruption('ECONNRESET'), false) + }) +}) diff --git a/tests/expected-counts.json b/tests/expected-counts.json index 5a0633ee..34c1e045 100644 --- a/tests/expected-counts.json +++ b/tests/expected-counts.json @@ -1,6 +1,6 @@ { - "tests": 1159, - "suites": 134, + "tests": 1175, + "suites": 136, "note": "Authoritative count. Verify with the per-file sum in AGENTS.md (\"The test gate\"). The -a on that grep is load-bearing: tool-prompt.test.js emits bytes that make grep call the stream binary, and without -a its whole summary line — 133 tests — is silently dropped from the sum.", "updated": "2026-09-16" } diff --git a/tests/sse-interrupt-metrics.test.js b/tests/sse-interrupt-metrics.test.js new file mode 100644 index 00000000..7849304e --- /dev/null +++ b/tests/sse-interrupt-metrics.test.js @@ -0,0 +1,67 @@ +const test = require('node:test') +const assert = require('node:assert/strict') +const { Readable } = require('node:stream') + +const { consumeSSEStream } = require('../src/utils/sse') + +// A mid-stream failure has to say how far the upstream got. The controller decides from +// it whether a retry is safe, and the [EGRESS] log needs the byte count to tell an early +// close (a few KB) from a late one (tens of KB) — the three live closes on 2026-09-16 were +// all late, 63–90 KiB in. + +const frame = (obj) => `data: ${JSON.stringify(obj)}\n\n` +const socketClose = () => Object.assign(new Error('other side closed'), { code: 'UND_ERR_SOCKET' }) + +test('a transport failure mid-stream carries the bytes and frames consumed so far', async () => { + const first = frame({ a: 1 }) + const failing = Readable.from((async function* () { + yield Buffer.from(first) + throw socketClose() + })()) + const seen = [] + await assert.rejects( + consumeSSEStream(failing, async (f) => { seen.push(f.data) }), + (err) => { + assert.equal(err.code, 'UND_ERR_SOCKET') + assert.equal(err.upstreamBytesRead, Buffer.byteLength(first)) + assert.equal(err.upstreamEventCount, 1) + return true + } + ) + assert.equal(seen.length, 1, 'the frame before the failure was still delivered') +}) + +test('a failure before any byte reports zero, not undefined', async () => { + const failing = Readable.from((async function* () { throw socketClose() })()) + await assert.rejects(consumeSSEStream(failing, async () => {}), (err) => { + assert.equal(err.upstreamBytesRead, 0) + assert.equal(err.upstreamEventCount, 0) + return true + }) +}) + +test('an onFrame rejection (upstream business error such as RateLimited) is annotated too', async () => { + const text = frame({ success: false, data: { code: 'RateLimited' } }) + await assert.rejects( + consumeSSEStream(Readable.from([text]), async () => { + throw Object.assign(new Error('quota'), { code: 'RateLimited' }) + }), + (err) => { + assert.equal(err.code, 'RateLimited') + assert.equal(err.upstreamBytesRead, Buffer.byteLength(text)) + assert.equal(err.upstreamEventCount, 1) + return true + } + ) +}) + +test('a clean stream reports bytesRead alongside eventCount; string and Buffer chunks count the same', async () => { + const text = frame({ a: 1 }) + frame({ b: 2 }) + 'data: [DONE]\n\n' + const asString = await consumeSSEStream(Readable.from([text]), async () => {}) + const asBuffer = await consumeSSEStream(Readable.from([Buffer.from(text)]), async () => {}) + assert.equal(asString.bytesRead, Buffer.byteLength(text)) + assert.equal(asBuffer.bytesRead, Buffer.byteLength(text)) + assert.equal(asString.eventCount, 3) + assert.equal(asString.sawDone, true) + assert.equal(asString.completed, true) +}) diff --git a/tests/upstream-quota-429.test.js b/tests/upstream-quota-429.test.js index 6fd96fe1..4b8f2356 100644 --- a/tests/upstream-quota-429.test.js +++ b/tests/upstream-quota-429.test.js @@ -37,9 +37,18 @@ const requestModule = require('../src/utils/request.js'); let upstreamFactory = null; /** La cuenta que el upstream dice haber usado. Sin esto no hay a quien culpar del gasto. */ let upstreamAccount = null; -requestModule.sendChatRequest = async () => (upstreamFactory - ? { status: true, response: upstreamFactory(), currentAccount: upstreamAccount } - : { status: false }); +requestModule.sendChatRequest = async (_body, options = {}) => { + // Como el rotador real: una cuenta excluida (ya quemada en este mismo request por el + // failover a mitad de stream) no vuelve a salir. Con una sola cuenta eso es "no hay + // alternativa", y el controlador debe entregar el fallo original. + const excluded = Array.isArray(options.excludeEmails) ? options.excludeEmails : []; + if (upstreamAccount?.email && excluded.includes(upstreamAccount.email)) { + return { status: false, response: null, message: 'offline test: no alternative account' }; + } + return upstreamFactory + ? { status: true, response: upstreamFactory(), currentAccount: upstreamAccount } + : { status: false }; +}; const { UpstreamResponseError, From 7ae0af0468f2b7e1ce721e3dd6ccd8d5754d230f Mon Sep 17 00:00:00 2001 From: PEDRO LOBATO CARCAMO Date: Wed, 16 Sep 2026 08:04:07 -0600 Subject: [PATCH 6/9] review: one transport classifier; the stream tags the account that failed - request.js#isRetryableNetworkError delegates to upstream-error.js#isTransportInterruption and only adds the two connect-phase codes (ECONNREFUSED, EAI_AGAIN). Same guard against HTTP responses, same message sniff, one list to maintain. proxy-helper's UNDICI_ERROR_CODES stays: it is a mapping onto Node codes, not a classifier. - anthropic.js: when the stream dies and no failover is left, the error carries failedAccountEmail (email only) so the handler's catch marks the account that actually served -- same idiom as chat.js. Drops the `let ctx = null` hoist. - anthropic.js: attemptsMade counts only turns the model answered; failover iterations skip the increment instead of decrementing after the fact. Tests: +1 (isRetryableNetworkError); two failover assertions now check failedAccountEmail. Gate 1175/136 -> 1176/137. Co-Authored-By: Claude Fable 5.1 --- src/controllers/anthropic.js | 25 ++++++++++++++-------- src/utils/request.js | 23 ++++++++------------ tests/anthropic-midstream-failover.test.js | 18 +++++++++++++++- tests/expected-counts.json | 4 ++-- 4 files changed, 44 insertions(+), 26 deletions(-) diff --git a/src/controllers/anthropic.js b/src/controllers/anthropic.js index 4f9a5d9b..dd4e4a73 100644 --- a/src/controllers/anthropic.js +++ b/src/controllers/anthropic.js @@ -1745,13 +1745,14 @@ const handleAnthropicStream = async (res, ctx, upstream) => { const maxAttempts = Math.max(1, Number(config.agentTurnMaxAttempts) || 1); let currentUpstream = upstream; + // Vueltas en las que el modelo llego a responder. Un failover no cuenta: el modelo aun no + // hablo, y el cupo de correccion de protocolo (maxAttempts) es para lo que SI dijo. let attemptsMade = 0; let retriedAfterVisibleText = false; let protocolRecoveryRetried = false; let failoverRetries = 0; for (;;) { - attemptsMade += 1; startAttempt(); try { @@ -1775,6 +1776,12 @@ const handleAnthropicStream = async (res, ctx, upstream) => { }); } if (!canFailover) { + // Quien servia cuando se cayo, para que el catch del handler marque ESA cuenta si el + // error es de cuota (tras un failover ya no es la del sorteo inicial). Solo el email: + // el error se loguea y no debe arrastrar el token. Mismo campo que recordFailedAccount. + if (e && typeof e === 'object' && !e.failedAccountEmail) { + e.failedAccountEmail = ctx.currentAccount?.email || null; + } logger.error('Anthropic 流式心跳包装失败', 'ANTHROPIC', '', e); throw e; } @@ -1800,10 +1807,9 @@ const handleAnthropicStream = async (res, ctx, upstream) => { currentUpstream = retryResp.response; // Stats y un eventual 429 posterior se atribuyen a quien sirvio de verdad. if (retryResp.currentAccount) ctx.currentAccount = retryResp.currentAccount; - // El failover no gasta cupo de correccion de protocolo: el modelo aun no ha hablado. - attemptsMade -= 1; continue; } + attemptsMade += 1; // 本轮收尾。解析器的尾巴属于这一轮,必须在判定之前放出来。文本通道截断之后例外: // 根本不 flush —— 解析器里压着的只是失控那一 push 的残余(半个触发器 / 半截负载), @@ -2720,9 +2726,6 @@ const handleAnthropicMessages = async (req, res) => { // Tambien fuera: el catch decide si olvidar un prefijo de historial reutilizado. let upstreamResp = null; let contextPrefixKey = null; - // Y el contexto del handler: un failover a mitad de stream cambia ctx.currentAccount por - // la cuenta que sirvio de verdad; el catch tiene que marcar ESA, no la del sorteo inicial. - let ctx = null; try { const compatibility = analyzeAnthropicCompatibility(req.body || {}); const compatibilityHeaders = buildAnthropicCompatibilityHeaders(compatibility); @@ -2762,7 +2765,7 @@ const handleAnthropicMessages = async (req, res) => { } const message_id = `msg_${generateUUID().replace(/-/g, '').slice(0, 24)}`; - ctx = { + const ctx = { message_id, model, hasTools, @@ -2793,9 +2796,13 @@ const handleAnthropicMessages = async (req, res) => { // devolviendo la misma cuenta agotada al sorteo, y la quema en cada vuelta. // Si el failover a mitad de stream ya paso la cuenta a cooldown (recordFailedAccount) // y luego no hubo otra cuenta a la que saltar, el error que llega aqui es el mismo: - // no se marca dos veces. + // no se marca dos veces. Tras un failover la cuenta que fallo no es la del sorteo + // inicial: el stream la deja en error.failedAccountEmail. Gemelo: chat.js. if (!error?.accountFailureRecorded) { - noteRateLimitedAccount(error, ctx?.currentAccount || currentAccount); + noteRateLimitedAccount( + error, + error?.failedAccountEmail ? { email: error.failedAccountEmail } : currentAccount + ); } // Un prefijo de historial reutilizado pudo ser la causa (file_id que Qwen ya no // reconoce): se olvida y el reintento del cliente hornea uno nuevo. Un 529 por diff --git a/src/utils/request.js b/src/utils/request.js index 6ff08c87..ccbec424 100644 --- a/src/utils/request.js +++ b/src/utils/request.js @@ -7,28 +7,22 @@ const { applyProxyToAxiosConfig, getChatBaseUrl } = require('./proxy-helper'); const { generateUUID, jitter } = require('./tools.js') const { uploadAgentContextFile, buildChatFileDescriptor } = require('./upload.js') const { buildRequestHeaders } = require('./header-profile') -const { ContextExternalizationError } = require('./upstream-error.js') +const { ContextExternalizationError, isTransportInterruption } = require('./upstream-error.js') const { contextPrefixCache, prefixMatches, canonicalHistoryHash } = require('./context-prefix-cache.js') const { TOOL_CALL_OPEN, LEDGER_HEADER, LEDGER_CAPTION, truncateToolHistoryLedger, stripRetainedThinking } = require('./agent-turn.js') -// 传输层(非 HTTP)错误码 — 这些重试的, HTTP 响应不重试 -const RETRYABLE_ERROR_CODES = new Set([ - 'ECONNRESET', - 'ECONNREFUSED', - 'ETIMEDOUT', - 'ECONNABORTED', - 'EAI_AGAIN' -]) +// 连接阶段失败(拒绝 / DNS 暂时不可用)。传输中断(socket 关闭、超时)的判定在 +// upstream-error.js#isTransportInterruption,这里只补连接前的两个码,不再各自维护一份。 +const CONNECT_FAILURE_CODES = new Set(['ECONNREFUSED', 'EAI_AGAIN']) const isRetryableNetworkError = (error) => { - if (!error) return false + if (!error || typeof error !== 'object') return false + if (isTransportInterruption(error)) return true // 已收到 HTTP 响应 = 上游回包了, 不是传输问题 if (error.response) return false - if (error.code && RETRYABLE_ERROR_CODES.has(error.code)) return true - if (typeof error.message === 'string' && error.message.includes('socket hang up')) return true - return false + return CONNECT_FAILURE_CODES.has(String(error.code || error.cause?.code || '')) } const delay = (ms) => new Promise(resolve => setTimeout(resolve, ms)) @@ -887,5 +881,6 @@ module.exports = { compactAgentContextFallback, externalizeOversizedAgentContext, buildPrefixReusePrompt, - invalidateContextPrefix + invalidateContextPrefix, + isRetryableNetworkError } diff --git a/tests/anthropic-midstream-failover.test.js b/tests/anthropic-midstream-failover.test.js index 96048b1c..f7b238af 100644 --- a/tests/anthropic-midstream-failover.test.js +++ b/tests/anthropic-midstream-failover.test.js @@ -23,6 +23,7 @@ const { Readable } = require('node:stream') const accountManager = require('../src/utils/account') const { handleAnthropicStream } = require('../src/controllers/anthropic.js') const { isTransportInterruption } = require('../src/utils/upstream-error.js') +const { isRetryableNetworkError } = require('../src/utils/request.js') after(() => { accountManager.destroy() }) @@ -122,7 +123,7 @@ describe('mid-stream failover before the first content block', () => { await assert.rejects( handleAnthropicStream(res, ctxFor(sendRequest), streamOf([frame('Half an ans')], socketClose())), - (err) => err.code === 'UND_ERR_SOCKET' + (err) => err.code === 'UND_ERR_SOCKET' && err.failedAccountEmail === 'first@example.com' ) assert.equal(sendRequest.calls.length, 0, 'no failover once the client has seen a block') assert.equal(emittedText(res.output), 'Half an ans', 'what was already sent stays sent') @@ -137,6 +138,7 @@ describe('mid-stream failover before the first content block', () => { await assert.rejects( handleAnthropicStream(res, ctxFor(sendRequest), streamOf([], socketClose())), (err) => err.code === 'UND_ERR_SOCKET' + && err.failedAccountEmail === 'second@example.com' ) assert.equal(sendRequest.calls.length, 1) }) @@ -197,3 +199,17 @@ describe('isTransportInterruption', () => { assert.equal(isTransportInterruption('ECONNRESET'), false) }) }) + +describe('isRetryableNetworkError (pre-response retry in sendRequest)', () => { + it('is isTransportInterruption plus the two connect-phase codes, with the same exclusions', () => { + for (const code of ['ECONNREFUSED', 'EAI_AGAIN', 'ECONNRESET', 'ETIMEDOUT', 'UND_ERR_SOCKET', 'EPIPE']) { + assert.equal(isRetryableNetworkError(Object.assign(new Error('x'), { code })), true, code) + } + assert.equal(isRetryableNetworkError(Object.assign(new TypeError('fetch failed'), { cause: { code: 'ECONNREFUSED' } })), true, 'connect code wrapped in cause') + assert.equal(isRetryableNetworkError(new Error('socket hang up')), true) + assert.equal(isRetryableNetworkError(Object.assign(new Error('refused'), { code: 'ECONNREFUSED', response: { status: 502 } })), false, 'an HTTP response is never a network error') + assert.equal(isRetryableNetworkError(Object.assign(new Error('canceled'), { code: 'ERR_CANCELED' })), false) + assert.equal(isRetryableNetworkError(Object.assign(new Error('quota'), { code: 'RateLimited' })), false) + assert.equal(isRetryableNetworkError(null), false) + }) +}) diff --git a/tests/expected-counts.json b/tests/expected-counts.json index 34c1e045..c15395e2 100644 --- a/tests/expected-counts.json +++ b/tests/expected-counts.json @@ -1,6 +1,6 @@ { - "tests": 1175, - "suites": 136, + "tests": 1176, + "suites": 137, "note": "Authoritative count. Verify with the per-file sum in AGENTS.md (\"The test gate\"). The -a on that grep is load-bearing: tool-prompt.test.js emits bytes that make grep call the stream binary, and without -a its whole summary line — 133 tests — is silently dropped from the sum.", "updated": "2026-09-16" } From 4b39af109a5579ea7a84d3d1feee78dd1df4d770 Mon Sep 17 00:00:00 2001 From: PEDRO LOBATO CARCAMO Date: Wed, 16 Sep 2026 10:12:41 -0600 Subject: [PATCH 7/9] lint: declare the usage tokens where they are assigned; fail the empty stream without a yield-less generator CI on main (928f816, PR #175) is red on four no-useless-assignment errors: `let promptTokens = 0` / `let completionTokens = 0` at the top of both Anthropic handlers are overwritten unconditionally after the upstream loop, so the zero is never read. Declare them `const` at the assignment instead. tests/sse-interrupt-metrics: the "failure before any byte" case built its stream from an async generator whose only statement is `throw`, which trips require-yield. A Readable that destroys itself on first read models the same thing (error before any data) without the generator. Co-Authored-By: Claude Fable 5.1 --- src/controllers/anthropic.js | 12 ++++-------- tests/sse-interrupt-metrics.test.js | 2 +- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/src/controllers/anthropic.js b/src/controllers/anthropic.js index dd4e4a73..8ce770ac 100644 --- a/src/controllers/anthropic.js +++ b/src/controllers/anthropic.js @@ -1304,8 +1304,6 @@ const handleAnthropicStream = async (res, ctx, upstream) => { let textBlockOpen = false; let thinkingBlockOpen = false; let thinkingSignature = null; - let promptTokens = 0; - let completionTokens = 0; let upstreamUsage = null; // 上游逐帧累计的 usage(DashScope 命名已归一化;null = 还没报) let upstreamFinishReason = null; let upstreamCompleted; @@ -2089,8 +2087,8 @@ const handleAnthropicStream = async (res, ctx, upstream) => { // 只对上游没报的字段补本地估算(早停的回合收不到尾部 usage 帧) const usage = reportUsage(upstreamUsage, () => createUsageObject(requestBody?.messages || '', completionContent), 'ANTHROPIC'); - promptTokens = usage.prompt_tokens; - completionTokens = usage.completion_tokens; + const promptTokens = usage.prompt_tokens; + const completionTokens = usage.completion_tokens; // Daily stats 累计——一次性归属主账户(见模块顶部 attributeChatUsage 注释) attributeChatUsage(ctx.currentAccount, promptTokens, completionTokens); @@ -2129,8 +2127,6 @@ const handleAnthropicNonStream = async (res, ctx, upstream) => { // 与流式分支同一条纪律,上一轮的泄漏已经重试过了。 let attemptThinkingContent = ''; let answerContent = ''; - let promptTokens = 0; - let completionTokens = 0; let upstreamUsage = null; // 上游逐帧累计的 usage(DashScope 命名已归一化;null = 还没报) let webSearchInfo = null; let upstreamFinishReason = null; @@ -2665,8 +2661,8 @@ const handleAnthropicNonStream = async (res, ctx, upstream) => { const nativeArgsText = nativeToolCalls.map(call => call.function.arguments || '').join(''); return createUsageObject(requestBody?.messages || '', thinkingContent + answerContent + nativeArgsText); }, 'ANTHROPIC'); - promptTokens = usage.prompt_tokens; - completionTokens = usage.completion_tokens; + const promptTokens = usage.prompt_tokens; + const completionTokens = usage.completion_tokens; const contentBlocks = []; if (thinkingContent && thinkingContent.trim()) { diff --git a/tests/sse-interrupt-metrics.test.js b/tests/sse-interrupt-metrics.test.js index 7849304e..7f05042b 100644 --- a/tests/sse-interrupt-metrics.test.js +++ b/tests/sse-interrupt-metrics.test.js @@ -32,7 +32,7 @@ test('a transport failure mid-stream carries the bytes and frames consumed so fa }) test('a failure before any byte reports zero, not undefined', async () => { - const failing = Readable.from((async function* () { throw socketClose() })()) + const failing = new Readable({ read() { this.destroy(socketClose()) } }) await assert.rejects(consumeSSEStream(failing, async () => {}), (err) => { assert.equal(err.upstreamBytesRead, 0) assert.equal(err.upstreamEventCount, 0) From 81b5526c8e0cc7b85351451ecf1174e64c0ed224 Mon Sep 17 00:00:00 2001 From: PEDRO LOBATO CARCAMO Date: Wed, 16 Sep 2026 10:23:16 -0600 Subject: [PATCH 8/9] parse: name the egress on transport failures of /files/parse and its status polls A socket hang-up or ECONNRESET on the parse POST (or on /files/parse/status) used to surface as a bare axios error: no hint of which proxy the request left through, so a broken WARP sidecar and a broken direct route looked the same in the logs. Wrap both calls with the same helper the OSS/STS path already uses: the error keeps its code, gains `egress` (protocol//host:port, credentials dropped) and the message ends with `(file_id, via )`. Docs: the proxy sections listed HTTP/HTTPS/SOCKS5 only; `socks5h://` has been accepted for a while and is what qwen-next actually runs, so list it and say what the `h` buys (DNS resolved at the proxy). Test: transport failure on the parse POST through `socks5h://user:s3cr3t@lohari-warp-qwen:9091` -> code ECONNRESET, egress `socks5h://lohari-warp-qwen:9091`, secret absent from the message, zero status polls. Suite blessed 1176 -> 1177. Co-Authored-By: Claude Fable 5.1 --- .env.example | 4 ++-- README-en.md | 6 +++--- README-ru.md | 6 +++--- README.md | 6 +++--- src/utils/proxy-helper.js | 5 +++-- src/utils/upload.js | 22 ++++++++++++++++------ tests/expected-counts.json | 2 +- tests/upload-parse-status.test.js | 23 ++++++++++++++++++++++- 8 files changed, 53 insertions(+), 21 deletions(-) diff --git a/.env.example b/.env.example index 61dbdc64..c00d2dbc 100644 --- a/.env.example +++ b/.env.example @@ -209,11 +209,11 @@ QWEN_CHAT_PROXY_URL= QWEN_CLI_PROXY_URL= # HTTP/HTTPS 代理配置 -# 支持 HTTP, HTTPS, SOCKS5 代理 +# 支持 HTTP, HTTPS, SOCKS5, SOCKS5H 代理(socks5h:// 由代理端解析 DNS) # 全局回退代理:当账号未配置专属代理时使用(账号级代理可在 dashboard 或 ACCOUNTS 中设置) # 优先级:account.proxy > PROXY_URL > 不使用代理 # HTTP/HTTPS proxy configuration -# Supports HTTP, HTTPS, SOCKS5 proxies +# Supports HTTP, HTTPS, SOCKS5, SOCKS5H proxies (socks5h:// resolves DNS at the proxy) # Global fallback proxy — used only when an account has no dedicated proxy # (per-account proxy can be set via the dashboard or ACCOUNTS env var) # Priority: account.proxy > PROXY_URL > no proxy diff --git a/README-en.md b/README-en.md index 83e0dbd8..b6b84069 100644 --- a/README-en.md +++ b/README-en.md @@ -37,7 +37,7 @@ Each account can be configured with its own outbound proxy, allowing multiple ac **Priority:** `account.proxy` > Global `PROXY_URL` > No proxy -**Supported Proxy Protocols:** HTTP / HTTPS / SOCKS5 (consistent with `PROXY_URL`) +**Supported Proxy Protocols:** HTTP / HTTPS / SOCKS5 / SOCKS5H (consistent with `PROXY_URL`) **Frontend Configuration (Recommended):** Open the management panel → Fill in the "Proxy Address" field when adding accounts, or click the "Modify Proxy" button on existing account cards. @@ -103,7 +103,7 @@ AGENT_TURN_MAX_TOOL_CALLS=24 # Anthropic path: text-channel tool_use cap per ag # 🌐 Proxy and Reverse Proxy Configuration QWEN_CHAT_PROXY_URL= # Custom Chat API reverse proxy URL (default: https://chat.qwen.ai) QWEN_CLI_PROXY_URL= # Custom CLI API reverse proxy URL (default: https://portal.qwen.ai) -PROXY_URL= # HTTP/HTTPS/SOCKS5 proxy address (example: http://127.0.0.1:7890) +PROXY_URL= # HTTP/HTTPS/SOCKS5/SOCKS5H proxy address (example: http://127.0.0.1:7890) # 🗄️ Data Storage DATA_SAVE_MODE=none # Data save mode (none/file/redis) @@ -132,7 +132,7 @@ CACHE_MODE=default # Image cache mode (default/file) | `AGENT_CONTEXT_LIVE_PROMPT_BYTES` | Maximum size of the tool protocol and current turn kept in the live request after context externalization | `49152` (48 KiB) | | `QWEN_CHAT_PROXY_URL` | Custom Chat API reverse proxy address | `https://your-proxy.com` | | `QWEN_CLI_PROXY_URL` | Custom CLI API reverse proxy address | `https://your-cli-proxy.com` | -| `PROXY_URL` | Outbound request proxy address, supports HTTP/HTTPS/SOCKS5 | `http://127.0.0.1:7890` | +| `PROXY_URL` | Outbound request proxy address, supports HTTP/HTTPS/SOCKS5/SOCKS5H | `http://127.0.0.1:7890` | | `DATA_SAVE_MODE` | Data persistence method | `none`/`file`/[redis](file://d:\Code\Qwen2API\src\utils\logger.js#L294-L296) | | `REDIS_URL` | Redis database connection address, use `rediss://` protocol when using TLS encryption | `redis://localhost:6379` or `rediss://xxx.upstash.io` | | `BATCH_LOGIN_CONCURRENCY` | Login concurrency during batch account addition, can be adjusted dynamically in frontend system settings | `5` | diff --git a/README-ru.md b/README-ru.md index 7d4e572b..03007ec8 100644 --- a/README-ru.md +++ b/README-ru.md @@ -57,7 +57,7 @@ QWEN_CHAT_PROXY_URL=http://127.0.0.1:8000/qwen # Адрес обратного **Приоритет:** `account.proxy` > глобальный `PROXY_URL` > без прокси -**Поддерживаемые протоколы:** HTTP / HTTPS / SOCKS5 (как у `PROXY_URL`) +**Поддерживаемые протоколы:** HTTP / HTTPS / SOCKS5 / SOCKS5H (как у `PROXY_URL`) **Через панель (рекомендуется):** Откройте dashboard → при добавлении аккаунта заполните поле «Proxy URL», либо нажмите кнопку «Изменить proxy» на карточке существующего аккаунта. @@ -123,7 +123,7 @@ AGENT_TURN_MAX_TOOL_CALLS=24 # Anthropic: лимит tool_use по тексто # 🌐 Прокси и обратный прокси QWEN_CHAT_PROXY_URL= # Пользовательский URL обратного прокси Chat API (по умолчанию: https://chat.qwen.ai) QWEN_CLI_PROXY_URL= # Пользовательский URL обратного прокси CLI API (по умолчанию: https://portal.qwen.ai) -PROXY_URL= # Адрес HTTP/HTTPS/SOCKS5 прокси (например: http://127.0.0.1:7890) +PROXY_URL= # Адрес HTTP/HTTPS/SOCKS5/SOCKS5H прокси (например: http://127.0.0.1:7890) # 🗄️ Хранение данных DATA_SAVE_MODE=none # Режим сохранения данных (none/file/redis) @@ -150,7 +150,7 @@ CACHE_MODE=default # Режим кэширования изображ | `AGENT_TURN_MAX_TOOL_CALLS` | Путь Anthropic: лимит блоков `tool_use` по текстовому каналу за один ход агента (4–256). Если модель «идёт вразнос» после нарративного `[TOOL CALL]` (сотни повторов одного вызова, выдуманная сессия целиком), upstream обрывается сразу после N-го принятого вызова, а принятые вызовы отдаются со `stop_reason=tool_use`; после уже принятого в более раннем delta вызова дубликат, отклонённый вызов или проза/размышление также обрывают ход | `24` | | `QWEN_CHAT_PROXY_URL` | Пользовательский адрес обратного прокси Chat API | `https://your-proxy.com` | | `QWEN_CLI_PROXY_URL` | Пользовательский адрес обратного прокси CLI API | `https://your-cli-proxy.com` | -| `PROXY_URL` | Адрес прокси для исходящих запросов, поддержка HTTP/HTTPS/SOCKS5 | `http://127.0.0.1:7890` | +| `PROXY_URL` | Адрес прокси для исходящих запросов, поддержка HTTP/HTTPS/SOCKS5/SOCKS5H | `http://127.0.0.1:7890` | | `DATA_SAVE_MODE` | Способ персистентного хранения данных | `none`/`file`/`redis` | | `REDIS_URL` | Адрес подключения к Redis, при использовании TLS-шифрования необходим протокол `rediss://` | `redis://localhost:6379` или `rediss://xxx.upstash.io` | | `BATCH_LOGIN_CONCURRENCY` | Параллельность входа при массовом добавлении аккаунтов, можно динамически менять в системных настройках фронтенда | `5` | diff --git a/README.md b/README.md index b85d07b1..1ed570c2 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ Qwen-Proxy 是一个将 `https://chat.qwen.ai` 和 `Qwen Code / Qwen Cli` 转换 **优先级:** `account.proxy` > 全局 `PROXY_URL` > 不使用代理 -**支持的代理协议:** HTTP / HTTPS / SOCKS5(与 `PROXY_URL` 一致) +**支持的代理协议:** HTTP / HTTPS / SOCKS5 / SOCKS5H(与 `PROXY_URL` 一致) **前端配置(推荐):** 打开管理面板 → 添加账号时填写 "代理地址" 字段,或在已有账号卡片上点击 "修改代理" 按钮。 @@ -108,7 +108,7 @@ AGENT_CONTEXT_LIVE_PROMPT_BYTES=49152 # 外置后仍内联保留的关键任 # 🌐 代理与反代配置 QWEN_CHAT_PROXY_URL= # 自定义 Chat API 反代URL (默认: https://chat.qwen.ai) QWEN_CLI_PROXY_URL= # 自定义 CLI API 反代URL (默认: https://portal.qwen.ai) -PROXY_URL= # HTTP/HTTPS/SOCKS5 代理地址 (例如: http://127.0.0.1:7890) +PROXY_URL= # HTTP/HTTPS/SOCKS5/SOCKS5H 代理地址 (例如: http://127.0.0.1:7890) # 🗄️ 数据存储 DATA_SAVE_MODE=none # 数据保存模式 (none/file/redis) @@ -140,7 +140,7 @@ CACHE_MODE=default # 图片缓存模式 (default/file) | `AGENT_CONTEXT_LIVE_PROMPT_BYTES` | 上下文外置后,实时请求中保留的工具协议、system/developer 指令、原始任务、最近工具进度和当前结果的最大大小 | `49152`(48 KiB) | | `QWEN_CHAT_PROXY_URL` | 自定义 Chat API 反代地址 | `https://your-proxy.com` | | `QWEN_CLI_PROXY_URL` | 自定义 CLI API 反代地址 | `https://your-cli-proxy.com` | -| `PROXY_URL` | 出站请求代理地址,支持 HTTP/HTTPS/SOCKS5 | `http://127.0.0.1:7890` | +| `PROXY_URL` | 出站请求代理地址,支持 HTTP/HTTPS/SOCKS5/SOCKS5H | `http://127.0.0.1:7890` | | `DATA_SAVE_MODE` | 数据持久化方式 | `none`/`file`/`redis` | | `REDIS_URL` | Redis 数据库连接地址,使用TLS加密时需使用 `rediss://` 协议 | `redis://localhost:6379` 或 `rediss://xxx.upstash.io` | | `BATCH_LOGIN_CONCURRENCY` | 批量添加账号时的登录并发数,可在前端系统设置中动态调整 | `5` | diff --git a/src/utils/proxy-helper.js b/src/utils/proxy-helper.js index 5e83602d..5ec0fc92 100644 --- a/src/utils/proxy-helper.js +++ b/src/utils/proxy-helper.js @@ -76,7 +76,8 @@ const getProxyTransport = (proxyAgent) => { const requestDispatcher = dispatcher.compose(interceptors.redirect({ maxRedirections: 20 }), interceptors.decompress()); // Never hand a Readable.toWeb/fromWeb-bridged body to a consumer: under Bun the bridge drops // undici's body error on a mid-body close, so the consumer hangs and the rejection escapes as a - // process crash (qwen-next died 6x in 2.5 h on 2026-09-16; tools/dev-probes/repro-bridge.js). + // process crash (qwen-next died 6x in 2.5 h on 2026-09-16; reproduced with a proxy that closes + // the socket mid-body). // Streams are returned as undici's own Node Readable; everything else is buffered right here. const fetch = async (url, options) => { const request = new globalThis.Request(url, options); @@ -133,7 +134,7 @@ const getProxyTransport = (proxyAgent) => { if (!validateStatus || validateStatus(axiosResponse.status)) return axiosResponse; throw new axios.AxiosError( `Request failed with status code ${axiosResponse.status}`, - [axios.AxiosError.ERR_BAD_REQUEST, axios.AxiosError.ERR_BAD_RESPONSE][Math.floor(axiosResponse.status / 100) - 4], + axiosResponse.status >= 500 ? axios.AxiosError.ERR_BAD_RESPONSE : axios.AxiosError.ERR_BAD_REQUEST, requestConfig, null, axiosResponse diff --git a/src/utils/upload.js b/src/utils/upload.js index eb8b3d45..995a709a 100644 --- a/src/utils/upload.js +++ b/src/utils/upload.js @@ -414,18 +414,28 @@ const parseUploadedTextFile = async (fileId, authToken, account, options = {}) = timeout: Math.max(1000, Number(options.timeoutMs) || 30000) }, account) - const parseResponse = await axios.post(`${baseUrl}/api/v2/files/parse`, { file_id: fileId }, requestConfig) + // Transport failures (dead proxy, reset, timeout) never reach the JSON checks in + // throwIfParseServiceFailed; tag them with the egress too, or a burnt proxy logs as a bare ECONNRESET. + const postViaEgress = async (path, body) => { + try { + return await axios.post(`${baseUrl}${path}`, body, requestConfig) + } catch (error) { + if (error && typeof error === 'object' && !error.egress) { + error.egress = egress + error.message = `${error.message} (${fileId}, via ${egress})` + } + throw error + } + } + + const parseResponse = await postViaEgress('/api/v2/files/parse', { file_id: fileId }) throwIfParseServiceFailed(parseResponse, fileId, egress) const maxAttempts = Math.max(1, Number(options.maxAttempts) || 30) const intervalMs = Math.max(50, Number(options.intervalMs) || 500) let lastStatus = '' for (let attempt = 1; attempt <= maxAttempts; attempt++) { - const response = await axios.post( - `${baseUrl}/api/v2/files/parse/status`, - { file_id_list: [fileId] }, - requestConfig - ) + const response = await postViaEgress('/api/v2/files/parse/status', { file_id_list: [fileId] }) throwIfParseServiceFailed(response, fileId, egress) const payload = unwrapApiData(response) const records = Array.isArray(payload) ? payload : (payload?.list || payload?.items || []) diff --git a/tests/expected-counts.json b/tests/expected-counts.json index c15395e2..07ea878a 100644 --- a/tests/expected-counts.json +++ b/tests/expected-counts.json @@ -1,5 +1,5 @@ { - "tests": 1176, + "tests": 1177, "suites": 137, "note": "Authoritative count. Verify with the per-file sum in AGENTS.md (\"The test gate\"). The -a on that grep is load-bearing: tool-prompt.test.js emits bytes that make grep call the stream binary, and without -a its whole summary line — 133 tests — is silently dropped from the sum.", "updated": "2026-09-16" diff --git a/tests/upload-parse-status.test.js b/tests/upload-parse-status.test.js index f1695ef2..4ffc94be 100644 --- a/tests/upload-parse-status.test.js +++ b/tests/upload-parse-status.test.js @@ -55,7 +55,10 @@ const axiosStub = { } }; } - if (url.endsWith('/api/v2/files/parse')) return parseResponse + if (url.endsWith('/api/v2/files/parse')) { + if (parseResponse instanceof Error) throw parseResponse + return parseResponse + } if (url.endsWith('/api/v2/files/parse/status')) { return statusQueue.length > 1 ? statusQueue.shift() : statusQueue[0] } @@ -119,6 +122,24 @@ test('parse POST answering with an error code fails before any status poll', asy assert.equal(statusCalls(), 0) }) +test('a transport failure on the parse POST names the egress, credentials dropped', async (context) => { + reset() + const account = { email: 'egress@example.invalid', proxy: 'socks5h://user:s3cr3t@lohari-warp-qwen:9091' } + context.after(() => invalidateProxyAgent(account.proxy)) + parseResponse = Object.assign(new Error('socket hang up'), { code: 'ECONNRESET' }) + await assert.rejects( + parseUploadedTextFile('f1', 'token', account, { intervalMs: 10 }), + (error) => { + assert.equal(error.code, 'ECONNRESET') + assert.equal(error.egress, 'socks5h://lohari-warp-qwen:9091') + assert.match(error.message, /socket hang up \(f1, via socks5h:\/\/lohari-warp-qwen:9091\)/) + assert.doesNotMatch(error.message, /s3cr3t/) + return true + } + ) + assert.equal(statusCalls(), 0) +}) + test('a genuinely pending parse still resolves once the file reports success', async () => { reset() statusQueue = [perFile('pending'), perFile('parsing'), perFile('success')] From dc54f997001b52a3035a47a5d807d82e8e15bbd5 Mon Sep 17 00:00:00 2001 From: PEDRO LOBATO CARCAMO Date: Wed, 16 Sep 2026 11:30:02 -0600 Subject: [PATCH 9/9] config: refuse to boot on an API_KEY with empty slots MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `docker-compose.next.yml` sets `API_KEY=${QWEN2API_ADMIN_KEY_V2},${QWEN2API_API_KEY}`. Deployed without the admin variable in scope it expands to `,sk-client…`; the parser filtered the empty slot away, so `apiKeys` held one key and `adminKey` silently became the CLIENT key. The dashboard then answered "invalid API key" to the real admin key — an auth symptom for what was a deploy bug. qwen-next lost its dashboard this way on 2026-09-16 (recreating with `doppler run --` restored both slots). Nobody writes an empty slot on purpose, so parseApiKeys now throws instead, naming how many slots are empty and which positions. An unset API_KEY is untouched: that is the separate already-handled "no keys configured" case. parseApiKeys takes the raw value as an argument (defaulting to process.env) and is exported so the refusal is testable. Gate 1177 -> 1181. Co-Authored-By: Claude Opus 5 (1M context) --- src/config/index.js | 23 +++++++++--- tests/config-api-key-slots.test.js | 58 ++++++++++++++++++++++++++++++ tests/expected-counts.json | 2 +- 3 files changed, 77 insertions(+), 6 deletions(-) create mode 100644 tests/config-api-key-slots.test.js diff --git a/src/config/index.js b/src/config/index.js index bc51cb0c..5c3bf4c6 100644 --- a/src/config/index.js +++ b/src/config/index.js @@ -5,16 +5,26 @@ dotenv.config() * 解析API_KEY环境变量,支持逗号分隔的多个key * @returns {Object} 包含apiKeys数组和adminKey的对象 */ -const parseApiKeys = () => { - const apiKeyEnv = process.env.API_KEY +const parseApiKeys = (apiKeyEnv = process.env.API_KEY) => { if (!apiKeyEnv) { return { apiKeys: [], adminKey: null } } - const keys = apiKeyEnv.split(',').map(key => key.trim()).filter(key => key.length > 0) + const slots = apiKeyEnv.split(',').map(key => key.trim()) + // 空槽位 = 变量没展开(`API_KEY=${A},${B}` 里 A 未定义就变成 `,B`)。旧代码把空槽 + // 过滤掉照常启动:admin key 静默变成了后面那个客户端 key,dashboard 用真正的 admin + // key 反而报 "invalid"。2026-09-16 qwen-next 就是这样丢掉后台访问的。没人会故意写 + // 出空槽,所以这里直接拒绝启动,而不是带着残缺的密钥表跑下去。 + const emptyAt = slots.map((key, index) => (key.length === 0 ? index + 1 : 0)).filter(Boolean) + if (emptyAt.length > 0) { + throw new Error( + `API_KEY 有 ${emptyAt.length} 个空槽位(第 ${emptyAt.join('、')} 位,共 ${slots.length} 位)——` + + '多半是环境变量没有展开。请检查部署时是否注入了全部密钥(例如 compose 的 ${...} 是否有值)。' + ) + } return { - apiKeys: keys, - adminKey: keys.length > 0 ? keys[0] : null + apiKeys: slots, + adminKey: slots.length > 0 ? slots[0] : null } } @@ -137,4 +147,7 @@ const config = { ) } +// 暴露解析器本身以便测试空槽位的拒绝逻辑(config 对象是模块级单例,无法重复解析)。 +config.parseApiKeys = parseApiKeys + module.exports = config diff --git a/tests/config-api-key-slots.test.js b/tests/config-api-key-slots.test.js new file mode 100644 index 00000000..cfba5d98 --- /dev/null +++ b/tests/config-api-key-slots.test.js @@ -0,0 +1,58 @@ +// An unexpanded API_KEY must stop the boot, not silently demote the admin key. +// +// `docker-compose.next.yml` carries `API_KEY=${QWEN2API_ADMIN_KEY_V2},${QWEN2API_API_KEY}`. +// Deployed without the admin variable in scope it expands to `,sk-client…`; the old parser +// filtered the empty slot away and promoted the CLIENT key to admin, so the dashboard +// answered "invalid API key" to the real admin key. That is how qwen-next lost its +// dashboard on 2026-09-16 — a config bug that only surfaced as an auth failure. +process.env.API_KEY = process.env.API_KEY || 'test-only-key' + +const test = require('node:test') +const assert = require('node:assert/strict') + +const { parseApiKeys } = require('../src/config/index.js') + +test('an empty slot is refused: an unexpanded variable never boots', () => { + // The exact shape the broken deploy produced. + assert.throws(() => parseApiKeys(',sk-client'), /API_KEY/) + // Trailing and middle slots too, and whitespace-only counts as empty. + assert.throws(() => parseApiKeys('sk-admin,'), /API_KEY/) + assert.throws(() => parseApiKeys('sk-admin,,sk-client'), /API_KEY/) + assert.throws(() => parseApiKeys('sk-admin, ,sk-client'), /API_KEY/) +}) + +test('the error names how many slots are empty and where, so the fix is obvious', () => { + assert.throws(() => parseApiKeys(',sk-client'), (error) => { + assert.match(error.message, /第 1 位/) + assert.match(error.message, /共 2 位/) + return true + }) + assert.throws(() => parseApiKeys(',sk-client,'), (error) => { + assert.match(error.message, /第 1、3 位/) + return true + }) +}) + +test('an unset API_KEY still yields an empty config, not a throw', () => { + // Boot without any key is a separate, already-handled case (the server logs and refuses + // requests); only a PARTIALLY expanded value is the deploy bug this guards. + // `undefined` is not testable here: it falls through to the parameter default, which + // reads the real process.env.API_KEY this file has to set to import the module at all. + for (const value of ['', null]) { + assert.deepEqual(parseApiKeys(value), { apiKeys: [], adminKey: null }) + } +}) + +test('well-formed values keep parsing exactly as before: slot 1 is the admin key', () => { + assert.deepEqual(parseApiKeys('sk-admin,sk-client'), { + apiKeys: ['sk-admin', 'sk-client'], + adminKey: 'sk-admin' + }) + // Surrounding whitespace is trimmed, as it always was. + assert.deepEqual(parseApiKeys(' sk-admin , sk-client '), { + apiKeys: ['sk-admin', 'sk-client'], + adminKey: 'sk-admin' + }) + // A single key is both the only key and the admin key. + assert.deepEqual(parseApiKeys('sk-only'), { apiKeys: ['sk-only'], adminKey: 'sk-only' }) +}) diff --git a/tests/expected-counts.json b/tests/expected-counts.json index 07ea878a..095af32e 100644 --- a/tests/expected-counts.json +++ b/tests/expected-counts.json @@ -1,5 +1,5 @@ { - "tests": 1177, + "tests": 1181, "suites": 137, "note": "Authoritative count. Verify with the per-file sum in AGENTS.md (\"The test gate\"). The -a on that grep is load-bearing: tool-prompt.test.js emits bytes that make grep call the stream binary, and without -a its whole summary line — 133 tests — is silently dropped from the sum.", "updated": "2026-09-16"