From c77497f54a87bf96e52348aede680a9cf7e41f5d Mon Sep 17 00:00:00 2001 From: Kyle Cox Date: Wed, 14 Oct 2026 23:58:22 +0000 Subject: [PATCH] fix(evolution): do not reuse stale CBOR chunk sizes on re-encode (#575) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A captured indefinite chunk layout was replayed over a changed byte or text string regardless of the new value's length: a shorter value was zero-padded to the old chunk total, a longer value lost its tail, and a text change could split a UTF-8 character — valid CBOR carrying different content, with no error. The layout is now reused only when it still fits: for bytes the chunk sizes must add up to the value's length; for text the same, and every chunk must end on a UTF-8 character boundary. Otherwise the encoder falls back to the default encoding for that string, as integer widths already do. Unchanged values and same-length aligned changes keep their captured layout byte for byte. Tests: 9 new checks in CBOR-with-format.test.ts (shorter/longer bytes and text, UTF-8 split at equal length, byte-identical controls). Full suite: 1687 passed / 75 skipped / 0 failed. --- .changeset/cbor-stale-chunk-sizes.md | 5 + packages/evolution/src/CBOR.ts | 107 +++++++++++------- .../evolution/test/CBOR-with-format.test.ts | 58 ++++++++++ 3 files changed, 128 insertions(+), 42 deletions(-) create mode 100644 .changeset/cbor-stale-chunk-sizes.md diff --git a/.changeset/cbor-stale-chunk-sizes.md b/.changeset/cbor-stale-chunk-sizes.md new file mode 100644 index 00000000..18a46673 --- /dev/null +++ b/.changeset/cbor-stale-chunk-sizes.md @@ -0,0 +1,5 @@ +--- +"@evolution-sdk/evolution": patch +--- + +Fix preservation-aware encoding reusing stale indefinite chunk sizes for byte and text strings. A captured chunk layout is now reused only when its chunk sizes add up to the new value's length — and, for text, only when every chunk ends on a UTF-8 character boundary. Otherwise the encoder falls back to the default encoding for that string, instead of zero-padding a shorter value, dropping the tail of a longer one, or splitting a UTF-8 character. diff --git a/packages/evolution/src/CBOR.ts b/packages/evolution/src/CBOR.ts index 26a595de..0c108e68 100644 --- a/packages/evolution/src/CBOR.ts +++ b/packages/evolution/src/CBOR.ts @@ -1281,31 +1281,40 @@ const encodeBytesSync = (value: Uint8Array, options: CodecOptions, fmt?: CBORFor if (stringEncoding !== undefined) { if (stringEncoding.tag === "indefinite") { const chunks = stringEncoding.chunks - let totalSize = 2 // 0x5f + 0xff - for (const chunk of chunks) { - totalSize += encodeIntHeader(2, BigInt(chunk.length), chunk.byteSize).length + chunk.length - } - const result = new Uint8Array(totalSize) - let pos = 0 - result[pos++] = 0x5f - let srcOffset = 0 - for (const chunk of chunks) { - const header = encodeIntHeader(2, BigInt(chunk.length), chunk.byteSize) - result.set(header, pos) - pos += header.length - result.set(value.subarray(srcOffset, srcOffset + chunk.length), pos) - pos += chunk.length - srcOffset += chunk.length + // Reuse the captured chunk layout only when it still fits the value: + // the chunk sizes must add up to the value's length. Otherwise the old + // sizes would pad or truncate the new value, so fall back to the + // default encoding below (as integer widths already do). + let chunkTotal = 0 + for (const chunk of chunks) chunkTotal += chunk.length + if (chunkTotal === length) { + let totalSize = 2 // 0x5f + 0xff + for (const chunk of chunks) { + totalSize += encodeIntHeader(2, BigInt(chunk.length), chunk.byteSize).length + chunk.length + } + const result = new Uint8Array(totalSize) + let pos = 0 + result[pos++] = 0x5f + let srcOffset = 0 + for (const chunk of chunks) { + const header = encodeIntHeader(2, BigInt(chunk.length), chunk.byteSize) + result.set(header, pos) + pos += header.length + result.set(value.subarray(srcOffset, srcOffset + chunk.length), pos) + pos += chunk.length + srcOffset += chunk.length + } + result[pos] = 0xff + return result } - result[pos] = 0xff + } else { + // Definite with specific byteSize + const header = encodeIntHeader(2, BigInt(length), stringEncoding.byteSize) + const result = new Uint8Array(header.length + length) + result.set(header, 0) + result.set(value, header.length) return result } - // Definite with specific byteSize - const header = encodeIntHeader(2, BigInt(length), stringEncoding.byteSize) - const result = new Uint8Array(header.length + length) - result.set(header, 0) - result.set(value, header.length) - return result } const useMinimal = options.mode !== "custom" || options.useMinimalEncoding @@ -1372,31 +1381,45 @@ const encodeTextSync = (value: string, options: CodecOptions, fmt?: CBORFormat): const utf8 = TEXT_ENCODER.encode(value) if (stringEncoding.tag === "indefinite") { const chunks = stringEncoding.chunks - let totalSize = 2 // 0x7f + 0xff + // Reuse the captured chunk layout only when it still fits the value: + // the chunk sizes must add up to the value's UTF-8 length and every + // chunk must end on a UTF-8 character boundary (the next byte is not + // a continuation byte). Otherwise fall back to the default encoding + // below, rather than padding, truncating, or splitting a character. + let chunkTotal = 0 + let chunksFit = true for (const chunk of chunks) { - totalSize += encodeIntHeader(3, BigInt(chunk.length), chunk.byteSize).length + chunk.length + chunkTotal += chunk.length + if (chunkTotal < utf8.length && (utf8[chunkTotal] & 0xc0) === 0x80) chunksFit = false } - const result = new Uint8Array(totalSize) - let pos = 0 - result[pos++] = 0x7f - let srcOffset = 0 - for (const chunk of chunks) { - const header = encodeIntHeader(3, BigInt(chunk.length), chunk.byteSize) - result.set(header, pos) - pos += header.length - result.set(utf8.subarray(srcOffset, srcOffset + chunk.length), pos) - pos += chunk.length - srcOffset += chunk.length + if (chunksFit && chunkTotal === utf8.length) { + let totalSize = 2 // 0x7f + 0xff + for (const chunk of chunks) { + totalSize += encodeIntHeader(3, BigInt(chunk.length), chunk.byteSize).length + chunk.length + } + const result = new Uint8Array(totalSize) + let pos = 0 + result[pos++] = 0x7f + let srcOffset = 0 + for (const chunk of chunks) { + const header = encodeIntHeader(3, BigInt(chunk.length), chunk.byteSize) + result.set(header, pos) + pos += header.length + result.set(utf8.subarray(srcOffset, srcOffset + chunk.length), pos) + pos += chunk.length + srcOffset += chunk.length + } + result[pos] = 0xff + return result } - result[pos] = 0xff + } else { + // Definite with specific byteSize + const header = encodeIntHeader(3, BigInt(utf8.length), stringEncoding.byteSize) + const result = new Uint8Array(header.length + utf8.length) + result.set(header, 0) + result.set(utf8, header.length) return result } - // Definite with specific byteSize - const header = encodeIntHeader(3, BigInt(utf8.length), stringEncoding.byteSize) - const result = new Uint8Array(header.length + utf8.length) - result.set(header, 0) - result.set(utf8, header.length) - return result } // Fast path for empty strings diff --git a/packages/evolution/test/CBOR-with-format.test.ts b/packages/evolution/test/CBOR-with-format.test.ts index a37fe75a..c04f6573 100644 --- a/packages/evolution/test/CBOR-with-format.test.ts +++ b/packages/evolution/test/CBOR-with-format.test.ts @@ -61,6 +61,64 @@ describe("CBOR WithFormat — format tree is the complete specification", () => }) }) +describe("CBOR WithFormat — stale chunk sizes are not reused (#575)", () => { + it("bytes: a shorter value under a chunked format decodes back to itself", () => { + const { format } = CBOR.fromCBORHexWithFormat("5f4201024103ff") // h'010203' as chunks 2 + 1 + const encoded = CBOR.toCBORHexWithFormat(new Uint8Array([0xaa]), format) + expect(CBOR.fromCBORHex(encoded)).toEqual(new Uint8Array([0xaa])) + }) + + it("bytes: a longer value under a chunked format decodes back to itself", () => { + const { format } = CBOR.fromCBORHexWithFormat("5f4201024103ff") + const value = new Uint8Array([1, 2, 3, 4, 5]) + const encoded = CBOR.toCBORHexWithFormat(value, format) + expect(CBOR.fromCBORHex(encoded)).toEqual(value) + }) + + it("bytes: an unchanged value under its own chunked format stays byte-identical", () => { + const hex = "5f4201024103ff" + const { format, value } = CBOR.fromCBORHexWithFormat(hex) + expect(CBOR.toCBORHexWithFormat(value, format)).toBe(hex) + }) + + it("bytes: a same-length changed value still reuses its chunk layout", () => { + const { format } = CBOR.fromCBORHexWithFormat("5f4201024103ff") + expect(CBOR.toCBORHexWithFormat(new Uint8Array([0xaa, 0xbb, 0xcc]), format)).toBe("5f42aabb41ccff") + }) + + it("text: a shorter string under a chunked format decodes back to itself", () => { + const { format } = CBOR.fromCBORHexWithFormat("7f626869616aff") // "hij" as chunks 2 + 1 + const encoded = CBOR.toCBORHexWithFormat("x", format) + expect(CBOR.fromCBORHex(encoded)).toBe("x") + }) + + it("text: a longer string under a chunked format decodes back to itself", () => { + const { format } = CBOR.fromCBORHexWithFormat("7f626869616aff") + const encoded = CBOR.toCBORHexWithFormat("hijk", format) + expect(CBOR.fromCBORHex(encoded)).toBe("hijk") + }) + + it("text: a string whose chunk split would break a UTF-8 character decodes back to itself", () => { + const { format } = CBOR.fromCBORHexWithFormat("7f626869616aff") + const encoded = CBOR.toCBORHexWithFormat("héllo", format) + expect(CBOR.fromCBORHex(encoded)).toBe("héllo") + }) + + it("text: a same-length string whose chunks would split a UTF-8 character falls back", () => { + const { format } = CBOR.fromCBORHexWithFormat("7f626869616aff") // chunks 2 + 1, total 3 bytes + // "xé" is 3 UTF-8 bytes (x + 2-byte é); the 2-byte first chunk would split é + const encoded = CBOR.toCBORHexWithFormat("xé", format) + expect(CBOR.fromCBORHex(encoded)).toBe("xé") + expect(encoded).not.toBe("7f6278c361a9ff") + }) + + it("text: an unchanged string under its own chunked format stays byte-identical", () => { + const hex = "7f626869616aff" + const { format, value } = CBOR.fromCBORHexWithFormat(hex) + expect(CBOR.toCBORHexWithFormat(value, format)).toBe(hex) + }) +}) + describe("CBOR WithFormat — hand-crafted format injection", () => { it("explicit uint format with byteSize:1 forces non-minimal encoding", () => { const fmt: CBOR.CBORFormat = { _tag: "uint", byteSize: 1 }