From 9e9a1391695719c13c7864fc2d7ef95c12a6c39f Mon Sep 17 00:00:00 2001 From: Kyle Cox Date: Thu, 15 Oct 2026 02:05:18 +0000 Subject: [PATCH] fix(evolution): preserve Plutus data encodings on re-encode (#576) Empty arrays and maps returned 80/a0 before the captured format was read, so an empty indefinite list or map and a wider length header were lost; the BoundedBytes branch dropped the captured format entirely, so a datum's byte-string header width and chunk layout were lost. The ledger hashes the original datum and redeemer bytes, so each change broke the script data hash or the transaction id. The empty fast paths now honour a captured indefinite length or header width, and encodeBoundedBytesSync receives the format: it keeps the header width and replays a chunk layout only when the chunk sizes add up to the value's length and no chunk is over 64 bytes, falling back to the default bounded encoding otherwise. Also adds the regression case for #580 part 2 (empty indefinite witness set) and corrects the CBOR guide's 'canonical by default' comment (#580 part 3). Tests: 1709 passed / 75 skipped / 0 failed (main baseline 1678 + 31 new). --- .changeset/plutus-data-encodings.md | 5 ++ docs/content/docs/encoding/cbor.mdx | 2 +- packages/evolution/src/CBOR.ts | 64 +++++++++++++++-- .../evolution/test/CBOR-with-format.test.ts | 70 +++++++++++++++++++ .../test/Transaction-with-format.test.ts | 61 ++++++++++++++++ 5 files changed, 194 insertions(+), 8 deletions(-) create mode 100644 .changeset/plutus-data-encodings.md diff --git a/.changeset/plutus-data-encodings.md b/.changeset/plutus-data-encodings.md new file mode 100644 index 00000000..08338b73 --- /dev/null +++ b/.changeset/plutus-data-encodings.md @@ -0,0 +1,5 @@ +--- +"@evolution-sdk/evolution": patch +--- + +Fix Plutus data encodings changing on re-encode. Empty arrays and maps now read their captured format before the empty fast paths, so an empty indefinite list or map — and an empty list with a wider length header — round-trips byte for byte, including an empty indefinite witness set. The `BoundedBytes` encoder now receives the captured format: it keeps the header width and replays a captured chunk layout only when the chunk sizes add up to the value's length and no chunk is over 64 bytes, so a stale layout is never carried into a datum. This keeps the script data hash and the transaction id stable for datums and redeemers in these forms. diff --git a/docs/content/docs/encoding/cbor.mdx b/docs/content/docs/encoding/cbor.mdx index 7a9bc68a..048e4d0d 100644 --- a/docs/content/docs/encoding/cbor.mdx +++ b/docs/content/docs/encoding/cbor.mdx @@ -34,7 +34,7 @@ if (CBOR.isArray(value)) { const len = value.length // 3 } -// Re-encode to hex (canonical by default) +// Re-encode to hex (the default keeps the original map order) const hex = CBOR.toCBORHex(value) // "83010203" // Or to bytes diff --git a/packages/evolution/src/CBOR.ts b/packages/evolution/src/CBOR.ts index 26a595de..64b73032 100644 --- a/packages/evolution/src/CBOR.ts +++ b/packages/evolution/src/CBOR.ts @@ -1072,7 +1072,7 @@ export const internalEncodeSync = (value: CBOR, options: CodecOptions = CML_DEFA "_tag" in value && (value as { _tag: unknown })._tag === "BoundedBytes" ) { - return encodeBoundedBytesSync((value as { _tag: "BoundedBytes"; bytes: Uint8Array }).bytes) + return encodeBoundedBytesSync((value as { _tag: "BoundedBytes"; bytes: Uint8Array }).bytes, resolvedFmt) } if ( typeof value === "object" && @@ -1232,8 +1232,50 @@ const writeChunkHeader = (buf: Uint8Array, pos: number, len: number): number => * depend on `CodecOptions` — the rule is always applied. */ const BOUNDED_BYTES_CHUNK_SIZE = 64 -const encodeBoundedBytesSync = (value: Uint8Array): Uint8Array => { +const encodeBoundedBytesSync = (value: Uint8Array, fmt?: CBORFormat): Uint8Array => { const length = value.length + // Replay the captured encoding when it still fits the value: a definite + // header keeps its width (values over 64 bytes cannot be definite under + // the bounded rule, so those fall through to chunking), and an indefinite + // chunk layout is kept only when its chunk sizes add up to the value's + // length and no chunk is over 64 bytes — a stale layout is never reused. + const encoding = fmt?._tag === "bytes" ? fmt.encoding : undefined + if (encoding !== undefined) { + if (encoding.tag === "indefinite") { + let total = 0 + let fits = true + for (const chunk of encoding.chunks) { + if (chunk.length > BOUNDED_BYTES_CHUNK_SIZE) fits = false + total += chunk.length + } + if (fits && total === length) { + let totalSize = 2 // 0x5f + 0xff + for (const chunk of encoding.chunks) { + totalSize += encodeIntHeader(2, BigInt(chunk.length), chunk.byteSize).length + chunk.length + } + const result = new Uint8Array(totalSize) + let pos = 0 + result[pos++] = 0x5f + let srcOffset = 0 + for (const chunk of encoding.chunks) { + const header = encodeIntHeader(2, BigInt(chunk.length), chunk.byteSize) + result.set(header, pos) + pos += header.length + result.set(value.subarray(srcOffset, srcOffset + chunk.length), pos) + pos += chunk.length + srcOffset += chunk.length + } + result[pos] = 0xff + return result + } + } else if (length <= BOUNDED_BYTES_CHUNK_SIZE) { + const header = encodeIntHeader(2, BigInt(length), encoding.byteSize) + const out = new Uint8Array(header.length + length) + out.set(header, 0) + out.set(value, header.length) + return out + } + } if (length === 0) return new Uint8Array([0x40]) if (length <= BOUNDED_BYTES_CHUNK_SIZE) { // Definite-length encoding @@ -1437,13 +1479,16 @@ const encodeTextSync = (value: string, options: CodecOptions, fmt?: CBORFormat): const encodeArraySync = (value: ReadonlyArray, options: CodecOptions, fmt?: CBORFormat): Uint8Array => { const length = value.length - // Fast path for empty arrays + const arrayFmt = fmt?._tag === "array" ? fmt : undefined + + // Empty arrays still carry a captured encoding: an indefinite length + // stays indefinite and a definite header keeps its width. if (length === 0) { + if (arrayFmt?.length?.tag === "indefinite") return new Uint8Array([0x9f, 0xff]) + if (arrayFmt?.length !== undefined) return encodeIntHeader(4, 0n, arrayFmt.length.byteSize) return new Uint8Array([0x80]) } - const arrayFmt = fmt?._tag === "array" ? fmt : undefined - // Use format metadata if available if (arrayFmt?.length !== undefined) { const items = new Array(length) @@ -1494,8 +1539,13 @@ const encodeMapEntriesSync = (pairs: Array<[CBOR, CBOR]>, options: CodecOptions, return encodeArraySync(pairArrays, options) } - // Fast path for empty maps - if (length === 0) return new Uint8Array([0xa0]) + // Empty maps still carry a captured encoding: an indefinite length + // stays indefinite and a definite header keeps its width. + if (length === 0) { + if (mapFmt?.length?.tag === "indefinite") return new Uint8Array([0xbf, 0xff]) + if (mapFmt?.length !== undefined) return encodeIntHeader(5, 0n, mapFmt.length.byteSize) + return new Uint8Array([0xa0]) + } // Encode each pair (with per-entry formats if format is available, otherwise canonical) let encodedPairs: Array<{ encodedKey: Uint8Array; encodedValue: Uint8Array }> diff --git a/packages/evolution/test/CBOR-with-format.test.ts b/packages/evolution/test/CBOR-with-format.test.ts index a37fe75a..e982ce40 100644 --- a/packages/evolution/test/CBOR-with-format.test.ts +++ b/packages/evolution/test/CBOR-with-format.test.ts @@ -14,6 +14,10 @@ const ENCODING_VECTORS = [ { name: "array indefinite length", hex: "9f0102ff" }, { name: "map indefinite length", hex: "bf01020304ff" }, { name: "tag non-minimal width (tag 24 as uint8)", hex: "d81800" }, + { name: "array empty indefinite length", hex: "9fff" }, + { name: "map empty indefinite length", hex: "bfff" }, + { name: "array empty 2-byte length header", hex: "9800" }, + { name: "uint written as a bignum (tag 2)", hex: "c24101" }, { name: "map non-canonical key order", hex: "a2036163016161" }, { name: "nested mixed encodings", hex: "a205a182000082d8798082186418c803d901028143010203" } ] as const @@ -84,6 +88,72 @@ describe("CBOR WithFormat — hand-crafted format injection", () => { expect(CBOR.toCBORHexWithFormat([1n, 2n], fmt)).toBe("9f0102ff") }) + it("empty array keeps a captured indefinite length", () => { + const fmt: CBOR.CBORFormat = { _tag: "array", length: { tag: "indefinite" }, children: [] } + expect(CBOR.toCBORHexWithFormat([], fmt)).toBe("9fff") + }) + + it("empty array keeps a captured 2-byte length header", () => { + const fmt: CBOR.CBORFormat = { _tag: "array", length: { tag: "definite", byteSize: 1 }, children: [] } + expect(CBOR.toCBORHexWithFormat([], fmt)).toBe("9800") + }) + + it("empty map keeps a captured indefinite length", () => { + const fmt: CBOR.CBORFormat = { _tag: "map", length: { tag: "indefinite" }, entries: [] } + expect(CBOR.toCBORHexWithFormat(new Map(), fmt)).toBe("bfff") + }) +}) + +describe("CBOR WithFormat — BoundedBytes captured encodings (#576)", () => { + it("keeps a captured 2-byte length header", () => { + const fmt: CBOR.CBORFormat = { _tag: "bytes", encoding: { tag: "definite", byteSize: 1 } } + const value = CBOR.BoundedBytes.make(new Uint8Array([0xcc, 0xcc, 0xcc])) + expect(CBOR.toCBORHexWithFormat(value, fmt)).toBe("5803cccccc") + }) + + it("keeps a captured chunk layout that still fits the value", () => { + const fmt: CBOR.CBORFormat = { + _tag: "bytes", + encoding: { + tag: "indefinite", + chunks: [ + { length: 2, byteSize: 0 }, + { length: 1, byteSize: 0 } + ] + } + } + const value = CBOR.BoundedBytes.make(new Uint8Array([1, 2, 3])) + expect(CBOR.toCBORHexWithFormat(value, fmt)).toBe("5f4201024103ff") + }) + + it("does not reuse a chunk layout whose sizes do not add up to the value length", () => { + const fmt: CBOR.CBORFormat = { + _tag: "bytes", + encoding: { + tag: "indefinite", + chunks: [ + { length: 2, byteSize: 0 }, + { length: 2, byteSize: 0 } + ] + } + } + const value = CBOR.BoundedBytes.make(new Uint8Array([1, 2, 3])) + // Stale layout (2 + 2 over a 3-byte value) falls back to the definite form. + expect(CBOR.toCBORHexWithFormat(value, fmt)).toBe("43010203") + }) + + it("does not reuse a chunk layout with a chunk over 64 bytes", () => { + const fmt: CBOR.CBORFormat = { + _tag: "bytes", + encoding: { tag: "indefinite", chunks: [{ length: 65, byteSize: 1 }] } + } + const value = CBOR.BoundedBytes.make(new Uint8Array(65).fill(1)) + // Falls back to the bounded rule: a 64-byte chunk plus a 1-byte chunk. + expect(CBOR.toCBORHexWithFormat(value, fmt)).toBe(`5f5840${"01".repeat(64)}4101ff`) + }) +}) + +describe("CBOR WithFormat — captured key order", () => { it("captured format round-trips correctly for map with non-canonical key order", () => { // key 3 (0x03) before key 1 (0x01) — non-canonical order const hex = "a2036163016161" diff --git a/packages/evolution/test/Transaction-with-format.test.ts b/packages/evolution/test/Transaction-with-format.test.ts index 87d64b2d..7eb52bbb 100644 --- a/packages/evolution/test/Transaction-with-format.test.ts +++ b/packages/evolution/test/Transaction-with-format.test.ts @@ -102,6 +102,67 @@ describe("Transaction WithFormat — property", () => { }) }) +// --------------------------------------------------------------------------- +// Plutus data encodings (#576) and the empty indefinite witness set (#580.2) +// --------------------------------------------------------------------------- + +// Body with one tagged input set, one legacy array-form output and a fee. +const DATUM_BODY_HEX = + "a300d9010281825820aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa0001818258390011111111111111111111111111111111111111111111111111111111222222222222222222222222222222222222222222222222222222221a000f4240021a0002a300" + +// Plutus data encodings the ledger accepts. Each one used to come back from +// a decode/encode round trip in different bytes (the bignum form already +// round-tripped on main; it stays here as a guard). +const PLUTUS_DATA_ENCODINGS = [ + { name: "constructor 0 with an empty indefinite list", hex: "d8799fff" }, + { name: "empty indefinite map", hex: "bfff" }, + { name: "constructor 0 with an empty list in a 2-byte header", hex: "d8799800" }, + { name: "1 written as a bignum", hex: "c24101" }, + { name: "3 bytes in a 2-byte length header", hex: "5803cccccc" }, + { name: "bytes in indefinite chunks", hex: "5f4201024103ff" }, + { name: "control: constructor 0 with a non-empty indefinite list", hex: "d8799f01ff" } +] as const + +const witnessDatumTxHex = (datumHex: string): string => `84${DATUM_BODY_HEX}a104d9010281${datumHex}f5f6` + +const inlineDatumTxHex = (datumHex: string): string => { + // Map-form output whose datum option (key 2) is [1, #6.24(datum bytes)]. + const header = (0x40 + datumHex.length / 2).toString(16) + const output = `a3005839001111111111111111111111111111111111111111111111111111111122222222222222222222222222222222222222222222222222222222011a000f4240028201d818${header}${datumHex}` + const body = `a300d9010281825820aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa000181${output}021a0002a300` + return `84${body}a0f5f6` +} + +describe("Transaction WithFormat — Plutus data encodings (#576)", () => { + it.each(PLUTUS_DATA_ENCODINGS)("witness datum round-trips exactly: $name", ({ hex }) => { + const txHex = witnessDatumTxHex(hex) + const tx = Transaction.fromCBORHex(txHex) + expect(Transaction.toCBORHex(tx)).toBe(txHex) + }) + + it.each(PLUTUS_DATA_ENCODINGS)("inline datum round-trips exactly: $name", ({ hex }) => { + const txHex = inlineDatumTxHex(hex) + const tx = Transaction.fromCBORHex(txHex) + expect(Transaction.toCBORHex(tx)).toBe(txHex) + }) + + it("preserves the body hash of a transaction with a chunked inline datum — verified via CML", () => { + const txHex = inlineDatumTxHex("5f4201024103ff") + const tx = Transaction.fromCBORHex(txHex) + const reEncoded = Transaction.toCBORHex(tx) + + const hashBefore = CML.hash_transaction(CML.Transaction.from_cbor_hex(txHex).body()).to_hex() + const hashAfter = CML.hash_transaction(CML.Transaction.from_cbor_hex(reEncoded).body()).to_hex() + expect(hashAfter).toBe(hashBefore) + }) + + it("round-trips an empty indefinite witness set exactly (#580 part 2)", () => { + const txHex = `84${DATUM_BODY_HEX}bffff5f6` + const tx = Transaction.fromCBORHex(txHex) + expect(Transaction.toCBORHex(tx)).toBe(txHex) + }) +}) + // --------------------------------------------------------------------------- // Helpers // ---------------------------------------------------------------------------