fix(evolution): preserve Plutus data encodings on re-encode (#576)

.changeset/plutus-data-encodings.md

@@ -0,0 +1,5 @@
+---
+"@evolution-sdk/evolution": patch
+---
+
+Fix Plutus data encodings changing on re-encode. Empty arrays and maps now read their captured format before the empty fast paths, so an empty indefinite list or map — and an empty list with a wider length header — round-trips byte for byte, including an empty indefinite witness set. The `BoundedBytes` encoder now receives the captured format: it keeps the header width and replays a captured chunk layout only when the chunk sizes add up to the value's length and no chunk is over 64 bytes, so a stale layout is never carried into a datum. This keeps the script data hash and the transaction id stable for datums and redeemers in these forms.

docs/content/docs/encoding/cbor.mdx

@@ -34,7 +34,7 @@ if (CBOR.isArray(value)) {
   const len = value.length // 3
 }
 
-// Re-encode to hex (canonical by default)
+// Re-encode to hex (the default keeps the original map order)
 const hex = CBOR.toCBORHex(value) // "83010203"
 
 // Or to bytes

packages/evolution/src/CBOR.ts

@@ -1072,7 +1072,7 @@ export const internalEncodeSync = (value: CBOR, options: CodecOptions = CML_DEFA
     "_tag" in value &&
     (value as { _tag: unknown })._tag === "BoundedBytes"
   ) {
-    return encodeBoundedBytesSync((value as { _tag: "BoundedBytes"; bytes: Uint8Array }).bytes)
+    return encodeBoundedBytesSync((value as { _tag: "BoundedBytes"; bytes: Uint8Array }).bytes, resolvedFmt)
   }
   if (
     typeof value === "object" &&
@@ -1232,8 +1232,50 @@ const writeChunkHeader = (buf: Uint8Array, pos: number, len: number): number =>
  * depend on `CodecOptions` — the rule is always applied.
  */
 const BOUNDED_BYTES_CHUNK_SIZE = 64
-const encodeBoundedBytesSync = (value: Uint8Array): Uint8Array => {
+const encodeBoundedBytesSync = (value: Uint8Array, fmt?: CBORFormat): Uint8Array => {
   const length = value.length
+  // Replay the captured encoding when it still fits the value: a definite
+  // header keeps its width (values over 64 bytes cannot be definite under
+  // the bounded rule, so those fall through to chunking), and an indefinite
+  // chunk layout is kept only when its chunk sizes add up to the value's
+  // length and no chunk is over 64 bytes — a stale layout is never reused.
+  const encoding = fmt?._tag === "bytes" ? fmt.encoding : undefined
+  if (encoding !== undefined) {
+    if (encoding.tag === "indefinite") {
+      let total = 0
+      let fits = true
+      for (const chunk of encoding.chunks) {
+        if (chunk.length > BOUNDED_BYTES_CHUNK_SIZE) fits = false
+        total += chunk.length
+      }
+      if (fits && total === length) {
+        let totalSize = 2 // 0x5f + 0xff
+        for (const chunk of encoding.chunks) {
+          totalSize += encodeIntHeader(2, BigInt(chunk.length), chunk.byteSize).length + chunk.length
+        }
+        const result = new Uint8Array(totalSize)
+        let pos = 0
+        result[pos++] = 0x5f
+        let srcOffset = 0
+        for (const chunk of encoding.chunks) {
+          const header = encodeIntHeader(2, BigInt(chunk.length), chunk.byteSize)
+          result.set(header, pos)
+          pos += header.length
+          result.set(value.subarray(srcOffset, srcOffset + chunk.length), pos)
+          pos += chunk.length
+          srcOffset += chunk.length
+        }
+        result[pos] = 0xff
+        return result
+      }
+    } else if (length <= BOUNDED_BYTES_CHUNK_SIZE) {
+      const header = encodeIntHeader(2, BigInt(length), encoding.byteSize)
+      const out = new Uint8Array(header.length + length)
+      out.set(header, 0)
+      out.set(value, header.length)
+      return out
+    }
+  }
   if (length === 0) return new Uint8Array([0x40])
   if (length <= BOUNDED_BYTES_CHUNK_SIZE) {
     // Definite-length encoding
@@ -1437,13 +1479,16 @@ const encodeTextSync = (value: string, options: CodecOptions, fmt?: CBORFormat):
 const encodeArraySync = (value: ReadonlyArray<CBOR>, options: CodecOptions, fmt?: CBORFormat): Uint8Array => {
   const length = value.length
 
-  // Fast path for empty arrays
+  const arrayFmt = fmt?._tag === "array" ? fmt : undefined
+
+  // Empty arrays still carry a captured encoding: an indefinite length
+  // stays indefinite and a definite header keeps its width.
   if (length === 0) {
+    if (arrayFmt?.length?.tag === "indefinite") return new Uint8Array([0x9f, 0xff])
+    if (arrayFmt?.length !== undefined) return encodeIntHeader(4, 0n, arrayFmt.length.byteSize)
     return new Uint8Array([0x80])
   }
 
-  const arrayFmt = fmt?._tag === "array" ? fmt : undefined
-
   // Use format metadata if available
   if (arrayFmt?.length !== undefined) {
     const items = new Array<Uint8Array>(length)
@@ -1494,8 +1539,13 @@ const encodeMapEntriesSync = (pairs: Array<[CBOR, CBOR]>, options: CodecOptions,
     return encodeArraySync(pairArrays, options)
   }
 
-  // Fast path for empty maps
-  if (length === 0) return new Uint8Array([0xa0])
+  // Empty maps still carry a captured encoding: an indefinite length
+  // stays indefinite and a definite header keeps its width.
+  if (length === 0) {
+    if (mapFmt?.length?.tag === "indefinite") return new Uint8Array([0xbf, 0xff])
+    if (mapFmt?.length !== undefined) return encodeIntHeader(5, 0n, mapFmt.length.byteSize)
+    return new Uint8Array([0xa0])
+  }
 
   // Encode each pair (with per-entry formats if format is available, otherwise canonical)
   let encodedPairs: Array<{ encodedKey: Uint8Array; encodedValue: Uint8Array }>

packages/evolution/test/CBOR-with-format.test.ts

@@ -14,6 +14,10 @@ const ENCODING_VECTORS = [
   { name: "array indefinite length", hex: "9f0102ff" },
   { name: "map indefinite length", hex: "bf01020304ff" },
   { name: "tag non-minimal width (tag 24 as uint8)", hex: "d81800" },
+  { name: "array empty indefinite length", hex: "9fff" },
+  { name: "map empty indefinite length", hex: "bfff" },
+  { name: "array empty 2-byte length header", hex: "9800" },
+  { name: "uint written as a bignum (tag 2)", hex: "c24101" },
   { name: "map non-canonical key order", hex: "a2036163016161" },
   { name: "nested mixed encodings", hex: "a205a182000082d8798082186418c803d901028143010203" }
 ] as const
@@ -84,6 +88,72 @@ describe("CBOR WithFormat — hand-crafted format injection", () => {
     expect(CBOR.toCBORHexWithFormat([1n, 2n], fmt)).toBe("9f0102ff")
   })
 
+  it("empty array keeps a captured indefinite length", () => {
+    const fmt: CBOR.CBORFormat = { _tag: "array", length: { tag: "indefinite" }, children: [] }
+    expect(CBOR.toCBORHexWithFormat([], fmt)).toBe("9fff")
+  })
+
+  it("empty array keeps a captured 2-byte length header", () => {
+    const fmt: CBOR.CBORFormat = { _tag: "array", length: { tag: "definite", byteSize: 1 }, children: [] }
+    expect(CBOR.toCBORHexWithFormat([], fmt)).toBe("9800")
+  })
+
+  it("empty map keeps a captured indefinite length", () => {
+    const fmt: CBOR.CBORFormat = { _tag: "map", length: { tag: "indefinite" }, entries: [] }
+    expect(CBOR.toCBORHexWithFormat(new Map(), fmt)).toBe("bfff")
+  })
+})
+
+describe("CBOR WithFormat — BoundedBytes captured encodings (#576)", () => {
+  it("keeps a captured 2-byte length header", () => {
+    const fmt: CBOR.CBORFormat = { _tag: "bytes", encoding: { tag: "definite", byteSize: 1 } }
+    const value = CBOR.BoundedBytes.make(new Uint8Array([0xcc, 0xcc, 0xcc]))
+    expect(CBOR.toCBORHexWithFormat(value, fmt)).toBe("5803cccccc")
+  })
+
+  it("keeps a captured chunk layout that still fits the value", () => {
+    const fmt: CBOR.CBORFormat = {
+      _tag: "bytes",
+      encoding: {
+        tag: "indefinite",
+        chunks: [
+          { length: 2, byteSize: 0 },
+          { length: 1, byteSize: 0 }
+        ]
+      }
+    }
+    const value = CBOR.BoundedBytes.make(new Uint8Array([1, 2, 3]))
+    expect(CBOR.toCBORHexWithFormat(value, fmt)).toBe("5f4201024103ff")
+  })
+
+  it("does not reuse a chunk layout whose sizes do not add up to the value length", () => {
+    const fmt: CBOR.CBORFormat = {
+      _tag: "bytes",
+      encoding: {
+        tag: "indefinite",
+        chunks: [
+          { length: 2, byteSize: 0 },
+          { length: 2, byteSize: 0 }
+        ]
+      }
+    }
+    const value = CBOR.BoundedBytes.make(new Uint8Array([1, 2, 3]))
+    // Stale layout (2 + 2 over a 3-byte value) falls back to the definite form.
+    expect(CBOR.toCBORHexWithFormat(value, fmt)).toBe("43010203")
+  })
+
+  it("does not reuse a chunk layout with a chunk over 64 bytes", () => {
+    const fmt: CBOR.CBORFormat = {
+      _tag: "bytes",
+      encoding: { tag: "indefinite", chunks: [{ length: 65, byteSize: 1 }] }
+    }
+    const value = CBOR.BoundedBytes.make(new Uint8Array(65).fill(1))
+    // Falls back to the bounded rule: a 64-byte chunk plus a 1-byte chunk.
+    expect(CBOR.toCBORHexWithFormat(value, fmt)).toBe(`5f5840${"01".repeat(64)}4101ff`)
+  })
+})
+
+describe("CBOR WithFormat — captured key order", () => {
   it("captured format round-trips correctly for map with non-canonical key order", () => {
     // key 3 (0x03) before key 1 (0x01) — non-canonical order
     const hex = "a2036163016161"

packages/evolution/test/Transaction-with-format.test.ts

@@ -102,6 +102,67 @@ describe("Transaction WithFormat — property", () => {
   })
 })
 
+// ---------------------------------------------------------------------------
+// Plutus data encodings (#576) and the empty indefinite witness set (#580.2)
+// ---------------------------------------------------------------------------
+
+// Body with one tagged input set, one legacy array-form output and a fee.
+const DATUM_BODY_HEX =
+  "a300d9010281825820aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa0001818258390011111111111111111111111111111111111111111111111111111111222222222222222222222222222222222222222222222222222222221a000f4240021a0002a300"
+
+// Plutus data encodings the ledger accepts. Each one used to come back from
+// a decode/encode round trip in different bytes (the bignum form already
+// round-tripped on main; it stays here as a guard).
+const PLUTUS_DATA_ENCODINGS = [
+  { name: "constructor 0 with an empty indefinite list", hex: "d8799fff" },
+  { name: "empty indefinite map", hex: "bfff" },
+  { name: "constructor 0 with an empty list in a 2-byte header", hex: "d8799800" },
+  { name: "1 written as a bignum", hex: "c24101" },
+  { name: "3 bytes in a 2-byte length header", hex: "5803cccccc" },
+  { name: "bytes in indefinite chunks", hex: "5f4201024103ff" },
+  { name: "control: constructor 0 with a non-empty indefinite list", hex: "d8799f01ff" }
+] as const
+
+const witnessDatumTxHex = (datumHex: string): string => `84${DATUM_BODY_HEX}a104d9010281${datumHex}f5f6`
+
+const inlineDatumTxHex = (datumHex: string): string => {
+  // Map-form output whose datum option (key 2) is [1, #6.24(datum bytes)].
+  const header = (0x40 + datumHex.length / 2).toString(16)
+  const output = `a3005839001111111111111111111111111111111111111111111111111111111122222222222222222222222222222222222222222222222222222222011a000f4240028201d818${header}${datumHex}`
+  const body = `a300d9010281825820aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa000181${output}021a0002a300`
+  return `84${body}a0f5f6`
+}
+
+describe("Transaction WithFormat — Plutus data encodings (#576)", () => {
+  it.each(PLUTUS_DATA_ENCODINGS)("witness datum round-trips exactly: $name", ({ hex }) => {
+    const txHex = witnessDatumTxHex(hex)
+    const tx = Transaction.fromCBORHex(txHex)
+    expect(Transaction.toCBORHex(tx)).toBe(txHex)
+  })
+
+  it.each(PLUTUS_DATA_ENCODINGS)("inline datum round-trips exactly: $name", ({ hex }) => {
+    const txHex = inlineDatumTxHex(hex)
+    const tx = Transaction.fromCBORHex(txHex)
+    expect(Transaction.toCBORHex(tx)).toBe(txHex)
+  })
+
+  it("preserves the body hash of a transaction with a chunked inline datum — verified via CML", () => {
+    const txHex = inlineDatumTxHex("5f4201024103ff")
+    const tx = Transaction.fromCBORHex(txHex)
+    const reEncoded = Transaction.toCBORHex(tx)
+
+    const hashBefore = CML.hash_transaction(CML.Transaction.from_cbor_hex(txHex).body()).to_hex()
+    const hashAfter = CML.hash_transaction(CML.Transaction.from_cbor_hex(reEncoded).body()).to_hex()
+    expect(hashAfter).toBe(hashBefore)
+  })
+
+  it("round-trips an empty indefinite witness set exactly (#580 part 2)", () => {
+    const txHex = `84${DATUM_BODY_HEX}bffff5f6`
+    const tx = Transaction.fromCBORHex(txHex)
+    expect(Transaction.toCBORHex(tx)).toBe(txHex)
+  })
+})
+
 // ---------------------------------------------------------------------------
 // Helpers
 // ---------------------------------------------------------------------------