fix(evolution): keep untagged sets untagged on re-encode (#574)

.changeset/untagged-sets-keep-form.md

@@ -0,0 +1,5 @@
+---
+"@evolution-sdk/evolution": patch
+---
+
+Transactions whose sets were decoded without tag 258 no longer gain the tag on re-encode. The captured format now decides whether the tag is written: when the format at a tag-258 position is an array node, the inner array is encoded with that node and the tag is omitted, so a plain round trip reproduces the input bytes exactly and the transaction id is unchanged. Witness sets holding Plutus data keep the untagged form too when a datum is added to an untagged datum set. With no captured format the tag is still written, so freshly built transactions do not change.

packages/evolution/src/CBOR.ts

@@ -1669,6 +1669,15 @@ const encodeRecordSync = (value: { readonly [key: string | number]: CBOR }, opti
 }
 
 const encodeTagSync = (tag: number, value: CBOR, options: CodecOptions, fmt?: CBORFormat): Uint8Array => {
+  // #574: a set the decoder read without tag 258 has an array node at this
+  // position in the captured format. The format decides whether the tag is
+  // written: encode the inner array with that node and omit the tag, so the
+  // nested choices the array node carries (length form, item widths) are
+  // replayed too. With no captured format — or a tag node — the tag is
+  // written as before, so freshly built transactions do not change.
+  if (tag === 258 && fmt?._tag === "array") {
+    return internalEncodeSync(value, options, fmt)
+  }
   const tagFmt = fmt?._tag === "tag" ? fmt : undefined
   // Use specific ByteSize from format metadata (pass child format even when width is canonical)
   if (tagFmt !== undefined) {

packages/evolution/src/TransactionWitnessSet.ts

@@ -799,10 +799,16 @@ const withAddedFormats = (decoded: CBOR.CBORFormat, fresh: CBOR.CBORFormat): CBO
       return { ...decoded, keyOrder, entries }
     }
     case "array": {
-      // A set decoded untagged is written tagged; it takes the tag format
-      // only when items are added
+      // A set decoded untagged keeps its untagged form when items are added
+      // (#574): the decoded items keep their formats and the added items
+      // take theirs from the fresh tree's inner array.
       if (fresh._tag === "tag" && fresh.child._tag === "array" && fresh.child.children.length > decoded.children.length) {
-        return { ...fresh, child: withAddedFormats(decoded, fresh.child) }
+        return {
+          ...decoded,
+          children: fresh.child.children.map((child, i) =>
+            i < decoded.children.length ? withAddedFormats(decoded.children[i], child) : child
+          )
+        }
       }
       if (fresh._tag !== "array") return decoded
       return {

packages/evolution/test/CBOR-with-format.test.ts

@@ -96,3 +96,43 @@ describe("CBOR WithFormat — hand-crafted format injection", () => {
     expect(CBOR.toCBORHex(value)).toBe(hex)
   })
 })
+
+describe("CBOR WithFormat — tag 258 with an untagged (array) captured format (#574)", () => {
+  const tagSet = () => CBOR.Tag.make({ tag: 258, value: [1n, 2n] }, { disableValidation: true })
+
+  it("omits the tag when the captured format is a definite array node", () => {
+    const fmt: CBOR.CBORFormat = { _tag: "array", children: [{ _tag: "uint" }, { _tag: "uint" }] }
+    expect(CBOR.toCBORHexWithFormat(tagSet(), fmt)).toBe("820102")
+  })
+
+  it("omits the tag and keeps the indefinite form when the captured format is an indefinite array node", () => {
+    const fmt: CBOR.CBORFormat = {
+      _tag: "array",
+      length: { tag: "indefinite" },
+      children: [{ _tag: "uint" }, { _tag: "uint" }]
+    }
+    expect(CBOR.toCBORHexWithFormat(tagSet(), fmt)).toBe("9f0102ff")
+  })
+
+  it("encodes the inner array with its captured node, so nested widths survive", () => {
+    // One input whose index 0 was written as 0x1800 (non-minimal uint8)
+    const value = CBOR.Tag.make({ tag: 258, value: [[new Uint8Array(32), 0n]] }, { disableValidation: true })
+    const fmt: CBOR.CBORFormat = {
+      _tag: "array",
+      children: [{ _tag: "array", children: [{ _tag: "bytes" }, { _tag: "uint", byteSize: 1 }] }]
+    }
+    expect(CBOR.toCBORHexWithFormat(value, fmt)).toBe(`81825820${"00".repeat(32)}1800`)
+  })
+
+  it("keeps the tag when the captured format is a tag node", () => {
+    const fmt: CBOR.CBORFormat = {
+      _tag: "tag",
+      child: { _tag: "array", children: [{ _tag: "uint" }, { _tag: "uint" }] }
+    }
+    expect(CBOR.toCBORHexWithFormat(tagSet(), fmt)).toBe("d90102820102")
+  })
+
+  it("keeps the tag when there is no captured format", () => {
+    expect(CBOR.toCBORHex(tagSet())).toBe("d90102820102")
+  })
+})

packages/evolution/test/Transaction-with-format.test.ts

@@ -117,3 +117,90 @@ function buildWalletWitnessHex(): string {
   wsMap.set(0n, CBOR.Tag.make({ tag: 258, value: [[new Uint8Array(32).fill(0xaa), new Uint8Array(64).fill(0xbb)]] }))
   return Buffer.from(CBOR.toCBORBytes(wsMap)).toString("hex")
 }
+
+// ---------------------------------------------------------------------------
+// Untagged sets (#574): a set decoded without tag 258 must not gain it on
+// re-encode — the body bytes change, so the tx id changes and witnesses
+// already collected no longer verify.
+// ---------------------------------------------------------------------------
+
+const HASH32 = "00".repeat(32)
+// One input: [bstr(32 zero bytes), index 0 written as 0x1800]
+const INPUT_1800 = `825820${HASH32}1800`
+const UNTAGGED_BODY_DEFINITE = `a30081${INPUT_1800}01800200`
+const UNTAGGED_BODY_INDEFINITE = `a3009f${INPUT_1800}ff01800200`
+const UNTAGGED_TX_DEFINITE = `84${UNTAGGED_BODY_DEFINITE}a0f5f6`
+const UNTAGGED_TX_INDEFINITE = `84${UNTAGGED_BODY_INDEFINITE}a0f5f6`
+
+describe("Transaction WithFormat — untagged sets (#574)", () => {
+  it("round-trips an untagged definite inputs set byte for byte (WithFormat)", () => {
+    const { format, value: tx } = Transaction.fromCBORHexWithFormat(UNTAGGED_TX_DEFINITE)
+    expect(Transaction.toCBORHexWithFormat(tx, format)).toBe(UNTAGGED_TX_DEFINITE)
+  })
+
+  it("round-trips an untagged definite inputs set byte for byte (cached default path)", () => {
+    const tx = Transaction.fromCBORHex(UNTAGGED_TX_DEFINITE)
+    expect(Transaction.toCBORHex(tx)).toBe(UNTAGGED_TX_DEFINITE)
+  })
+
+  it("round-trips an untagged indefinite inputs set byte for byte", () => {
+    const { format, value: tx } = Transaction.fromCBORHexWithFormat(UNTAGGED_TX_INDEFINITE)
+    expect(Transaction.toCBORHexWithFormat(tx, format)).toBe(UNTAGGED_TX_INDEFINITE)
+  })
+
+  it("keeps the body hash of an untagged transaction — verified via CML", () => {
+    const { format, value: tx } = Transaction.fromCBORHexWithFormat(UNTAGGED_TX_DEFINITE)
+    const reEncoded = Transaction.toCBORHexWithFormat(tx, format)
+    const hashBefore = CML.hash_transaction(CML.Transaction.from_cbor_hex(UNTAGGED_TX_DEFINITE).body()).to_hex()
+    const hashAfter = CML.hash_transaction(CML.Transaction.from_cbor_hex(reEncoded).body()).to_hex()
+    expect(hashAfter).toBe(hashBefore)
+  })
+
+  it("round-trips untagged collateral, required-signer, reference-input and certificate sets", () => {
+    const collateralInput = `825820${"33".repeat(32)}01`
+    const signer = `581c${"22".repeat(28)}`
+    const referenceInput = `825820${"44".repeat(32)}02`
+    // Stake registration certificate: [0, [0, key hash]]
+    const certificate = `82008200581c${"11".repeat(28)}`
+    const body =
+      `a70081${INPUT_1800}01800200` +
+      `0481${certificate}` +
+      `0d81${collateralInput}` +
+      `0e81${signer}` +
+      `1281${referenceInput}`
+    const txHex = `84${body}a0f5f6`
+    const { format, value: tx } = Transaction.fromCBORHexWithFormat(txHex)
+    expect(Transaction.toCBORHexWithFormat(tx, format)).toBe(txHex)
+  })
+
+  it("round-trips an untagged vkey witness set byte for byte", () => {
+    const witness = `a10081825820${"aa".repeat(32)}5840${"bb".repeat(64)}`
+    const txHex = `84${NON_CANONICAL_BODY_HEX}${witness}f5f6`
+    const { format, value: tx } = Transaction.fromCBORHexWithFormat(txHex)
+    expect(Transaction.toCBORHexWithFormat(tx, format)).toBe(txHex)
+  })
+
+  it("addVKeyWitnessesHex leaves an untagged transaction's body bytes unchanged", () => {
+    const signedHex = Transaction.addVKeyWitnessesHex(UNTAGGED_TX_DEFINITE, buildWalletWitnessHex())
+    expect(signedHex).toContain(UNTAGGED_BODY_DEFINITE)
+    const hashBefore = CML.hash_transaction(CML.Transaction.from_cbor_hex(UNTAGGED_TX_DEFINITE).body()).to_hex()
+    const hashAfter = CML.hash_transaction(CML.Transaction.from_cbor_hex(signedHex).body()).to_hex()
+    expect(hashAfter).toBe(hashBefore)
+  })
+
+  it("a witness appended to an untagged witness set keeps the untagged form", () => {
+    const first = `825820${"aa".repeat(32)}5840${"bb".repeat(64)}`
+    const txHex = `84${NON_CANONICAL_BODY_HEX}a10081${first}f5f6`
+    const walletWsHex = `a100d9010281825820${"cc".repeat(32)}5840${"dd".repeat(64)}`
+    const signedHex = Transaction.addVKeyWitnessesHex(txHex, walletWsHex)
+    const second = `825820${"cc".repeat(32)}5840${"dd".repeat(64)}`
+    expect(signedHex).toBe(`84${NON_CANONICAL_BODY_HEX}a10082${first}${second}f5f6`)
+  })
+
+  it("control: tagged sets keep the tag, and a freshly built transaction is unchanged", () => {
+    const { format, value: tx } = Transaction.fromCBORHexWithFormat(NON_CANONICAL_TX_HEX)
+    expect(Transaction.toCBORHexWithFormat(tx, format)).toBe(NON_CANONICAL_TX_HEX)
+    const fresh = Transaction.fromCBORHex(NON_CANONICAL_TX_HEX)
+    expect(Transaction.toCBORHex(fresh)).toBe(NON_CANONICAL_TX_HEX)
+  })
+})

packages/evolution/test/TransactionWitnessSet.DataOptions.test.ts

@@ -477,12 +477,13 @@ describe("a datum or redeemer added to a decoded transaction", () => {
   })
 
   it("a datum added to an untagged datum set is written in the data layout", () => {
-    // The set is written tagged, as the encoder writes every datum set
+    // The set keeps the untagged form it was decoded with (#574); the added
+    // datum is written in the data layout
     const untaggedHex = "d8799f0102ff"
     const { format, value: tx } = Transaction.fromCBORHexWithFormat(txHex(`a10481${untaggedHex}`))
     const extended = withWitnessSet(tx, { plutusData: [...tx.witnessSet.plutusData!, datum] })
     expect(Transaction.toCBORHexWithFormat(extended, format)).toBe(
-      txHex(datumsOnlyWitnessSetHex([untaggedHex, Data.toCBORHex(datum)]))
+      txHex(`a10482${untaggedHex}${Data.toCBORHex(datum)}`)
     )
   })