fix(evolution): hash a decoded datum over its original bytes (#397 part 2)

.changeset/data-datum-hash-original-bytes.md

@@ -0,0 +1,5 @@
+---
+"@evolution-sdk/evolution": patch
+---
+
+`Data.toDatumHash` now hashes a decoded datum over the bytes it was decoded from — the hash the ledger computes — instead of over a re-encode of the value. `Data.fromCBORBytes` and `Data.fromCBORHex` remember a copy of the source bytes for object datums decoded with the default codec options (a WeakMap, mirroring Transaction's format cache), and `toDatumHash` replays them under the default options. A datum whose source bytes differ from the module's own encoding — a definite-length list, an indefinite-length map, a non-minimal integer or length header — previously hashed to a value the node never computed. Bare integer datums are bigints, which cannot be WeakMap keys, and still hash their re-encode.

packages/evolution/src/Data.ts

@@ -1109,23 +1109,51 @@ export const toCBORBytes = (data: Data, options: CBOR.CodecOptions = DEFAULT_CBO
 export const toCBORHex = (data: Data, options: CBOR.CodecOptions = DEFAULT_CBOR_OPTIONS): string =>
   Bytes.toHex(toCBORBytes(data, options))
 
+/**
+ * Internal cache that associates a decoded datum with a copy of the bytes it
+ * was decoded from. The ledger hashes a datum over its original bytes, so
+ * {@link toDatumHash} replays these bytes instead of hashing a re-encode,
+ * which can differ from the source (definite vs indefinite containers,
+ * non-minimal integer or length headers). Mirrors the format cache in
+ * Transaction.ts.
+ *
+ * Only object datums can be WeakMap keys: a bare integer datum is a bigint
+ * and falls back to hashing its re-encode. The WeakMap ensures the bytes are
+ * garbage-collected with the datum.
+ */
+const sourceBytesCache = new WeakMap<object, Uint8Array>()
+
+// `DEFAULT_CBOR_OPTIONS` passed explicitly acts as the default
+const isDefault = (options: CBOR.CodecOptions): boolean => options === DEFAULT_CBOR_OPTIONS
+
+const cacheSourceBytes = (data: Data, bytes: Uint8Array): Data => {
+  if (typeof data === "object" && data !== null) sourceBytesCache.set(data, bytes.slice())
+  return data
+}
+
 /**
  * Decode PlutusData from CBOR bytes
  *
  * @since 2.0.0
  * @category transformation
  */
-export const fromCBORBytes = (bytes: Uint8Array, options: CBOR.CodecOptions = DEFAULT_CBOR_OPTIONS): Data =>
-  Schema.decodeSync(FromCBORBytes(options))(bytes)
+export const fromCBORBytes = (bytes: Uint8Array, options: CBOR.CodecOptions = DEFAULT_CBOR_OPTIONS): Data => {
+  const data = Schema.decodeSync(FromCBORBytes(options))(bytes)
+  // Non-default options signal the caller wants the re-encode on later
+  // hashing, so skip the cache — as Transaction.fromCBORBytes does.
+  return isDefault(options) ? cacheSourceBytes(data, bytes) : data
+}
 
 /**
  * Decode PlutusData from CBOR hex string
  *
  * @since 2.0.0
  * @category transformation
  */
-export const fromCBORHex = (hex: string, options: CBOR.CodecOptions = DEFAULT_CBOR_OPTIONS): Data =>
-  Schema.decodeSync(FromCBORHex(options))(hex)
+export const fromCBORHex = (hex: string, options: CBOR.CodecOptions = DEFAULT_CBOR_OPTIONS): Data => {
+  const data = Schema.decodeSync(FromCBORHex(options))(hex)
+  return isDefault(options) ? cacheSourceBytes(data, Bytes.fromHex(hex)) : data
+}
 
 /**
  * Create a schema that transforms from a custom type to Data and provides CBOR encoding
@@ -1149,7 +1177,10 @@ export const withSchema = <A, I extends Data>(
 
 /**
  * Compute the hash of PlutusData using blake2b-256 over its CBOR encoding.
- * Defaults to PLUTUS_DATA_OPTIONS, the node layout.
+ * Defaults to PLUTUS_DATA_OPTIONS, the node layout. A datum decoded by
+ * {@link fromCBORBytes} or {@link fromCBORHex} with the default options
+ * hashes the bytes it was decoded from, as the ledger does, even when those
+ * bytes are not the encoding this module would write for the value.
  *
  * @since 2.0.0
  * @category hashing
@@ -1174,7 +1205,11 @@ export const toDatumHash = (
   data: Data,
   options: CBOR.CodecOptions = DEFAULT_CBOR_OPTIONS
 ): DatumHash.DatumHash => {
-  const bytes = toCBORBytes(data, options)
+  // A datum decoded with the default options hashes the bytes it was decoded
+  // from — the hash the ledger computes — not a re-encode of the value.
+  const cached =
+    isDefault(options) && typeof data === "object" && data !== null ? sourceBytesCache.get(data) : undefined
+  const bytes = cached ?? toCBORBytes(data, options)
   const digest = blake2b(bytes, { dkLen: 32 })
   return new DatumHash.DatumHash({ hash: digest })
 }

packages/evolution/test/Data.DatumHashOriginalBytes.test.ts

@@ -0,0 +1,79 @@
+import { blake2b } from "@noble/hashes/blake2.js"
+import { describe, expect, it } from "vitest"
+
+import * as Bytes from "../src/Bytes.js"
+import * as Data from "../src/Data.js"
+
+// #397 part 2: toDatumHash must hash the bytes a datum was decoded from, not a
+// re-encode — the ledger hashes the original bytes. The re-encode column below
+// is the Plutus layout this module writes (indefinite lists, definite maps,
+// minimal integers), probed on main for each vector.
+const hashHex = (hex: string): string => Bytes.toHex(blake2b(Bytes.fromHex(hex), { dkLen: 32 }))
+
+describe("Data.toDatumHash over original bytes (#397)", () => {
+  it("hashes a list holding a non-minimally encoded integer from its original bytes", () => {
+    const original = "811a0000000a" // [10], definite, 10 in a 4-byte integer header
+    const data = Data.fromCBORHex(original)
+    expect(Data.toCBORHex(data)).toBe("9f0aff") // the re-encode really is different
+    expect(Bytes.toHex(Data.toDatumHash(data).hash)).toBe(hashHex(original))
+  })
+
+  it("hashes a definite-length list from its original bytes", () => {
+    const original = "820102" // [1, 2] definite; the Plutus layout writes lists indefinite
+    const data = Data.fromCBORHex(original)
+    expect(Data.toCBORHex(data)).toBe("9f0102ff")
+    expect(Bytes.toHex(Data.toDatumHash(data).hash)).toBe(hashHex(original))
+  })
+
+  it("hashes a byte string with a non-minimal length header from its original bytes", () => {
+    const original = "5802aabb" // bytes(2) written with a 1-byte length header
+    const data = Data.fromCBORHex(original)
+    expect(Data.toCBORHex(data)).toBe("42aabb")
+    expect(Bytes.toHex(Data.toDatumHash(data).hash)).toBe(hashHex(original))
+  })
+
+  it("hashes a map with a non-minimally encoded value from its original bytes", () => {
+    const original = "a1011a00000002" // {1: 2} with 2 in a 4-byte integer header
+    const data = Data.fromCBORHex(original)
+    expect(Data.toCBORHex(data)).toBe("a10102")
+    expect(Bytes.toHex(Data.toDatumHash(data).hash)).toBe(hashHex(original))
+  })
+
+  it("hashes an indefinite-length map from its original bytes", () => {
+    const original = "bf0102ff" // {1: 2} indefinite; the Plutus layout writes maps definite
+    const data = Data.fromCBORHex(original)
+    expect(Data.toCBORHex(data)).toBe("a10102")
+    expect(Bytes.toHex(Data.toDatumHash(data).hash)).toBe(hashHex(original))
+  })
+
+  it("hashes bytes decoded through fromCBORBytes from their original bytes", () => {
+    const original = "811a0000000a"
+    const data = Data.fromCBORBytes(Bytes.fromHex(original))
+    expect(Bytes.toHex(Data.toDatumHash(data).hash)).toBe(hashHex(original))
+  })
+
+  it("a datum already in the Plutus layout hashes the same decoded and rebuilt", () => {
+    const layout = "9f0aff" // [10] as this module writes it
+    const decoded = Data.fromCBORHex(layout)
+    expect(Bytes.toHex(Data.toDatumHash(decoded).hash)).toBe(hashHex(layout))
+    expect(Bytes.toHex(Data.toDatumHash([10n]).hash)).toBe(hashHex(layout))
+  })
+
+  it("a locally built datum still hashes its own encoding", () => {
+    const data = Data.constr(0n, [1n, 2n])
+    expect(Bytes.toHex(Data.toDatumHash(data).hash)).toBe(hashHex(Data.toCBORHex(data)))
+  })
+
+  it("a bare integer datum falls back to its re-encode (bigint cannot be WeakMap-keyed)", () => {
+    const data = Data.fromCBORHex("1a0000000a") // 10 as a bare top-level integer
+    expect(data).toBe(10n)
+    expect(Bytes.toHex(Data.toDatumHash(data).hash)).toBe(hashHex("0a"))
+  })
+
+  it("a non-default codec option re-encodes instead of replaying the original bytes", () => {
+    const original = "811a0000000a"
+    const data = Data.fromCBORHex(original)
+    const custom = { ...Data.DEFAULT_CBOR_OPTIONS }
+    expect(Bytes.toHex(Data.toDatumHash(data, custom).hash)).toBe(hashHex("9f0aff"))
+  })
+})