fix(evolution): do not reuse stale CBOR chunk sizes on re-encode (#575)

.changeset/cbor-stale-chunk-sizes.md

@@ -0,0 +1,5 @@
+---
+"@evolution-sdk/evolution": patch
+---
+
+Fix preservation-aware encoding reusing stale indefinite chunk sizes for byte and text strings. A captured chunk layout is now reused only when its chunk sizes add up to the new value's length — and, for text, only when every chunk ends on a UTF-8 character boundary. Otherwise the encoder falls back to the default encoding for that string, instead of zero-padding a shorter value, dropping the tail of a longer one, or splitting a UTF-8 character.

packages/evolution/src/CBOR.ts

@@ -1281,31 +1281,40 @@ const encodeBytesSync = (value: Uint8Array, options: CodecOptions, fmt?: CBORFor
   if (stringEncoding !== undefined) {
     if (stringEncoding.tag === "indefinite") {
       const chunks = stringEncoding.chunks
-      let totalSize = 2 // 0x5f + 0xff
-      for (const chunk of chunks) {
-        totalSize += encodeIntHeader(2, BigInt(chunk.length), chunk.byteSize).length + chunk.length
-      }
-      const result = new Uint8Array(totalSize)
-      let pos = 0
-      result[pos++] = 0x5f
-      let srcOffset = 0
-      for (const chunk of chunks) {
-        const header = encodeIntHeader(2, BigInt(chunk.length), chunk.byteSize)
-        result.set(header, pos)
-        pos += header.length
-        result.set(value.subarray(srcOffset, srcOffset + chunk.length), pos)
-        pos += chunk.length
-        srcOffset += chunk.length
+      // Reuse the captured chunk layout only when it still fits the value:
+      // the chunk sizes must add up to the value's length. Otherwise the old
+      // sizes would pad or truncate the new value, so fall back to the
+      // default encoding below (as integer widths already do).
+      let chunkTotal = 0
+      for (const chunk of chunks) chunkTotal += chunk.length
+      if (chunkTotal === length) {
+        let totalSize = 2 // 0x5f + 0xff
+        for (const chunk of chunks) {
+          totalSize += encodeIntHeader(2, BigInt(chunk.length), chunk.byteSize).length + chunk.length
+        }
+        const result = new Uint8Array(totalSize)
+        let pos = 0
+        result[pos++] = 0x5f
+        let srcOffset = 0
+        for (const chunk of chunks) {
+          const header = encodeIntHeader(2, BigInt(chunk.length), chunk.byteSize)
+          result.set(header, pos)
+          pos += header.length
+          result.set(value.subarray(srcOffset, srcOffset + chunk.length), pos)
+          pos += chunk.length
+          srcOffset += chunk.length
+        }
+        result[pos] = 0xff
+        return result
       }
-      result[pos] = 0xff
+    } else {
+      // Definite with specific byteSize
+      const header = encodeIntHeader(2, BigInt(length), stringEncoding.byteSize)
+      const result = new Uint8Array(header.length + length)
+      result.set(header, 0)
+      result.set(value, header.length)
       return result
     }
-    // Definite with specific byteSize
-    const header = encodeIntHeader(2, BigInt(length), stringEncoding.byteSize)
-    const result = new Uint8Array(header.length + length)
-    result.set(header, 0)
-    result.set(value, header.length)
-    return result
   }
 
   const useMinimal = options.mode !== "custom" || options.useMinimalEncoding
@@ -1372,31 +1381,45 @@ const encodeTextSync = (value: string, options: CodecOptions, fmt?: CBORFormat):
     const utf8 = TEXT_ENCODER.encode(value)
     if (stringEncoding.tag === "indefinite") {
       const chunks = stringEncoding.chunks
-      let totalSize = 2 // 0x7f + 0xff
+      // Reuse the captured chunk layout only when it still fits the value:
+      // the chunk sizes must add up to the value's UTF-8 length and every
+      // chunk must end on a UTF-8 character boundary (the next byte is not
+      // a continuation byte). Otherwise fall back to the default encoding
+      // below, rather than padding, truncating, or splitting a character.
+      let chunkTotal = 0
+      let chunksFit = true
       for (const chunk of chunks) {
-        totalSize += encodeIntHeader(3, BigInt(chunk.length), chunk.byteSize).length + chunk.length
+        chunkTotal += chunk.length
+        if (chunkTotal < utf8.length && (utf8[chunkTotal] & 0xc0) === 0x80) chunksFit = false
       }
-      const result = new Uint8Array(totalSize)
-      let pos = 0
-      result[pos++] = 0x7f
-      let srcOffset = 0
-      for (const chunk of chunks) {
-        const header = encodeIntHeader(3, BigInt(chunk.length), chunk.byteSize)
-        result.set(header, pos)
-        pos += header.length
-        result.set(utf8.subarray(srcOffset, srcOffset + chunk.length), pos)
-        pos += chunk.length
-        srcOffset += chunk.length
+      if (chunksFit && chunkTotal === utf8.length) {
+        let totalSize = 2 // 0x7f + 0xff
+        for (const chunk of chunks) {
+          totalSize += encodeIntHeader(3, BigInt(chunk.length), chunk.byteSize).length + chunk.length
+        }
+        const result = new Uint8Array(totalSize)
+        let pos = 0
+        result[pos++] = 0x7f
+        let srcOffset = 0
+        for (const chunk of chunks) {
+          const header = encodeIntHeader(3, BigInt(chunk.length), chunk.byteSize)
+          result.set(header, pos)
+          pos += header.length
+          result.set(utf8.subarray(srcOffset, srcOffset + chunk.length), pos)
+          pos += chunk.length
+          srcOffset += chunk.length
+        }
+        result[pos] = 0xff
+        return result
       }
-      result[pos] = 0xff
+    } else {
+      // Definite with specific byteSize
+      const header = encodeIntHeader(3, BigInt(utf8.length), stringEncoding.byteSize)
+      const result = new Uint8Array(header.length + utf8.length)
+      result.set(header, 0)
+      result.set(utf8, header.length)
       return result
     }
-    // Definite with specific byteSize
-    const header = encodeIntHeader(3, BigInt(utf8.length), stringEncoding.byteSize)
-    const result = new Uint8Array(header.length + utf8.length)
-    result.set(header, 0)
-    result.set(utf8, header.length)
-    return result
   }
 
   // Fast path for empty strings

packages/evolution/test/CBOR-with-format.test.ts

@@ -61,6 +61,64 @@ describe("CBOR WithFormat — format tree is the complete specification", () =>
   })
 })
 
+describe("CBOR WithFormat — stale chunk sizes are not reused (#575)", () => {
+  it("bytes: a shorter value under a chunked format decodes back to itself", () => {
+    const { format } = CBOR.fromCBORHexWithFormat("5f4201024103ff") // h'010203' as chunks 2 + 1
+    const encoded = CBOR.toCBORHexWithFormat(new Uint8Array([0xaa]), format)
+    expect(CBOR.fromCBORHex(encoded)).toEqual(new Uint8Array([0xaa]))
+  })
+
+  it("bytes: a longer value under a chunked format decodes back to itself", () => {
+    const { format } = CBOR.fromCBORHexWithFormat("5f4201024103ff")
+    const value = new Uint8Array([1, 2, 3, 4, 5])
+    const encoded = CBOR.toCBORHexWithFormat(value, format)
+    expect(CBOR.fromCBORHex(encoded)).toEqual(value)
+  })
+
+  it("bytes: an unchanged value under its own chunked format stays byte-identical", () => {
+    const hex = "5f4201024103ff"
+    const { format, value } = CBOR.fromCBORHexWithFormat(hex)
+    expect(CBOR.toCBORHexWithFormat(value, format)).toBe(hex)
+  })
+
+  it("bytes: a same-length changed value still reuses its chunk layout", () => {
+    const { format } = CBOR.fromCBORHexWithFormat("5f4201024103ff")
+    expect(CBOR.toCBORHexWithFormat(new Uint8Array([0xaa, 0xbb, 0xcc]), format)).toBe("5f42aabb41ccff")
+  })
+
+  it("text: a shorter string under a chunked format decodes back to itself", () => {
+    const { format } = CBOR.fromCBORHexWithFormat("7f626869616aff") // "hij" as chunks 2 + 1
+    const encoded = CBOR.toCBORHexWithFormat("x", format)
+    expect(CBOR.fromCBORHex(encoded)).toBe("x")
+  })
+
+  it("text: a longer string under a chunked format decodes back to itself", () => {
+    const { format } = CBOR.fromCBORHexWithFormat("7f626869616aff")
+    const encoded = CBOR.toCBORHexWithFormat("hijk", format)
+    expect(CBOR.fromCBORHex(encoded)).toBe("hijk")
+  })
+
+  it("text: a string whose chunk split would break a UTF-8 character decodes back to itself", () => {
+    const { format } = CBOR.fromCBORHexWithFormat("7f626869616aff")
+    const encoded = CBOR.toCBORHexWithFormat("héllo", format)
+    expect(CBOR.fromCBORHex(encoded)).toBe("héllo")
+  })
+
+  it("text: a same-length string whose chunks would split a UTF-8 character falls back", () => {
+    const { format } = CBOR.fromCBORHexWithFormat("7f626869616aff") // chunks 2 + 1, total 3 bytes
+    // "xé" is 3 UTF-8 bytes (x + 2-byte é); the 2-byte first chunk would split é
+    const encoded = CBOR.toCBORHexWithFormat("xé", format)
+    expect(CBOR.fromCBORHex(encoded)).toBe("xé")
+    expect(encoded).not.toBe("7f6278c361a9ff")
+  })
+
+  it("text: an unchanged string under its own chunked format stays byte-identical", () => {
+    const hex = "7f626869616aff"
+    const { format, value } = CBOR.fromCBORHexWithFormat(hex)
+    expect(CBOR.toCBORHexWithFormat(value, format)).toBe(hex)
+  })
+})
+
 describe("CBOR WithFormat — hand-crafted format injection", () => {
   it("explicit uint format with byteSize:1 forces non-minimal encoding", () => {
     const fmt: CBOR.CBORFormat = { _tag: "uint", byteSize: 1 }