diff --git a/packages/pointers/src/dereference/index.test.ts b/packages/pointers/src/dereference/index.test.ts index b151462844..1751bfaf3e 100644 --- a/packages/pointers/src/dereference/index.test.ts +++ b/packages/pointers/src/dereference/index.test.ts @@ -202,6 +202,49 @@ describe("dereference", () => { }); }); + it("resolves .length lookups of $this inside list items", async () => { + // mirrors the shape of the `struct Record[] memory` schema example, + // where each item's offset is computed from its own length + const pointer: Pointer = { + list: { + count: 3, + each: "item-index", + is: { + name: "item", + location: "memory", + offset: { + $sum: [64, { $product: ["item-index", { ".length": "$this" }] }], + }, + length: 32, + }, + }, + }; + + const cursor = await dereference(pointer); + + const { regions } = await cursor.view(state); + + expect(regions.map(({ offset }) => offset!.asUint())).toEqual([ + 64n, + 96n, + 128n, + ]); + }); + + it("throws an error on a self-referential $this lookup", async () => { + const pointer: Pointer = { + location: "memory", + offset: 0, + length: { ".length": "$this" }, + }; + + const cursor = await dereference(pointer); + + await expect(cursor.view(state)).rejects.toThrow( + "Circular reference detected: $this.length", + ); + }); + it("throws an error on circular reference", async () => { const pointer: Pointer = { location: "memory", diff --git a/packages/pointers/src/read.test.ts b/packages/pointers/src/read.test.ts index c30f88498f..8cee2f9c28 100644 --- a/packages/pointers/src/read.test.ts +++ b/packages/pointers/src/read.test.ts @@ -237,3 +237,222 @@ describe("read", () => { ); }); }); + +describe("read (segment carry)", () => { + // each slot `n` holds 32 bytes, all equal to `0xa0 + n`, so that the + // bytes of a result identify which slot(s) they came from + const wordFor = (index: bigint): Uint8Array => + new Uint8Array(32).fill(0xa0 + Number(index)); + + const sliced = ( + word: Uint8Array, + slice: Machine.State.Slice | undefined, + ): Data => + Data.fromBytes( + slice + ? word.slice(Number(slice.offset), Number(slice.offset + slice.length)) + : word, + ); + + let options: ReadOptions; + + beforeEach(() => { + const words: Machine.State.Words = { + read: vitest.fn(async ({ slot, slice }) => + sliced(wordFor(slot.asUint()), slice), + ), + }; + + const state: Machine.State = { + stack: { + length: 50n, + peek: vitest.fn(async ({ depth, slice }) => + sliced(wordFor(depth), slice), + ), + }, + storage: words, + transient: { + read: vitest.fn(async ({ slot, slice }) => + sliced(wordFor(slot.asUint()), slice), + ), + }, + } as unknown as Machine.State; + + options = { state }; + }); + + const bytes = (...runs: [number, number][]): Data => + Data.fromBytes( + new Uint8Array( + runs.flatMap(([byte, count]) => new Array(count).fill(byte)), + ), + ); + + it("treats offset $wordsize as byte 0 of the next slot", async () => { + const region: Cursor.Region = { + location: "storage", + slot: Data.fromNumber(0), + offset: Data.fromNumber(32), + length: Data.fromNumber(4), + }; + + const result = await read(region, options); + + expect(options.state.storage.read).toHaveBeenCalledTimes(1); + expect(options.state.storage.read).toHaveBeenCalledWith({ + slot: Data.fromNumber(1), + slice: { offset: 0n, length: 4n }, + }); + expect(result).toEqual(bytes([0xa1, 4])); + }); + + it("carries an offset beyond $wordsize into a later slot", async () => { + const region: Cursor.Region = { + location: "storage", + slot: Data.fromNumber(5), + offset: Data.fromNumber(40), + length: Data.fromNumber(8), + }; + + const result = await read(region, options); + + expect(options.state.storage.read).toHaveBeenCalledTimes(1); + expect(options.state.storage.read).toHaveBeenCalledWith({ + slot: Data.fromNumber(6), + slice: { offset: 8n, length: 8n }, + }); + expect(result).toEqual(bytes([0xa6, 8])); + }); + + it("concatenates a range that spans a slot boundary", async () => { + const region: Cursor.Region = { + location: "storage", + slot: Data.fromNumber(0), + offset: Data.fromNumber(28), + length: Data.fromNumber(8), + }; + + const result = await read(region, options); + + expect(options.state.storage.read).toHaveBeenCalledTimes(2); + expect(options.state.storage.read).toHaveBeenNthCalledWith(1, { + slot: Data.fromNumber(0), + slice: { offset: 28n, length: 4n }, + }); + expect(options.state.storage.read).toHaveBeenNthCalledWith(2, { + slot: Data.fromNumber(1), + slice: { offset: 0n, length: 4n }, + }); + expect(result).toEqual(bytes([0xa0, 4], [0xa1, 4])); + }); + + it("concatenates a range that spans three slots", async () => { + const region: Cursor.Region = { + location: "storage", + slot: Data.fromNumber(0), + offset: Data.fromNumber(16), + length: Data.fromNumber(64), + }; + + const result = await read(region, options); + + expect(options.state.storage.read).toHaveBeenCalledTimes(3); + expect(options.state.storage.read).toHaveBeenNthCalledWith(1, { + slot: Data.fromNumber(0), + slice: { offset: 16n, length: 16n }, + }); + expect(options.state.storage.read).toHaveBeenNthCalledWith(2, { + slot: Data.fromNumber(1), + slice: { offset: 0n, length: 32n }, + }); + expect(options.state.storage.read).toHaveBeenNthCalledWith(3, { + slot: Data.fromNumber(2), + slice: { offset: 0n, length: 16n }, + }); + expect(result).toEqual(bytes([0xa0, 16], [0xa1, 32], [0xa2, 16])); + }); + + it("defaults length to the end of the slot it begins in", async () => { + const region: Cursor.Region = { + location: "storage", + slot: Data.fromNumber(0), + offset: Data.fromNumber(40), + }; + + const result = await read(region, options); + + expect(options.state.storage.read).toHaveBeenCalledTimes(1); + expect(options.state.storage.read).toHaveBeenCalledWith({ + slot: Data.fromNumber(1), + slice: { offset: 8n, length: 24n }, + }); + expect(result).toEqual(bytes([0xa1, 24])); + }); + + it("reads nothing for a zero-length segment", async () => { + const region: Cursor.Region = { + location: "storage", + slot: Data.fromNumber(0), + offset: Data.fromNumber(0), + length: Data.fromNumber(0), + }; + + const result = await read(region, options); + + expect(options.state.storage.read).not.toHaveBeenCalled(); + expect(result).toEqual(Data.zero()); + }); + + it("preserves the width of the given slot", async () => { + const slot = Data.fromHex(`0x${"00".repeat(31)}07`); + const region: Cursor.Region = { + location: "storage", + slot, + offset: Data.fromNumber(32), + length: Data.fromNumber(1), + }; + + await read(region, options); + + expect(options.state.storage.read).toHaveBeenCalledWith({ + slot: Data.fromHex(`0x${"00".repeat(31)}08`), + slice: { offset: 0n, length: 1n }, + }); + }); + + it("applies carry to transient storage", async () => { + const region: Cursor.Region = { + location: "transient", + slot: Data.fromNumber(0), + offset: Data.fromNumber(30), + length: Data.fromNumber(4), + }; + + const result = await read(region, options); + + expect(options.state.transient.read).toHaveBeenCalledTimes(2); + expect(result).toEqual(bytes([0xa0, 2], [0xa1, 2])); + }); + + it("applies carry to stack depth", async () => { + const region: Cursor.Region = { + location: "stack", + slot: Data.fromNumber(2), + offset: Data.fromNumber(30), + length: Data.fromNumber(4), + }; + + const result = await read(region, options); + + expect(options.state.stack.peek).toHaveBeenCalledTimes(2); + expect(options.state.stack.peek).toHaveBeenNthCalledWith(1, { + depth: 2n, + slice: { offset: 30n, length: 2n }, + }); + expect(options.state.stack.peek).toHaveBeenNthCalledWith(2, { + depth: 3n, + slice: { offset: 0n, length: 2n }, + }); + expect(result).toEqual(bytes([0xa2, 2], [0xa3, 2])); + }); +}); diff --git a/packages/pointers/src/read.ts b/packages/pointers/src/read.ts index 742a596fe2..2dfed67a6e 100644 --- a/packages/pointers/src/read.ts +++ b/packages/pointers/src/read.ts @@ -16,19 +16,14 @@ export async function read( switch (location) { case "stack": { - const { - slot, - offset = 0n, - length = 32n, - } = withPropertiesAsUints(["slot", "offset", "length"], region); - - return await state.stack.peek({ - depth: slot, - slice: { - offset, - length, - }, - }); + const { slot, offset, length } = withPropertiesAsUints( + ["slot", "offset", "length"], + region, + ); + + return await readSegment({ offset, length }, (carry, slice) => + state.stack.peek({ depth: slot + carry, slice }), + ); } case "memory": { const { offset, length } = withPropertiesAsUints( @@ -45,18 +40,14 @@ export async function read( } case "storage": { const { slot } = region; - const { offset = 0n, length = 32n } = withPropertiesAsUints( + const { offset, length } = withPropertiesAsUints( ["offset", "length"], region, ); - return await state.storage.read({ - slot, - slice: { - offset, - length, - }, - }); + return await readSegment({ offset, length }, (carry, slice) => + state.storage.read({ slot: slotAfter(slot, carry), slice }), + ); } case "calldata": { const { offset, length } = withPropertiesAsUints( @@ -76,18 +67,14 @@ export async function read( } case "transient": { const { slot } = region; - const { offset = 0n, length = 32n } = withPropertiesAsUints( + const { offset, length } = withPropertiesAsUints( ["offset", "length"], region, ); - return await state.transient.read({ - slot, - slice: { - offset, - length, - }, - }); + return await readSegment({ offset, length }, (carry, slice) => + state.transient.read({ slot: slotAfter(slot, carry), slice }), + ); } case "code": { const { offset, length } = withPropertiesAsUints( @@ -100,6 +87,64 @@ export async function read( } } +const wordsize = 32n; + +/** + * Read a segment of bytes from a word-addressed data location, where + * `offset` may meet or exceed the word size (carrying into subsequent + * words) and `length` may run across word boundaries (concatenating + * sequentially-addressed words). + * + * Given offset `n`, the segment begins at byte `n mod $wordsize` of the + * word `floor(n / $wordsize)` words after the one specified. When `length` + * is omitted, the segment ends at the end of the word in which it begins. + * + * `readWord(carry, slice)` reads bytes from the word `carry` words after + * the one specified. + */ +async function readSegment( + { offset = 0n, length }: { offset?: bigint; length?: bigint }, + readWord: (carry: bigint, slice: Machine.State.Slice) => Promise, +): Promise { + const startCarry = offset / wordsize; + const startByte = offset % wordsize; + const totalLength = length ?? wordsize - startByte; + + if (totalLength === 0n) { + return Data.zero(); + } + + const endByte = startByte + totalLength; + const wordCount = (endByte + wordsize - 1n) / wordsize; + + const words: Data[] = []; + for (let index = 0n; index < wordCount; index++) { + const from = index === 0n ? startByte : 0n; + const to = index === wordCount - 1n ? endByte - index * wordsize : wordsize; + + words.push( + await readWord(startCarry + index, { + offset: from, + length: to - from, + }), + ); + } + + return Data.zero().concat(...words); +} + +/** + * Compute the slot `carry` slots after `slot`, keeping at least the width + * of the given slot data. + */ +function slotAfter(slot: Data, carry: bigint): Data { + if (carry === 0n) { + return slot; + } + + return Data.fromUint(slot.asUint() + carry).padUntilAtLeast(slot.length); +} + type DataProperties = { [K in keyof Cursor.Region & ("slot" | "offset" | "length")]: Cursor.Region[K]; diff --git a/packages/pointers/src/test-cases.ts b/packages/pointers/src/test-cases.ts index ad726d4ec2..2e03ad173b 100644 --- a/packages/pointers/src/test-cases.ts +++ b/packages/pointers/src/test-cases.ts @@ -55,7 +55,7 @@ const structStorageTest: ObserveTraceTest<{ }), expectedValues: [ - { x: 0, y: 0, salt: "0x" }, + { x: 0, y: 0, salt: "0x00000000" }, { x: 5, y: 8, salt: "0xdeadbeef" }, { x: 1, y: 2, salt: "0xfeedface" }, ], @@ -75,27 +75,29 @@ const structStorageTest: ObserveTraceTest<{ }, }; -const stringStorageTest: ObserveTraceTest = { - pointer: findExamplePointer("string-storage-contract-variable-slot"), +const stringStorageCompileOptions = singleSourceCompilation({ + path: "StringStorage.sol", + contractName: "StringStorage", + content: `contract StringStorage { + string storedString; + bool done; - compileOptions: singleSourceCompilation({ - path: "StringStorage.sol", - contractName: "StringStorage", - content: `contract StringStorage { - string storedString; - bool done; + event Done(); - event Done(); + constructor() { + storedString = "hello world"; + storedString = "solidity storage is a fun lesson in endianness"; - constructor() { - storedString = "hello world"; - storedString = "solidity storage is a fun lesson in endianness"; - - done = true; - } + done = true; } - `, - }), + } + `, +}); + +const stringStorageTest: ObserveTraceTest = { + pointer: findExamplePointer("string-storage-contract-variable-slot"), + + compileOptions: stringStorageCompileOptions, expectedValues: [ "", @@ -117,6 +119,35 @@ const stringStorageTest: ObserveTraceTest = { }, }; +/** + * the companion `string storage` example expresses the long-string body as + * a single region whose `length` runs across storage slots (see the segment + * addressing scheme). It models only the long form, so this test observes + * the trace only while the variable is unset or holds a long string. + */ +const multiSlotStringStorageTest: ObserveTraceTest = { + pointer: findExamplePointer("string-storage-slot"), + + compileOptions: stringStorageCompileOptions, + + expectedValues: ["", "solidity storage is a fun lesson in endianness"], + + async observe({ regions, read }: Cursor.View): Promise { + const [string] = regions.named("string"); + + return new TextDecoder().decode(await read(string)); + }, + + // solc marks long strings by an odd length word; the example does not + // model the short form, whose length word would decode as a huge length + async shouldObserve(state) { + const lengthData = await state.storage.read({ slot: Data.zero() }); + const lengthWord = lengthData.asUint(); + + return lengthWord === 0n || lengthWord % 2n === 1n; + }, +}; + const uint256ArrayMemoryTest: ObserveTraceTest = { pointer: findExamplePointer("uint256-array-memory-pointer-slot"), compileOptions: singleSourceCompilation({ @@ -218,5 +249,6 @@ const uint256ArrayMemoryTest: ObserveTraceTest = { export const observeTraceTests = { "struct storage": structStorageTest, "string storage": stringStorageTest, + "string storage (multi-slot)": multiSlotStringStorageTest, "uint256[] memory": uint256ArrayMemoryTest, }; diff --git a/packages/pointers/test/ganache.ts b/packages/pointers/test/ganache.ts index 01065612ae..1cc105b970 100644 --- a/packages/pointers/test/ganache.ts +++ b/packages/pointers/test/ganache.ts @@ -124,7 +124,9 @@ function makeWords(slots: StructLog["storage"]): Machine.State.Words { const rawHex = slots[slot.resizeTo(32).toHex().slice(2) as keyof typeof slots]; - const data = Data.fromHex(`0x${rawHex}`); + // slots untouched by the transaction so far are absent from the + // struct log; treat them as zero + const data = Data.fromHex(`0x${rawHex ?? "00".repeat(32)}`); return new Data(data.slice(Number(offset), Number(offset + length))); }, diff --git a/packages/web/docs/core-schemas/pointers/regions.mdx b/packages/web/docs/core-schemas/pointers/regions.mdx index 1c9026671d..c1e9d95fb9 100644 --- a/packages/web/docs/core-schemas/pointers/regions.mdx +++ b/packages/web/docs/core-schemas/pointers/regions.mdx @@ -31,12 +31,18 @@ Regions in these locations use `offset` and `length`: ### Slot-based locations **Storage**, **transient storage**, and **stack** are organized in 32-byte -slots. Regions use `slot`. - -For storage and transient storage, values that don't fill a full slot can -specify sub-slot positioning with `offset` and `length` within the slot — -useful for packed storage, such as a 20-byte address held at byte 12 of a -slot. +slots. Regions use `slot`, and may narrow or extend the byte range with +`offset` and `length`, both measured in bytes relative to the start of `slot`. + +A segment may begin at any non-negative `offset`; an offset at or past +`$wordsize` carries into later slots, so byte `$wordsize` of slot `p` is byte +`0` of slot `p + 1`. Its `length` may likewise run past the end of the slot, +in which case the segment is the concatenation of consecutive slots. When +`length` is omitted, the segment ends at the end of the slot in which it +begins. This covers packed storage, such as a 20-byte address held at byte 12 +of a slot, as well as data spanning many slots — the multi-slot +`string storage` example in the [pointer schema](/spec/pointer) reads a long +string's bytes as a single region. ## Location-specific details diff --git a/schemas/pointer.schema.yaml b/schemas/pointer.schema.yaml index cf0ec1683a..fbf8c0f9de 100644 --- a/schemas/pointer.schema.yaml +++ b/schemas/pointer.schema.yaml @@ -146,7 +146,7 @@ examples: - .length: "array-count" - $product: - "item-index" - - .length: "struct-pointer" + - .length: $this length: $wordsize # following that pointer leads to the region corresponding to @@ -259,3 +259,39 @@ examples: offset: 0 length: $difference: ["string-length", "previous-length"] + + - # example `string storage` (long form) as a single multi-slot region + # + # this is the same long-string body as the previous example, collapsed + # to one region. Because a segment's length may run across slots (see + # the segment addressing scheme), the whole string is a single region + # beginning at "start-slot"; no per-slot list or last-slot trim is + # needed, and the compiler emits far less. The per-slot list form above + # remains useful when a consumer wants a distinct region per slot. + define: + "string-storage-slot": 0 + in: + group: + - name: "long-string-length-data" + location: storage + slot: "string-storage-slot" + offset: 0 + length: $wordsize + + - define: + "string-length": + $quotient: + - $difference: + - $read: "long-string-length-data" + - 1 + - 2 + + "start-slot": + $keccak256: + - $wordsized: "string-storage-slot" + in: + name: "string" + location: storage + slot: "start-slot" + offset: 0 + length: "string-length" diff --git a/schemas/pointer/expression.schema.yaml b/schemas/pointer/expression.schema.yaml index 21a1ff1612..1d22e73c5d 100644 --- a/schemas/pointer/expression.schema.yaml +++ b/schemas/pointer/expression.schema.yaml @@ -165,6 +165,9 @@ $defs: - const: "$this" description: | Indicates a reference to the region containing this expression. + A property lookup via `$this` (e.g. `{ ".length": "$this" }`) must + not be circular: the referenced property must be resolvable without + depending on the value currently being defined. Keccak256: title: Keccak256 hash diff --git a/schemas/pointer/scheme/segment.schema.yaml b/schemas/pointer/scheme/segment.schema.yaml index 447356ed35..487795bf50 100644 --- a/schemas/pointer/scheme/segment.schema.yaml +++ b/schemas/pointer/scheme/segment.schema.yaml @@ -28,9 +28,17 @@ properties: `0`, indicating that the segment begins at the start of the specified slot. - This field's expression must resolve to a value _n_ such that - 0 ≤ _n_ \< `$wordsize` (i.e., the offset **must** - begin inside the slot). + This field's expression must resolve to a non-negative value. It is + **not** bounded by the word size: an offset that meets or exceeds + `$wordsize` carries into subsequent slots. Given a `slot` value `p` + and an `offset` value `n`, the segment begins at byte + `n mod $wordsize` of slot `p + floor(n / $wordsize)`. (Equivalently, + byte `{ "offset": "$wordsize" }` of slot `p` is byte `0` of slot + `p + 1`, consistent with the multi-slot note above.) Emitters may + therefore chain byte sums across a slot boundary without decomposing + into slot and byte components themselves; a resolver recovers the + effective slot and byte by division and remainder against + `$wordsize`. $ref: "schema:ethdebug/format/pointer/expression" default: 0 length: @@ -38,17 +46,20 @@ properties: The length of the bytes range this segment represents. This field is **optional**. If unspecified, its default value indicates - that the segment ends at the end of the slot. + that the segment ends at the end of the slot in which it begins (after + applying any `offset` carry). If this field has value larger than the default value, i.e., if the segment extends beyond the last byte in the slot, then this segment is defined to be the concatenation of the sequentially-addressed slot(s) - following following the slot specified. + following the slot specified. $ref: "schema:ethdebug/format/pointer/expression" default: $difference: - $wordsize - - .offset: $this + - $remainder: + - .offset: $this + - $wordsize required: - slot @@ -60,3 +71,9 @@ examples: $product: - $wordsize - 3 + # a carry example: an offset at or beyond `$wordsize` addresses a later + # slot. Here `offset: $wordsize` is byte 0 of slot 1, so this segment is + # the 4 bytes beginning there. + - slot: 0 + offset: $wordsize + length: 4