diff --git a/commands/metrics/cbor_gen.maps.go b/commands/metrics/cbor_gen.maps.go index b6eff36..a24f655 100644 --- a/commands/metrics/cbor_gen.maps.go +++ b/commands/metrics/cbor_gen.maps.go @@ -309,7 +309,7 @@ func (t *SampleOK) MarshalCBOR(w io.Writer) error { } } - // t.Samples (metrics.SampleSet) (struct) + // t.Samples ([]metrics.SampleItem) (slice) if len("samples") > 8192 { return xerrors.Errorf("Value in field \"samples\" was too long") } @@ -321,9 +321,19 @@ func (t *SampleOK) MarshalCBOR(w io.Writer) error { return err } - if err := t.Samples.MarshalCBOR(cw); err != nil { + if len(t.Samples) > 8192 { + return xerrors.Errorf("Slice value in field t.Samples was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.Samples))); err != nil { return err } + for _, v := range t.Samples { + if err := v.MarshalCBOR(cw); err != nil { + return err + } + + } return nil } @@ -446,15 +456,44 @@ func (t *SampleOK) UnmarshalCBOR(r io.Reader) (err error) { t.Window = int64(extraI) } - // t.Samples (metrics.SampleSet) (struct) + // t.Samples ([]metrics.SampleItem) (slice) case "samples": - { + maj, extra, err = cr.ReadHeader() + if err != nil { + return err + } - if err := t.Samples.UnmarshalCBOR(cr); err != nil { - return xerrors.Errorf("unmarshaling t.Samples: %w", err) - } + if extra > 8192 { + return fmt.Errorf("t.Samples: array too large (%d)", extra) + } + if maj != cbg.MajArray { + return fmt.Errorf("expected cbor array") + } + + if extra > 0 { + t.Samples = make([]SampleItem, extra) + } + + for i := 0; i < int(extra); i++ { + { + var maj byte + var extra uint64 + var err error + _ = maj + _ = extra + _ = err + + { + + if err := t.Samples[i].UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.Samples[i]: %w", err) + } + + } + + } } default: diff --git a/commands/metrics/codec.go b/commands/metrics/codec.go deleted file mode 100644 index a876110..0000000 --- a/commands/metrics/codec.go +++ /dev/null @@ -1,129 +0,0 @@ -//go:build !codegen - -package metrics - -import ( - "io" - - jsg "github.com/alanshaw/dag-json-gen" - "github.com/fil-forge/libforge/commands/internal/codec" - "github.com/fil-forge/ucantone/did" - cbg "github.com/whyrusleeping/cbor-gen" - xerrors "golang.org/x/xerrors" -) - -// SampleSet has a hand-written codec because cbor-gen / dag-json-gen do not -// support DID-keyed maps. The CBOR and DAG-JSON primitives live in the shared -// internal codec package. - -func (t SampleSet) MarshalCBOR(w io.Writer) error { - cw := cbg.NewCborWriter(w) - if err := codec.WriteCborMapHeader(cw, len(t.Entries)); err != nil { - return err - } - for _, k := range codec.SortedDIDs(t.Entries) { - if err := codec.WriteCborString(cw, k.String()); err != nil { - return err - } - samples := t.Entries[k] - if err := codec.WriteCborArrayHeader(cw, len(samples)); err != nil { - return err - } - for i := range samples { - if err := samples[i].MarshalCBOR(cw); err != nil { - return err - } - } - } - return nil -} - -func (t *SampleSet) UnmarshalCBOR(r io.Reader) error { - cr := cbg.NewCborReader(r) - n, err := codec.ReadCborMapHeader(cr) - if err != nil { - return err - } - m := make(map[did.DID][]SampleItem, n) - for i := uint64(0); i < n; i++ { - ks, err := codec.ReadCborString(cr) - if err != nil { - return err - } - k, err := did.Parse(ks) - if err != nil { - return xerrors.Errorf("parsing provider did %q: %w", ks, err) - } - sn, err := codec.ReadCborArrayHeader(cr) - if err != nil { - return err - } - samples := make([]SampleItem, sn) - for j := uint64(0); j < sn; j++ { - if err := samples[j].UnmarshalCBOR(cr); err != nil { - return err - } - } - m[k] = samples - } - *t = SampleSet{Entries: m} - return nil -} - -func (t SampleSet) MarshalDagJSON(w io.Writer) error { - jw := jsg.NewDagJsonWriter(w) - if err := jw.WriteObjectOpen(); err != nil { - return err - } - for i, k := range codec.SortedDIDs(t.Entries) { - if err := codec.WriteJSONKey(jw, k.String(), i > 0); err != nil { - return err - } - if err := jw.WriteArrayOpen(); err != nil { - return err - } - samples := t.Entries[k] - for j := range samples { - if j > 0 { - if err := jw.WriteComma(); err != nil { - return err - } - } - if err := samples[j].MarshalDagJSON(jw); err != nil { - return err - } - } - if err := jw.WriteArrayClose(); err != nil { - return err - } - } - return jw.WriteObjectClose() -} - -func (t *SampleSet) UnmarshalDagJSON(r io.Reader) error { - jr := jsg.NewDagJsonReader(r) - m := map[did.DID][]SampleItem{} - err := codec.ReadJSONObject(jr, func(ks string) error { - k, err := did.Parse(ks) - if err != nil { - return xerrors.Errorf("parsing provider did %q: %w", ks, err) - } - samples, err := codec.ReadJSONArray(jr, func() (SampleItem, error) { - var s SampleItem - if err := s.UnmarshalDagJSON(jr); err != nil { - return SampleItem{}, err - } - return s, nil - }) - if err != nil { - return err - } - m[k] = samples - return nil - }) - if err != nil { - return err - } - *t = SampleSet{Entries: m} - return nil -} diff --git a/commands/metrics/codec_test.go b/commands/metrics/codec_test.go index 1c3b328..79ef1a8 100644 --- a/commands/metrics/codec_test.go +++ b/commands/metrics/codec_test.go @@ -9,7 +9,6 @@ import ( "testing" "github.com/fil-forge/libforge/commands/metrics" - "github.com/fil-forge/ucantone/did" "github.com/stretchr/testify/require" ) @@ -46,97 +45,44 @@ func TestSampleArgumentsRoundTrip(t *testing.T) { require.Equal(t, `{"from":1700000000,"to":1700003600,"window":3600}`, roundTrip(t, in)) } -// provider is a fixed DID so the expected DAG-JSON below is stable. -var provider = did.MustParse("did:web:provider.example") - func TestSampleOKRoundTrip(t *testing.T) { in := &metrics.SampleOK{ From: 1700000000, To: 1700007200, Window: 3600, - Samples: metrics.SampleSet{Entries: map[did.DID][]metrics.SampleItem{ - provider: { - {Timestamp: 1700003600, BytesStored: 1024, BytesIngested: 1024}, - {Timestamp: 1700007200, BytesStored: 512, BytesIngested: 0}, - }, - }}, + Samples: []metrics.SampleItem{ + {Timestamp: 1700003600, BytesStored: 1024, BytesIngested: 1024}, + {Timestamp: 1700007200, BytesStored: 512, BytesIngested: 0}, + }, } // SampleItem is tuple encoded: [timestamp, bytesStored, bytesIngested]. require.Equal(t, - `{"from":1700000000,"samples":{"did:web:provider.example":[`+ + `{"from":1700000000,"samples":[`+ `[1700003600,1024,1024],`+ `[1700007200,512,0]`+ - `]},"to":1700007200,"window":3600}`, - roundTrip(t, in)) -} - -// A space provisioned with several providers carries a series each, on one -// shared bucket grid. The keys sort, so the encoding is deterministic. -func TestSampleOKSeveralProvidersRoundTrip(t *testing.T) { - a := did.MustParse("did:web:a.example") - b := did.MustParse("did:web:b.example") - in := &metrics.SampleOK{ - From: 1700000000, - To: 1700003600, - Window: 3600, - Samples: metrics.SampleSet{Entries: map[did.DID][]metrics.SampleItem{ - b: {{Timestamp: 1700003600, BytesStored: 2, BytesIngested: 2}}, - a: {{Timestamp: 1700003600, BytesStored: 1, BytesIngested: 1}}, - }}, - } - require.Equal(t, - `{"from":1700000000,"samples":{`+ - `"did:web:a.example":[[1700003600,1,1]],`+ - `"did:web:b.example":[[1700003600,2,2]]`+ - `},"to":1700003600,"window":3600}`, + `],"to":1700007200,"window":3600}`, roundTrip(t, in)) } -// A range entirely in the future carries no samples. An absent set and an -// empty one are the same thing on the wire, so these assert the encoding and +// A range entirely in the future carries no samples. An absent series and an +// empty one are the same thing on the wire, so this asserts the encoding and // the decoded content rather than round-trip identity. -func TestSampleOKEmptySetRoundTrip(t *testing.T) { - in := &metrics.SampleOK{From: 1700000000, To: 1700000000, Window: 3600} - - var jb bytes.Buffer - require.NoError(t, in.MarshalDagJSON(&jb)) - require.Equal(t, `{"from":1700000000,"samples":{},"to":1700000000,"window":3600}`, jb.String()) - - var fromJSON metrics.SampleOK - require.NoError(t, fromJSON.UnmarshalDagJSON(bytes.NewReader(jb.Bytes()))) - require.Empty(t, fromJSON.Samples.Entries) - - var cb bytes.Buffer - require.NoError(t, in.MarshalCBOR(&cb)) - var fromCBOR metrics.SampleOK - require.NoError(t, fromCBOR.UnmarshalCBOR(bytes.NewReader(cb.Bytes()))) - require.Empty(t, fromCBOR.Samples.Entries) -} - -// A provider present with nothing to report keeps its key and an empty series. func TestSampleOKEmptySeriesRoundTrip(t *testing.T) { - in := &metrics.SampleOK{ - From: 1700000000, To: 1700000000, Window: 3600, - Samples: metrics.SampleSet{Entries: map[did.DID][]metrics.SampleItem{provider: {}}}, - } + in := &metrics.SampleOK{From: 1700000000, To: 1700000000, Window: 3600} var jb bytes.Buffer require.NoError(t, in.MarshalDagJSON(&jb)) - require.Equal(t, - `{"from":1700000000,"samples":{"did:web:provider.example":[]},"to":1700000000,"window":3600}`, - jb.String()) + require.Equal(t, `{"from":1700000000,"samples":[],"to":1700000000,"window":3600}`, jb.String()) var fromJSON metrics.SampleOK require.NoError(t, fromJSON.UnmarshalDagJSON(bytes.NewReader(jb.Bytes()))) - require.Len(t, fromJSON.Samples.Entries, 1) - require.Empty(t, fromJSON.Samples.Entries[provider]) + require.Empty(t, fromJSON.Samples) var cb bytes.Buffer require.NoError(t, in.MarshalCBOR(&cb)) var fromCBOR metrics.SampleOK require.NoError(t, fromCBOR.UnmarshalCBOR(bytes.NewReader(cb.Bytes()))) - require.Len(t, fromCBOR.Samples.Entries, 1) - require.Empty(t, fromCBOR.Samples.Entries[provider]) + require.Empty(t, fromCBOR.Samples) } // 768 samples is a 32 day range at hourly granularity, the largest series @@ -156,14 +102,14 @@ func TestSampleOKLargeSeriesRoundTrip(t *testing.T) { BytesIngested: 1 << 20, }) } - in.Samples = metrics.SampleSet{Entries: map[did.DID][]metrics.SampleItem{provider: samples}} + in.Samples = samples in.To = in.From + count*hour var cb bytes.Buffer require.NoError(t, in.MarshalCBOR(&cb)) var out metrics.SampleOK require.NoError(t, out.UnmarshalCBOR(bytes.NewReader(cb.Bytes()))) - require.Len(t, out.Samples.Entries[provider], count) + require.Len(t, out.Samples, count) require.Equal(t, *in, out) t.Logf("%d samples encode to %d bytes of CBOR", count, cb.Len()) } diff --git a/commands/metrics/gen/main.go b/commands/metrics/gen/main.go index 1a03f15..cfa471c 100644 --- a/commands/metrics/gen/main.go +++ b/commands/metrics/gen/main.go @@ -23,8 +23,6 @@ func tag(path string) { } func main() { - // SampleSet has a hand-written codec (see codec.go) because cbor-gen / - // dag-json-gen do not support DID-keyed maps. mapModels := []any{ metrics.SampleArguments{}, metrics.SampleOK{}, diff --git a/commands/metrics/json_gen.maps.go b/commands/metrics/json_gen.maps.go index 5af6d89..f3e15e6 100644 --- a/commands/metrics/json_gen.maps.go +++ b/commands/metrics/json_gen.maps.go @@ -219,7 +219,7 @@ func (t *SampleOK) MarshalDagJSON(w io.Writer) error { } } - // t.Samples (metrics.SampleSet) (struct) + // t.Samples ([]metrics.SampleItem) (slice) if len("samples") > 8192 { return fmt.Errorf("string in field \"samples\" was too long") } @@ -229,9 +229,27 @@ func (t *SampleOK) MarshalDagJSON(w io.Writer) error { if err := jw.WriteObjectColon(); err != nil { return err } - if err := t.Samples.MarshalDagJSON(jw); err != nil { - return fmt.Errorf("marshaling field t.Samples: %w", err) + if len(t.Samples) > 8192 { + return fmt.Errorf("slice value in field t.Samples was too long") } + + if err := jw.WriteArrayOpen(); err != nil { + return fmt.Errorf("writing array open for field t.Samples: %w", err) + } + for i, v := range t.Samples { + if i > 0 { + if err := jw.WriteComma(); err != nil { + return fmt.Errorf("writing comma for field t.Samples: %w", err) + } + } + if err := v.MarshalDagJSON(jw); err != nil { + return fmt.Errorf("marshaling field v: %w", err) + } + } + if err := jw.WriteArrayClose(); err != nil { + return fmt.Errorf("writing array close for field t.Samples: %w", err) + } + written = true if written { if err := jw.WriteComma(); err != nil { @@ -327,11 +345,46 @@ func (t *SampleOK) UnmarshalDagJSON(r io.Reader) (err error) { } - // t.Samples (metrics.SampleSet) (struct) + // t.Samples ([]metrics.SampleItem) (slice) case "samples": + { + + if err := jr.ReadArrayOpen(); err != nil { + return fmt.Errorf("reading array open for field t.Samples: %w", err) + } + + close, err := jr.PeekArrayClose() + if err != nil { + return fmt.Errorf("peeking array close for field t.Samples: %w", err) + } + if close { + if err := jr.ReadArrayClose(); err != nil { + return fmt.Errorf("reading array close for field t.Samples: %w", err) + } + + } else { + for i := 0; i < 8192; i++ { + item := make([]SampleItem, 1) + + if err := item[0].UnmarshalDagJSON(jr); err != nil { + return fmt.Errorf("unmarshaling item[0]: %w", err) + } + + t.Samples = append(t.Samples, item[0]) + + close, err := jr.ReadArrayCloseOrComma() + if err != nil { + return fmt.Errorf("reading array close or comma for field t.Samples: %w", err) + } + if close { + break + } + if i == 8192-1 { + return fmt.Errorf("reading array for field t.Samples: slice too large") + } + } + } - if err := t.Samples.UnmarshalDagJSON(jr); err != nil { - return fmt.Errorf("unmarshaling t.Samples: %w", err) } // t.To (int64) (int64) diff --git a/commands/metrics/types.go b/commands/metrics/types.go index 5a192b9..8ce4b44 100644 --- a/commands/metrics/types.go +++ b/commands/metrics/types.go @@ -1,15 +1,8 @@ // Package metrics defines the usage metering capabilities. The space the // samples describe is the invocation subject, so it never appears in the // arguments. -// -// SampleSet is a DID-keyed map. cbor-gen / dag-json-gen only generate -// string-keyed maps, so it has a hand-written codec (see codec.go) and is -// defined as a struct wrapping the map so that structs embedding it can still -// be generated as usual. package metrics -import "github.com/fil-forge/ucantone/did" - // SampleArguments is the argument shape of `/metrics/sample`. It asks for a // usage time series for the invocation subject, covering [From, To) in buckets // of Window seconds. @@ -32,9 +25,9 @@ type SampleArguments struct { Window int64 `cborgen:"window" dagjsongen:"window"` } -// SampleOK is the success return for `/metrics/sample`. Each series in Samples -// holds exactly one entry per bucket in [From, To), ordered by ascending -// Timestamp, with no gaps. +// SampleOK is the success return for `/metrics/sample`. Samples holds exactly +// one entry per bucket in [From, To), ordered by ascending Timestamp, with no +// gaps. // // From, To and Window restate the range the samples cover. From and Window // always equal the request's. To is the requested To clamped to the service's @@ -58,27 +51,8 @@ type SampleOK struct { To int64 `cborgen:"to" dagjsongen:"to"` // Window is the width of one bucket, in seconds. Window int64 `cborgen:"window" dagjsongen:"window"` - // Samples is the series recorded against each of the space's storage - // providers. - Samples SampleSet `cborgen:"samples" dagjsongen:"samples"` -} - -// SampleSet maps a storage provider DID to the series recorded against it. -// Every series shares one bucket grid: the same count, the same timestamps, -// covering [From, To). -// -// Usage is recorded per provider, so a space provisioned with several holds -// several series. They describe the same stored bytes from each provider's -// side rather than parts of a whole, so a consumer picks the provider it cares -// about; adding them together would count the same bytes once per provider. -// -// It is a struct wrapping the map (rather than a named map type) so that -// structs embedding it can be generated by cbor-gen / dag-json-gen, which -// delegate to a struct field's MarshalCBOR but cannot generate a DID-keyed map -// inline. On the wire it encodes as the bare map, not as an object with an -// "entries" key. -type SampleSet struct { - Entries map[did.DID][]SampleItem + // Samples is one entry per bucket, ascending by Timestamp. + Samples []SampleItem `cborgen:"samples" dagjsongen:"samples"` } // SampleItem is one bucket of the series.