From b0a438d66406890e15134e47b5bb4105be4c1ba3 Mon Sep 17 00:00:00 2001 From: Sung-Kyu Yoo Date: Sun, 11 Oct 2026 18:30:41 +0900 Subject: [PATCH 1/3] feat(azure): complete Phase 3 pilot with staged blocks, conditional uploads, and metadata - Expand OpenAPI model with PutBlock, PutBlockList, and SetBlobMetadata operations - Fix codegen non-determinism by sorting schemas and properties in openapi_parser - Support conditional requests (If-None-Match, If-Match) in FileStore and Provider - Implement chunked block staging and atomic block list commitment with path traversal guards - Support user-defined blob metadata and preserve lowercase x-ms-meta-* response headers - Expand azure-storage-blob compatibility tests covering new operations - Sync docs/roadmap.md, docs/services/azure.md, and docs/coverage.md --- api/models/azure-blob-storage.json | 191 +++++++++++ docs/coverage.md | 4 +- docs/roadmap.md | 17 +- docs/services/azure.md | 26 +- internal/codegen/openapi_parser.go | 20 +- internal/codegen/openapi_parser_test.go | 25 ++ internal/gateway/router.go | 6 +- internal/generated/blob/base_provider.go | 12 + internal/generated/blob/errors.go | 34 ++ internal/generated/blob/router.go | 3 + internal/generated/blob/types.go | 33 +- internal/generated/compat/services.json | 5 +- .../generated/crudregistry/registry_gen.go | 3 + internal/generated/fidelity/manifest_gen.go | 3 + internal/services/blob/provider.go | 145 +++++++- internal/services/blob/provider_test.go | 119 +++++++ internal/services/blob/store.go | 323 ++++++++++++++++-- test/compatibility/test_azure_blob.py | 105 +++++- 18 files changed, 1037 insertions(+), 37 deletions(-) diff --git a/api/models/azure-blob-storage.json b/api/models/azure-blob-storage.json index 78856e0..a7d3d87 100644 --- a/api/models/azure-blob-storage.json +++ b/api/models/azure-blob-storage.json @@ -207,6 +207,158 @@ } } }, + "/{container}/{blob+}?comp=block": { + "put": { + "operationId": "PutBlock", + "description": "Uploads the specified block to the blob's uncommitted block list", + "parameters": [ + { + "name": "container", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "blob+", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "comp", + "in": "query", + "required": true, + "schema": { + "type": "string", + "enum": ["block"] + } + }, + { + "name": "blockid", + "in": "query", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "201": { + "description": "Block uploaded successfully" + }, + "404": { + "description": "Container not found", + "content": { + "application/xml": { + "schema": { + "$ref": "#/components/schemas/ContainerNotFound" + } + } + } + } + } + } + }, + "/{container}/{blob+}?comp=blocklist": { + "put": { + "operationId": "PutBlockList", + "description": "Commits a sequence of blocks that make up the blob", + "parameters": [ + { + "name": "container", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "blob+", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "comp", + "in": "query", + "required": true, + "schema": { + "type": "string", + "enum": ["blocklist"] + } + } + ], + "responses": { + "201": { + "description": "Block list committed successfully" + }, + "404": { + "description": "Container not found", + "content": { + "application/xml": { + "schema": { + "$ref": "#/components/schemas/ContainerNotFound" + } + } + } + } + } + } + }, + "/{container}/{blob+}?comp=metadata": { + "put": { + "operationId": "SetBlobMetadata", + "description": "Sets user-defined metadata on the specified blob", + "parameters": [ + { + "name": "container", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "blob+", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "comp", + "in": "query", + "required": true, + "schema": { + "type": "string", + "enum": ["metadata"] + } + } + ], + "responses": { + "200": { + "description": "Metadata set successfully" + }, + "404": { + "description": "Blob or Container not found", + "content": { + "application/xml": { + "schema": { + "$ref": "#/components/schemas/BlobNotFound" + } + } + } + } + } + } + }, "/{container}/{blob+}": { "put": { "operationId": "PutBlob", @@ -453,6 +605,45 @@ "BlobNotFound": { "type": "object", "x-error-status": 404 + }, + "BlobAlreadyExists": { + "type": "object", + "x-error-status": 409 + }, + "ConditionNotMet": { + "type": "object", + "x-error-status": 412 + }, + "PutBlockInput": { + "type": "object", + "properties": { + "Container": { "type": "string" }, + "Blob": { "type": "string" }, + "BlockId": { "type": "string" } + } + }, + "PutBlockOutput": { + "type": "object" + }, + "PutBlockListInput": { + "type": "object", + "properties": { + "Container": { "type": "string" }, + "Blob": { "type": "string" } + } + }, + "PutBlockListOutput": { + "type": "object" + }, + "SetBlobMetadataInput": { + "type": "object", + "properties": { + "Container": { "type": "string" }, + "Blob": { "type": "string" } + } + }, + "SetBlobMetadataOutput": { + "type": "object" } } } diff --git a/docs/coverage.md b/docs/coverage.md index 0321629..03bdc8b 100644 --- a/docs/coverage.md +++ b/docs/coverage.md @@ -17,9 +17,9 @@ denominators, because [routing and depth are two targets](#the-target): | Tier | Serving target | All registered | |---|---|---| | `hand-verified` | 4,620 | 4,651 | -| `auto-crud` | 7,529 | 14,082 | +| `auto-crud` | 7,532 | 14,085 | | `unimplemented` | 267 | 477 | -| **total known** | **12,416** | **19,210** | +| **total known** | **12,419** | **19,213** | | **hand-verified share** | **37.2%** | **24.2%** | The **serving target** column is the depth promise: every registered service diff --git a/docs/roadmap.md b/docs/roadmap.md index e179e1a..3686c4c 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -92,7 +92,7 @@ Phase 2 deliberately shipped **no** non-AWS service, no second `ModelSource`, an no signature verification. Those are Phase 3 and beyond; Phase 2's job was to make each of them an addition rather than a fork. -## Phase 3 — First non-AWS service (next, v2.0, exploratory) +## Phase 3 — First non-AWS service (complete, v2.0 groundwork) Validate the multi-CSP architecture with one well-scoped pilot. @@ -102,6 +102,17 @@ Validate the multi-CSP architecture with one well-scoped pilot. - [x] Compatibility tests against `azure-sdk-for-python` - [x] Documentation pattern for multi-CSP service docs +Phase 3 execution completed: Azure Blob Storage (`blob`) is implemented as the first +non-AWS pilot under `providers.azure.services.blob`. The OpenAPI model in `api/models/` +compiles through `OpenAPISource` into deterministic IR and generated routers, types, and +error shapes. The storage engine (`internal/services/blob`) provides local filesystem-backed +persistence supporting container CRUD, BlockBlob CRUD, staged chunk uploads (`PutBlock`, +`PutBlockList`), conditional requests (`If-None-Match: *`, `If-Match`), custom metadata +(`x-ms-meta-*`), and byte-range downloads. The gateway router and auth adapters validate +Azure Shared Key and emulator signatures, and integration tests verify complete end-to-end +compatibility against `azure-sdk-for-python` in CI. See [Azure Services](services/azure.md) +for configuration and local limits. + ## Phase 4 — Breadth expansion (v2.x+) - [ ] More Azure services — Queue Storage, Table Storage, Cosmos DB @@ -128,6 +139,6 @@ Validate the multi-CSP architecture with one well-scoped pilot. | Version | Focus | |---------|-------| | 0.x | AWS services, unstable API | -| 1.x ← current | AWS depth, stable plugin API, multi-CSP groundwork | -| 2.x | Multi-CSP architecture, Azure pilot | +| 1.x | AWS depth, stable plugin API, multi-CSP groundwork | +| 2.x ← current | Multi-CSP architecture, Azure pilot | | 3.x+ | Broad CSP coverage, community-owned providers | diff --git a/docs/services/azure.md b/docs/services/azure.md index 3a3646f..be68440 100644 --- a/docs/services/azure.md +++ b/docs/services/azure.md @@ -16,9 +16,12 @@ DevCloud expands its local development runtime to include Microsoft Azure servic | Containers | `DeleteContainer` | `DELETE /{container}?restype=container` | Returns `202 Accepted` or `404 ContainerNotFound` | | Containers | `GetContainerProperties` | `GET /{container}?restype=container` | Returns lease state and metadata headers | | Containers | `ListContainers` | `GET /?comp=list` | Returns XML `EnumerationResults` listing all containers | -| Blobs | `PutBlob` | `PUT /{container}/{blob}` | Creates or replaces a BlockBlob; stores payload and computes MD5 | +| Blobs | `PutBlob` | `PUT /{container}/{blob}` | Creates or replaces a BlockBlob; supports preconditions (`If-None-Match: *`, `If-Match`), stores payload and computes MD5 | +| Blobs | `PutBlock` | `PUT /{container}/{blob}?comp=block&blockid={id}` | Stages an uncommitted block for chunked/streaming blob uploads | +| Blobs | `PutBlockList` | `PUT /{container}/{blob}?comp=blocklist` | Commits staged blocks in sequence to materialize or update a BlockBlob | +| Blobs | `SetBlobMetadata` | `PUT /{container}/{blob}?comp=metadata` | Updates user-defined `x-ms-meta-*` headers on the blob | | Blobs | `GetBlob` | `GET /{container}/{blob}` | Downloads blob content; supports `Range` and `x-ms-range` requests | -| Blobs | `GetBlobProperties` | `HEAD /{container}/{blob}` | Returns headers (`x-ms-blob-type: BlockBlob`, size, ETag, Content-MD5) | +| Blobs | `GetBlobProperties` | `HEAD /{container}/{blob}` | Returns headers (`x-ms-blob-type: BlockBlob`, size, ETag, Content-MD5, `x-ms-meta-*`) | | Blobs | `DeleteBlob` | `DELETE /{container}/{blob}` | Permanently removes the blob | | Blobs | `ListBlobs` | `GET /{container}?restype=container&comp=list` | Returns XML `EnumerationResults` listing blobs in the container | @@ -76,4 +79,23 @@ print(content.decode("utf-8")) # 4. List blobs for blob in container_client.list_blobs(): print(f"Blob: {blob.name}, Size: {blob.size}") + +# 5. Conditional upload (fail if exists) +from azure.core.exceptions import ResourceExistsError +try: + blob_client.upload_blob(b"will not overwrite", overwrite=False) +except ResourceExistsError: + print("Blob already exists!") + +# 6. Chunked upload via stage_block & commit_block_list +from azure.storage.blob import BlobBlock +chunk_client = container_client.get_blob_client("large.bin") +chunk_client.stage_block(block_id="AAAAAA==", data=b"Part 1 ") +chunk_client.stage_block(block_id="AQIDBA==", data=b"Part 2!") +chunk_client.commit_block_list([BlobBlock(block_id="AAAAAA=="), BlobBlock(block_id="AQIDBA==")]) + +# 7. Metadata management +blob_client.set_blob_metadata(metadata={"env": "local", "pilot": "phase3"}) +props = blob_client.get_blob_properties() +print(f"Metadata: {props.metadata}") ``` diff --git a/internal/codegen/openapi_parser.go b/internal/codegen/openapi_parser.go index 19c316b..043d4f4 100644 --- a/internal/codegen/openapi_parser.go +++ b/internal/codegen/openapi_parser.go @@ -172,8 +172,15 @@ func ParseOpenAPIJSON(data []byte) (*ir.Model, error) { } } - // Parse component schemas - for name, schema := range doc.Components.Schemas { + // Parse component schemas in deterministic order + schemaNames := make([]string, 0, len(doc.Components.Schemas)) + for name := range doc.Components.Schemas { + schemaNames = append(schemaNames, name) + } + sort.Strings(schemaNames) + + for _, name := range schemaNames { + schema := doc.Components.Schemas[name] shape := &ir.Shape{ Name: name, Type: ir.ShapeStructure, @@ -186,7 +193,14 @@ func ParseOpenAPIJSON(data []byte) (*ir.Model, error) { } } - for propName, prop := range schema.Properties { + propNames := make([]string, 0, len(schema.Properties)) + for propName := range schema.Properties { + propNames = append(propNames, propName) + } + sort.Strings(propNames) + + for _, propName := range propNames { + prop := schema.Properties[propName] goType := "string" switch prop.Type { case "integer": diff --git a/internal/codegen/openapi_parser_test.go b/internal/codegen/openapi_parser_test.go index 66df92e..21ef09b 100644 --- a/internal/codegen/openapi_parser_test.go +++ b/internal/codegen/openapi_parser_test.go @@ -34,6 +34,9 @@ func TestParseOpenAPIJSON(t *testing.T) { "DeleteContainer": "DELETE /{container}?restype=container", "GetContainerProperties": "GET /{container}?restype=container", "ListBlobs": "GET /{container}?restype=container&comp=list", + "PutBlock": "PUT /{container}/{blob+}?comp=block", + "PutBlockList": "PUT /{container}/{blob+}?comp=blocklist", + "SetBlobMetadata": "PUT /{container}/{blob+}?comp=metadata", "PutBlob": "PUT /{container}/{blob+}", "GetBlob": "GET /{container}/{blob+}", "GetBlobProperties": "HEAD /{container}/{blob+}", @@ -65,4 +68,26 @@ func TestParseOpenAPIJSON(t *testing.T) { require.True(t, ok) require.NotNil(t, errShape.ErrorTrait) assert.Equal(t, 404, errShape.ErrorTrait.HTTPStatus) + + existsShape, ok := model.Shapes["BlobAlreadyExists"] + require.True(t, ok) + require.NotNil(t, existsShape.ErrorTrait) + assert.Equal(t, 409, existsShape.ErrorTrait.HTTPStatus) + + condShape, ok := model.Shapes["ConditionNotMet"] + require.True(t, ok) + require.NotNil(t, condShape.ErrorTrait) + assert.Equal(t, 412, condShape.ErrorTrait.HTTPStatus) + + // Verify determinism: parsing multiple times produces identical member ordering + model2, err := ParseOpenAPIJSON(data) + require.NoError(t, err) + for shapeName, s1 := range model.Shapes { + s2 := model2.Shapes[shapeName] + require.NotNil(t, s2) + require.Equal(t, len(s1.Members), len(s2.Members)) + for idx := range s1.Members { + assert.Equal(t, s1.Members[idx].Name, s2.Members[idx].Name) + } + } } diff --git a/internal/gateway/router.go b/internal/gateway/router.go index 0646e3f..2f9ea53 100644 --- a/internal/gateway/router.go +++ b/internal/gateway/router.go @@ -139,7 +139,11 @@ func (sr *ServiceRouter) ServeHTTP(w http.ResponseWriter, r *http.Request) { // Write response headers from the plugin. for k, v := range resp.Headers { - w.Header().Set(k, v) + if strings.HasPrefix(strings.ToLower(k), "x-ms-") { + w.Header()[strings.ToLower(k)] = []string{v} + } else { + w.Header().Set(k, v) + } } ct := resp.ContentType if ct == "" { diff --git a/internal/generated/blob/base_provider.go b/internal/generated/blob/base_provider.go index b840841..c426bef 100644 --- a/internal/generated/blob/base_provider.go +++ b/internal/generated/blob/base_provider.go @@ -31,6 +31,18 @@ func (b *BaseProvider) ListContainers(ctx context.Context, input *ListContainers return nil, fmt.Errorf("ListContainers: %w", ErrNotImplemented) } +func (b *BaseProvider) PutBlock(ctx context.Context, input *PutBlockInput) (*PutBlockOutput, error) { + return nil, fmt.Errorf("PutBlock: %w", ErrNotImplemented) +} + +func (b *BaseProvider) PutBlockList(ctx context.Context, input *PutBlockListInput) (*PutBlockListOutput, error) { + return nil, fmt.Errorf("PutBlockList: %w", ErrNotImplemented) +} + +func (b *BaseProvider) SetBlobMetadata(ctx context.Context, input *SetBlobMetadataInput) (*SetBlobMetadataOutput, error) { + return nil, fmt.Errorf("SetBlobMetadata: %w", ErrNotImplemented) +} + func (b *BaseProvider) DeleteBlob(ctx context.Context, input *DeleteBlobInput) (*DeleteBlobOutput, error) { return nil, fmt.Errorf("DeleteBlob: %w", ErrNotImplemented) } diff --git a/internal/generated/blob/errors.go b/internal/generated/blob/errors.go index a4efa0f..d076c19 100644 --- a/internal/generated/blob/errors.go +++ b/internal/generated/blob/errors.go @@ -4,6 +4,23 @@ package blob import "fmt" +// BlobAlreadyExists is returned with HTTP status 409. +type BlobAlreadyExists struct { + Message string +} + +func (e *BlobAlreadyExists) Error() string { + return fmt.Sprintf("BlobAlreadyExists: %s", e.Message) +} + +func (e *BlobAlreadyExists) ErrorCode() string { + return "BlobAlreadyExists" +} + +func (e *BlobAlreadyExists) HTTPStatus() int { + return 409 +} + // BlobNotFound is returned with HTTP status 404. type BlobNotFound struct { Message string @@ -21,6 +38,23 @@ func (e *BlobNotFound) HTTPStatus() int { return 404 } +// ConditionNotMet is returned with HTTP status 412. +type ConditionNotMet struct { + Message string +} + +func (e *ConditionNotMet) Error() string { + return fmt.Sprintf("ConditionNotMet: %s", e.Message) +} + +func (e *ConditionNotMet) ErrorCode() string { + return "ConditionNotMet" +} + +func (e *ConditionNotMet) HTTPStatus() int { + return 412 +} + // ContainerAlreadyExists is returned with HTTP status 409. type ContainerAlreadyExists struct { Message string diff --git a/internal/generated/blob/router.go b/internal/generated/blob/router.go index 44fd831..c27aeb2 100644 --- a/internal/generated/blob/router.go +++ b/internal/generated/blob/router.go @@ -17,6 +17,9 @@ var OperationRoutes = []OperationRoute{ {Method: "DELETE", Pattern: "/{container}?restype=container", Operation: "DeleteContainer"}, {Method: "GET", Pattern: "/{container}?restype=container", Operation: "GetContainerProperties"}, {Method: "GET", Pattern: "/?comp=list", Operation: "ListContainers"}, + {Method: "PUT", Pattern: "/{container}/{blob+}?comp=block", Operation: "PutBlock"}, + {Method: "PUT", Pattern: "/{container}/{blob+}?comp=blocklist", Operation: "PutBlockList"}, + {Method: "PUT", Pattern: "/{container}/{blob+}?comp=metadata", Operation: "SetBlobMetadata"}, {Method: "DELETE", Pattern: "/{container}/{blob+}", Operation: "DeleteBlob"}, {Method: "GET", Pattern: "/{container}/{blob+}", Operation: "GetBlob"}, {Method: "HEAD", Pattern: "/{container}/{blob+}", Operation: "GetBlobProperties"}, diff --git a/internal/generated/blob/types.go b/internal/generated/blob/types.go index ad4deb8..14927fa 100644 --- a/internal/generated/blob/types.go +++ b/internal/generated/blob/types.go @@ -10,8 +10,8 @@ type CreateContainerOutput struct { } type DeleteBlobInput struct { - Container string `json:"container" xml:"Container"` Blob string `json:"blob" xml:"Blob"` + Container string `json:"container" xml:"Container"` } type DeleteBlobOutput struct { @@ -25,16 +25,16 @@ type DeleteContainerOutput struct { } type GetBlobInput struct { - Container string `json:"container" xml:"Container"` Blob string `json:"blob" xml:"Blob"` + Container string `json:"container" xml:"Container"` } type GetBlobOutput struct { } type GetBlobPropertiesInput struct { - Container string `json:"container" xml:"Container"` Blob string `json:"blob" xml:"Blob"` + Container string `json:"container" xml:"Container"` } type GetBlobPropertiesOutput struct { @@ -61,9 +61,34 @@ type ListContainersOutput struct { } type PutBlobInput struct { - Container string `json:"container" xml:"Container"` Blob string `json:"blob" xml:"Blob"` + Container string `json:"container" xml:"Container"` } type PutBlobOutput struct { } + +type PutBlockInput struct { + Blob string `json:"blob" xml:"Blob"` + BlockId string `json:"blockId" xml:"BlockId"` + Container string `json:"container" xml:"Container"` +} + +type PutBlockListInput struct { + Blob string `json:"blob" xml:"Blob"` + Container string `json:"container" xml:"Container"` +} + +type PutBlockListOutput struct { +} + +type PutBlockOutput struct { +} + +type SetBlobMetadataInput struct { + Blob string `json:"blob" xml:"Blob"` + Container string `json:"container" xml:"Container"` +} + +type SetBlobMetadataOutput struct { +} diff --git a/internal/generated/compat/services.json b/internal/generated/compat/services.json index 49cbbcd..6de8661 100644 --- a/internal/generated/compat/services.json +++ b/internal/generated/compat/services.json @@ -2325,7 +2325,10 @@ "GetContainerProperties", "ListBlobs", "ListContainers", - "PutBlob" + "PutBlob", + "PutBlock", + "PutBlockList", + "SetBlobMetadata" ] }, "braket": { diff --git a/internal/generated/crudregistry/registry_gen.go b/internal/generated/crudregistry/registry_gen.go index 5d7c4d0..fa471cc 100644 --- a/internal/generated/crudregistry/registry_gen.go +++ b/internal/generated/crudregistry/registry_gen.go @@ -2143,6 +2143,9 @@ func init() { "DeleteContainer": {Verb: "Delete", Resource: "Container", OutputListKey: "", OutputItemKey: "", Method: "DELETE", URI: "/{container}?restype=container"}, "GetContainerProperties": {Verb: "Get", Resource: "ContainerProperty", OutputListKey: "", OutputItemKey: "", Method: "GET", URI: "/{container}?restype=container"}, "ListContainers": {Verb: "List", Resource: "Container", OutputListKey: "", OutputItemKey: "", Method: "GET", URI: "/?comp=list"}, + "PutBlock": {Verb: "Update", Resource: "Block", OutputListKey: "", OutputItemKey: "", Method: "PUT", URI: "/{container}/{blob+}?comp=block"}, + "PutBlockList": {Verb: "Update", Resource: "BlockList", OutputListKey: "", OutputItemKey: "", Method: "PUT", URI: "/{container}/{blob+}?comp=blocklist"}, + "SetBlobMetadata": {Verb: "Update", Resource: "BlobMetadata", OutputListKey: "", OutputItemKey: "", Method: "PUT", URI: "/{container}/{blob+}?comp=metadata"}, "DeleteBlob": {Verb: "Delete", Resource: "Blob", OutputListKey: "", OutputItemKey: "", Method: "DELETE", URI: "/{container}/{blob+}"}, "GetBlob": {Verb: "Get", Resource: "Blob", OutputListKey: "", OutputItemKey: "", Method: "GET", URI: "/{container}/{blob+}"}, "GetBlobProperties": {Verb: "Get", Resource: "BlobProperty", OutputListKey: "", OutputItemKey: "", Method: "HEAD", URI: "/{container}/{blob+}"}, diff --git a/internal/generated/fidelity/manifest_gen.go b/internal/generated/fidelity/manifest_gen.go index dd3777c..0dae539 100644 --- a/internal/generated/fidelity/manifest_gen.go +++ b/internal/generated/fidelity/manifest_gen.go @@ -2293,6 +2293,9 @@ var Services = map[string]Service{ "ListBlobs": TierHandVerified, "ListContainers": TierHandVerified, "PutBlob": TierHandVerified, + "PutBlock": TierHandVerified, + "PutBlockList": TierHandVerified, + "SetBlobMetadata": TierHandVerified, }}, "braket": {Protocol: "rest-json", ModelBacked: true, EngineWired: true, Operations: map[string]Tier{ "CancelJob": TierAutoCRUD, diff --git a/internal/services/blob/provider.go b/internal/services/blob/provider.go index c8c4289..7588c40 100644 --- a/internal/services/blob/provider.go +++ b/internal/services/blob/provider.go @@ -9,6 +9,7 @@ import ( "encoding/xml" "errors" "fmt" + "io" "net/http" "strings" "time" @@ -43,6 +44,24 @@ func (p *Provider) Shutdown(ctx context.Context) error { return nil } +func extractMetadata(h http.Header) map[string]string { + meta := make(map[string]string) + for k, vv := range h { + lower := strings.ToLower(k) + if strings.HasPrefix(lower, "x-ms-meta-") && len(vv) > 0 { + metaKey := strings.TrimPrefix(lower, "x-ms-meta-") + meta[metaKey] = vv[0] + } + } + return meta +} + +func writeMetadataHeaders(headers map[string]string, meta map[string]string) { + for k, v := range meta { + headers["x-ms-meta-"+strings.ToLower(k)] = v + } +} + func randomRequestID() string { b := make([]byte, 16) _, _ = rand.Read(b) @@ -189,6 +208,7 @@ func (p *Provider) HandleRequest(ctx context.Context, _ string, req *http.Reques "x-ms-default-encryption-scope": "$account-encryption-key", "x-ms-deny-encryption-scope-override": "false", } + writeMetadataHeaders(headers, info.Metadata) return p.reply(req, http.StatusOK, headers, nil), nil case "ListContainers": @@ -234,10 +254,98 @@ func (p *Provider) HandleRequest(ctx context.Context, _ string, req *http.Reques container := params["container"] blob := params["blob"] contentType := req.Header.Get("Content-Type") - info, err := p.store.PutBlob(account, container, blob, contentType, req.Body) + opts := PutBlobOptions{ + ContentType: contentType, + IfNoneMatch: req.Header.Get("If-None-Match"), + IfMatch: req.Header.Get("If-Match"), + Metadata: extractMetadata(req.Header), + } + info, err := p.store.PutBlobWithOptions(account, container, blob, opts, req.Body) + if errors.Is(err, ErrContainerNotFound) { + return p.errorReply(req, http.StatusNotFound, "ContainerNotFound", "The specified container does not exist."), nil + } + if errors.Is(err, ErrBlobAlreadyExists) { + return p.errorReply(req, http.StatusConflict, "BlobAlreadyExists", "The specified blob already exists."), nil + } + if errors.Is(err, ErrConditionNotMet) { + return p.errorReply(req, http.StatusPreconditionFailed, "ConditionNotMet", "The condition specified in the conditional header(s) was not met."), nil + } + if errors.Is(err, ErrInvalidName) { + return p.errorReply(req, http.StatusBadRequest, "InvalidResourceName", "The specified resource name contains invalid characters."), nil + } + if err != nil { + return p.errorReply(req, http.StatusInternalServerError, "InternalError", err.Error()), nil + } + headers := map[string]string{ + "ETag": info.ETag, + "Last-Modified": info.LastModified.Format(http.TimeFormat), + "Content-MD5": info.MD5, + "x-ms-request-server-encrypted": "true", + } + return p.reply(req, http.StatusCreated, headers, nil), nil + + case "PutBlock": + container := params["container"] + blob := params["blob"] + blockID := req.URL.Query().Get("blockid") + if blockID == "" { + return p.errorReply(req, http.StatusBadRequest, "InvalidQueryParameterValue", "Value for one of the query parameters specified in the request is invalid."), nil + } + err := p.store.StageBlock(account, container, blob, blockID, req.Body) + if errors.Is(err, ErrContainerNotFound) { + return p.errorReply(req, http.StatusNotFound, "ContainerNotFound", "The specified container does not exist."), nil + } + if errors.Is(err, ErrInvalidName) { + return p.errorReply(req, http.StatusBadRequest, "InvalidResourceName", "The specified resource name contains invalid characters."), nil + } + if err != nil { + return p.errorReply(req, http.StatusInternalServerError, "InternalError", err.Error()), nil + } + headers := map[string]string{ + "x-ms-request-server-encrypted": "true", + } + return p.reply(req, http.StatusCreated, headers, nil), nil + + case "PutBlockList": + container := params["container"] + blob := params["blob"] + contentType := req.Header.Get("Content-Type") + bodyBytes, err := io.ReadAll(req.Body) + if err != nil { + return p.errorReply(req, http.StatusBadRequest, "InvalidXmlDocument", "XML specified is not syntactically valid."), nil + } + var blockListReq XMLBlockListRequest + if len(bodyBytes) > 0 { + if err := xml.Unmarshal(bodyBytes, &blockListReq); err != nil { + return p.errorReply(req, http.StatusBadRequest, "InvalidXmlDocument", "XML specified is not syntactically valid."), nil + } + } + var blockIDs []string + for _, item := range blockListReq.Blocks { + id := strings.TrimSpace(item.ID) + if id != "" { + blockIDs = append(blockIDs, id) + } + } + opts := PutBlobOptions{ + ContentType: contentType, + IfNoneMatch: req.Header.Get("If-None-Match"), + IfMatch: req.Header.Get("If-Match"), + Metadata: extractMetadata(req.Header), + } + info, err := p.store.CommitBlockList(account, container, blob, blockIDs, opts) if errors.Is(err, ErrContainerNotFound) { return p.errorReply(req, http.StatusNotFound, "ContainerNotFound", "The specified container does not exist."), nil } + if errors.Is(err, ErrInvalidBlock) { + return p.errorReply(req, http.StatusBadRequest, "InvalidBlockList", "The specified block list is invalid."), nil + } + if errors.Is(err, ErrBlobAlreadyExists) { + return p.errorReply(req, http.StatusConflict, "BlobAlreadyExists", "The specified blob already exists."), nil + } + if errors.Is(err, ErrConditionNotMet) { + return p.errorReply(req, http.StatusPreconditionFailed, "ConditionNotMet", "The condition specified in the conditional header(s) was not met."), nil + } if errors.Is(err, ErrInvalidName) { return p.errorReply(req, http.StatusBadRequest, "InvalidResourceName", "The specified resource name contains invalid characters."), nil } @@ -252,6 +360,29 @@ func (p *Provider) HandleRequest(ctx context.Context, _ string, req *http.Reques } return p.reply(req, http.StatusCreated, headers, nil), nil + case "SetBlobMetadata": + container := params["container"] + blob := params["blob"] + metadata := extractMetadata(req.Header) + info, err := p.store.SetBlobMetadata(account, container, blob, metadata) + if errors.Is(err, ErrContainerNotFound) { + return p.errorReply(req, http.StatusNotFound, "ContainerNotFound", "The specified container does not exist."), nil + } + if errors.Is(err, ErrBlobNotFound) { + return p.errorReply(req, http.StatusNotFound, "BlobNotFound", "The specified blob does not exist."), nil + } + if errors.Is(err, ErrInvalidName) { + return p.errorReply(req, http.StatusBadRequest, "InvalidResourceName", "The specified resource name contains invalid characters."), nil + } + if err != nil { + return p.errorReply(req, http.StatusInternalServerError, "InternalError", err.Error()), nil + } + headers := map[string]string{ + "ETag": info.ETag, + "Last-Modified": info.LastModified.Format(http.TimeFormat), + } + return p.reply(req, http.StatusOK, headers, nil), nil + case "GetBlob": container := params["container"] blob := params["blob"] @@ -281,6 +412,7 @@ func (p *Provider) HandleRequest(ctx context.Context, _ string, req *http.Reques "x-ms-creation-time": info.CreatedAt.Format(http.TimeFormat), "Accept-Ranges": "bytes", } + writeMetadataHeaders(headers, info.Metadata) rangeHeader := req.Header.Get("x-ms-range") if rangeHeader == "" { @@ -350,6 +482,7 @@ func (p *Provider) HandleRequest(ctx context.Context, _ string, req *http.Reques "x-ms-access-tier": "Hot", "x-ms-creation-time": info.CreatedAt.Format(http.TimeFormat), } + writeMetadataHeaders(headers, info.Metadata) return p.reply(req, http.StatusOK, headers, nil), nil case "DeleteBlob": @@ -506,6 +639,16 @@ type XMLBlobProperties struct { ServerEncrypted string `xml:"ServerEncrypted"` } +type XMLBlockItem struct { + XMLName xml.Name + ID string `xml:",chardata"` +} + +type XMLBlockListRequest struct { + XMLName xml.Name `xml:"BlockList"` + Blocks []XMLBlockItem `xml:",any"` +} + func init() { plugin.DefaultRegistry.Register("blob", func() plugin.ServicePlugin { return &Provider{} diff --git a/internal/services/blob/provider_test.go b/internal/services/blob/provider_test.go index 3089dcd..1e064fb 100644 --- a/internal/services/blob/provider_test.go +++ b/internal/services/blob/provider_test.go @@ -215,3 +215,122 @@ func TestBlobProvider_VirtualDirectory(t *testing.T) { assert.Equal(t, http.StatusOK, resp.StatusCode) assert.Equal(t, payload, resp.Body) } + +func TestBlobProvider_ConditionalPutBlob(t *testing.T) { + p := newTestProvider(t) + ctx := context.Background() + + // Create container + reqCreate := httptest.NewRequest(http.MethodPut, "/devstoreaccount1/cond-container?restype=container", nil) + resp, err := p.HandleRequest(ctx, "", reqCreate) + require.NoError(t, err) + assert.Equal(t, http.StatusCreated, resp.StatusCode) + + // 1. Put blob with If-None-Match: * (should succeed since blob does not exist) + reqPut1 := httptest.NewRequest(http.MethodPut, "/devstoreaccount1/cond-container/item.txt", bytes.NewReader([]byte("first"))) + reqPut1.Header.Set("If-None-Match", "*") + respPut1, err := p.HandleRequest(ctx, "", reqPut1) + require.NoError(t, err) + assert.Equal(t, http.StatusCreated, respPut1.StatusCode) + etag := respPut1.Headers["ETag"] + assert.NotEmpty(t, etag) + + // 2. Put blob again with If-None-Match: * (should fail with 409 BlobAlreadyExists) + reqPutDup := httptest.NewRequest(http.MethodPut, "/devstoreaccount1/cond-container/item.txt", bytes.NewReader([]byte("duplicate"))) + reqPutDup.Header.Set("If-None-Match", "*") + respPutDup, err := p.HandleRequest(ctx, "", reqPutDup) + require.NoError(t, err) + assert.Equal(t, http.StatusConflict, respPutDup.StatusCode) + assert.Equal(t, "BlobAlreadyExists", respPutDup.Headers["x-ms-error-code"]) + + // 3. Put blob with valid If-Match (should succeed) + reqPutMatch := httptest.NewRequest(http.MethodPut, "/devstoreaccount1/cond-container/item.txt", bytes.NewReader([]byte("updated"))) + reqPutMatch.Header.Set("If-Match", etag) + respPutMatch, err := p.HandleRequest(ctx, "", reqPutMatch) + require.NoError(t, err) + assert.Equal(t, http.StatusCreated, respPutMatch.StatusCode) + + // 4. Put blob with invalid If-Match (should fail with 412 ConditionNotMet) + reqPutBadMatch := httptest.NewRequest(http.MethodPut, "/devstoreaccount1/cond-container/item.txt", bytes.NewReader([]byte("bad"))) + reqPutBadMatch.Header.Set("If-Match", "\"0xINVALID\"") + respPutBadMatch, err := p.HandleRequest(ctx, "", reqPutBadMatch) + require.NoError(t, err) + assert.Equal(t, http.StatusPreconditionFailed, respPutBadMatch.StatusCode) + assert.Equal(t, "ConditionNotMet", respPutBadMatch.Headers["x-ms-error-code"]) +} + +func TestBlobProvider_BlockStagingAndCommit(t *testing.T) { + p := newTestProvider(t) + ctx := context.Background() + + // Create container + reqCreate := httptest.NewRequest(http.MethodPut, "/devstoreaccount1/block-container?restype=container", nil) + resp, err := p.HandleRequest(ctx, "", reqCreate) + require.NoError(t, err) + assert.Equal(t, http.StatusCreated, resp.StatusCode) + + // Stage block 1 + reqStage1 := httptest.NewRequest(http.MethodPut, "/devstoreaccount1/block-container/chunked.txt?comp=block&blockid=YmxvY2sx", bytes.NewReader([]byte("Hello "))) + resp1, err := p.HandleRequest(ctx, "", reqStage1) + require.NoError(t, err) + assert.Equal(t, http.StatusCreated, resp1.StatusCode) + + // Stage block 2 + reqStage2 := httptest.NewRequest(http.MethodPut, "/devstoreaccount1/block-container/chunked.txt?comp=block&blockid=YmxvY2sy", bytes.NewReader([]byte("World!"))) + resp2, err := p.HandleRequest(ctx, "", reqStage2) + require.NoError(t, err) + assert.Equal(t, http.StatusCreated, resp2.StatusCode) + + // Commit block list + blockListXML := `YmxvY2sxYmxvY2sy` + reqCommit := httptest.NewRequest(http.MethodPut, "/devstoreaccount1/block-container/chunked.txt?comp=blocklist", bytes.NewReader([]byte(blockListXML))) + respCommit, err := p.HandleRequest(ctx, "", reqCommit) + require.NoError(t, err) + assert.Equal(t, http.StatusCreated, respCommit.StatusCode) + + // Download blob and verify concatenated content + reqGet := httptest.NewRequest(http.MethodGet, "/devstoreaccount1/block-container/chunked.txt", nil) + respGet, err := p.HandleRequest(ctx, "", reqGet) + require.NoError(t, err) + assert.Equal(t, http.StatusOK, respGet.StatusCode) + assert.Equal(t, []byte("Hello World!"), respGet.Body) +} + +func TestBlobProvider_CustomMetadata(t *testing.T) { + p := newTestProvider(t) + ctx := context.Background() + + // Create container + reqCreate := httptest.NewRequest(http.MethodPut, "/devstoreaccount1/meta-container?restype=container", nil) + resp, err := p.HandleRequest(ctx, "", reqCreate) + require.NoError(t, err) + assert.Equal(t, http.StatusCreated, resp.StatusCode) + + // Put blob with metadata + reqPut := httptest.NewRequest(http.MethodPut, "/devstoreaccount1/meta-container/doc.txt", bytes.NewReader([]byte("meta payload"))) + reqPut.Header.Set("x-ms-meta-author", "antigravity") + reqPut.Header.Set("x-ms-meta-environment", "staging") + respPut, err := p.HandleRequest(ctx, "", reqPut) + require.NoError(t, err) + assert.Equal(t, http.StatusCreated, respPut.StatusCode) + + // Get blob properties and verify metadata + reqHead := httptest.NewRequest(http.MethodHead, "/devstoreaccount1/meta-container/doc.txt", nil) + respHead, err := p.HandleRequest(ctx, "", reqHead) + require.NoError(t, err) + assert.Equal(t, http.StatusOK, respHead.StatusCode) + assert.Equal(t, "antigravity", respHead.Headers["x-ms-meta-author"]) + assert.Equal(t, "staging", respHead.Headers["x-ms-meta-environment"]) + + // Update metadata via SetBlobMetadata + reqSetMeta := httptest.NewRequest(http.MethodPut, "/devstoreaccount1/meta-container/doc.txt?comp=metadata", nil) + reqSetMeta.Header.Set("x-ms-meta-author", "updated-user") + respSetMeta, err := p.HandleRequest(ctx, "", reqSetMeta) + require.NoError(t, err) + assert.Equal(t, http.StatusOK, respSetMeta.StatusCode) + + // Re-check properties + respHead2, err := p.HandleRequest(ctx, "", reqHead) + require.NoError(t, err) + assert.Equal(t, "updated-user", respHead2.Headers["x-ms-meta-author"]) +} diff --git a/internal/services/blob/store.go b/internal/services/blob/store.go index d040f71..a7fc613 100644 --- a/internal/services/blob/store.go +++ b/internal/services/blob/store.go @@ -20,27 +20,39 @@ var ( ErrContainerNotFound = errors.New("ContainerNotFound") ErrContainerAlreadyExists = errors.New("ContainerAlreadyExists") ErrBlobNotFound = errors.New("BlobNotFound") + ErrBlobAlreadyExists = errors.New("BlobAlreadyExists") + ErrConditionNotMet = errors.New("ConditionNotMet") ErrInvalidName = errors.New("InvalidName") + ErrInvalidBlock = errors.New("InvalidBlock") ) type ContainerInfo struct { - Name string `json:"name"` - Account string `json:"account"` - CreatedAt time.Time `json:"createdAt"` - ETag string `json:"etag"` + Name string `json:"name"` + Account string `json:"account"` + CreatedAt time.Time `json:"createdAt"` + ETag string `json:"etag"` + Metadata map[string]string `json:"metadata,omitempty"` } type BlobInfo struct { - Name string `json:"name"` - Container string `json:"container"` - Account string `json:"account"` - Size int64 `json:"size"` - ContentType string `json:"contentType"` - MD5 string `json:"md5"` - ETag string `json:"etag"` - CreatedAt time.Time `json:"createdAt"` - LastModified time.Time `json:"lastModified"` - BlobType string `json:"blobType"` + Name string `json:"name"` + Container string `json:"container"` + Account string `json:"account"` + Size int64 `json:"size"` + ContentType string `json:"contentType"` + MD5 string `json:"md5"` + ETag string `json:"etag"` + CreatedAt time.Time `json:"createdAt"` + LastModified time.Time `json:"lastModified"` + BlobType string `json:"blobType"` + Metadata map[string]string `json:"metadata,omitempty"` +} + +type PutBlobOptions struct { + ContentType string + IfNoneMatch string + IfMatch string + Metadata map[string]string } type FileStore struct { @@ -259,6 +271,10 @@ func (fs *FileStore) ListContainers(account string) ([]ContainerInfo, error) { } func (fs *FileStore) PutBlob(account, container, blob string, contentType string, r io.Reader) (*BlobInfo, error) { + return fs.PutBlobWithOptions(account, container, blob, PutBlobOptions{ContentType: contentType}, r) +} + +func (fs *FileStore) PutBlobWithOptions(account, container, blob string, opts PutBlobOptions, r io.Reader) (*BlobInfo, error) { fs.mu.Lock() defer fs.mu.Unlock() @@ -276,6 +292,23 @@ func (fs *FileStore) PutBlob(account, container, blob string, contentType string return nil, err } + var existing *BlobInfo + if existingData, err := os.ReadFile(metaPath); err == nil { + var ex BlobInfo + if json.Unmarshal(existingData, &ex) == nil { + existing = &ex + } + } + + if opts.IfNoneMatch == "*" && existing != nil { + return nil, ErrBlobAlreadyExists + } + if opts.IfMatch != "" { + if existing == nil || existing.ETag != opts.IfMatch { + return nil, ErrConditionNotMet + } + } + if err := os.MkdirAll(filepath.Dir(dataPath), 0755); err != nil { return nil, err } @@ -316,30 +349,202 @@ func (fs *FileStore) PutBlob(account, container, blob string, contentType string md5Sum := hex.EncodeToString(hasher.Sum(nil)) now := time.Now().UTC() etag := fmt.Sprintf("\"0x%X\"", now.UnixNano()) + contentType := opts.ContentType if contentType == "" { contentType = "application/octet-stream" } - // Check existing metadata to preserve creation time createdAt := now + if existing != nil && !existing.CreatedAt.IsZero() { + createdAt = existing.CreatedAt + } + + info := &BlobInfo{ + Name: blob, + Container: container, + Account: account, + Size: size, + ContentType: contentType, + MD5: md5Sum, + ETag: etag, + CreatedAt: createdAt, + LastModified: now, + BlobType: "BlockBlob", + Metadata: opts.Metadata, + } + + metaBytes, err := json.Marshal(info) + if err != nil { + return nil, err + } + if err := os.WriteFile(metaPath, metaBytes, 0644); err != nil { + return nil, err + } + return info, nil +} + +func (fs *FileStore) StageBlock(account, container, blob, blockID string, r io.Reader) error { + fs.mu.Lock() + defer fs.mu.Unlock() + + containerDir, err := fs.safeContainerDir(account, container) + if err != nil { + return err + } + if _, err := os.Stat(filepath.Join(containerDir, "container.json")); os.IsNotExist(err) { + return ErrContainerNotFound + } + if blob == "" || blockID == "" { + return ErrInvalidName + } + + blocksBase := filepath.Clean(filepath.Join(containerDir, "blocks")) + _ = os.MkdirAll(blocksBase, 0755) + + blobKey := hex.EncodeToString([]byte(blob)) + blocksDir := filepath.Clean(filepath.Join(blocksBase, blobKey)) + if !strings.HasPrefix(blocksDir, blocksBase+string(filepath.Separator)) { + return ErrInvalidName + } + if err := os.MkdirAll(blocksDir, 0755); err != nil { + return err + } + + blockFile := filepath.Clean(filepath.Join(blocksDir, hex.EncodeToString([]byte(blockID))+".block")) + if !strings.HasPrefix(blockFile, blocksDir+string(filepath.Separator)) { + return ErrInvalidName + } + + f, err := os.Create(blockFile) + if err != nil { + return err + } + _, err = io.Copy(f, r) + _ = f.Close() + return err +} + +func (fs *FileStore) CommitBlockList(account, container, blob string, blockIDs []string, opts PutBlobOptions) (*BlobInfo, error) { + fs.mu.Lock() + defer fs.mu.Unlock() + + containerDir, err := fs.safeContainerDir(account, container) + if err != nil { + return nil, err + } + if _, err := os.Stat(filepath.Join(containerDir, "container.json")); os.IsNotExist(err) { + return nil, ErrContainerNotFound + } + + dataPath, metaPath, err := fs.safeBlobPaths(account, container, blob) + if err != nil { + return nil, err + } + + var existing *BlobInfo if existingData, err := os.ReadFile(metaPath); err == nil { - var existing BlobInfo - if json.Unmarshal(existingData, &existing) == nil && !existing.CreatedAt.IsZero() { - createdAt = existing.CreatedAt + var ex BlobInfo + if json.Unmarshal(existingData, &ex) == nil { + existing = &ex + } + } + + if opts.IfNoneMatch == "*" && existing != nil { + return nil, ErrBlobAlreadyExists + } + if opts.IfMatch != "" { + if existing == nil || existing.ETag != opts.IfMatch { + return nil, ErrConditionNotMet } } + blocksBase := filepath.Clean(filepath.Join(containerDir, "blocks")) + blobKey := hex.EncodeToString([]byte(blob)) + blocksDir := filepath.Clean(filepath.Join(blocksBase, blobKey)) + + for _, bid := range blockIDs { + bf := filepath.Clean(filepath.Join(blocksDir, hex.EncodeToString([]byte(bid))+".block")) + if _, err := os.Stat(bf); os.IsNotExist(err) { + return nil, ErrInvalidBlock + } + } + + if err := os.MkdirAll(filepath.Dir(dataPath), 0755); err != nil { + return nil, err + } + if err := os.MkdirAll(filepath.Dir(metaPath), 0755); err != nil { + return nil, err + } + + tmpFile := filepath.Clean(dataPath + ".tmp") + dataDir := filepath.Clean(filepath.Join(containerDir, "data")) + dataPrefix := dataDir + string(filepath.Separator) + if !strings.HasPrefix(tmpFile, dataPrefix) { + return nil, ErrInvalidName + } + relTmp, err := filepath.Rel(dataDir, tmpFile) + if err != nil || !filepath.IsLocal(relTmp) || relTmp == "." { + return nil, ErrInvalidName + } + + outFile, err := os.Create(tmpFile) + if err != nil { + return nil, err + } + + hasher := md5.New() + mw := io.MultiWriter(outFile, hasher) + var totalSize int64 + for _, bid := range blockIDs { + bf := filepath.Clean(filepath.Join(blocksDir, hex.EncodeToString([]byte(bid))+".block")) + bfHandle, err := os.Open(bf) + if err != nil { + _ = outFile.Close() + _ = os.Remove(tmpFile) + return nil, err + } + n, err := io.Copy(mw, bfHandle) + _ = bfHandle.Close() + if err != nil { + _ = outFile.Close() + _ = os.Remove(tmpFile) + return nil, err + } + totalSize += n + } + _ = outFile.Close() + + if err := os.Rename(tmpFile, dataPath); err != nil { + _ = os.Remove(tmpFile) + return nil, err + } + _ = os.RemoveAll(blocksDir) + + md5Sum := hex.EncodeToString(hasher.Sum(nil)) + now := time.Now().UTC() + etag := fmt.Sprintf("\"0x%X\"", now.UnixNano()) + contentType := opts.ContentType + if contentType == "" { + contentType = "application/octet-stream" + } + + createdAt := now + if existing != nil && !existing.CreatedAt.IsZero() { + createdAt = existing.CreatedAt + } + info := &BlobInfo{ Name: blob, Container: container, Account: account, - Size: size, + Size: totalSize, ContentType: contentType, MD5: md5Sum, ETag: etag, CreatedAt: createdAt, LastModified: now, BlobType: "BlockBlob", + Metadata: opts.Metadata, } metaBytes, err := json.Marshal(info) @@ -352,6 +557,86 @@ func (fs *FileStore) PutBlob(account, container, blob string, contentType string return info, nil } +func (fs *FileStore) SetBlobMetadata(account, container, blob string, metadata map[string]string) (*BlobInfo, error) { + fs.mu.Lock() + defer fs.mu.Unlock() + + containerDir, err := fs.safeContainerDir(account, container) + if err != nil { + return nil, err + } + if _, err := os.Stat(filepath.Join(containerDir, "container.json")); os.IsNotExist(err) { + return nil, ErrContainerNotFound + } + + _, metaPath, err := fs.safeBlobPaths(account, container, blob) + if err != nil { + return nil, err + } + + metaBytes, err := os.ReadFile(metaPath) + if err != nil { + if os.IsNotExist(err) { + return nil, ErrBlobNotFound + } + return nil, err + } + + var info BlobInfo + if err := json.Unmarshal(metaBytes, &info); err != nil { + return nil, err + } + + info.Metadata = metadata + now := time.Now().UTC() + info.LastModified = now + info.ETag = fmt.Sprintf("\"0x%X\"", now.UnixNano()) + + newBytes, err := json.Marshal(&info) + if err != nil { + return nil, err + } + if err := os.WriteFile(metaPath, newBytes, 0644); err != nil { + return nil, err + } + return &info, nil +} + +func (fs *FileStore) SetContainerMetadata(account, container string, metadata map[string]string) (*ContainerInfo, error) { + fs.mu.Lock() + defer fs.mu.Unlock() + + containerDir, err := fs.safeContainerDir(account, container) + if err != nil { + return nil, err + } + metaFile := filepath.Join(containerDir, "container.json") + data, err := os.ReadFile(metaFile) + if err != nil { + if os.IsNotExist(err) { + return nil, ErrContainerNotFound + } + return nil, err + } + + var info ContainerInfo + if err := json.Unmarshal(data, &info); err != nil { + return nil, err + } + + info.Metadata = metadata + info.ETag = fmt.Sprintf("\"0x%X\"", time.Now().UTC().UnixNano()) + + newBytes, err := json.Marshal(&info) + if err != nil { + return nil, err + } + if err := os.WriteFile(metaFile, newBytes, 0644); err != nil { + return nil, err + } + return &info, nil +} + func (fs *FileStore) GetBlob(account, container, blob string) (*BlobInfo, []byte, error) { fs.mu.RLock() defer fs.mu.RUnlock() diff --git a/test/compatibility/test_azure_blob.py b/test/compatibility/test_azure_blob.py index c5d4610..411b9d8 100644 --- a/test/compatibility/test_azure_blob.py +++ b/test/compatibility/test_azure_blob.py @@ -1,5 +1,5 @@ from azure.core.exceptions import ResourceExistsError, ResourceNotFoundError -from azure.storage.blob import BlobServiceClient +from azure.storage.blob import BlobBlock, BlobServiceClient from conftest import DEVCLOUD_URL import pytest @@ -82,3 +82,106 @@ def test_blob_crud_lifecycle(azure_blob_client): finally: # Clean up container client.delete_container(container_name) + + +def test_blob_conditional_upload(azure_blob_client): + client = azure_blob_client + container_name = "test-blob-conditional" + container_client = client.create_container(container_name) + + try: + blob_name = "conditional.txt" + blob_client = container_client.get_blob_client(blob_name) + + # 1. Upload initial blob with overwrite=False + blob_client.upload_blob(b"initial data", overwrite=False) + + # 2. Attempting upload with overwrite=False raises ResourceExistsError (If-None-Match: *) + with pytest.raises(ResourceExistsError): + blob_client.upload_blob(b"new data", overwrite=False) + + # 3. Uploading with overwrite=True succeeds + blob_client.upload_blob(b"overwritten data", overwrite=True) + assert blob_client.download_blob().readall() == b"overwritten data" + finally: + client.delete_container(container_name) + + +def test_blob_staged_block_upload(azure_blob_client): + client = azure_blob_client + container_name = "test-blob-staged-blocks" + container_client = client.create_container(container_name) + + try: + blob_name = "chunked.bin" + blob_client = container_client.get_blob_client(blob_name) + + block1_id = "AAAAAA==" + block2_id = "AQIDBA==" + part1 = b"DevCloud Azure Blob " + part2 = b"Staged Blocks Verified!" + + # 1. Stage block 1 and block 2 + blob_client.stage_block(block_id=block1_id, data=part1) + blob_client.stage_block(block_id=block2_id, data=part2) + + # 2. Commit block list + blob_client.commit_block_list( + [BlobBlock(block_id=block1_id), BlobBlock(block_id=block2_id)] + ) + + # 3. Verify downloaded content is full concatenated payload + downloaded = blob_client.download_blob().readall() + assert downloaded == part1 + part2 + finally: + client.delete_container(container_name) + + +def test_blob_custom_metadata(azure_blob_client): + client = azure_blob_client + container_name = "test-blob-metadata" + container_client = client.create_container(container_name) + + try: + blob_name = "report.pdf" + blob_client = container_client.get_blob_client(blob_name) + initial_meta = {"author": "devcloud", "phase": "3"} + + # 1. Upload blob with metadata + blob_client.upload_blob(b"sample report", metadata=initial_meta) + props = blob_client.get_blob_properties() + assert props.metadata.get("author") == "devcloud" + assert props.metadata.get("phase") == "3" + + # 2. Update metadata + updated_meta = { + "author": "devcloud", + "phase": "3-complete", + "status": "verified", + } + blob_client.set_blob_metadata(metadata=updated_meta) + + # 3. Verify updated metadata + props2 = blob_client.get_blob_properties() + assert props2.metadata.get("phase") == "3-complete" + assert props2.metadata.get("status") == "verified" + finally: + client.delete_container(container_name) + + +def test_blob_byte_range_download(azure_blob_client): + client = azure_blob_client + container_name = "test-blob-range" + container_client = client.create_container(container_name) + + try: + blob_name = "stream.txt" + blob_client = container_client.get_blob_client(blob_name) + payload = b"0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ" + blob_client.upload_blob(payload) + + # Download range (offset 10, length 5 -> "ABCDE") + chunk = blob_client.download_blob(offset=10, length=5).readall() + assert chunk == b"ABCDE" + finally: + client.delete_container(container_name) From a8781d2c020698c99440736e4b37aea5b5e28334 Mon Sep 17 00:00:00 2001 From: Sung-Kyu Yoo Date: Sun, 11 Oct 2026 18:31:24 +0900 Subject: [PATCH 2/3] chore: add changelog fragment for #192 --- changes/unreleased/Added-20261011-183100.yaml | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 changes/unreleased/Added-20261011-183100.yaml diff --git a/changes/unreleased/Added-20261011-183100.yaml b/changes/unreleased/Added-20261011-183100.yaml new file mode 100644 index 0000000..f3d17c0 --- /dev/null +++ b/changes/unreleased/Added-20261011-183100.yaml @@ -0,0 +1,4 @@ +kind: Added +body: Complete Phase 3 Azure Blob Storage pilot with staged block uploads, conditional requests, and custom metadata. +custom: + Issue: 192 From ebb9e3d8fca80d6d61e18b1ec8de5583735636d6 Mon Sep 17 00:00:00 2001 From: Sung-Kyu Yoo Date: Sun, 11 Oct 2026 18:40:28 +0900 Subject: [PATCH 3/3] docs: sync coverage.md operation tiers with manifest after hand-verified operations added --- docs/coverage.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/coverage.md b/docs/coverage.md index 03bdc8b..09c7716 100644 --- a/docs/coverage.md +++ b/docs/coverage.md @@ -16,8 +16,8 @@ denominators, because [routing and depth are two targets](#the-target): | Tier | Serving target | All registered | |---|---|---| -| `hand-verified` | 4,620 | 4,651 | -| `auto-crud` | 7,532 | 14,085 | +| `hand-verified` | 4,623 | 4,654 | +| `auto-crud` | 7,529 | 14,082 | | `unimplemented` | 267 | 477 | | **total known** | **12,419** | **19,213** | | **hand-verified share** | **37.2%** | **24.2%** |