Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
191 changes: 191 additions & 0 deletions api/models/azure-blob-storage.json
Original file line number Diff line number Diff line change
Expand Up @@ -207,6 +207,158 @@
}
}
},
"/{container}/{blob+}?comp=block": {
"put": {
"operationId": "PutBlock",
"description": "Uploads the specified block to the blob's uncommitted block list",
"parameters": [
{
"name": "container",
"in": "path",
"required": true,
"schema": {
"type": "string"
}
},
{
"name": "blob+",
"in": "path",
"required": true,
"schema": {
"type": "string"
}
},
{
"name": "comp",
"in": "query",
"required": true,
"schema": {
"type": "string",
"enum": ["block"]
}
},
{
"name": "blockid",
"in": "query",
"required": true,
"schema": {
"type": "string"
}
}
],
"responses": {
"201": {
"description": "Block uploaded successfully"
},
"404": {
"description": "Container not found",
"content": {
"application/xml": {
"schema": {
"$ref": "#/components/schemas/ContainerNotFound"
}
}
}
}
}
}
},
"/{container}/{blob+}?comp=blocklist": {
"put": {
"operationId": "PutBlockList",
"description": "Commits a sequence of blocks that make up the blob",
"parameters": [
{
"name": "container",
"in": "path",
"required": true,
"schema": {
"type": "string"
}
},
{
"name": "blob+",
"in": "path",
"required": true,
"schema": {
"type": "string"
}
},
{
"name": "comp",
"in": "query",
"required": true,
"schema": {
"type": "string",
"enum": ["blocklist"]
}
}
],
"responses": {
"201": {
"description": "Block list committed successfully"
},
"404": {
"description": "Container not found",
"content": {
"application/xml": {
"schema": {
"$ref": "#/components/schemas/ContainerNotFound"
}
}
}
}
}
}
},
"/{container}/{blob+}?comp=metadata": {
"put": {
"operationId": "SetBlobMetadata",
"description": "Sets user-defined metadata on the specified blob",
"parameters": [
{
"name": "container",
"in": "path",
"required": true,
"schema": {
"type": "string"
}
},
{
"name": "blob+",
"in": "path",
"required": true,
"schema": {
"type": "string"
}
},
{
"name": "comp",
"in": "query",
"required": true,
"schema": {
"type": "string",
"enum": ["metadata"]
}
}
],
"responses": {
"200": {
"description": "Metadata set successfully"
},
"404": {
"description": "Blob or Container not found",
"content": {
"application/xml": {
"schema": {
"$ref": "#/components/schemas/BlobNotFound"
}
}
}
}
}
}
},
"/{container}/{blob+}": {
"put": {
"operationId": "PutBlob",
Expand Down Expand Up @@ -453,6 +605,45 @@
"BlobNotFound": {
"type": "object",
"x-error-status": 404
},
"BlobAlreadyExists": {
"type": "object",
"x-error-status": 409
},
"ConditionNotMet": {
"type": "object",
"x-error-status": 412
},
"PutBlockInput": {
"type": "object",
"properties": {
"Container": { "type": "string" },
"Blob": { "type": "string" },
"BlockId": { "type": "string" }
}
},
"PutBlockOutput": {
"type": "object"
},
"PutBlockListInput": {
"type": "object",
"properties": {
"Container": { "type": "string" },
"Blob": { "type": "string" }
}
},
"PutBlockListOutput": {
"type": "object"
},
"SetBlobMetadataInput": {
"type": "object",
"properties": {
"Container": { "type": "string" },
"Blob": { "type": "string" }
}
},
"SetBlobMetadataOutput": {
"type": "object"
}
}
}
Expand Down
4 changes: 4 additions & 0 deletions changes/unreleased/Added-20261011-183100.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
kind: Added
body: Complete Phase 3 Azure Blob Storage pilot with staged block uploads, conditional requests, and custom metadata.
custom:
Issue: 192
4 changes: 2 additions & 2 deletions docs/coverage.md
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,10 @@ denominators, because [routing and depth are two targets](#the-target):

| Tier | Serving target | All registered |
|---|---|---|
| `hand-verified` | 4,620 | 4,651 |
| `hand-verified` | 4,623 | 4,654 |
| `auto-crud` | 7,529 | 14,082 |
| `unimplemented` | 267 | 477 |
| **total known** | **12,416** | **19,210** |
| **total known** | **12,419** | **19,213** |
| **hand-verified share** | **37.2%** | **24.2%** |

The **serving target** column is the depth promise: every registered service
Expand Down
17 changes: 14 additions & 3 deletions docs/roadmap.md
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ Phase 2 deliberately shipped **no** non-AWS service, no second `ModelSource`, an
no signature verification. Those are Phase 3 and beyond; Phase 2's job was to make
each of them an addition rather than a fork.

## Phase 3 — First non-AWS service (next, v2.0, exploratory)
## Phase 3 — First non-AWS service (complete, v2.0 groundwork)

Validate the multi-CSP architecture with one well-scoped pilot.

Expand All @@ -102,6 +102,17 @@ Validate the multi-CSP architecture with one well-scoped pilot.
- [x] Compatibility tests against `azure-sdk-for-python`
- [x] Documentation pattern for multi-CSP service docs

Phase 3 execution completed: Azure Blob Storage (`blob`) is implemented as the first
non-AWS pilot under `providers.azure.services.blob`. The OpenAPI model in `api/models/`
compiles through `OpenAPISource` into deterministic IR and generated routers, types, and
error shapes. The storage engine (`internal/services/blob`) provides local filesystem-backed
persistence supporting container CRUD, BlockBlob CRUD, staged chunk uploads (`PutBlock`,
`PutBlockList`), conditional requests (`If-None-Match: *`, `If-Match`), custom metadata
(`x-ms-meta-*`), and byte-range downloads. The gateway router and auth adapters validate
Azure Shared Key and emulator signatures, and integration tests verify complete end-to-end
compatibility against `azure-sdk-for-python` in CI. See [Azure Services](services/azure.md)
for configuration and local limits.

## Phase 4 — Breadth expansion (v2.x+)

- [ ] More Azure services — Queue Storage, Table Storage, Cosmos DB
Expand All @@ -128,6 +139,6 @@ Validate the multi-CSP architecture with one well-scoped pilot.
| Version | Focus |
|---------|-------|
| 0.x | AWS services, unstable API |
| 1.x ← current | AWS depth, stable plugin API, multi-CSP groundwork |
| 2.x | Multi-CSP architecture, Azure pilot |
| 1.x | AWS depth, stable plugin API, multi-CSP groundwork |
| 2.x ← current | Multi-CSP architecture, Azure pilot |
| 3.x+ | Broad CSP coverage, community-owned providers |
26 changes: 24 additions & 2 deletions docs/services/azure.md
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,12 @@ DevCloud expands its local development runtime to include Microsoft Azure servic
| Containers | `DeleteContainer` | `DELETE /{container}?restype=container` | Returns `202 Accepted` or `404 ContainerNotFound` |
| Containers | `GetContainerProperties` | `GET /{container}?restype=container` | Returns lease state and metadata headers |
| Containers | `ListContainers` | `GET /?comp=list` | Returns XML `EnumerationResults` listing all containers |
| Blobs | `PutBlob` | `PUT /{container}/{blob}` | Creates or replaces a BlockBlob; stores payload and computes MD5 |
| Blobs | `PutBlob` | `PUT /{container}/{blob}` | Creates or replaces a BlockBlob; supports preconditions (`If-None-Match: *`, `If-Match`), stores payload and computes MD5 |
| Blobs | `PutBlock` | `PUT /{container}/{blob}?comp=block&blockid={id}` | Stages an uncommitted block for chunked/streaming blob uploads |
| Blobs | `PutBlockList` | `PUT /{container}/{blob}?comp=blocklist` | Commits staged blocks in sequence to materialize or update a BlockBlob |
| Blobs | `SetBlobMetadata` | `PUT /{container}/{blob}?comp=metadata` | Updates user-defined `x-ms-meta-*` headers on the blob |
| Blobs | `GetBlob` | `GET /{container}/{blob}` | Downloads blob content; supports `Range` and `x-ms-range` requests |
| Blobs | `GetBlobProperties` | `HEAD /{container}/{blob}` | Returns headers (`x-ms-blob-type: BlockBlob`, size, ETag, Content-MD5) |
| Blobs | `GetBlobProperties` | `HEAD /{container}/{blob}` | Returns headers (`x-ms-blob-type: BlockBlob`, size, ETag, Content-MD5, `x-ms-meta-*`) |
| Blobs | `DeleteBlob` | `DELETE /{container}/{blob}` | Permanently removes the blob |
| Blobs | `ListBlobs` | `GET /{container}?restype=container&comp=list` | Returns XML `EnumerationResults` listing blobs in the container |

Expand Down Expand Up @@ -76,4 +79,23 @@ print(content.decode("utf-8"))
# 4. List blobs
for blob in container_client.list_blobs():
print(f"Blob: {blob.name}, Size: {blob.size}")

# 5. Conditional upload (fail if exists)
from azure.core.exceptions import ResourceExistsError
try:
blob_client.upload_blob(b"will not overwrite", overwrite=False)
except ResourceExistsError:
print("Blob already exists!")

# 6. Chunked upload via stage_block & commit_block_list
from azure.storage.blob import BlobBlock
chunk_client = container_client.get_blob_client("large.bin")
chunk_client.stage_block(block_id="AAAAAA==", data=b"Part 1 ")
chunk_client.stage_block(block_id="AQIDBA==", data=b"Part 2!")
chunk_client.commit_block_list([BlobBlock(block_id="AAAAAA=="), BlobBlock(block_id="AQIDBA==")])

# 7. Metadata management
blob_client.set_blob_metadata(metadata={"env": "local", "pilot": "phase3"})
props = blob_client.get_blob_properties()
print(f"Metadata: {props.metadata}")
```
20 changes: 17 additions & 3 deletions internal/codegen/openapi_parser.go
Original file line number Diff line number Diff line change
Expand Up @@ -172,8 +172,15 @@ func ParseOpenAPIJSON(data []byte) (*ir.Model, error) {
}
}

// Parse component schemas
for name, schema := range doc.Components.Schemas {
// Parse component schemas in deterministic order
schemaNames := make([]string, 0, len(doc.Components.Schemas))
for name := range doc.Components.Schemas {
schemaNames = append(schemaNames, name)
}
sort.Strings(schemaNames)

for _, name := range schemaNames {
schema := doc.Components.Schemas[name]
shape := &ir.Shape{
Name: name,
Type: ir.ShapeStructure,
Expand All @@ -186,7 +193,14 @@ func ParseOpenAPIJSON(data []byte) (*ir.Model, error) {
}
}

for propName, prop := range schema.Properties {
propNames := make([]string, 0, len(schema.Properties))
for propName := range schema.Properties {
propNames = append(propNames, propName)
}
sort.Strings(propNames)

for _, propName := range propNames {
prop := schema.Properties[propName]
goType := "string"
switch prop.Type {
case "integer":
Expand Down
25 changes: 25 additions & 0 deletions internal/codegen/openapi_parser_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,9 @@ func TestParseOpenAPIJSON(t *testing.T) {
"DeleteContainer": "DELETE /{container}?restype=container",
"GetContainerProperties": "GET /{container}?restype=container",
"ListBlobs": "GET /{container}?restype=container&comp=list",
"PutBlock": "PUT /{container}/{blob+}?comp=block",
"PutBlockList": "PUT /{container}/{blob+}?comp=blocklist",
"SetBlobMetadata": "PUT /{container}/{blob+}?comp=metadata",
"PutBlob": "PUT /{container}/{blob+}",
"GetBlob": "GET /{container}/{blob+}",
"GetBlobProperties": "HEAD /{container}/{blob+}",
Expand Down Expand Up @@ -65,4 +68,26 @@ func TestParseOpenAPIJSON(t *testing.T) {
require.True(t, ok)
require.NotNil(t, errShape.ErrorTrait)
assert.Equal(t, 404, errShape.ErrorTrait.HTTPStatus)

existsShape, ok := model.Shapes["BlobAlreadyExists"]
require.True(t, ok)
require.NotNil(t, existsShape.ErrorTrait)
assert.Equal(t, 409, existsShape.ErrorTrait.HTTPStatus)

condShape, ok := model.Shapes["ConditionNotMet"]
require.True(t, ok)
require.NotNil(t, condShape.ErrorTrait)
assert.Equal(t, 412, condShape.ErrorTrait.HTTPStatus)

// Verify determinism: parsing multiple times produces identical member ordering
model2, err := ParseOpenAPIJSON(data)
require.NoError(t, err)
for shapeName, s1 := range model.Shapes {
s2 := model2.Shapes[shapeName]
require.NotNil(t, s2)
require.Equal(t, len(s1.Members), len(s2.Members))
for idx := range s1.Members {
assert.Equal(t, s1.Members[idx].Name, s2.Members[idx].Name)
}
}
}
6 changes: 5 additions & 1 deletion internal/gateway/router.go
Original file line number Diff line number Diff line change
Expand Up @@ -139,7 +139,11 @@ func (sr *ServiceRouter) ServeHTTP(w http.ResponseWriter, r *http.Request) {

// Write response headers from the plugin.
for k, v := range resp.Headers {
w.Header().Set(k, v)
if strings.HasPrefix(strings.ToLower(k), "x-ms-") {
w.Header()[strings.ToLower(k)] = []string{v}
} else {
w.Header().Set(k, v)
}
}
ct := resp.ContentType
if ct == "" {
Expand Down
Loading
Loading