diff --git a/api/models/azure-blob-storage.json b/api/models/azure-blob-storage.json index f9f2ca29..78856e01 100644 --- a/api/models/azure-blob-storage.json +++ b/api/models/azure-blob-storage.json @@ -2,17 +2,457 @@ "openapi": "3.0.0", "info": { "title": "Azure Blob Storage", - "version": "2023-01-03" + "version": "2023-11-03", + "description": "Azure Blob Storage Data Plane REST API" }, "paths": { + "/": { + "get": { + "operationId": "ListContainers", + "description": "Lists all containers in the storage account", + "parameters": [ + { + "name": "comp", + "in": "query", + "required": true, + "schema": { + "type": "string", + "enum": ["list"] + } + } + ], + "responses": { + "200": { + "description": "List of containers", + "content": { + "application/xml": { + "schema": { + "$ref": "#/components/schemas/ListContainersOutput" + } + } + } + } + } + } + }, "/{container}": { "put": { "operationId": "CreateContainer", + "description": "Creates a new container", + "parameters": [ + { + "name": "container", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "restype", + "in": "query", + "required": true, + "schema": { + "type": "string", + "enum": ["container"] + } + } + ], "responses": { "201": { "description": "Container created" + }, + "409": { + "description": "Container already exists", + "content": { + "application/xml": { + "schema": { + "$ref": "#/components/schemas/ContainerAlreadyExists" + } + } + } } } + }, + "delete": { + "operationId": "DeleteContainer", + "description": "Deletes the specified container", + "parameters": [ + { + "name": "container", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "restype", + "in": "query", + "required": true, + "schema": { + "type": "string", + "enum": ["container"] + } + } + ], + "responses": { + "202": { + "description": "Container deleted" + }, + "404": { + "description": "Container not found", + "content": { + "application/xml": { + "schema": { + "$ref": "#/components/schemas/ContainerNotFound" + } + } + } + } + } + }, + "get": { + "operationId": "GetContainerProperties", + "description": "Gets properties for the specified container, or lists blobs if comp=list query is provided", + "parameters": [ + { + "name": "container", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "restype", + "in": "query", + "required": true, + "schema": { + "type": "string", + "enum": ["container"] + } + } + ], + "responses": { + "200": { + "description": "Container properties retrieved" + }, + "404": { + "description": "Container not found", + "content": { + "application/xml": { + "schema": { + "$ref": "#/components/schemas/ContainerNotFound" + } + } + } + } + } + } + }, + "/{container}?restype=container&comp=list": { + "get": { + "operationId": "ListBlobs", + "description": "Lists blobs in the specified container", + "parameters": [ + { + "name": "container", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "restype", + "in": "query", + "required": true, + "schema": { + "type": "string", + "enum": ["container"] + } + }, + { + "name": "comp", + "in": "query", + "required": true, + "schema": { + "type": "string", + "enum": ["list"] + } + } + ], + "responses": { + "200": { + "description": "List of blobs", + "content": { + "application/xml": { + "schema": { + "$ref": "#/components/schemas/ListBlobsOutput" + } + } + } + }, + "404": { + "description": "Container not found", + "content": { + "application/xml": { + "schema": { + "$ref": "#/components/schemas/ContainerNotFound" + } + } + } + } + } + } + }, + "/{container}/{blob+}": { + "put": { + "operationId": "PutBlob", + "description": "Uploads a block blob to the container", + "parameters": [ + { + "name": "container", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "blob+", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "201": { + "description": "Blob uploaded successfully" + }, + "404": { + "description": "Container not found", + "content": { + "application/xml": { + "schema": { + "$ref": "#/components/schemas/ContainerNotFound" + } + } + } + } + } + }, + "get": { + "operationId": "GetBlob", + "description": "Downloads the specified blob content", + "parameters": [ + { + "name": "container", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "blob+", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Blob content" + }, + "404": { + "description": "Blob or Container not found", + "content": { + "application/xml": { + "schema": { + "$ref": "#/components/schemas/BlobNotFound" + } + } + } + } + } + }, + "head": { + "operationId": "GetBlobProperties", + "description": "Gets properties for the specified blob", + "parameters": [ + { + "name": "container", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "blob+", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Blob properties" + }, + "404": { + "description": "Blob not found", + "content": { + "application/xml": { + "schema": { + "$ref": "#/components/schemas/BlobNotFound" + } + } + } + } + } + }, + "delete": { + "operationId": "DeleteBlob", + "description": "Deletes the specified blob", + "parameters": [ + { + "name": "container", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "blob+", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "202": { + "description": "Blob deleted" + }, + "404": { + "description": "Blob not found", + "content": { + "application/xml": { + "schema": { + "$ref": "#/components/schemas/BlobNotFound" + } + } + } + } + } + } + } + }, + "components": { + "schemas": { + "CreateContainerInput": { + "type": "object", + "properties": { + "Container": { "type": "string" } + } + }, + "CreateContainerOutput": { + "type": "object" + }, + "DeleteContainerInput": { + "type": "object", + "properties": { + "Container": { "type": "string" } + } + }, + "DeleteContainerOutput": { + "type": "object" + }, + "GetContainerPropertiesInput": { + "type": "object", + "properties": { + "Container": { "type": "string" } + } + }, + "GetContainerPropertiesOutput": { + "type": "object" + }, + "ListContainersInput": { + "type": "object" + }, + "ListContainersOutput": { + "type": "object" + }, + "ListBlobsInput": { + "type": "object", + "properties": { + "Container": { "type": "string" } + } + }, + "ListBlobsOutput": { + "type": "object" + }, + "PutBlobInput": { + "type": "object", + "properties": { + "Container": { "type": "string" }, + "Blob": { "type": "string" } + } + }, + "PutBlobOutput": { + "type": "object" + }, + "GetBlobInput": { + "type": "object", + "properties": { + "Container": { "type": "string" }, + "Blob": { "type": "string" } + } + }, + "GetBlobOutput": { + "type": "object" + }, + "GetBlobPropertiesInput": { + "type": "object", + "properties": { + "Container": { "type": "string" }, + "Blob": { "type": "string" } + } + }, + "GetBlobPropertiesOutput": { + "type": "object" + }, + "DeleteBlobInput": { + "type": "object", + "properties": { + "Container": { "type": "string" }, + "Blob": { "type": "string" } + } + }, + "DeleteBlobOutput": { + "type": "object" + }, + "ContainerNotFound": { + "type": "object", + "x-error-status": 404 + }, + "ContainerAlreadyExists": { + "type": "object", + "x-error-status": 409 + }, + "BlobNotFound": { + "type": "object", + "x-error-status": 404 } } } diff --git a/docs/architecture.md b/docs/architecture.md index d6f86341..33ac4972 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -193,3 +193,8 @@ touched an *addition* rather than an edit: The plugin system, protocol detector and storage abstractions stay CSP-agnostic by convention rather than enforcement — `ProtocolType` is an open string type, not an enum, for exactly this reason. + +Phase 3 established the first non-AWS pilot (**Azure Blob Storage**) exercising all +four seams: OpenAPI models in `api/models/` parsed via `OpenAPISource`, namespaced +configuration under `providers.azure.services.blob`, `ProviderScoped` plugin registration, +and `AzureSharedKey` authentication. diff --git a/docs/coverage.md b/docs/coverage.md index 9afea740..03216296 100644 --- a/docs/coverage.md +++ b/docs/coverage.md @@ -16,10 +16,10 @@ denominators, because [routing and depth are two targets](#the-target): | Tier | Serving target | All registered | |---|---|---| -| `hand-verified` | 4,611 | 4,642 | -| `auto-crud` | 7,530 | 14,083 | +| `hand-verified` | 4,620 | 4,651 | +| `auto-crud` | 7,529 | 14,082 | | `unimplemented` | 267 | 477 | -| **total known** | **12,408** | **19,202** | +| **total known** | **12,416** | **19,210** | | **hand-verified share** | **37.2%** | **24.2%** | The **serving target** column is the depth promise: every registered service diff --git a/docs/services/azure.md b/docs/services/azure.md index 62825855..3a3646f7 100644 --- a/docs/services/azure.md +++ b/docs/services/azure.md @@ -1,16 +1,79 @@ # Azure Services -DevCloud is expanding its local development capabilities to include Microsoft Azure services, starting with Azure Blob Storage. This represents the beginning of our multi-CSP architectural vision. +DevCloud expands its local development runtime to include Microsoft Azure services, starting with Azure Blob Storage as the Phase 3 multi-CSP pilot. ## Supported Services | Service | Supported | Local limits | |---|---|---| -| Blob Storage | Initializing | Mock engine returning generic responses; strict semantic verification in progress. | +| Blob Storage (`blob`) | Supported (Phase 3 pilot) | Local filesystem-backed FileStore under `data/blob`. Supports container CRUD, BlockBlob CRUD, listing/enumeration, and byte-range downloads. | -## Credentials -DevCloud accepts Azure Shared Key credentials without validating signatures. For local development, use any dummy account key (or the standard local development key): +## Operations + +| Area | Operation | Method & URI | Notes | +|---|---|---|---| +| Containers | `CreateContainer` | `PUT /{container}?restype=container` | Returns `201 Created` or `409 ContainerAlreadyExists` | +| Containers | `DeleteContainer` | `DELETE /{container}?restype=container` | Returns `202 Accepted` or `404 ContainerNotFound` | +| Containers | `GetContainerProperties` | `GET /{container}?restype=container` | Returns lease state and metadata headers | +| Containers | `ListContainers` | `GET /?comp=list` | Returns XML `EnumerationResults` listing all containers | +| Blobs | `PutBlob` | `PUT /{container}/{blob}` | Creates or replaces a BlockBlob; stores payload and computes MD5 | +| Blobs | `GetBlob` | `GET /{container}/{blob}` | Downloads blob content; supports `Range` and `x-ms-range` requests | +| Blobs | `GetBlobProperties` | `HEAD /{container}/{blob}` | Returns headers (`x-ms-blob-type: BlockBlob`, size, ETag, Content-MD5) | +| Blobs | `DeleteBlob` | `DELETE /{container}/{blob}` | Permanently removes the blob | +| Blobs | `ListBlobs` | `GET /{container}?restype=container&comp=list` | Returns XML `EnumerationResults` listing blobs in the container | + +## Configuration + +In `devcloud.yaml`, Azure services are configured under the namespaced `providers.azure` block: + +```yaml +server: + port: 4747 + +providers: + azure: + services: + blob: + enabled: true + data_dir: ./data/blob +``` + +## Credentials & Connection Strings + +DevCloud accepts Azure Shared Key, SharedKeyLite, and Azurite emulator credentials without signature validation: ```bash export AZURE_STORAGE_CONNECTION_STRING="DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:4747/devstoreaccount1;" ``` + +## Python SDK Quickstart + +Using `azure-storage-blob`: + +```python +from azure.storage.blob import BlobServiceClient + +connection_string = ( + "DefaultEndpointsProtocol=http;" + "AccountName=devstoreaccount1;" + "AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;" + "BlobEndpoint=http://127.0.0.1:4747/devstoreaccount1;" +) + +client = BlobServiceClient.from_connection_string(connection_string) + +# 1. Create a container +container_client = client.create_container("my-container") + +# 2. Upload a BlockBlob +blob_client = container_client.get_blob_client("notes.txt") +blob_client.upload_blob(b"Hello from local DevCloud Azure Blob Storage!") + +# 3. Download the blob +content = blob_client.download_blob().readall() +print(content.decode("utf-8")) + +# 4. List blobs +for blob in container_client.list_blobs(): + print(f"Blob: {blob.name}, Size: {blob.size}") +``` diff --git a/internal/auth/azure_shared_key.go b/internal/auth/azure_shared_key.go index ebc0fe78..511f228e 100644 --- a/internal/auth/azure_shared_key.go +++ b/internal/auth/azure_shared_key.go @@ -13,18 +13,53 @@ func (AzureSharedKey) Provider() string { return "azure" } func (s AzureSharedKey) Authenticate(r *http.Request) (Identity, bool) { authz := r.Header.Get("Authorization") - if !strings.HasPrefix(authz, "SharedKey ") { - return Identity{}, false + if strings.HasPrefix(authz, "SharedKey ") { + parts := strings.SplitN(authz[10:], ":", 2) + if len(parts) == 2 { + return Identity{ + Provider: s.Provider(), + AccessKeyID: parts[0], + Service: "blob", + }, true + } + } + if strings.HasPrefix(authz, "SharedKeyLite ") { + parts := strings.SplitN(authz[14:], ":", 2) + if len(parts) == 2 { + return Identity{ + Provider: s.Provider(), + AccessKeyID: parts[0], + Service: "blob", + }, true + } + } + + q := r.URL.Query() + if q.Get("sig") != "" && (q.Get("se") != "" || q.Get("sp") != "" || q.Get("sv") != "") { + account := "devstoreaccount1" + trimmed := strings.Trim(r.URL.Path, "/") + if parts := strings.Split(trimmed, "/"); len(parts) > 0 && parts[0] != "" { + account = parts[0] + } + return Identity{ + Provider: s.Provider(), + AccessKeyID: account, + Service: "blob", + }, true } - parts := strings.SplitN(authz[10:], ":", 2) - if len(parts) != 2 { - return Identity{}, false + if r.Header.Get("x-ms-version") != "" || strings.HasPrefix(r.URL.Path, "/devstoreaccount1") { + account := "devstoreaccount1" + trimmed := strings.Trim(r.URL.Path, "/") + if parts := strings.Split(trimmed, "/"); len(parts) > 0 && parts[0] != "" { + account = parts[0] + } + return Identity{ + Provider: s.Provider(), + AccessKeyID: account, + Service: "blob", + }, true } - return Identity{ - Provider: s.Provider(), - AccessKeyID: parts[0], - Service: "blob", - }, true + return Identity{}, false } diff --git a/internal/auth/azure_shared_key_test.go b/internal/auth/azure_shared_key_test.go new file mode 100644 index 00000000..4b6c933f --- /dev/null +++ b/internal/auth/azure_shared_key_test.go @@ -0,0 +1,57 @@ +// SPDX-License-Identifier: Apache-2.0 + +package auth + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestAzureSharedKey_Authenticate(t *testing.T) { + adapter := AzureSharedKey{} + + t.Run("SharedKey header", func(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, "/mycontainer", nil) + req.Header.Set("Authorization", "SharedKey devstoreaccount1:abcdef==") + id, ok := adapter.Authenticate(req) + assert.True(t, ok) + assert.Equal(t, "azure", id.Provider) + assert.Equal(t, "devstoreaccount1", id.AccessKeyID) + assert.Equal(t, "blob", id.Service) + }) + + t.Run("SharedKeyLite header", func(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, "/mycontainer", nil) + req.Header.Set("Authorization", "SharedKeyLite myaccount:secret123==") + id, ok := adapter.Authenticate(req) + assert.True(t, ok) + assert.Equal(t, "azure", id.Provider) + assert.Equal(t, "myaccount", id.AccessKeyID) + }) + + t.Run("SAS query parameters", func(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, "/myaccount/mycontainer?sig=sig123&se=2026-10-10", nil) + id, ok := adapter.Authenticate(req) + assert.True(t, ok) + assert.Equal(t, "azure", id.Provider) + assert.Equal(t, "myaccount", id.AccessKeyID) + }) + + t.Run("x-ms-version header", func(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, "/devstoreaccount1/mycontainer", nil) + req.Header.Set("x-ms-version", "2023-11-03") + id, ok := adapter.Authenticate(req) + assert.True(t, ok) + assert.Equal(t, "azure", id.Provider) + assert.Equal(t, "devstoreaccount1", id.AccessKeyID) + }) + + t.Run("unrecognized request", func(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, "/bucket/key", nil) + _, ok := adapter.Authenticate(req) + assert.False(t, ok) + }) +} diff --git a/internal/codegen/openapi_parser.go b/internal/codegen/openapi_parser.go index 617388b9..19c316b8 100644 --- a/internal/codegen/openapi_parser.go +++ b/internal/codegen/openapi_parser.go @@ -5,43 +5,243 @@ package codegen import ( "encoding/json" "fmt" + "sort" + "strings" + "unicode" "github.com/skyoo2003/devcloud/internal/codegen/ir" ) -// ParseOpenAPIJSON provides a proof-of-concept OpenAPI parser. -// In a full implementation, this would read the paths, components/schemas, -// and translate them into ir.Model, ir.Operation, and ir.Shape. +type openAPIDoc struct { + OpenAPI string `json:"openapi"` + Info struct { + Title string `json:"title"` + Version string `json:"version"` + Description string `json:"description"` + ServiceID string `json:"x-service-id"` + Provider string `json:"x-provider"` + } `json:"info"` + Paths map[string]map[string]json.RawMessage `json:"paths"` + Components struct { + Schemas map[string]openAPISchema `json:"schemas"` + } `json:"components"` +} + +type openAPIOperation struct { + OperationID string `json:"operationId"` + Description string `json:"description"` + Parameters []openAPIParam `json:"parameters"` + Responses map[string]json.RawMessage `json:"responses"` + HTTPUri string `json:"x-http-uri"` +} + +type openAPIParam struct { + Name string `json:"name"` + In string `json:"in"` + Required bool `json:"required"` + Schema *openAPISchema `json:"schema"` +} + +type openAPISchema struct { + Type string `json:"type"` + Enum []string `json:"enum"` + ErrorStatus int `json:"x-error-status"` + Properties map[string]openAPIProperty `json:"properties"` +} + +type openAPIProperty struct { + Type string `json:"type"` +} + +// ParseOpenAPIJSON translates an OpenAPI 3.0/3.1 JSON model into the provider-neutral IR. func ParseOpenAPIJSON(data []byte) (*ir.Model, error) { - var doc struct { - Info struct { - Title string `json:"title"` - } `json:"info"` - } + var doc openAPIDoc if err := json.Unmarshal(data, &doc); err != nil { - return nil, fmt.Errorf("failed to parse OpenAPI json: %v", err) + return nil, fmt.Errorf("failed to parse OpenAPI json: %w", err) + } + + provider := doc.Info.Provider + if provider == "" { + provider = "azure" + } + + serviceID := doc.Info.ServiceID + if serviceID == "" { + titleLower := strings.ToLower(doc.Info.Title) + if strings.Contains(titleLower, "blob") { + serviceID = "blob" + } else { + serviceID = strings.ReplaceAll(titleLower, " ", "") + } } - // This is a stub returning a minimal model for proof-of-concept. - // We hardcode the azure Blob service ID to satisfy the pipeline until - // full mapping is implemented. model := &ir.Model{ - Provider: "azure", + Provider: provider, ServiceName: doc.Info.Title, - ServiceID: "blob", - Protocol: "rest-json", - Operations: []ir.Operation{ - { - Name: "CreateContainer", - InputName: "CreateContainerInput", - OutputName: "CreateContainerOutput", - }, - }, - Shapes: map[string]*ir.Shape{ - "CreateContainerInput": {Name: "CreateContainerInput", Type: ir.ShapeStructure}, - "CreateContainerOutput": {Name: "CreateContainerOutput", Type: ir.ShapeStructure}, - }, + ServiceID: serviceID, + Protocol: "rest-xml", + Operations: nil, + Shapes: make(map[string]*ir.Shape), + } + + httpMethods := map[string]bool{ + "get": true, + "put": true, + "post": true, + "delete": true, + "head": true, + "patch": true, + "options": true, } + // Parse paths and operations + pathKeys := make([]string, 0, len(doc.Paths)) + for p := range doc.Paths { + pathKeys = append(pathKeys, p) + } + sort.Strings(pathKeys) + + for _, pathKey := range pathKeys { + methods := doc.Paths[pathKey] + methodKeys := make([]string, 0, len(methods)) + for m := range methods { + if httpMethods[strings.ToLower(m)] { + methodKeys = append(methodKeys, m) + } + } + sort.Strings(methodKeys) + + for _, method := range methodKeys { + rawOp := methods[method] + var op openAPIOperation + if err := json.Unmarshal(rawOp, &op); err != nil { + return nil, fmt.Errorf("failed to parse operation for %s %s: %w", method, pathKey, err) + } + + opName := op.OperationID + if opName == "" { + opName = fmt.Sprintf("%s%s", titleCase(method), sanitizeName(pathKey)) + } + + uri := op.HTTPUri + if uri == "" { + uri = pathKey + if !strings.Contains(uri, "?") { + var queryConstraints []string + for _, param := range op.Parameters { + if param.In == "query" && param.Required { + if param.Schema != nil && len(param.Schema.Enum) > 0 { + queryConstraints = append(queryConstraints, fmt.Sprintf("%s=%s", param.Name, param.Schema.Enum[0])) + } else { + queryConstraints = append(queryConstraints, param.Name) + } + } + } + if len(queryConstraints) > 0 { + uri = uri + "?" + strings.Join(queryConstraints, "&") + } + } + } + + inputName := opName + "Input" + outputName := opName + "Output" + + irOp := ir.Operation{ + Name: opName, + InputName: inputName, + OutputName: outputName, + HTTPMethod: strings.ToUpper(method), + HTTPUri: uri, + } + + model.Operations = append(model.Operations, irOp) + + // Ensure input and output shapes exist + if _, exists := model.Shapes[inputName]; !exists { + model.Shapes[inputName] = &ir.Shape{ + Name: inputName, + Type: ir.ShapeStructure, + } + } + if _, exists := model.Shapes[outputName]; !exists { + model.Shapes[outputName] = &ir.Shape{ + Name: outputName, + Type: ir.ShapeStructure, + } + } + } + } + + // Parse component schemas + for name, schema := range doc.Components.Schemas { + shape := &ir.Shape{ + Name: name, + Type: ir.ShapeStructure, + } + + if schema.ErrorStatus > 0 { + shape.ErrorTrait = &ir.ErrorTrait{ + Type: "client", + HTTPStatus: schema.ErrorStatus, + } + } + + for propName, prop := range schema.Properties { + goType := "string" + switch prop.Type { + case "integer": + goType = "int" + case "boolean": + goType = "bool" + } + shape.Members = append(shape.Members, ir.Member{ + Name: propName, + TargetName: propName, + GoType: goType, + }) + } + + model.Shapes[name] = shape + } + + // Sort operations: + // Routes with more query constraints must be ordered before routes with fewer + // query constraints so httproute.Match resolves the more specific route first. + sort.Slice(model.Operations, func(i, j int) bool { + qI := queryParamsCount(model.Operations[i].HTTPUri) + qJ := queryParamsCount(model.Operations[j].HTTPUri) + if qI != qJ { + return qI > qJ + } + return model.Operations[i].Name < model.Operations[j].Name + }) + return model, nil } + +func queryParamsCount(uri string) int { + parts := strings.Split(uri, "?") + if len(parts) < 2 || parts[1] == "" { + return 0 + } + return strings.Count(parts[1], "&") + 1 +} + +func sanitizeName(s string) string { + var b strings.Builder + for _, r := range s { + if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') { + b.WriteRune(r) + } + } + return b.String() +} + +func titleCase(s string) string { + if s == "" { + return "" + } + r := []rune(s) + r[0] = unicode.ToUpper(r[0]) + return string(r) +} diff --git a/internal/codegen/openapi_parser_test.go b/internal/codegen/openapi_parser_test.go new file mode 100644 index 00000000..66df92ea --- /dev/null +++ b/internal/codegen/openapi_parser_test.go @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: Apache-2.0 + +package codegen + +import ( + "os" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestParseOpenAPIJSON(t *testing.T) { + data, err := os.ReadFile("../../api/models/azure-blob-storage.json") + require.NoError(t, err) + + model, err := ParseOpenAPIJSON(data) + require.NoError(t, err) + + assert.Equal(t, "azure", model.Provider) + assert.Equal(t, "blob", model.ServiceID) + assert.Equal(t, "Azure Blob Storage", model.ServiceName) + assert.Equal(t, "rest-xml", model.Protocol) + + // Ensure all 9 operations are parsed + opMap := make(map[string]string) + for _, op := range model.Operations { + opMap[op.Name] = op.HTTPMethod + " " + op.HTTPUri + } + + expectedOps := map[string]string{ + "ListContainers": "GET /?comp=list", + "CreateContainer": "PUT /{container}?restype=container", + "DeleteContainer": "DELETE /{container}?restype=container", + "GetContainerProperties": "GET /{container}?restype=container", + "ListBlobs": "GET /{container}?restype=container&comp=list", + "PutBlob": "PUT /{container}/{blob+}", + "GetBlob": "GET /{container}/{blob+}", + "GetBlobProperties": "HEAD /{container}/{blob+}", + "DeleteBlob": "DELETE /{container}/{blob+}", + } + + for opName, expectedRoute := range expectedOps { + route, ok := opMap[opName] + assert.True(t, ok, "operation %s must exist", opName) + assert.Equal(t, expectedRoute, route, "operation %s route mismatch", opName) + } + + // Verify route ordering: ListBlobs (2 query params) must precede GetContainerProperties (1 query param) + listBlobsIdx := -1 + getContainerPropsIdx := -1 + for i, op := range model.Operations { + if op.Name == "ListBlobs" { + listBlobsIdx = i + } + if op.Name == "GetContainerProperties" { + getContainerPropsIdx = i + } + } + assert.True(t, listBlobsIdx != -1 && getContainerPropsIdx != -1) + assert.True(t, listBlobsIdx < getContainerPropsIdx, "ListBlobs must precede GetContainerProperties for route matching") + + // Verify error shapes + errShape, ok := model.Shapes["ContainerNotFound"] + require.True(t, ok) + require.NotNil(t, errShape.ErrorTrait) + assert.Equal(t, 404, errShape.ErrorTrait.HTTPStatus) +} diff --git a/internal/config/config.go b/internal/config/config.go index 9095ead7..2ea5c0ca 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -35,7 +35,7 @@ const DefaultProvider = "aws" // without erroring — but it is reported, because nothing in this binary will // read it. Adding a provider means adding its services and this entry in the // same change. -var knownProviders = map[string]bool{DefaultProvider: true} +var knownProviders = map[string]bool{DefaultProvider: true, "azure": true} type Config struct { Server ServerConfig `yaml:"server"` diff --git a/internal/config/config_test.go b/internal/config/config_test.go index d068bcf4..b9962d82 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -110,14 +110,25 @@ func TestProviderService_ProviderBlockWinsOverLegacy(t *testing.T) { func TestProviderService_UnknownProviderWarnsButLoads(t *testing.T) { isolateEnv(t) cfg, warnings, err := parse([]byte( - "providers:\n azure:\n services:\n blob:\n enabled: true\n")) + "providers:\n gcp:\n services:\n gcs:\n enabled: true\n")) require.NoError(t, err) require.Len(t, warnings, 1) - assert.Contains(t, warnings[0], "providers.azure") + assert.Contains(t, warnings[0], "providers.gcp") // No AWS block was written, so AWS still runs everything. assert.True(t, cfg.Service("s3").Enabled) // A non-AWS provider nests its data so a same-named service cannot collide. + assert.Equal(t, filepath.Join("./data", "gcp", "gcs"), + cfg.ProviderService("gcp", "gcs").DataDir) +} + +func TestProviderService_KnownAzureProviderLoadsWithoutWarning(t *testing.T) { + isolateEnv(t) + cfg, warnings, err := parse([]byte( + "providers:\n azure:\n services:\n blob:\n enabled: true\n")) + require.NoError(t, err) + assert.Empty(t, warnings) + assert.True(t, cfg.ProviderService("azure", "blob").Enabled) assert.Equal(t, filepath.Join("./data", "azure", "blob"), cfg.ProviderService("azure", "blob").DataDir) } diff --git a/internal/gateway/protocol.go b/internal/gateway/protocol.go index ce7a8bc9..c4e88da4 100644 --- a/internal/gateway/protocol.go +++ b/internal/gateway/protocol.go @@ -23,6 +23,9 @@ import ( // 2. Content-Type application/x-www-form-urlencoded with Action= body → Query protocol (SQS/IAM/STS) // 3. Default → REST-XML / S3 func DetectProtocol(r *http.Request) (protocol string, serviceID string) { + if isAzureRequest(r) { + return "rest-xml", "blob" + } if serviceFromSigV4(r) == "s3express" { return "rest-xml", "s3" } @@ -461,3 +464,23 @@ func serviceFromQueryRequest(r *http.Request, body string) string { } return "sqs" } + +func isAzureRequest(r *http.Request) bool { + if id, ok := auth.FromContext(r.Context()); ok && id.Provider == "azure" { + return true + } + if id, ok := (auth.AzureSharedKey{}).Authenticate(r); ok && id.Provider == "azure" { + return true + } + if r.Header.Get("x-ms-version") != "" || r.Header.Get("x-ms-date") != "" { + return true + } + if strings.HasPrefix(r.URL.Path, "/devstoreaccount1") { + return true + } + authz := r.Header.Get("Authorization") + if strings.HasPrefix(authz, "SharedKey ") || strings.HasPrefix(authz, "SharedKeyLite ") { + return true + } + return false +} diff --git a/internal/gateway/protocol_test.go b/internal/gateway/protocol_test.go index 6be8c2f1..0331ae4d 100644 --- a/internal/gateway/protocol_test.go +++ b/internal/gateway/protocol_test.go @@ -217,3 +217,28 @@ func TestDetectProtocol_Query_STS(t *testing.T) { assert.Equal(t, "query", proto) assert.Equal(t, "sts", service) } + +func TestDetectProtocol_AzureBlob(t *testing.T) { + t.Run("SharedKey auth header", func(t *testing.T) { + req := httptest.NewRequest("PUT", "/mycontainer?restype=container", nil) + req.Header.Set("Authorization", "SharedKey devstoreaccount1:abcdef==") + proto, service := DetectProtocol(req) + assert.Equal(t, "rest-xml", proto) + assert.Equal(t, "blob", service) + }) + + t.Run("x-ms-version header", func(t *testing.T) { + req := httptest.NewRequest("GET", "/devstoreaccount1?comp=list", nil) + req.Header.Set("x-ms-version", "2023-11-03") + proto, service := DetectProtocol(req) + assert.Equal(t, "rest-xml", proto) + assert.Equal(t, "blob", service) + }) + + t.Run("devstoreaccount1 path prefix", func(t *testing.T) { + req := httptest.NewRequest("GET", "/devstoreaccount1/container/blob.txt", nil) + proto, service := DetectProtocol(req) + assert.Equal(t, "rest-xml", proto) + assert.Equal(t, "blob", service) + }) +} diff --git a/internal/gateway/router.go b/internal/gateway/router.go index f1e58896..0646e3f0 100644 --- a/internal/gateway/router.go +++ b/internal/gateway/router.go @@ -7,6 +7,7 @@ import ( "encoding/json" "encoding/xml" "errors" + "fmt" "io" "mime" "net/http" @@ -48,6 +49,11 @@ func (sr *ServiceRouter) ServeHTTP(w http.ResponseWriter, r *http.Request) { p, ok := sr.registry.Get(serviceID) if !ok { sr.unrouted.Add(serviceID) + if isAzureRequest(r) { + writeAzureError(w, http.StatusBadRequest, "UnknownService", + "The requested service is not available: "+serviceID) + return + } writeAWSError(w, protocol, http.StatusBadRequest, "UnknownService", "The requested service is not available: "+serviceID) return @@ -97,6 +103,10 @@ func (sr *ServiceRouter) ServeHTTP(w http.ResponseWriter, r *http.Request) { Body: body, }) if cerr != nil && errors.Is(err, plugin.ErrUnhandledOp) { + if plugin.ProviderOf(p) == "azure" { + writeAzureError(w, http.StatusBadRequest, "InvalidAction", "unknown action: "+op) + return + } writeAWSError(w, protocol, http.StatusBadRequest, "InvalidAction", "unknown action: "+op) return } @@ -106,6 +116,10 @@ func (sr *ServiceRouter) ServeHTTP(w http.ResponseWriter, r *http.Request) { } if err != nil { + if plugin.ProviderOf(p) == "azure" { + writeAzureError(w, http.StatusInternalServerError, "InternalError", err.Error()) + return + } writeAWSError(w, protocol, http.StatusInternalServerError, "InternalError", err.Error()) return } @@ -302,3 +316,11 @@ func writeAWSError(w http.ResponseWriter, protocol string, status int, code, mes _, _ = w.Write([]byte(xml.Header)) _, _ = w.Write(body) } + +func writeAzureError(w http.ResponseWriter, status int, code, message string) { + w.Header().Set("Content-Type", "application/xml") + w.Header().Set("x-ms-error-code", code) + w.WriteHeader(status) + _, _ = fmt.Fprintf(w, "\n%s%s", + code, message) +} diff --git a/internal/generated/blob/base_provider.go b/internal/generated/blob/base_provider.go index 4a43c044..b840841d 100644 --- a/internal/generated/blob/base_provider.go +++ b/internal/generated/blob/base_provider.go @@ -11,6 +11,38 @@ var ErrNotImplemented = fmt.Errorf("operation not implemented") type BaseProvider struct{} +func (b *BaseProvider) ListBlobs(ctx context.Context, input *ListBlobsInput) (*ListBlobsOutput, error) { + return nil, fmt.Errorf("ListBlobs: %w", ErrNotImplemented) +} + func (b *BaseProvider) CreateContainer(ctx context.Context, input *CreateContainerInput) (*CreateContainerOutput, error) { return nil, fmt.Errorf("CreateContainer: %w", ErrNotImplemented) } + +func (b *BaseProvider) DeleteContainer(ctx context.Context, input *DeleteContainerInput) (*DeleteContainerOutput, error) { + return nil, fmt.Errorf("DeleteContainer: %w", ErrNotImplemented) +} + +func (b *BaseProvider) GetContainerProperties(ctx context.Context, input *GetContainerPropertiesInput) (*GetContainerPropertiesOutput, error) { + return nil, fmt.Errorf("GetContainerProperties: %w", ErrNotImplemented) +} + +func (b *BaseProvider) ListContainers(ctx context.Context, input *ListContainersInput) (*ListContainersOutput, error) { + return nil, fmt.Errorf("ListContainers: %w", ErrNotImplemented) +} + +func (b *BaseProvider) DeleteBlob(ctx context.Context, input *DeleteBlobInput) (*DeleteBlobOutput, error) { + return nil, fmt.Errorf("DeleteBlob: %w", ErrNotImplemented) +} + +func (b *BaseProvider) GetBlob(ctx context.Context, input *GetBlobInput) (*GetBlobOutput, error) { + return nil, fmt.Errorf("GetBlob: %w", ErrNotImplemented) +} + +func (b *BaseProvider) GetBlobProperties(ctx context.Context, input *GetBlobPropertiesInput) (*GetBlobPropertiesOutput, error) { + return nil, fmt.Errorf("GetBlobProperties: %w", ErrNotImplemented) +} + +func (b *BaseProvider) PutBlob(ctx context.Context, input *PutBlobInput) (*PutBlobOutput, error) { + return nil, fmt.Errorf("PutBlob: %w", ErrNotImplemented) +} diff --git a/internal/generated/blob/errors.go b/internal/generated/blob/errors.go index 3127d42d..a4efa0f5 100644 --- a/internal/generated/blob/errors.go +++ b/internal/generated/blob/errors.go @@ -1,3 +1,56 @@ // Code generated by devcloud codegen. DO NOT EDIT. // SPDX-License-Identifier: Apache-2.0 package blob + +import "fmt" + +// BlobNotFound is returned with HTTP status 404. +type BlobNotFound struct { + Message string +} + +func (e *BlobNotFound) Error() string { + return fmt.Sprintf("BlobNotFound: %s", e.Message) +} + +func (e *BlobNotFound) ErrorCode() string { + return "BlobNotFound" +} + +func (e *BlobNotFound) HTTPStatus() int { + return 404 +} + +// ContainerAlreadyExists is returned with HTTP status 409. +type ContainerAlreadyExists struct { + Message string +} + +func (e *ContainerAlreadyExists) Error() string { + return fmt.Sprintf("ContainerAlreadyExists: %s", e.Message) +} + +func (e *ContainerAlreadyExists) ErrorCode() string { + return "ContainerAlreadyExists" +} + +func (e *ContainerAlreadyExists) HTTPStatus() int { + return 409 +} + +// ContainerNotFound is returned with HTTP status 404. +type ContainerNotFound struct { + Message string +} + +func (e *ContainerNotFound) Error() string { + return fmt.Sprintf("ContainerNotFound: %s", e.Message) +} + +func (e *ContainerNotFound) ErrorCode() string { + return "ContainerNotFound" +} + +func (e *ContainerNotFound) HTTPStatus() int { + return 404 +} diff --git a/internal/generated/blob/router.go b/internal/generated/blob/router.go index d0b2d36d..44fd831b 100644 --- a/internal/generated/blob/router.go +++ b/internal/generated/blob/router.go @@ -12,7 +12,15 @@ type OperationRoute = httproute.Route // OperationRoutes is the list of all routes for this service. var OperationRoutes = []OperationRoute{ - {Method: "", Pattern: "", Operation: "CreateContainer"}, + {Method: "GET", Pattern: "/{container}?restype=container&comp=list", Operation: "ListBlobs"}, + {Method: "PUT", Pattern: "/{container}?restype=container", Operation: "CreateContainer"}, + {Method: "DELETE", Pattern: "/{container}?restype=container", Operation: "DeleteContainer"}, + {Method: "GET", Pattern: "/{container}?restype=container", Operation: "GetContainerProperties"}, + {Method: "GET", Pattern: "/?comp=list", Operation: "ListContainers"}, + {Method: "DELETE", Pattern: "/{container}/{blob+}", Operation: "DeleteBlob"}, + {Method: "GET", Pattern: "/{container}/{blob+}", Operation: "GetBlob"}, + {Method: "HEAD", Pattern: "/{container}/{blob+}", Operation: "GetBlobProperties"}, + {Method: "PUT", Pattern: "/{container}/{blob+}", Operation: "PutBlob"}, } // MatchOperation returns the operation name and extracted path parameters for diff --git a/internal/generated/blob/types.go b/internal/generated/blob/types.go index 60736606..ad4deb84 100644 --- a/internal/generated/blob/types.go +++ b/internal/generated/blob/types.go @@ -3,7 +3,67 @@ package blob type CreateContainerInput struct { + Container string `json:"container" xml:"Container"` } type CreateContainerOutput struct { } + +type DeleteBlobInput struct { + Container string `json:"container" xml:"Container"` + Blob string `json:"blob" xml:"Blob"` +} + +type DeleteBlobOutput struct { +} + +type DeleteContainerInput struct { + Container string `json:"container" xml:"Container"` +} + +type DeleteContainerOutput struct { +} + +type GetBlobInput struct { + Container string `json:"container" xml:"Container"` + Blob string `json:"blob" xml:"Blob"` +} + +type GetBlobOutput struct { +} + +type GetBlobPropertiesInput struct { + Container string `json:"container" xml:"Container"` + Blob string `json:"blob" xml:"Blob"` +} + +type GetBlobPropertiesOutput struct { +} + +type GetContainerPropertiesInput struct { + Container string `json:"container" xml:"Container"` +} + +type GetContainerPropertiesOutput struct { +} + +type ListBlobsInput struct { + Container string `json:"container" xml:"Container"` +} + +type ListBlobsOutput struct { +} + +type ListContainersInput struct { +} + +type ListContainersOutput struct { +} + +type PutBlobInput struct { + Container string `json:"container" xml:"Container"` + Blob string `json:"blob" xml:"Blob"` +} + +type PutBlobOutput struct { +} diff --git a/internal/generated/compat/services.json b/internal/generated/compat/services.json index 36d780da..49cbbcd1 100644 --- a/internal/generated/compat/services.json +++ b/internal/generated/compat/services.json @@ -2315,9 +2315,17 @@ ] }, "blob": { - "protocol": "rest-json", + "protocol": "rest-xml", "servedOps": [ - "CreateContainer" + "CreateContainer", + "DeleteBlob", + "DeleteContainer", + "GetBlob", + "GetBlobProperties", + "GetContainerProperties", + "ListBlobs", + "ListContainers", + "PutBlob" ] }, "braket": { diff --git a/internal/generated/crudregistry/registry_gen.go b/internal/generated/crudregistry/registry_gen.go index 89a1cec1..5d7c4d09 100644 --- a/internal/generated/crudregistry/registry_gen.go +++ b/internal/generated/crudregistry/registry_gen.go @@ -2138,7 +2138,15 @@ func init() { "UpdatePricingRule": {Verb: "Update", Resource: "PricingRule", OutputListKey: "", OutputItemKey: "Tiering", Method: "PUT", URI: "/update-pricing-rule"}, }) crud.Register("blob", map[string]crud.OpMeta{ - "CreateContainer": {Verb: "Create", Resource: "Container", OutputListKey: "", OutputItemKey: ""}, + "ListBlobs": {Verb: "List", Resource: "Blob", OutputListKey: "", OutputItemKey: "", Method: "GET", URI: "/{container}?restype=container&comp=list"}, + "CreateContainer": {Verb: "Create", Resource: "Container", OutputListKey: "", OutputItemKey: "", Method: "PUT", URI: "/{container}?restype=container"}, + "DeleteContainer": {Verb: "Delete", Resource: "Container", OutputListKey: "", OutputItemKey: "", Method: "DELETE", URI: "/{container}?restype=container"}, + "GetContainerProperties": {Verb: "Get", Resource: "ContainerProperty", OutputListKey: "", OutputItemKey: "", Method: "GET", URI: "/{container}?restype=container"}, + "ListContainers": {Verb: "List", Resource: "Container", OutputListKey: "", OutputItemKey: "", Method: "GET", URI: "/?comp=list"}, + "DeleteBlob": {Verb: "Delete", Resource: "Blob", OutputListKey: "", OutputItemKey: "", Method: "DELETE", URI: "/{container}/{blob+}"}, + "GetBlob": {Verb: "Get", Resource: "Blob", OutputListKey: "", OutputItemKey: "", Method: "GET", URI: "/{container}/{blob+}"}, + "GetBlobProperties": {Verb: "Get", Resource: "BlobProperty", OutputListKey: "", OutputItemKey: "", Method: "HEAD", URI: "/{container}/{blob+}"}, + "PutBlob": {Verb: "Update", Resource: "Blob", OutputListKey: "", OutputItemKey: "", Method: "PUT", URI: "/{container}/{blob+}"}, }) crud.Register("braket", map[string]crud.OpMeta{ "CancelJob": {Verb: "Toggle", Resource: "Job", OutputListKey: "", OutputItemKey: "", Method: "PUT", URI: "/job/{jobArn}/cancel"}, diff --git a/internal/generated/fidelity/manifest_gen.go b/internal/generated/fidelity/manifest_gen.go index bcd9c2e6..dd3777c8 100644 --- a/internal/generated/fidelity/manifest_gen.go +++ b/internal/generated/fidelity/manifest_gen.go @@ -2283,8 +2283,16 @@ var Services = map[string]Service{ "UpdatePricingPlan": TierAutoCRUD, "UpdatePricingRule": TierAutoCRUD, }}, - "blob": {Protocol: "rest-json", ModelBacked: true, EngineWired: true, Operations: map[string]Tier{ - "CreateContainer": TierAutoCRUD, + "blob": {Protocol: "rest-xml", ModelBacked: true, EngineWired: true, Operations: map[string]Tier{ + "CreateContainer": TierHandVerified, + "DeleteBlob": TierHandVerified, + "DeleteContainer": TierHandVerified, + "GetBlob": TierHandVerified, + "GetBlobProperties": TierHandVerified, + "GetContainerProperties": TierHandVerified, + "ListBlobs": TierHandVerified, + "ListContainers": TierHandVerified, + "PutBlob": TierHandVerified, }}, "braket": {Protocol: "rest-json", ModelBacked: true, EngineWired: true, Operations: map[string]Tier{ "CancelJob": TierAutoCRUD, diff --git a/internal/services/blob/provider.go b/internal/services/blob/provider.go index ca687399..c8c42890 100644 --- a/internal/services/blob/provider.go +++ b/internal/services/blob/provider.go @@ -4,8 +4,16 @@ package blob import ( "context" + "crypto/rand" + "encoding/hex" + "encoding/xml" + "errors" + "fmt" "net/http" + "strings" + "time" + "github.com/skyoo2003/devcloud/internal/auth" generated "github.com/skyoo2003/devcloud/internal/generated/blob" "github.com/skyoo2003/devcloud/internal/plugin" ) @@ -14,14 +22,20 @@ import ( type Provider struct { generated.BaseProvider dataDir string + store *FileStore } func (p *Provider) ServiceID() string { return "blob" } func (p *Provider) ServiceName() string { return "Azure Blob Storage" } -func (p *Provider) Protocol() plugin.ProtocolType { return plugin.ProtocolRESTJSON } +func (p *Provider) Protocol() plugin.ProtocolType { return plugin.ProtocolRESTXML } +func (p *Provider) Provider() string { return "azure" } func (p *Provider) Init(cfg plugin.PluginConfig) error { p.dataDir = cfg.DataDir + if p.dataDir == "" { + p.dataDir = "./data/blob" + } + p.store = NewFileStore(p.dataDir) return nil } @@ -29,21 +43,467 @@ func (p *Provider) Shutdown(ctx context.Context) error { return nil } -// HandleRequest implements nothing by hand and says so, which is what hands the -// request to the generic CRUD engine (see docs/crud-engine.md). A scaffolded -// service therefore serves its CRUD-shaped operations from the moment it is -// generated; anything the engine cannot classify still returns an honest -// InvalidAction rather than a fabricated success. -// -// Declining any other way — including generated.ErrNotImplemented — is a plain -// refusal the gateway never routes to the engine, leaving the service -// registered, routed, and serving zero operations. -func (p *Provider) HandleRequest(ctx context.Context, op string, req *http.Request) (*plugin.Response, error) { +func randomRequestID() string { + b := make([]byte, 16) + _, _ = rand.Read(b) + return hex.EncodeToString(b) +} + +func (p *Provider) reply(req *http.Request, status int, headers map[string]string, body []byte) *plugin.Response { + if headers == nil { + headers = make(map[string]string) + } + headers["x-ms-request-id"] = randomRequestID() + version := req.Header.Get("x-ms-version") + if version == "" { + version = "2023-11-03" + } + headers["x-ms-version"] = version + headers["Date"] = time.Now().UTC().Format(http.TimeFormat) + + ct := headers["Content-Type"] + if ct == "" { + ct = "application/octet-stream" + } + return &plugin.Response{ + StatusCode: status, + Headers: headers, + Body: body, + ContentType: ct, + } +} + +func (p *Provider) errorReply(req *http.Request, status int, code, message string) *plugin.Response { + headers := map[string]string{ + "x-ms-error-code": code, + "Content-Type": "application/xml", + "x-ms-request-id": randomRequestID(), + "Date": time.Now().UTC().Format(http.TimeFormat), + } + version := req.Header.Get("x-ms-version") + if version == "" { + version = "2023-11-03" + } + headers["x-ms-version"] = version + + body := fmt.Sprintf("\n%s%s\nRequestId:%s\nTime:%s", + code, message, headers["x-ms-request-id"], time.Now().UTC().Format(time.RFC3339Nano)) + + return &plugin.Response{ + StatusCode: status, + Headers: headers, + Body: []byte(body), + ContentType: "application/xml", + } +} + +func (p *Provider) HandleRequest(ctx context.Context, _ string, req *http.Request) (*plugin.Response, error) { + account := "devstoreaccount1" + if id, ok := auth.FromContext(ctx); ok && id.AccessKeyID != "" { + account = id.AccessKeyID + } + + // Normalize URI: strip leading account name prefix if present + uri := req.URL.RequestURI() + for _, accPrefix := range []string{"/" + account, "/devstoreaccount1"} { + if strings.HasPrefix(uri, accPrefix+"/") { + uri = uri[len(accPrefix):] + break + } else if strings.HasPrefix(uri, accPrefix+"?") { + uri = "/" + uri[len(accPrefix):] + break + } else if uri == accPrefix { + uri = "/" + break + } + } + + op, params := generated.MatchOperation(req.Method, uri) + if op == "" { + return nil, plugin.ErrUnhandledOp + } + + if container, ok := params["container"]; ok && container != "" { + if container == "." || strings.Contains(container, "..") || strings.Contains(container, "/") || strings.Contains(container, "\\") { + return p.errorReply(req, http.StatusBadRequest, "InvalidResourceName", "The specified resource name contains invalid characters."), nil + } + } + if blob, ok := params["blob"]; ok && blob != "" { + if blob == "." || strings.Contains(blob, "..") || strings.Contains(blob, "\\") || strings.HasPrefix(blob, "/") { + return p.errorReply(req, http.StatusBadRequest, "InvalidResourceName", "The specified resource name contains invalid characters."), nil + } + } + + switch op { + case "CreateContainer": + container := params["container"] + info, err := p.store.CreateContainer(account, container) + if errors.Is(err, ErrContainerAlreadyExists) { + return p.errorReply(req, http.StatusConflict, "ContainerAlreadyExists", "The specified container already exists."), nil + } + if errors.Is(err, ErrInvalidName) { + return p.errorReply(req, http.StatusBadRequest, "InvalidResourceName", "The specified resource name contains invalid characters."), nil + } + if err != nil { + return p.errorReply(req, http.StatusInternalServerError, "InternalError", err.Error()), nil + } + headers := map[string]string{ + "ETag": info.ETag, + "Last-Modified": info.CreatedAt.Format(http.TimeFormat), + } + return p.reply(req, http.StatusCreated, headers, nil), nil + + case "DeleteContainer": + container := params["container"] + err := p.store.DeleteContainer(account, container) + if errors.Is(err, ErrContainerNotFound) { + return p.errorReply(req, http.StatusNotFound, "ContainerNotFound", "The specified container does not exist."), nil + } + if errors.Is(err, ErrInvalidName) { + return p.errorReply(req, http.StatusBadRequest, "InvalidResourceName", "The specified resource name contains invalid characters."), nil + } + if err != nil { + return p.errorReply(req, http.StatusInternalServerError, "InternalError", err.Error()), nil + } + return p.reply(req, http.StatusAccepted, nil, nil), nil + + case "GetContainerProperties": + container := params["container"] + info, err := p.store.GetContainer(account, container) + if errors.Is(err, ErrContainerNotFound) { + return p.errorReply(req, http.StatusNotFound, "ContainerNotFound", "The specified container does not exist."), nil + } + if errors.Is(err, ErrInvalidName) { + return p.errorReply(req, http.StatusBadRequest, "InvalidResourceName", "The specified resource name contains invalid characters."), nil + } + if err != nil { + return p.errorReply(req, http.StatusInternalServerError, "InternalError", err.Error()), nil + } + headers := map[string]string{ + "ETag": info.ETag, + "Last-Modified": info.CreatedAt.Format(http.TimeFormat), + "x-ms-lease-status": "unlocked", + "x-ms-lease-state": "available", + "x-ms-has-immutability-policy": "false", + "x-ms-has-legal-hold": "false", + "x-ms-default-encryption-scope": "$account-encryption-key", + "x-ms-deny-encryption-scope-override": "false", + } + return p.reply(req, http.StatusOK, headers, nil), nil + + case "ListContainers": + containers, err := p.store.ListContainers(account) + if errors.Is(err, ErrInvalidName) { + return p.errorReply(req, http.StatusBadRequest, "InvalidResourceName", "The specified resource name contains invalid characters."), nil + } + if err != nil { + return p.errorReply(req, http.StatusInternalServerError, "InternalError", err.Error()), nil + } + res := XMLContainerEnumerationResults{ + ServiceEndpoint: fmt.Sprintf("http://%s/%s/", req.Host, account), + Containers: XMLContainersWrapper{ + Container: make([]XMLContainerItem, 0, len(containers)), + }, + } + for _, c := range containers { + res.Containers.Container = append(res.Containers.Container, XMLContainerItem{ + Name: c.Name, + Properties: XMLContainerProperties{ + LastModified: c.CreatedAt.Format(http.TimeFormat), + Etag: c.ETag, + LeaseStatus: "unlocked", + LeaseState: "available", + DefaultEncryptionScope: "$account-encryption-key", + DenyEncryptionScopeOverride: "false", + HasImmutabilityPolicy: "false", + HasLegalHold: "false", + }, + }) + } + xmlBytes, err := xml.MarshalIndent(res, "", " ") + if err != nil { + return p.errorReply(req, http.StatusInternalServerError, "InternalError", err.Error()), nil + } + body := append([]byte(xml.Header), xmlBytes...) + headers := map[string]string{ + "Content-Type": "application/xml", + } + return p.reply(req, http.StatusOK, headers, body), nil + + case "PutBlob": + container := params["container"] + blob := params["blob"] + contentType := req.Header.Get("Content-Type") + info, err := p.store.PutBlob(account, container, blob, contentType, req.Body) + if errors.Is(err, ErrContainerNotFound) { + return p.errorReply(req, http.StatusNotFound, "ContainerNotFound", "The specified container does not exist."), nil + } + if errors.Is(err, ErrInvalidName) { + return p.errorReply(req, http.StatusBadRequest, "InvalidResourceName", "The specified resource name contains invalid characters."), nil + } + if err != nil { + return p.errorReply(req, http.StatusInternalServerError, "InternalError", err.Error()), nil + } + headers := map[string]string{ + "ETag": info.ETag, + "Last-Modified": info.LastModified.Format(http.TimeFormat), + "Content-MD5": info.MD5, + "x-ms-request-server-encrypted": "true", + } + return p.reply(req, http.StatusCreated, headers, nil), nil + + case "GetBlob": + container := params["container"] + blob := params["blob"] + info, data, err := p.store.GetBlob(account, container, blob) + if errors.Is(err, ErrContainerNotFound) { + return p.errorReply(req, http.StatusNotFound, "ContainerNotFound", "The specified container does not exist."), nil + } + if errors.Is(err, ErrBlobNotFound) { + return p.errorReply(req, http.StatusNotFound, "BlobNotFound", "The specified blob does not exist."), nil + } + if errors.Is(err, ErrInvalidName) { + return p.errorReply(req, http.StatusBadRequest, "InvalidResourceName", "The specified resource name contains invalid characters."), nil + } + if err != nil { + return p.errorReply(req, http.StatusInternalServerError, "InternalError", err.Error()), nil + } + headers := map[string]string{ + "Content-Type": info.ContentType, + "ETag": info.ETag, + "Last-Modified": info.LastModified.Format(http.TimeFormat), + "Content-MD5": info.MD5, + "x-ms-blob-type": "BlockBlob", + "x-ms-lease-status": "unlocked", + "x-ms-lease-state": "available", + "x-ms-server-encrypted": "true", + "x-ms-access-tier": "Hot", + "x-ms-creation-time": info.CreatedAt.Format(http.TimeFormat), + "Accept-Ranges": "bytes", + } + + rangeHeader := req.Header.Get("x-ms-range") + if rangeHeader == "" { + rangeHeader = req.Header.Get("Range") + } + + if rangeHeader != "" && info.Size > 0 { + start := int64(0) + end := info.Size - 1 + trimmed := strings.TrimPrefix(rangeHeader, "bytes=") + parts := strings.Split(trimmed, "-") + if len(parts) >= 1 && parts[0] != "" { + _, _ = fmt.Sscanf(parts[0], "%d", &start) + } + if len(parts) >= 2 && parts[1] != "" { + _, _ = fmt.Sscanf(parts[1], "%d", &end) + } + if start < 0 { + start = 0 + } + if end >= info.Size { + end = info.Size - 1 + } + if start > end { + start = end + } + slicedData := data[start : end+1] + headers["Content-Range"] = fmt.Sprintf("bytes %d-%d/%d", start, end, info.Size) + headers["Content-Length"] = fmt.Sprintf("%d", len(slicedData)) + return p.reply(req, http.StatusPartialContent, headers, slicedData), nil + } + + headers["Content-Length"] = fmt.Sprintf("%d", info.Size) + if info.Size > 0 { + headers["Content-Range"] = fmt.Sprintf("bytes 0-%d/%d", info.Size-1, info.Size) + } else { + headers["Content-Range"] = "bytes */0" + } + return p.reply(req, http.StatusOK, headers, data), nil + + case "GetBlobProperties": + container := params["container"] + blob := params["blob"] + info, err := p.store.GetBlobProperties(account, container, blob) + if errors.Is(err, ErrContainerNotFound) { + return p.errorReply(req, http.StatusNotFound, "ContainerNotFound", "The specified container does not exist."), nil + } + if errors.Is(err, ErrBlobNotFound) { + return p.errorReply(req, http.StatusNotFound, "BlobNotFound", "The specified blob does not exist."), nil + } + if errors.Is(err, ErrInvalidName) { + return p.errorReply(req, http.StatusBadRequest, "InvalidResourceName", "The specified resource name contains invalid characters."), nil + } + if err != nil { + return p.errorReply(req, http.StatusInternalServerError, "InternalError", err.Error()), nil + } + headers := map[string]string{ + "Content-Type": info.ContentType, + "Content-Length": fmt.Sprintf("%d", info.Size), + "ETag": info.ETag, + "Last-Modified": info.LastModified.Format(http.TimeFormat), + "Content-MD5": info.MD5, + "x-ms-blob-type": "BlockBlob", + "x-ms-lease-status": "unlocked", + "x-ms-lease-state": "available", + "x-ms-server-encrypted": "true", + "x-ms-access-tier": "Hot", + "x-ms-creation-time": info.CreatedAt.Format(http.TimeFormat), + } + return p.reply(req, http.StatusOK, headers, nil), nil + + case "DeleteBlob": + container := params["container"] + blob := params["blob"] + err := p.store.DeleteBlob(account, container, blob) + if errors.Is(err, ErrContainerNotFound) { + return p.errorReply(req, http.StatusNotFound, "ContainerNotFound", "The specified container does not exist."), nil + } + if errors.Is(err, ErrBlobNotFound) { + return p.errorReply(req, http.StatusNotFound, "BlobNotFound", "The specified blob does not exist."), nil + } + if errors.Is(err, ErrInvalidName) { + return p.errorReply(req, http.StatusBadRequest, "InvalidResourceName", "The specified resource name contains invalid characters."), nil + } + if err != nil { + return p.errorReply(req, http.StatusInternalServerError, "InternalError", err.Error()), nil + } + headers := map[string]string{ + "x-ms-delete-type-permanent": "true", + } + return p.reply(req, http.StatusAccepted, headers, nil), nil + + case "ListBlobs": + container := params["container"] + blobs, err := p.store.ListBlobs(account, container) + if errors.Is(err, ErrContainerNotFound) { + return p.errorReply(req, http.StatusNotFound, "ContainerNotFound", "The specified container does not exist."), nil + } + if errors.Is(err, ErrInvalidName) { + return p.errorReply(req, http.StatusBadRequest, "InvalidResourceName", "The specified resource name contains invalid characters."), nil + } + if err != nil { + return p.errorReply(req, http.StatusInternalServerError, "InternalError", err.Error()), nil + } + res := XMLBlobEnumerationResults{ + ServiceEndpoint: fmt.Sprintf("http://%s/%s/", req.Host, account), + ContainerName: container, + Blobs: XMLBlobsWrapper{ + Blob: make([]XMLBlobItem, 0, len(blobs)), + }, + } + for _, b := range blobs { + res.Blobs.Blob = append(res.Blobs.Blob, XMLBlobItem{ + Name: b.Name, + Properties: XMLBlobProperties{ + CreationTime: b.CreatedAt.Format(http.TimeFormat), + LastModified: b.LastModified.Format(http.TimeFormat), + Etag: b.ETag, + ContentLength: b.Size, + ContentType: b.ContentType, + ContentMD5: b.MD5, + BlobType: "BlockBlob", + AccessTier: "Hot", + AccessTierInferred: "true", + LeaseStatus: "unlocked", + LeaseState: "available", + ServerEncrypted: "true", + }, + }) + } + xmlBytes, err := xml.MarshalIndent(res, "", " ") + if err != nil { + return p.errorReply(req, http.StatusInternalServerError, "InternalError", err.Error()), nil + } + body := append([]byte(xml.Header), xmlBytes...) + headers := map[string]string{ + "Content-Type": "application/xml", + } + return p.reply(req, http.StatusOK, headers, body), nil + } + return nil, plugin.ErrUnhandledOp } func (p *Provider) ListResources(ctx context.Context) ([]plugin.Resource, error) { - return []plugin.Resource{}, nil + account := "devstoreaccount1" + containers, err := p.store.ListContainers(account) + if err != nil { + return []plugin.Resource{}, nil + } + res := make([]plugin.Resource, 0, len(containers)) + for _, c := range containers { + res = append(res, plugin.Resource{ + Type: "container", + ID: c.Name, + Name: c.Name, + }) + } + return res, nil +} + +type XMLContainerEnumerationResults struct { + XMLName xml.Name `xml:"EnumerationResults"` + ServiceEndpoint string `xml:"ServiceEndpoint,attr"` + Containers XMLContainersWrapper `xml:"Containers"` + NextMarker string `xml:"NextMarker"` +} + +type XMLContainersWrapper struct { + Container []XMLContainerItem `xml:"Container"` +} + +type XMLContainerItem struct { + Name string `xml:"Name"` + Properties XMLContainerProperties `xml:"Properties"` +} + +type XMLContainerProperties struct { + LastModified string `xml:"Last-Modified"` + Etag string `xml:"Etag"` + LeaseStatus string `xml:"LeaseStatus"` + LeaseState string `xml:"LeaseState"` + DefaultEncryptionScope string `xml:"DefaultEncryptionScope"` + DenyEncryptionScopeOverride string `xml:"DenyEncryptionScopeOverride"` + HasImmutabilityPolicy string `xml:"HasImmutabilityPolicy"` + HasLegalHold string `xml:"HasLegalHold"` +} + +type XMLBlobEnumerationResults struct { + XMLName xml.Name `xml:"EnumerationResults"` + ServiceEndpoint string `xml:"ServiceEndpoint,attr"` + ContainerName string `xml:"ContainerName,attr"` + Blobs XMLBlobsWrapper `xml:"Blobs"` + NextMarker string `xml:"NextMarker"` +} + +type XMLBlobsWrapper struct { + Blob []XMLBlobItem `xml:"Blob"` +} + +type XMLBlobItem struct { + Name string `xml:"Name"` + Properties XMLBlobProperties `xml:"Properties"` +} + +type XMLBlobProperties struct { + CreationTime string `xml:"Creation-Time"` + LastModified string `xml:"Last-Modified"` + Etag string `xml:"Etag"` + ContentLength int64 `xml:"Content-Length"` + ContentType string `xml:"Content-Type"` + ContentEncoding string `xml:"Content-Encoding"` + ContentLanguage string `xml:"Content-Language"` + ContentCRC64 string `xml:"Content-CRC64"` + ContentMD5 string `xml:"Content-MD5"` + CacheControl string `xml:"Cache-Control"` + ContentDisposition string `xml:"Content-Disposition"` + BlobType string `xml:"BlobType"` + AccessTier string `xml:"AccessTier"` + AccessTierInferred string `xml:"AccessTierInferred"` + LeaseStatus string `xml:"LeaseStatus"` + LeaseState string `xml:"LeaseState"` + ServerEncrypted string `xml:"ServerEncrypted"` } func init() { diff --git a/internal/services/blob/provider_test.go b/internal/services/blob/provider_test.go new file mode 100644 index 00000000..3089dcd3 --- /dev/null +++ b/internal/services/blob/provider_test.go @@ -0,0 +1,217 @@ +// SPDX-License-Identifier: Apache-2.0 + +package blob + +import ( + "bytes" + "context" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/skyoo2003/devcloud/internal/plugin" +) + +func newTestProvider(t *testing.T) *Provider { + t.Helper() + dir, err := os.MkdirTemp("", "devcloud-blob-test-*") + require.NoError(t, err) + t.Cleanup(func() { _ = os.RemoveAll(dir) }) + + p := &Provider{} + err = p.Init(plugin.PluginConfig{DataDir: filepath.Join(dir, "blob")}) + require.NoError(t, err) + return p +} + +func TestBlobProvider_Metadata(t *testing.T) { + p := newTestProvider(t) + assert.Equal(t, "blob", p.ServiceID()) + assert.Equal(t, "Azure Blob Storage", p.ServiceName()) + assert.Equal(t, plugin.ProtocolRESTXML, p.Protocol()) + assert.Equal(t, "azure", plugin.ProviderOf(p)) +} + +func TestBlobProvider_Lifecycle(t *testing.T) { + p := newTestProvider(t) + ctx := context.Background() + + // 1. Create Container + reqCreate := httptest.NewRequest(http.MethodPut, "/devstoreaccount1/mycontainer?restype=container", nil) + resp, err := p.HandleRequest(ctx, "", reqCreate) + require.NoError(t, err) + assert.Equal(t, http.StatusCreated, resp.StatusCode) + assert.NotEmpty(t, resp.Headers["ETag"]) + assert.Equal(t, "2023-11-03", resp.Headers["x-ms-version"]) + + // Duplicate create should return 409 + respDup, err := p.HandleRequest(ctx, "", reqCreate) + require.NoError(t, err) + assert.Equal(t, http.StatusConflict, respDup.StatusCode) + assert.Equal(t, "ContainerAlreadyExists", respDup.Headers["x-ms-error-code"]) + + // 2. Get Container Properties + reqGetProps := httptest.NewRequest(http.MethodGet, "/devstoreaccount1/mycontainer?restype=container", nil) + respProps, err := p.HandleRequest(ctx, "", reqGetProps) + require.NoError(t, err) + assert.Equal(t, http.StatusOK, respProps.StatusCode) + assert.Equal(t, "available", respProps.Headers["x-ms-lease-state"]) + + // 3. List Containers + reqListCont := httptest.NewRequest(http.MethodGet, "/devstoreaccount1?comp=list", nil) + respListCont, err := p.HandleRequest(ctx, "", reqListCont) + require.NoError(t, err) + assert.Equal(t, http.StatusOK, respListCont.StatusCode) + assert.Contains(t, string(respListCont.Body), "mycontainer") + + // 4. Put Blob + payload := []byte("Hello, Azure Blob Storage from DevCloud!") + reqPutBlob := httptest.NewRequest(http.MethodPut, "/devstoreaccount1/mycontainer/hello.txt", bytes.NewReader(payload)) + reqPutBlob.Header.Set("Content-Type", "text/plain") + respPutBlob, err := p.HandleRequest(ctx, "", reqPutBlob) + require.NoError(t, err) + assert.Equal(t, http.StatusCreated, respPutBlob.StatusCode) + assert.NotEmpty(t, respPutBlob.Headers["ETag"]) + assert.NotEmpty(t, respPutBlob.Headers["Content-MD5"]) + + // 5. Get Blob + reqGetBlob := httptest.NewRequest(http.MethodGet, "/devstoreaccount1/mycontainer/hello.txt", nil) + respGetBlob, err := p.HandleRequest(ctx, "", reqGetBlob) + require.NoError(t, err) + assert.Equal(t, http.StatusOK, respGetBlob.StatusCode) + assert.Equal(t, "text/plain", respGetBlob.Headers["Content-Type"]) + assert.Equal(t, payload, respGetBlob.Body) + + // 6. Get Blob Properties (HEAD) + reqHeadBlob := httptest.NewRequest(http.MethodHead, "/devstoreaccount1/mycontainer/hello.txt", nil) + respHeadBlob, err := p.HandleRequest(ctx, "", reqHeadBlob) + require.NoError(t, err) + assert.Equal(t, http.StatusOK, respHeadBlob.StatusCode) + assert.Equal(t, "BlockBlob", respHeadBlob.Headers["x-ms-blob-type"]) + assert.Empty(t, respHeadBlob.Body) + + // 7. List Blobs + reqListBlobs := httptest.NewRequest(http.MethodGet, "/devstoreaccount1/mycontainer?restype=container&comp=list", nil) + respListBlobs, err := p.HandleRequest(ctx, "", reqListBlobs) + require.NoError(t, err) + assert.Equal(t, http.StatusOK, respListBlobs.StatusCode) + assert.Contains(t, string(respListBlobs.Body), "hello.txt") + + // 8. Delete Blob + reqDelBlob := httptest.NewRequest(http.MethodDelete, "/devstoreaccount1/mycontainer/hello.txt", nil) + respDelBlob, err := p.HandleRequest(ctx, "", reqDelBlob) + require.NoError(t, err) + assert.Equal(t, http.StatusAccepted, respDelBlob.StatusCode) + + // Get deleted blob should return 404 + respGetDel, err := p.HandleRequest(ctx, "", reqGetBlob) + require.NoError(t, err) + assert.Equal(t, http.StatusNotFound, respGetDel.StatusCode) + assert.Equal(t, "BlobNotFound", respGetDel.Headers["x-ms-error-code"]) + + // 9. Delete Container + reqDelCont := httptest.NewRequest(http.MethodDelete, "/devstoreaccount1/mycontainer?restype=container", nil) + respDelCont, err := p.HandleRequest(ctx, "", reqDelCont) + require.NoError(t, err) + assert.Equal(t, http.StatusAccepted, respDelCont.StatusCode) + + // Get deleted container should return 404 + respGetDelCont, err := p.HandleRequest(ctx, "", reqGetProps) + require.NoError(t, err) + assert.Equal(t, http.StatusNotFound, respGetDelCont.StatusCode) + assert.Equal(t, "ContainerNotFound", respGetDelCont.Headers["x-ms-error-code"]) +} + +func TestBlobProvider_ListResources(t *testing.T) { + p := newTestProvider(t) + ctx := context.Background() + + _, err := p.store.CreateContainer("devstoreaccount1", "res-container-1") + require.NoError(t, err) + _, err = p.store.CreateContainer("devstoreaccount1", "res-container-2") + require.NoError(t, err) + + resources, err := p.ListResources(ctx) + require.NoError(t, err) + assert.Len(t, resources, 2) +} + +func TestFileStore_PathTraversal(t *testing.T) { + store := NewFileStore(t.TempDir()) + + // Account traversal rejected + _, err := store.CreateContainer("../escape", "cont") + assert.ErrorIs(t, err, ErrInvalidName) + + _, err = store.CreateContainer("account/sub", "cont") + assert.ErrorIs(t, err, ErrInvalidName) + + _, err = store.ListContainers("../escape") + assert.ErrorIs(t, err, ErrInvalidName) + + // Container traversal rejected + for _, badContainer := range []string{"..", "../escape", "a/b", "a\\b", ".", "/abs"} { + _, err := store.CreateContainer("devstoreaccount1", badContainer) + assert.ErrorIs(t, err, ErrInvalidName, "container %q should be rejected", badContainer) + + err = store.DeleteContainer("devstoreaccount1", badContainer) + assert.ErrorIs(t, err, ErrInvalidName, "container %q should be rejected", badContainer) + + _, err = store.GetContainer("devstoreaccount1", badContainer) + assert.ErrorIs(t, err, ErrInvalidName, "container %q should be rejected", badContainer) + } + + // Create valid container + _, err = store.CreateContainer("devstoreaccount1", "validcont") + require.NoError(t, err) + + // Blob traversal rejected + for _, badBlob := range []string{"..", "../escape", "../../secret", "/etc/passwd", "a/../../escape", "a\\b"} { + _, err := store.PutBlob("devstoreaccount1", "validcont", badBlob, "text/plain", bytes.NewReader([]byte("data"))) + assert.ErrorIs(t, err, ErrInvalidName, "blob %q should be rejected", badBlob) + + _, _, err = store.GetBlob("devstoreaccount1", "validcont", badBlob) + assert.ErrorIs(t, err, ErrInvalidName, "blob %q should be rejected", badBlob) + + _, err = store.GetBlobProperties("devstoreaccount1", "validcont", badBlob) + assert.ErrorIs(t, err, ErrInvalidName, "blob %q should be rejected", badBlob) + + err = store.DeleteBlob("devstoreaccount1", "validcont", badBlob) + assert.ErrorIs(t, err, ErrInvalidName, "blob %q should be rejected", badBlob) + } + + // Valid nested blob accepted + bi, err := store.PutBlob("devstoreaccount1", "validcont", "a/b/c.txt", "text/plain", bytes.NewReader([]byte("ok"))) + require.NoError(t, err) + assert.Equal(t, "a/b/c.txt", bi.Name) +} + +func TestBlobProvider_VirtualDirectory(t *testing.T) { + p := newTestProvider(t) + ctx := context.Background() + + // Create container + reqCreate := httptest.NewRequest(http.MethodPut, "/devstoreaccount1/vdir-container?restype=container", nil) + resp, err := p.HandleRequest(ctx, "", reqCreate) + require.NoError(t, err) + assert.Equal(t, http.StatusCreated, resp.StatusCode) + + // Put blob with virtual directory + payload := []byte("nested content") + reqPut := httptest.NewRequest(http.MethodPut, "/devstoreaccount1/vdir-container/folder/subfolder/file.txt", bytes.NewReader(payload)) + resp, err = p.HandleRequest(ctx, "", reqPut) + require.NoError(t, err) + assert.Equal(t, http.StatusCreated, resp.StatusCode) + + // Get blob with virtual directory + reqGet := httptest.NewRequest(http.MethodGet, "/devstoreaccount1/vdir-container/folder/subfolder/file.txt", nil) + resp, err = p.HandleRequest(ctx, "", reqGet) + require.NoError(t, err) + assert.Equal(t, http.StatusOK, resp.StatusCode) + assert.Equal(t, payload, resp.Body) +} diff --git a/internal/services/blob/store.go b/internal/services/blob/store.go new file mode 100644 index 00000000..d040f711 --- /dev/null +++ b/internal/services/blob/store.go @@ -0,0 +1,489 @@ +// SPDX-License-Identifier: Apache-2.0 + +package blob + +import ( + "crypto/md5" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "sync" + "time" +) + +var ( + ErrContainerNotFound = errors.New("ContainerNotFound") + ErrContainerAlreadyExists = errors.New("ContainerAlreadyExists") + ErrBlobNotFound = errors.New("BlobNotFound") + ErrInvalidName = errors.New("InvalidName") +) + +type ContainerInfo struct { + Name string `json:"name"` + Account string `json:"account"` + CreatedAt time.Time `json:"createdAt"` + ETag string `json:"etag"` +} + +type BlobInfo struct { + Name string `json:"name"` + Container string `json:"container"` + Account string `json:"account"` + Size int64 `json:"size"` + ContentType string `json:"contentType"` + MD5 string `json:"md5"` + ETag string `json:"etag"` + CreatedAt time.Time `json:"createdAt"` + LastModified time.Time `json:"lastModified"` + BlobType string `json:"blobType"` +} + +type FileStore struct { + mu sync.RWMutex + baseDir string +} + +func NewFileStore(baseDir string) *FileStore { + abs, err := filepath.Abs(baseDir) + if err != nil { + abs = filepath.Clean(baseDir) + } + cleaned := filepath.Clean(abs) + _ = os.MkdirAll(cleaned, 0755) + return &FileStore{baseDir: cleaned} +} + +func (fs *FileStore) safeContainerDir(account, container string) (string, error) { + if account == "" || container == "" || account == "." || container == "." { + return "", ErrInvalidName + } + if strings.Contains(account, "..") || strings.Contains(account, "/") || strings.Contains(account, "\\") { + return "", ErrInvalidName + } + if strings.Contains(container, "..") || strings.Contains(container, "/") || strings.Contains(container, "\\") { + return "", ErrInvalidName + } + if !filepath.IsLocal(account) || !filepath.IsLocal(container) { + return "", ErrInvalidName + } + + dir := filepath.Clean(filepath.Join(fs.baseDir, account, container)) + basePrefix := fs.baseDir + string(filepath.Separator) + if !strings.HasPrefix(dir, basePrefix) { + return "", ErrInvalidName + } + rel, err := filepath.Rel(fs.baseDir, dir) + if err != nil || !filepath.IsLocal(rel) || rel == "." { + return "", ErrInvalidName + } + return dir, nil +} + +func (fs *FileStore) safeBlobPaths(account, container, blob string) (dataPath, metaPath string, err error) { + containerDir, err := fs.safeContainerDir(account, container) + if err != nil { + return "", "", err + } + if blob == "" || strings.Contains(blob, "..") || strings.Contains(blob, "\\") || strings.HasPrefix(blob, "/") { + return "", "", ErrInvalidName + } + if !filepath.IsLocal(blob) || blob == "." { + return "", "", ErrInvalidName + } + + cleanBlob := filepath.Clean(blob) + if strings.Contains(cleanBlob, "..") || strings.HasPrefix(cleanBlob, "/") || cleanBlob == "." { + return "", "", ErrInvalidName + } + if !filepath.IsLocal(cleanBlob) { + return "", "", ErrInvalidName + } + + dataDir := filepath.Clean(filepath.Join(containerDir, "data")) + dataPath = filepath.Clean(filepath.Join(dataDir, cleanBlob)) + dataPrefix := dataDir + string(filepath.Separator) + if !strings.HasPrefix(dataPath, dataPrefix) { + return "", "", ErrInvalidName + } + relData, err := filepath.Rel(dataDir, dataPath) + if err != nil || !filepath.IsLocal(relData) || relData == "." { + return "", "", ErrInvalidName + } + + metaDir := filepath.Clean(filepath.Join(containerDir, "meta")) + metaPath = filepath.Clean(filepath.Join(metaDir, cleanBlob+".json")) + metaPrefix := metaDir + string(filepath.Separator) + if !strings.HasPrefix(metaPath, metaPrefix) { + return "", "", ErrInvalidName + } + relMeta, err := filepath.Rel(metaDir, metaPath) + if err != nil || !filepath.IsLocal(relMeta) || relMeta == "." { + return "", "", ErrInvalidName + } + + return dataPath, metaPath, nil +} + +func (fs *FileStore) CreateContainer(account, container string) (*ContainerInfo, error) { + fs.mu.Lock() + defer fs.mu.Unlock() + + dir, err := fs.safeContainerDir(account, container) + if err != nil { + return nil, err + } + + metaFile := filepath.Join(dir, "container.json") + if _, err := os.Stat(metaFile); err == nil { + return nil, ErrContainerAlreadyExists + } + + if err := os.MkdirAll(filepath.Join(dir, "data"), 0755); err != nil { + return nil, err + } + if err := os.MkdirAll(filepath.Join(dir, "meta"), 0755); err != nil { + return nil, err + } + + now := time.Now().UTC() + etag := fmt.Sprintf("\"0x%X\"", now.UnixNano()) + info := &ContainerInfo{ + Name: container, + Account: account, + CreatedAt: now, + ETag: etag, + } + + data, err := json.Marshal(info) + if err != nil { + return nil, err + } + if err := os.WriteFile(metaFile, data, 0644); err != nil { + return nil, err + } + return info, nil +} + +func (fs *FileStore) DeleteContainer(account, container string) error { + fs.mu.Lock() + defer fs.mu.Unlock() + + dir, err := fs.safeContainerDir(account, container) + if err != nil { + return err + } + + metaFile := filepath.Join(dir, "container.json") + if _, err := os.Stat(metaFile); os.IsNotExist(err) { + return ErrContainerNotFound + } + + return os.RemoveAll(dir) +} + +func (fs *FileStore) GetContainer(account, container string) (*ContainerInfo, error) { + fs.mu.RLock() + defer fs.mu.RUnlock() + + dir, err := fs.safeContainerDir(account, container) + if err != nil { + return nil, err + } + + metaFile := filepath.Join(dir, "container.json") + data, err := os.ReadFile(metaFile) + if err != nil { + if os.IsNotExist(err) { + return nil, ErrContainerNotFound + } + return nil, err + } + + var info ContainerInfo + if err := json.Unmarshal(data, &info); err != nil { + return nil, err + } + return &info, nil +} + +func (fs *FileStore) ListContainers(account string) ([]ContainerInfo, error) { + fs.mu.RLock() + defer fs.mu.RUnlock() + + if account == "" || account == "." || strings.Contains(account, "..") || strings.Contains(account, "/") || strings.Contains(account, "\\") { + return nil, ErrInvalidName + } + if !filepath.IsLocal(account) { + return nil, ErrInvalidName + } + + accountDir := filepath.Clean(filepath.Join(fs.baseDir, account)) + basePrefix := fs.baseDir + string(filepath.Separator) + if !strings.HasPrefix(accountDir, basePrefix) { + return nil, ErrInvalidName + } + rel, err := filepath.Rel(fs.baseDir, accountDir) + if err != nil || !filepath.IsLocal(rel) || rel == "." { + return nil, ErrInvalidName + } + + entries, err := os.ReadDir(accountDir) + if err != nil { + if os.IsNotExist(err) { + return []ContainerInfo{}, nil + } + return nil, err + } + + var list []ContainerInfo + for _, entry := range entries { + if !entry.IsDir() { + continue + } + metaFile := filepath.Join(accountDir, entry.Name(), "container.json") + data, err := os.ReadFile(metaFile) + if err != nil { + continue + } + var info ContainerInfo + if err := json.Unmarshal(data, &info); err == nil { + list = append(list, info) + } + } + return list, nil +} + +func (fs *FileStore) PutBlob(account, container, blob string, contentType string, r io.Reader) (*BlobInfo, error) { + fs.mu.Lock() + defer fs.mu.Unlock() + + // Ensure container exists + containerDir, err := fs.safeContainerDir(account, container) + if err != nil { + return nil, err + } + if _, err := os.Stat(filepath.Join(containerDir, "container.json")); os.IsNotExist(err) { + return nil, ErrContainerNotFound + } + + dataPath, metaPath, err := fs.safeBlobPaths(account, container, blob) + if err != nil { + return nil, err + } + + if err := os.MkdirAll(filepath.Dir(dataPath), 0755); err != nil { + return nil, err + } + if err := os.MkdirAll(filepath.Dir(metaPath), 0755); err != nil { + return nil, err + } + + // Read content and compute MD5 + tmpFile := filepath.Clean(dataPath + ".tmp") + dataDir := filepath.Clean(filepath.Join(containerDir, "data")) + dataPrefix := dataDir + string(filepath.Separator) + if !strings.HasPrefix(tmpFile, dataPrefix) { + return nil, ErrInvalidName + } + relTmp, err := filepath.Rel(dataDir, tmpFile) + if err != nil || !filepath.IsLocal(relTmp) || relTmp == "." { + return nil, ErrInvalidName + } + f, err := os.Create(tmpFile) + if err != nil { + return nil, err + } + + hasher := md5.New() + mw := io.MultiWriter(f, hasher) + size, err := io.Copy(mw, r) + _ = f.Close() + if err != nil { + _ = os.Remove(tmpFile) + return nil, err + } + + if err := os.Rename(tmpFile, dataPath); err != nil { + _ = os.Remove(tmpFile) + return nil, err + } + + md5Sum := hex.EncodeToString(hasher.Sum(nil)) + now := time.Now().UTC() + etag := fmt.Sprintf("\"0x%X\"", now.UnixNano()) + if contentType == "" { + contentType = "application/octet-stream" + } + + // Check existing metadata to preserve creation time + createdAt := now + if existingData, err := os.ReadFile(metaPath); err == nil { + var existing BlobInfo + if json.Unmarshal(existingData, &existing) == nil && !existing.CreatedAt.IsZero() { + createdAt = existing.CreatedAt + } + } + + info := &BlobInfo{ + Name: blob, + Container: container, + Account: account, + Size: size, + ContentType: contentType, + MD5: md5Sum, + ETag: etag, + CreatedAt: createdAt, + LastModified: now, + BlobType: "BlockBlob", + } + + metaBytes, err := json.Marshal(info) + if err != nil { + return nil, err + } + if err := os.WriteFile(metaPath, metaBytes, 0644); err != nil { + return nil, err + } + return info, nil +} + +func (fs *FileStore) GetBlob(account, container, blob string) (*BlobInfo, []byte, error) { + fs.mu.RLock() + defer fs.mu.RUnlock() + + containerDir, err := fs.safeContainerDir(account, container) + if err != nil { + return nil, nil, err + } + if _, err := os.Stat(filepath.Join(containerDir, "container.json")); os.IsNotExist(err) { + return nil, nil, ErrContainerNotFound + } + + dataPath, metaPath, err := fs.safeBlobPaths(account, container, blob) + if err != nil { + return nil, nil, err + } + + metaBytes, err := os.ReadFile(metaPath) + if err != nil { + if os.IsNotExist(err) { + return nil, nil, ErrBlobNotFound + } + return nil, nil, err + } + + var info BlobInfo + if err := json.Unmarshal(metaBytes, &info); err != nil { + return nil, nil, err + } + + data, err := os.ReadFile(dataPath) + if err != nil { + if os.IsNotExist(err) { + return nil, nil, ErrBlobNotFound + } + return nil, nil, err + } + return &info, data, nil +} + +func (fs *FileStore) GetBlobProperties(account, container, blob string) (*BlobInfo, error) { + fs.mu.RLock() + defer fs.mu.RUnlock() + + containerDir, err := fs.safeContainerDir(account, container) + if err != nil { + return nil, err + } + if _, err := os.Stat(filepath.Join(containerDir, "container.json")); os.IsNotExist(err) { + return nil, ErrContainerNotFound + } + + _, metaPath, err := fs.safeBlobPaths(account, container, blob) + if err != nil { + return nil, err + } + + metaBytes, err := os.ReadFile(metaPath) + if err != nil { + if os.IsNotExist(err) { + return nil, ErrBlobNotFound + } + return nil, err + } + + var info BlobInfo + if err := json.Unmarshal(metaBytes, &info); err != nil { + return nil, err + } + return &info, nil +} + +func (fs *FileStore) DeleteBlob(account, container, blob string) error { + fs.mu.Lock() + defer fs.mu.Unlock() + + containerDir, err := fs.safeContainerDir(account, container) + if err != nil { + return err + } + if _, err := os.Stat(filepath.Join(containerDir, "container.json")); os.IsNotExist(err) { + return ErrContainerNotFound + } + + dataPath, metaPath, err := fs.safeBlobPaths(account, container, blob) + if err != nil { + return err + } + + if _, err := os.Stat(metaPath); os.IsNotExist(err) { + return ErrBlobNotFound + } + + _ = os.Remove(dataPath) + _ = os.Remove(metaPath) + return nil +} + +func (fs *FileStore) ListBlobs(account, container string) ([]BlobInfo, error) { + fs.mu.RLock() + defer fs.mu.RUnlock() + + containerDir, err := fs.safeContainerDir(account, container) + if err != nil { + return nil, err + } + if _, err := os.Stat(filepath.Join(containerDir, "container.json")); os.IsNotExist(err) { + return nil, ErrContainerNotFound + } + + metaDir := filepath.Join(containerDir, "meta") + if _, err := os.Stat(metaDir); os.IsNotExist(err) { + return []BlobInfo{}, nil + } + + var blobs []BlobInfo + err = filepath.Walk(metaDir, func(path string, info os.FileInfo, err error) error { + if err != nil || info.IsDir() || !strings.HasSuffix(path, ".json") { + return nil + } + data, readErr := os.ReadFile(path) + if readErr != nil { + return nil + } + var bi BlobInfo + if json.Unmarshal(data, &bi) == nil { + blobs = append(blobs, bi) + } + return nil + }) + if err != nil { + return nil, err + } + return blobs, nil +} diff --git a/test/compatibility/test_azure_blob.py b/test/compatibility/test_azure_blob.py index 75735f43..c5d4610d 100644 --- a/test/compatibility/test_azure_blob.py +++ b/test/compatibility/test_azure_blob.py @@ -1,13 +1,84 @@ +from azure.core.exceptions import ResourceExistsError, ResourceNotFoundError from azure.storage.blob import BlobServiceClient +from conftest import DEVCLOUD_URL +import pytest -def test_blob_create_container(): - # Proof of concept test against DevCloud's Azure mock - # This uses local HTTP endpoint with a well-known development account key - _ = BlobServiceClient( - account_url="http://127.0.0.1:4747/devstoreaccount1", - credential="DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:4747/devstoreaccount1;", +@pytest.fixture +def azure_blob_client(devcloud_server): + endpoint = f"{DEVCLOUD_URL}/devstoreaccount1" + conn_str = ( + "DefaultEndpointsProtocol=http;" + "AccountName=devstoreaccount1;" + "AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;" + f"BlobEndpoint={endpoint};" ) - # The actual implementation of the mock will be expanded in a later phase. - # For now, we just want to ensure we can instantiate the client and hit the endpoint. - pass + return BlobServiceClient.from_connection_string(conn_str) + + +def test_blob_container_lifecycle(azure_blob_client): + client = azure_blob_client + container_name = "test-container-lifecycle" + + # 1. Create container + container_client = client.create_container(container_name) + assert container_client.container_name == container_name + + # 2. Duplicate create raises ResourceExistsError + with pytest.raises(ResourceExistsError): + client.create_container(container_name) + + # 3. Get container properties + props = container_client.get_container_properties() + assert props.name == container_name + assert props.etag is not None + + # 4. List containers + container_names = [c.name for c in client.list_containers()] + assert container_name in container_names + + # 5. Delete container + client.delete_container(container_name) + + # 6. Accessing deleted container raises ResourceNotFoundError + with pytest.raises(ResourceNotFoundError): + container_client.get_container_properties() + + +def test_blob_crud_lifecycle(azure_blob_client): + client = azure_blob_client + container_name = "test-blob-crud" + container_client = client.create_container(container_name) + + try: + blob_name = "docs/sample.txt" + blob_client = container_client.get_blob_client(blob_name) + payload = b"DevCloud Azure Blob Storage Phase 3 Test Payload" + + # 1. Upload BlockBlob + blob_client.upload_blob(payload) + + # 2. Get Blob Properties + props = blob_client.get_blob_properties() + assert props.name == blob_name + assert props.size == len(payload) + assert props.blob_type == "BlockBlob" + + # 3. Download Blob and verify data + download_stream = blob_client.download_blob() + downloaded = download_stream.readall() + assert downloaded == payload + + # 4. List Blobs in container + blob_names = [b.name for b in container_client.list_blobs()] + assert blob_name in blob_names + + # 5. Delete Blob + blob_client.delete_blob() + + # 6. Accessing deleted blob raises ResourceNotFoundError + with pytest.raises(ResourceNotFoundError): + blob_client.get_blob_properties() + finally: + # Clean up container + client.delete_container(container_name)