diff --git a/README.md b/README.md index b99b4797..eb2ad737 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ DevCloud is an **on-ramp to the cloud**, not a replacement for it. The goal is t ## Features -- **432 AWS services registered, 432 serving at least one operation** — every service AWS publishes is routed, so no SDK call escapes to a billable account. Depth is a separate, smaller promise — see [coverage.md](docs/coverage.md) for both targets. +- **437 AWS services registered, 437 serving at least one operation** — every service AWS publishes is routed, so no SDK call escapes to a billable account. Depth is a separate, smaller promise — see [coverage.md](docs/coverage.md) for both targets. - **boto3-compatible** — a 1,530-test suite runs in CI (`make test-compat`) across every registered service. Unsupported operations return a clean AWS error, never a false success. - **Cross-service integration** — CloudFormation provisioning, DynamoDB Streams → Lambda, EventBridge targets, S3 → Lambda - **Smithy-driven codegen** — Go types, routers and error catalogues generated from AWS models, with a weekly sync workflow that keeps them current diff --git a/api/models/azure-cosmos-db.json b/api/models/azure-cosmos-db.json new file mode 100644 index 00000000..d78ee038 --- /dev/null +++ b/api/models/azure-cosmos-db.json @@ -0,0 +1,335 @@ +{ + "openapi": "3.0.0", + "info": { + "title": "Azure Cosmos DB", + "version": "2018-12-31", + "description": "Azure Cosmos DB SQL API REST API", + "x-service-id": "cosmosdb", + "x-provider": "azure", + "x-protocol": "rest-json" + }, + "paths": { + "/dbs": { + "get": { + "operationId": "ListDatabases", + "description": "Lists all databases in the account", + "responses": { + "200": { + "description": "List of databases" + } + } + }, + "post": { + "operationId": "CreateDatabase", + "description": "Creates a new database", + "responses": { + "201": { + "description": "Database created" + } + } + } + }, + "/dbs/{db}": { + "get": { + "operationId": "GetDatabase", + "description": "Retrieves a database by ID", + "parameters": [ + { + "name": "db", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Database retrieved" + } + } + }, + "delete": { + "operationId": "DeleteDatabase", + "description": "Deletes the specified database", + "parameters": [ + { + "name": "db", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "204": { + "description": "Database deleted" + } + } + } + }, + "/dbs/{db}/colls": { + "get": { + "operationId": "ListContainers", + "description": "Lists containers in the database", + "parameters": [ + { + "name": "db", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "List of containers" + } + } + }, + "post": { + "operationId": "CreateContainer", + "description": "Creates a new container", + "parameters": [ + { + "name": "db", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "201": { + "description": "Container created" + } + } + } + }, + "/dbs/{db}/colls/{coll}": { + "get": { + "operationId": "GetContainer", + "description": "Retrieves container properties", + "parameters": [ + { + "name": "db", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "coll", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Container retrieved" + } + } + }, + "delete": { + "operationId": "DeleteContainer", + "description": "Deletes the container", + "parameters": [ + { + "name": "db", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "coll", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "204": { + "description": "Container deleted" + } + } + } + }, + "/dbs/{db}/colls/{coll}/docs": { + "get": { + "operationId": "ListDocuments", + "description": "Lists documents in the container", + "parameters": [ + { + "name": "db", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "coll", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "List of documents" + } + } + }, + "post": { + "operationId": "CreateDocument", + "description": "Creates a new document", + "parameters": [ + { + "name": "db", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "coll", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "201": { + "description": "Document created" + } + } + } + }, + "/dbs/{db}/colls/{coll}/docs/{doc}": { + "get": { + "operationId": "GetDocument", + "description": "Retrieves a document", + "parameters": [ + { + "name": "db", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "coll", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "doc", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Document retrieved" + } + } + }, + "put": { + "operationId": "ReplaceDocument", + "description": "Replaces an existing document", + "parameters": [ + { + "name": "db", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "coll", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "doc", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Document replaced" + } + } + }, + "delete": { + "operationId": "DeleteDocument", + "description": "Deletes the document", + "parameters": [ + { + "name": "db", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "coll", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "doc", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "204": { + "description": "Document deleted" + } + } + } + } + }, + "components": { + "schemas": {} + } +} diff --git a/api/models/azure-queue-storage.json b/api/models/azure-queue-storage.json new file mode 100644 index 00000000..9b09efd1 --- /dev/null +++ b/api/models/azure-queue-storage.json @@ -0,0 +1,257 @@ +{ + "openapi": "3.0.0", + "info": { + "title": "Azure Queue Storage", + "version": "2023-11-03", + "description": "Azure Queue Storage REST API", + "x-service-id": "azurequeue", + "x-provider": "azure", + "x-protocol": "rest-xml" + }, + "paths": { + "/?comp=list": { + "get": { + "operationId": "ListQueues", + "description": "Lists all queues in the storage account", + "parameters": [ + { + "name": "comp", + "in": "query", + "required": true, + "schema": { + "type": "string", + "enum": ["list"] + } + } + ], + "responses": { + "200": { + "description": "List of queues" + } + } + } + }, + "/{queue}": { + "put": { + "operationId": "CreateQueue", + "description": "Creates a queue", + "parameters": [ + { + "name": "queue", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "201": { + "description": "Queue created" + }, + "204": { + "description": "Queue already exists" + } + } + }, + "delete": { + "operationId": "DeleteQueue", + "description": "Deletes the queue", + "parameters": [ + { + "name": "queue", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "204": { + "description": "Queue deleted" + } + } + } + }, + "/{queue}?comp=metadata": { + "get": { + "operationId": "GetQueueMetadata", + "description": "Retrieves queue metadata and approximate message count", + "parameters": [ + { + "name": "queue", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "comp", + "in": "query", + "required": true, + "schema": { + "type": "string", + "enum": ["metadata"] + } + } + ], + "responses": { + "200": { + "description": "Queue metadata" + } + } + }, + "put": { + "operationId": "SetQueueMetadata", + "description": "Sets user-defined metadata on the queue", + "parameters": [ + { + "name": "queue", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "comp", + "in": "query", + "required": true, + "schema": { + "type": "string", + "enum": ["metadata"] + } + } + ], + "responses": { + "204": { + "description": "Metadata updated" + } + } + } + }, + "/{queue}/messages": { + "post": { + "operationId": "PutMessage", + "description": "Adds a message to the queue", + "parameters": [ + { + "name": "queue", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "201": { + "description": "Message created" + } + } + }, + "get": { + "operationId": "GetMessages", + "description": "Retrieves messages from the queue", + "parameters": [ + { + "name": "queue", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Queue messages list" + } + } + }, + "delete": { + "operationId": "ClearMessages", + "description": "Deletes all messages from the queue", + "parameters": [ + { + "name": "queue", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "204": { + "description": "Queue cleared" + } + } + } + }, + "/{queue}/messages?peekonly=true": { + "get": { + "operationId": "PeekMessages", + "description": "Retrieves messages without altering visibility", + "parameters": [ + { + "name": "queue", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "peekonly", + "in": "query", + "required": true, + "schema": { + "type": "string", + "enum": ["true"] + } + } + ], + "responses": { + "200": { + "description": "Queue messages list" + } + } + } + }, + "/{queue}/messages/{messageid}": { + "delete": { + "operationId": "DeleteMessage", + "description": "Deletes the specified message", + "parameters": [ + { + "name": "queue", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "messageid", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "204": { + "description": "Message deleted" + } + } + } + } + }, + "components": { + "schemas": {} + } +} diff --git a/api/models/azure-table-storage.json b/api/models/azure-table-storage.json new file mode 100644 index 00000000..9d183fd5 --- /dev/null +++ b/api/models/azure-table-storage.json @@ -0,0 +1,247 @@ +{ + "openapi": "3.0.0", + "info": { + "title": "Azure Table Storage", + "version": "2023-11-03", + "description": "Azure Table Storage REST API", + "x-service-id": "azuretable", + "x-provider": "azure", + "x-protocol": "rest-json" + }, + "paths": { + "/Tables": { + "get": { + "operationId": "QueryTables", + "description": "Queries all tables in the account", + "responses": { + "200": { + "description": "Table query result" + } + } + }, + "post": { + "operationId": "CreateTable", + "description": "Creates a new table", + "responses": { + "201": { + "description": "Table created" + }, + "204": { + "description": "Table created without content" + } + } + } + }, + "/Tables('{table}')": { + "delete": { + "operationId": "DeleteTable", + "description": "Deletes the specified table", + "parameters": [ + { + "name": "table", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "204": { + "description": "Table deleted" + } + } + } + }, + "/{table}()": { + "get": { + "operationId": "QueryEntities", + "description": "Queries entities in the table", + "parameters": [ + { + "name": "table", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Entity query results" + } + } + } + }, + "/{table}": { + "post": { + "operationId": "InsertEntity", + "description": "Inserts a new entity", + "parameters": [ + { + "name": "table", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "201": { + "description": "Entity created" + }, + "204": { + "description": "Entity created without content" + } + } + } + }, + "/{table}(PartitionKey='{partitionKey}',RowKey='{rowKey}')": { + "get": { + "operationId": "GetEntity", + "description": "Retrieves an entity by PartitionKey and RowKey", + "parameters": [ + { + "name": "table", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "partitionKey", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "rowKey", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Entity returned" + } + } + }, + "put": { + "operationId": "UpdateEntity", + "description": "Replaces an entity", + "parameters": [ + { + "name": "table", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "partitionKey", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "rowKey", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "204": { + "description": "Entity updated" + } + } + }, + "merge": { + "operationId": "MergeEntity", + "description": "Merges properties into an entity", + "parameters": [ + { + "name": "table", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "partitionKey", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "rowKey", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "204": { + "description": "Entity merged" + } + } + }, + "delete": { + "operationId": "DeleteEntity", + "description": "Deletes an entity", + "parameters": [ + { + "name": "table", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "partitionKey", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "rowKey", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "204": { + "description": "Entity deleted" + } + } + } + } + }, + "components": { + "schemas": {} + } +} diff --git a/api/models/devcloud-identity.json b/api/models/devcloud-identity.json new file mode 100644 index 00000000..5e2806ce --- /dev/null +++ b/api/models/devcloud-identity.json @@ -0,0 +1,68 @@ +{ + "openapi": "3.0.0", + "info": { + "title": "DevCloud Federated Identity", + "version": "v1", + "description": "DevCloud Federated Identity & Cross-CSP Playground", + "x-service-id": "identity", + "x-provider": "devcloud", + "x-protocol": "rest-json" + }, + "paths": { + "/devcloud/identity/.well-known/openid-configuration": { + "get": { + "operationId": "GetOpenIDConfiguration", + "description": "Retrieves OIDC discovery document", + "responses": { + "200": { + "description": "OIDC discovery configuration" + } + } + } + }, + "/devcloud/identity/jwks.json": { + "get": { + "operationId": "GetJWKS", + "description": "Retrieves public JSON Web Key Set", + "responses": { + "200": { + "description": "Public JWKS keys" + } + } + } + }, + "/devcloud/identity/token": { + "post": { + "operationId": "MintToken", + "description": "Mints a simulated OIDC RS256 JWT", + "responses": { + "200": { + "description": "Minted token" + } + } + } + }, + "/devcloud/identity/azure/oauth2/v2.0/token": { + "post": { + "operationId": "ExchangeAzureToken", + "description": "Exchanges subject assertion for Azure AD Bearer token", + "responses": { + "200": { + "description": "Azure AD token" + } + } + } + }, + "/devcloud/identity/gcp/v1/token": { + "post": { + "operationId": "ExchangeGCPToken", + "description": "Exchanges subject token for Google OAuth2 access token", + "responses": { + "200": { + "description": "GCP access token" + } + } + } + } + } +} diff --git a/api/models/gcp-storage.json b/api/models/gcp-storage.json new file mode 100644 index 00000000..fe27277e --- /dev/null +++ b/api/models/gcp-storage.json @@ -0,0 +1,306 @@ +{ + "openapi": "3.0.0", + "info": { + "title": "Google Cloud Storage", + "version": "v1", + "description": "Google Cloud Storage JSON API v1", + "x-service-id": "gcs", + "x-provider": "gcp", + "x-protocol": "rest-json" + }, + "paths": { + "/storage/v1/b": { + "get": { + "operationId": "ListBuckets", + "description": "Retrieves a list of buckets for a given project", + "parameters": [ + { + "name": "project", + "in": "query", + "required": false, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "List of buckets", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BucketList" + } + } + } + } + } + }, + "post": { + "operationId": "InsertBucket", + "description": "Creates a new bucket", + "parameters": [ + { + "name": "project", + "in": "query", + "required": false, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Created bucket metadata", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Bucket" + } + } + } + } + } + } + }, + "/storage/v1/b/{bucket}": { + "get": { + "operationId": "GetBucket", + "description": "Returns metadata for the specified bucket", + "parameters": [ + { + "name": "bucket", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Bucket metadata", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Bucket" + } + } + } + } + } + }, + "delete": { + "operationId": "DeleteBucket", + "description": "Permanently deletes an empty bucket", + "parameters": [ + { + "name": "bucket", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "204": { + "description": "Bucket deleted" + } + } + } + }, + "/storage/v1/b/{bucket}/o": { + "get": { + "operationId": "ListObjects", + "description": "Retrieves a list of objects matching the criteria", + "parameters": [ + { + "name": "bucket", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "prefix", + "in": "query", + "required": false, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "List of objects", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ObjectList" + } + } + } + } + } + } + }, + "/storage/v1/b/{bucket}/o/{object+}": { + "get": { + "operationId": "GetObject", + "description": "Retrieves an object or its metadata", + "parameters": [ + { + "name": "bucket", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "object", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "alt", + "in": "query", + "required": false, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Object metadata or media stream" + } + } + }, + "delete": { + "operationId": "DeleteObject", + "description": "Deletes an object and its metadata", + "parameters": [ + { + "name": "bucket", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "object", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "204": { + "description": "Object deleted" + } + } + } + }, + "/upload/storage/v1/b/{bucket}/o": { + "post": { + "operationId": "InsertObject", + "description": "Stores a new object and metadata", + "parameters": [ + { + "name": "bucket", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "name", + "in": "query", + "required": false, + "schema": { + "type": "string" + } + }, + { + "name": "uploadType", + "in": "query", + "required": false, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Object resource", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Object" + } + } + } + } + } + } + } + }, + "components": { + "schemas": { + "Bucket": { + "type": "object", + "properties": { + "kind": { "type": "string" }, + "id": { "type": "string" }, + "name": { "type": "string" }, + "projectNumber": { "type": "string" }, + "location": { "type": "string" }, + "storageClass": { "type": "string" }, + "timeCreated": { "type": "string" }, + "updated": { "type": "string" }, + "etag": { "type": "string" } + } + }, + "BucketList": { + "type": "object", + "properties": { + "kind": { "type": "string" }, + "items": { "type": "string" } + } + }, + "Object": { + "type": "object", + "properties": { + "kind": { "type": "string" }, + "id": { "type": "string" }, + "name": { "type": "string" }, + "bucket": { "type": "string" }, + "size": { "type": "string" }, + "contentType": { "type": "string" }, + "md5Hash": { "type": "string" }, + "crc32c": { "type": "string" }, + "etag": { "type": "string" }, + "timeCreated": { "type": "string" }, + "updated": { "type": "string" } + } + }, + "ObjectList": { + "type": "object", + "properties": { + "kind": { "type": "string" }, + "items": { "type": "string" } + } + } + } + } +} diff --git a/cmd/devcloud/imports.go b/cmd/devcloud/imports.go index 8d2bf401..4c78bcfe 100644 --- a/cmd/devcloud/imports.go +++ b/cmd/devcloud/imports.go @@ -39,6 +39,8 @@ import ( _ "github.com/skyoo2003/devcloud/internal/services/auditmanager" _ "github.com/skyoo2003/devcloud/internal/services/autoscaling" _ "github.com/skyoo2003/devcloud/internal/services/autoscalingplans" + _ "github.com/skyoo2003/devcloud/internal/services/azurequeue" + _ "github.com/skyoo2003/devcloud/internal/services/azuretable" _ "github.com/skyoo2003/devcloud/internal/services/b2bi" _ "github.com/skyoo2003/devcloud/internal/services/backup" _ "github.com/skyoo2003/devcloud/internal/services/backupgateway" @@ -114,6 +116,7 @@ import ( _ "github.com/skyoo2003/devcloud/internal/services/connectparticipant" _ "github.com/skyoo2003/devcloud/internal/services/controlcatalog" _ "github.com/skyoo2003/devcloud/internal/services/controltower" + _ "github.com/skyoo2003/devcloud/internal/services/cosmosdb" _ "github.com/skyoo2003/devcloud/internal/services/costandusagereportservice" _ "github.com/skyoo2003/devcloud/internal/services/costexplorer" _ "github.com/skyoo2003/devcloud/internal/services/costoptimizationhub" @@ -173,6 +176,7 @@ import ( _ "github.com/skyoo2003/devcloud/internal/services/fsx" _ "github.com/skyoo2003/devcloud/internal/services/gamelift" _ "github.com/skyoo2003/devcloud/internal/services/gameliftstreams" + _ "github.com/skyoo2003/devcloud/internal/services/gcs" _ "github.com/skyoo2003/devcloud/internal/services/geomaps" _ "github.com/skyoo2003/devcloud/internal/services/geoplaces" _ "github.com/skyoo2003/devcloud/internal/services/georoutes" @@ -188,6 +192,7 @@ import ( _ "github.com/skyoo2003/devcloud/internal/services/healthlake" _ "github.com/skyoo2003/devcloud/internal/services/iam" _ "github.com/skyoo2003/devcloud/internal/services/iamtoolbox" + _ "github.com/skyoo2003/devcloud/internal/services/identity" _ "github.com/skyoo2003/devcloud/internal/services/identitystore" _ "github.com/skyoo2003/devcloud/internal/services/imagebuilder" _ "github.com/skyoo2003/devcloud/internal/services/inspector" diff --git a/cmd/devcloud/main.go b/cmd/devcloud/main.go index 18486ca2..2f98059e 100644 --- a/cmd/devcloud/main.go +++ b/cmd/devcloud/main.go @@ -27,7 +27,7 @@ import ( // dependency order — sts borrows iam's store, so iam precedes it. A failure // here is fatal; every other registered service only warns. var initOrder = []string{ - "s3", "sqs", "dynamodb", "iam", "sts", "lambda", + "s3", "sqs", "dynamodb", "iam", "sts", "identity", "lambda", "kms", "sns", "secretsmanager", "ssm", "cloudwatchlogs", "cloudwatch", "eventbridge", "ec2", "ecs", "ecr", "route53", "acm", } diff --git a/docs/README.md b/docs/README.md index f73c908b..14b3d736 100644 --- a/docs/README.md +++ b/docs/README.md @@ -13,7 +13,7 @@ | Page | What it covers | |---|---| -| [Coverage](coverage.md) | 432 registered / 432 serving — the routing and depth targets, and what each promises | +| [Coverage](coverage.md) | 437 registered / 437 serving — the routing and depth targets, and what each promises | | [Compatibility Policy](compatibility-policy.md) | What v1.0 guarantees across 1.x, what it does not, and how deprecation works | | [Fidelity Manifest](fidelity-manifest.md) | Per-operation tiers: how much to trust any given call | | [CRUD Engine](crud-engine.md) | How engine-served operations behave, and where they stop | diff --git a/docs/architecture.md b/docs/architecture.md index 33ac4972..512a1b3f 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -198,3 +198,9 @@ Phase 3 established the first non-AWS pilot (**Azure Blob Storage**) exercising four seams: OpenAPI models in `api/models/` parsed via `OpenAPISource`, namespaced configuration under `providers.azure.services.blob`, `ProviderScoped` plugin registration, and `AzureSharedKey` authentication. + +Phase 4 expanded multi-CSP breadth to **Google Cloud Platform** (`providers.gcp.services.gcs`) +with OAuth2 Bearer token authentication (`GCPAuth`), expanded **Azure Storage & Database** +capabilities (Queue Storage `azurequeue`, Table Storage `azuretable`, and Cosmos DB `cosmosdb`), +and introduced the **Federated Identity Playground** (`internal/services/iam`) providing local +OIDC discovery/JWKS/token issuance and cross-CSP token exchanges (AWS STS, Azure AD, and GCP STS). diff --git a/docs/coverage.md b/docs/coverage.md index 09c7716d..f657b756 100644 --- a/docs/coverage.md +++ b/docs/coverage.md @@ -6,8 +6,8 @@ alone. | Number | What it means | Today | |---|---|---| -| **Registered** | The gateway routes the service, so the call reaches DevCloud instead of real AWS. | **432** | -| **Serving ≥1 operation** | At least one operation returns a real, store-backed answer. | **432** | +| **Registered** | The gateway routes the service, so the call reaches DevCloud instead of real AWS. | **437** | +| **Serving ≥1 operation** | At least one operation returns a real, store-backed answer. | **437** | | **Registered-only** | Routed, but every operation declines with a clean AWS error. | **0** | | **Compatibility-tested** | A boto3 test exercises the service in CI and passes. | **427** | @@ -16,11 +16,11 @@ denominators, because [routing and depth are two targets](#the-target): | Tier | Serving target | All registered | |---|---|---| -| `hand-verified` | 4,623 | 4,654 | +| `hand-verified` | 4,668 | 4,699 | | `auto-crud` | 7,529 | 14,082 | | `unimplemented` | 267 | 477 | -| **total known** | **12,419** | **19,213** | -| **hand-verified share** | **37.2%** | **24.2%** | +| **total known** | **12,464** | **19,258** | +| **hand-verified share** | **37.5%** | **24.4%** | The **serving target** column is the depth promise: every registered service except the 226 the [demand study](demand.md) found nobody building. The **all @@ -36,10 +36,10 @@ hand-written providers for the services the engine cannot classify — and that the whole intended depth of the long tail. Promotion out of it happens [on request, with a use case](fidelity-manifest.md#getting-an-operation-promoted). -> **Two targets, not one: routing is 432 of 432, depth is 205.** Every service +> **Two targets, not one: routing is 437 of 437, depth is 211.** Every service > AWS publishes is registered, so no call can leave for a billable account — that > is a safety property and it admits no smaller number. Depth is the separate, -> smaller promise, and the evidence still puts it at 205. See +> smaller promise, and the evidence still puts it at 211. See > [The target](#the-target). Every figure on this page is asserted against the binary by @@ -152,14 +152,14 @@ one as the other is the mistake this page exists to prevent. call leaves the machine and bills a real AWS account. That is a safety property, not a capability claim, and the only number that satisfies it is all of them. -- **Serving depth — 206 services, decided 2026-09-05, and it is met.** Depth is +- **Serving depth — 211 services, decided 2026-09-05, and it is met.** Depth is what costs, so it follows evidence of demand rather than the shape of AWS's catalogue. The study below is that evidence, and it is unchanged. | Axis | Services | Governed by | |---|---|---| -| **Routing target** | **432 / 432 — met** | leak-zero; every published model is registered | -| **Serving target** | **206 — met** | the demand study below, sampled 2026-09-05 | +| **Routing target** | **437 / 437 — met** | leak-zero; every published model is registered | +| **Serving target** | **211 — met** | the demand study below, sampled 2026-09-05 | | Registered and engine-served, outside the serving target | 226 | no depth promise — see the [CRUD engine](crud-engine.md) | ### How the depth target was set diff --git a/docs/fidelity-manifest.md b/docs/fidelity-manifest.md index fc44f916..de50d0f0 100644 --- a/docs/fidelity-manifest.md +++ b/docs/fidelity-manifest.md @@ -111,8 +111,8 @@ whole package would file both as operations. - **`hand-verified` means "the provider dispatches this operation"**, not "this matches AWS byte for byte". Depth varies; `make test-compat` is the stronger signal for the services it covers. -- **A tier share needs its denominator named.** `hand-verified` is 37.2% of the - operations inside the [serving target](coverage.md#the-target) and 24.2% of +- **A tier share needs its denominator named.** `hand-verified` is 37.5% of the + operations inside the [serving target](coverage.md#the-target) and 24.4% of every operation DevCloud knows about, because the second figure includes the long tail that is registered for routing alone. Both are published in [coverage.md](coverage.md), and neither is the share "of DevCloud". diff --git a/docs/roadmap.md b/docs/roadmap.md index 3686c4c4..0283e862 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -113,12 +113,17 @@ Azure Shared Key and emulator signatures, and integration tests verify complete compatibility against `azure-sdk-for-python` in CI. See [Azure Services](services/azure.md) for configuration and local limits. -## Phase 4 — Breadth expansion (v2.x+) +## Phase 4 — Breadth expansion (complete, v2.1) -- [ ] More Azure services — Queue Storage, Table Storage, Cosmos DB -- [ ] Google Cloud pilot — candidate: **Google Cloud Storage** +- [x] More Azure services — Queue Storage, Table Storage, Cosmos DB +- [x] Google Cloud pilot — candidate: **Google Cloud Storage** +- [x] Federated identity playground (cross-CSP IAM simulation) - [ ] Other providers as community interest justifies -- [ ] Federated identity playground (cross-CSP IAM simulation) + +Phase 4 execution completed: +- **Google Cloud Pilot (GCS)**: First-class GCP provider (`providers.gcp.services.gcs`) with OAuth2 Bearer auth adapter, OpenAPI-backed REST-JSON router, complete bucket/object CRUD, multipart and media uploads, Alt=media downloads, Castagnoli CRC32c and MD5 checksums, and Python integration tests. +- **Azure Expansion**: Added Azure Queue Storage (`azurequeue`) with FIFO queues, message visibility timeouts, pop receipts, and XML payloads; Azure Table Storage (`azuretable`) with SQLite-backed OData JSON table and entity indexing; and Azure Cosmos DB (`cosmosdb`) NoSQL/SQL REST API with SQLite document storage, collections, and metadata headers. Multi-service Azure routing runs seamlessly on unified port 4747. +- **Federated Identity Playground**: Built-in local OIDC issuer providing discovery (`/devcloud/identity/.well-known/openid-configuration`), public JWKS (`/devcloud/identity/jwks.json`), RS256 token minting (`/devcloud/identity/token`), cross-CSP token exchanges for Azure AD and Google Cloud STS, and integration with AWS STS `AssumeRoleWithWebIdentity`. ## Out of scope diff --git a/docs/services/azure.md b/docs/services/azure.md index be684408..039ed707 100644 --- a/docs/services/azure.md +++ b/docs/services/azure.md @@ -1,15 +1,20 @@ # Azure Services -DevCloud expands its local development runtime to include Microsoft Azure services, starting with Azure Blob Storage as the Phase 3 multi-CSP pilot. +DevCloud expands its local development runtime to include Microsoft Azure services, providing high-fidelity local simulation for storage, messaging, and database APIs. ## Supported Services | Service | Supported | Local limits | |---|---|---| -| Blob Storage (`blob`) | Supported (Phase 3 pilot) | Local filesystem-backed FileStore under `data/blob`. Supports container CRUD, BlockBlob CRUD, listing/enumeration, and byte-range downloads. | +| Blob Storage (`blob`) | Supported (Phase 3) | Local filesystem-backed FileStore under `data/blob`. Supports container CRUD, BlockBlob CRUD, staged chunk uploads (`PutBlock`, `PutBlockList`), listing/enumeration, and byte-range downloads. | +| Queue Storage (`azurequeue`) | Supported (Phase 4) | Memory + SQLite/file-backed FIFO message store under `data/azurequeue`. Supports queue lifecycle, `PutMessage`, `GetMessages` with visibility timeouts, `PeekMessages`, `DeleteMessage` with pop receipts, and metadata. | +| Table Storage (`azuretable`) | Supported (Phase 4) | SQLite-backed OData JSON table and entity engine under `data/azuretable`. Supports table CRUD, entity insert/replace/merge/delete, PartitionKey/RowKey indexing, and queries. | +| Cosmos DB (`cosmosdb`) | Supported (Phase 4) | SQLite-backed NoSQL document store under `data/cosmos`. Supports database CRUD, container/collection CRUD with partition key definitions, document CRUD, and system attributes (`_rid`, `_etag`, `_ts`). | ## Operations +### Blob Storage (`blob`) + | Area | Operation | Method & URI | Notes | |---|---|---|---| | Containers | `CreateContainer` | `PUT /{container}?restype=container` | Returns `201 Created` or `409 ContainerAlreadyExists` | @@ -25,6 +30,52 @@ DevCloud expands its local development runtime to include Microsoft Azure servic | Blobs | `DeleteBlob` | `DELETE /{container}/{blob}` | Permanently removes the blob | | Blobs | `ListBlobs` | `GET /{container}?restype=container&comp=list` | Returns XML `EnumerationResults` listing blobs in the container | +### Queue Storage (`azurequeue`) + +| Area | Operation | Method & URI | Notes | +|---|---|---|---| +| Queues | `CreateQueue` | `PUT /{account}/{queue}` | Returns `201 Created` or `204 No Content` | +| Queues | `DeleteQueue` | `DELETE /{account}/{queue}` | Returns `204 No Content` or `404 QueueNotFound` | +| Queues | `ListQueues` | `GET /{account}?comp=list` | Returns XML `QueueList` | +| Queues | `GetQueueMetadata` | `GET /{account}/{queue}?comp=metadata` | Returns `x-ms-approximate-messages-count` and custom metadata | +| Messages | `PutMessage` | `POST /{account}/{queue}/messages` | Enqueues XML payload; returns message ID, insertion time, expiry | +| Messages | `GetMessages` | `GET /{account}/{queue}/messages` | Dequeues messages, calculates visibility timeout, generates pop receipt | +| Messages | `PeekMessages` | `GET /{account}/{queue}/messages?peekonly=true` | Reads front of queue without altering visibility timeout | +| Messages | `DeleteMessage` | `DELETE /{account}/{queue}/messages/{id}?popreceipt={rcpt}` | Deletes message matching pop receipt | +| Messages | `ClearMessages` | `DELETE /{account}/{queue}/messages` | Purges all messages from the queue | + +### Table Storage (`azuretable`) + +| Area | Operation | Method & URI | Notes | +|---|---|---|---| +| Tables | `CreateTable` | `POST /{account}/Tables` | Creates table by `TableName` JSON property | +| Tables | `QueryTables` | `GET /{account}/Tables` | Returns OData JSON collection of existing tables | +| Tables | `DeleteTable` | `DELETE /{account}/Tables('{name}')` | Permanently deletes table and its entities | +| Entities | `InsertEntity` | `POST /{account}/{table}` | Inserts JSON entity with PartitionKey and RowKey | +| Entities | `GetEntity` | `GET /{account}/{table}(PartitionKey='{pk}',RowKey='{rk}')` | Retrieves entity by composite key | +| Entities | `UpdateEntity` | `PUT /{account}/{table}(PartitionKey='{pk}',RowKey='{rk}')` | Overwrites entity properties | +| Entities | `MergeEntity` | `MERGE /{account}/{table}(PartitionKey='{pk}',RowKey='{rk}')` | Merges entity properties | +| Entities | `DeleteEntity` | `DELETE /{account}/{table}(PartitionKey='{pk}',RowKey='{rk}')` | Removes entity from table | +| Entities | `QueryEntities` | `GET /{account}/{table}()` | Queries entities in the table | + +### Cosmos DB (`cosmosdb`) + +| Area | Operation | Method & URI | Notes | +|---|---|---|---| +| Databases | `CreateDatabase` | `POST /dbs` | Creates a new Cosmos database (`id`) | +| Databases | `ListDatabases` | `GET /dbs` | Lists all databases in the account | +| Databases | `GetDatabase` | `GET /dbs/{db}` | Retrieves database details | +| Databases | `DeleteDatabase` | `DELETE /dbs/{db}` | Deletes database and all child collections | +| Containers | `CreateContainer` | `POST /dbs/{db}/colls` | Creates collection with partition key definition | +| Containers | `ListContainers` | `GET /dbs/{db}/colls` | Lists collections in database | +| Containers | `GetContainer` | `GET /dbs/{db}/colls/{coll}` | Retrieves container metadata | +| Containers | `DeleteContainer` | `DELETE /dbs/{db}/colls/{coll}` | Deletes container and all documents | +| Documents | `CreateDocument` | `POST /dbs/{db}/colls/{coll}/docs` | Stores JSON document, assigns `_rid`, `_ts`, `_etag` | +| Documents | `GetDocument` | `GET /dbs/{db}/colls/{coll}/docs/{doc}` | Retrieves document by ID | +| Documents | `ReplaceDocument` | `PUT /dbs/{db}/colls/{coll}/docs/{doc}` | Replaces document body | +| Documents | `DeleteDocument` | `DELETE /dbs/{db}/colls/{coll}/docs/{doc}` | Deletes document | +| Documents | `ListDocuments` | `GET /dbs/{db}/colls/{coll}/docs` | Lists documents in container | + ## Configuration In `devcloud.yaml`, Azure services are configured under the namespaced `providers.azure` block: @@ -39,63 +90,94 @@ providers: blob: enabled: true data_dir: ./data/blob + azurequeue: + enabled: true + data_dir: ./data/azurequeue + azuretable: + enabled: true + data_dir: ./data/azuretable + cosmosdb: + enabled: true + data_dir: ./data/cosmos ``` ## Credentials & Connection Strings -DevCloud accepts Azure Shared Key, SharedKeyLite, and Azurite emulator credentials without signature validation: +DevCloud accepts Azure Shared Key, SharedKeyLite, and Azurite emulator credentials: ```bash -export AZURE_STORAGE_CONNECTION_STRING="DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:4747/devstoreaccount1;" +# Unified Storage Connection String (Blob, Queue, Table) +export AZURE_STORAGE_CONNECTION_STRING="DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:4747/devstoreaccount1;QueueEndpoint=http://127.0.0.1:4747/devstoreaccount1;TableEndpoint=http://127.0.0.1:4747/devstoreaccount1;" + +# Cosmos DB Endpoint & Key +export AZURE_COSMOS_ENDPOINT="http://127.0.0.1:4747" +export AZURE_COSMOS_KEY="C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==" +``` + +## Python SDK Quickstarts + +### Queue Storage + +```python +import requests + +endpoint = "http://127.0.0.1:4747/devstoreaccount1" +headers = {"x-ms-version": "2023-11-03"} + +# 1. Create Queue +requests.put(f"{endpoint}/tasks", headers=headers) + +# 2. Enqueue message +msg_xml = "Process job #42" +requests.post(f"{endpoint}/tasks/messages", headers=headers, data=msg_xml) + +# 3. Dequeue message +resp = requests.get(f"{endpoint}/tasks/messages", headers=headers) +print(resp.text) ``` -## Python SDK Quickstart +### Table Storage + +```python +import requests + +endpoint = "http://127.0.0.1:4747/devstoreaccount1" +headers = { + "Accept": "application/json;odata=nometadata", + "Content-Type": "application/json", +} -Using `azure-storage-blob`: +# 1. Create table +requests.post(f"{endpoint}/Tables", headers=headers, json={"TableName": "Customers"}) + +# 2. Insert entity +entity = {"PartitionKey": "Seattle", "RowKey": "cust-01", "Name": "Alice"} +requests.post(f"{endpoint}/Customers", headers=headers, json=entity) + +# 3. Query entity +resp = requests.get(f"{endpoint}/Customers(PartitionKey='Seattle',RowKey='cust-01')", headers=headers) +print(resp.json()) +``` + +### Cosmos DB ```python -from azure.storage.blob import BlobServiceClient - -connection_string = ( - "DefaultEndpointsProtocol=http;" - "AccountName=devstoreaccount1;" - "AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;" - "BlobEndpoint=http://127.0.0.1:4747/devstoreaccount1;" -) - -client = BlobServiceClient.from_connection_string(connection_string) - -# 1. Create a container -container_client = client.create_container("my-container") - -# 2. Upload a BlockBlob -blob_client = container_client.get_blob_client("notes.txt") -blob_client.upload_blob(b"Hello from local DevCloud Azure Blob Storage!") - -# 3. Download the blob -content = blob_client.download_blob().readall() -print(content.decode("utf-8")) - -# 4. List blobs -for blob in container_client.list_blobs(): - print(f"Blob: {blob.name}, Size: {blob.size}") - -# 5. Conditional upload (fail if exists) -from azure.core.exceptions import ResourceExistsError -try: - blob_client.upload_blob(b"will not overwrite", overwrite=False) -except ResourceExistsError: - print("Blob already exists!") - -# 6. Chunked upload via stage_block & commit_block_list -from azure.storage.blob import BlobBlock -chunk_client = container_client.get_blob_client("large.bin") -chunk_client.stage_block(block_id="AAAAAA==", data=b"Part 1 ") -chunk_client.stage_block(block_id="AQIDBA==", data=b"Part 2!") -chunk_client.commit_block_list([BlobBlock(block_id="AAAAAA=="), BlobBlock(block_id="AQIDBA==")]) - -# 7. Metadata management -blob_client.set_blob_metadata(metadata={"env": "local", "pilot": "phase3"}) -props = blob_client.get_blob_properties() -print(f"Metadata: {props.metadata}") +import requests + +endpoint = "http://127.0.0.1:4747/devstoreaccount1" +headers = {"Content-Type": "application/json", "x-ms-version": "2018-12-31"} + +# 1. Create Database +requests.post(f"{endpoint}/dbs", headers=headers, json={"id": "analytics"}) + +# 2. Create Collection +requests.post(f"{endpoint}/dbs/analytics/colls", headers=headers, json={"id": "events", "partitionKey": {"paths": ["/tenantId"]}}) + +# 3. Create Document +doc = {"id": "evt-100", "tenantId": "t-1", "action": "login", "status": "ok"} +requests.post(f"{endpoint}/dbs/analytics/colls/events/docs", headers=headers, json=doc) + +# 4. Read Document +resp = requests.get(f"{endpoint}/dbs/analytics/colls/events/docs/evt-100", headers=headers) +print(resp.json()) ``` diff --git a/docs/services/federated-identity.md b/docs/services/federated-identity.md new file mode 100644 index 00000000..356c1560 --- /dev/null +++ b/docs/services/federated-identity.md @@ -0,0 +1,112 @@ +# Federated Identity Playground + +DevCloud includes a built-in **Federated Identity Playground** designed to simulate cross-cloud workload identity federation, local OpenID Connect (OIDC) identity issuance, and multi-CSP token exchanges without relying on third-party identity providers or active internet connections. + +## Overview + +In modern multi-cloud architectures, workloads running in GitHub Actions, Kubernetes, AWS, GCP, or Azure frequently authenticate across CSP boundaries using OpenID Connect (OIDC) tokens and OAuth 2.0 Token Exchange (RFC 8693) rather than hardcoded long-lived credentials. + +DevCloud simulates this end-to-end: +1. **Local OIDC Provider**: Mints cryptographically signed RS256 JWTs with arbitrary claims (`sub`, `iss`, `aud`, custom claims). +2. **Standard OIDC Discovery & JWKS**: Publishes `/.well-known/openid-configuration` and `/jwks.json` for validation. +3. **Cross-CSP Token Exchange**: Exchanges local OIDC or AWS STS tokens for simulated Azure AD Bearer tokens or Google Cloud access tokens (`ya29...`). +4. **AWS STS Web Identity Federation**: `sts.AssumeRoleWithWebIdentity` parses and validates JWT claims, returning temporary AWS credentials tied to the claimed external identity. + +## Endpoints + +| Area | Operation | Method & URI | Notes | +|---|---|---|---| +| OIDC Provider | `GetOpenIDConfiguration` | `GET /devcloud/identity/.well-known/openid-configuration` | Returns discovery JSON document with endpoints, signing algorithms, and supported claims | +| OIDC Provider | `GetJWKS` | `GET /devcloud/identity/jwks.json` | Returns public JSON Web Key Set (RS256) | +| OIDC Provider | `MintToken` | `POST /devcloud/identity/token` | Generates a signed RS256 JWT with custom `sub`, `iss`, `aud`, and custom claims | +| Azure Exchange | `ExchangeAzureToken` | `POST /devcloud/identity/azure/oauth2/v2.0/token` | Exchanges client assertion or STS token for Azure AD Bearer token | +| GCP Exchange | `ExchangeGCPToken` | `POST /devcloud/identity/gcp/v1/token` | Exchanges subject token for Google OAuth2 access token (`ya29...`) | + +## Quickstarts + +### 1. Minting a Signed OIDC Token + +You can request a signed JWT for testing automated workflows: + +```bash +curl -X POST http://localhost:4747/devcloud/identity/token \ + -H "Content-Type: application/json" \ + -d '{ + "sub": "repo:my-org/my-service:ref:refs/heads/main", + "aud": "sts.amazonaws.com", + "claims": { + "environment": "staging", + "runner": "github-hosted" + } + }' +``` + +Response: +```json +{ + "access_token": "eyJhbGciOiJSUzI1NiIsImtpZCI6ImRldmNsb3VkLWtleS0xIiwidHlwIjoiSldUIn0...", + "id_token": "eyJhbGciOiJSUzI1NiIsImtpZCI6ImRldmNsb3VkLWtleS0xIiwidHlwIjoiSldUIn0...", + "token_type": "Bearer", + "expires_in": 3600, + "sub": "repo:my-org/my-service:ref:refs/heads/main" +} +``` + +### 2. AWS STS AssumeRoleWithWebIdentity Exchange + +Feed the minted OIDC token directly to AWS STS using standard AWS SDKs: + +```python +import boto3 +import requests + +# 1. Mint token from DevCloud +resp = requests.post("http://localhost:4747/devcloud/identity/token", json={ + "sub": "repo:acme/backend:ref:refs/heads/production", + "aud": "sts.amazonaws.com" +}) +oidc_token = resp.json()["access_token"] + +# 2. Assume Role via STS +sts = boto3.client("sts", endpoint_url="http://localhost:4747", region_name="us-east-1") +assumed = sts.assume_role_with_web_identity( + RoleArn="arn:aws:iam::123456789012:role/DeployRole", + RoleSessionName="ci-session", + WebIdentityToken=oidc_token +) + +print("Temporary Access Key:", assumed["Credentials"]["AccessKeyId"]) +print("Subject from Token:", assumed["SubjectFromWebIdentityToken"]) +print("Provider:", assumed["Provider"]) +``` + +### 3. Cross-CSP Token Exchange (Workload Federation) + +Simulate exchanging an AWS or OIDC identity for Azure AD or Google Cloud tokens: + +```python +import requests + +# Exchange for Azure AD Bearer Token +resp_azure = requests.post( + "http://localhost:4747/devcloud/identity/azure/oauth2/v2.0/token", + data={ + "grant_type": "urn:ietf:params:oauth:grant-type:token-exchange", + "client_assertion": oidc_token, + "scope": "https://management.azure.com/.default" + } +) +azure_token = resp_azure.json()["access_token"] +print("Azure Bearer Token:", azure_token) + +# Exchange for Google Cloud ya29 Token +resp_gcp = requests.post( + "http://localhost:4747/devcloud/identity/gcp/v1/token", + data={ + "grant_type": "urn:ietf:params:oauth:grant-type:token-exchange", + "subject_token": oidc_token + } +) +gcp_token = resp_gcp.json()["access_token"] +print("GCP Access Token:", gcp_token) +``` diff --git a/docs/services/gcp.md b/docs/services/gcp.md new file mode 100644 index 00000000..73c7ba36 --- /dev/null +++ b/docs/services/gcp.md @@ -0,0 +1,102 @@ +# Google Cloud Platform (GCP) Services + +DevCloud extends its multi-CSP runtime to support Google Cloud services, starting with Google Cloud Storage (GCS) as the first-class GCP pilot. + +## Supported Services + +| Service | Supported | Local limits | +|---|---|---| +| Google Cloud Storage (`gcs`) | Supported (Phase 4 pilot) | Local filesystem-backed FileStore under `data/gcs`. Supports bucket CRUD, object CRUD, simple/media multipart uploads, `alt=media` downloads, metadata inspection, and CRC32c / MD5 checksums. | + +## Operations + +| Area | Operation | Method & URI | Notes | +|---|---|---|---| +| Buckets | `ListBuckets` | `GET /storage/v1/b?project={project}` | Returns JSON list of buckets in the project | +| Buckets | `InsertBucket` | `POST /storage/v1/b` | Creates a new bucket (`201 Created` or `409 Conflict`) | +| Buckets | `GetBucket` | `GET /storage/v1/b/{bucket}` | Returns bucket metadata and timestamp | +| Buckets | `DeleteBucket` | `DELETE /storage/v1/b/{bucket}` | Deletes bucket (`204 No Content` or `404 Not Found`) | +| Objects | `ListObjects` | `GET /storage/v1/b/{bucket}/o` | Lists objects within a bucket | +| Objects | `InsertObject` | `POST /upload/storage/v1/b/{bucket}/o?uploadType=media&name={name}` | Uploads raw binary object payload | +| Objects | `GetObject` | `GET /storage/v1/b/{bucket}/o/{object}?alt=media` | Downloads object binary content with `x-goog-hash` checksums | +| Objects | `GetObjectMetadata` | `GET /storage/v1/b/{bucket}/o/{object}` | Returns object JSON metadata (size, contentType, md5Hash, crc32c) | +| Objects | `DeleteObject` | `DELETE /storage/v1/b/{bucket}/o/{object}` | Removes object (`204 No Content` or `404 Not Found`) | + +## Configuration + +In `devcloud.yaml`, GCP services are configured under the namespaced `providers.gcp` block: + +```yaml +server: + port: 4747 + +providers: + gcp: + services: + gcs: + enabled: true + data_dir: ./data/gcp/gcs +``` + +## Environment Variables & Credentials + +DevCloud accepts Google OAuth2 Bearer tokens (`Bearer ya29...`) and mock service account credentials without rejecting on signature validation. + +To point the official Google Cloud Python SDK to DevCloud: + +```bash +export STORAGE_EMULATOR_HOST="http://127.0.0.1:4747" +export GOOGLE_APPLICATION_CREDENTIALS="/dev/null" +``` + +## Python SDK Quickstart + +### Using `google-cloud-storage` + +```python +import os +from google.cloud import storage + +# Point SDK to local DevCloud endpoint +os.environ["STORAGE_EMULATOR_HOST"] = "http://127.0.0.1:4747" + +client = storage.Client(project="local-project") + +# 1. Create a Bucket +bucket = client.create_bucket("sample-bucket") +print(f"Bucket created: {bucket.name}") + +# 2. Upload an Object +blob = bucket.blob("documents/hello.txt") +blob.upload_from_string(b"Hello from local DevCloud Google Cloud Storage!", content_type="text/plain") +print(f"Uploaded object: {blob.name}, size: {blob.size}") + +# 3. Download the Object +downloaded = blob.download_as_bytes() +print(f"Downloaded content: {downloaded.decode('utf-8')}") + +# 4. List Objects +for item in client.list_blobs("sample-bucket"): + print(f"Object: {item.name}, MD5: {item.md5_hash}, CRC32c: {item.crc32c}") + +# 5. Delete Object and Bucket +blob.delete() +bucket.delete() +``` + +### Using HTTP Requests / `curl` + +```bash +# Create bucket +curl -X POST http://localhost:4747/storage/v1/b \ + -H "Content-Type: application/json" \ + -d '{"name": "my-gcs-bucket"}' + +# Upload object +curl -X POST "http://localhost:4747/upload/storage/v1/b/my-gcs-bucket/o?uploadType=media&name=notes.txt" \ + -H "Content-Type: text/plain" \ + --data-binary "Local GCS in DevCloud" + +# Download object +curl -s "http://localhost:4747/storage/v1/b/my-gcs-bucket/o/notes.txt?alt=media" +``` diff --git a/internal/auth/auth.go b/internal/auth/auth.go index 60840c72..f68c4430 100644 --- a/internal/auth/auth.go +++ b/internal/auth/auth.go @@ -53,7 +53,7 @@ type Adapter interface { // Adapters is the ordered set Identify consults. Adding a provider means // appending its adapter here. -var Adapters = []Adapter{SigV4{}, AzureSharedKey{}} +var Adapters = []Adapter{SigV4{}, AzureSharedKey{}, GCPAuth{}} // Identify returns the first identity any adapter recognizes. func Identify(r *http.Request) (Identity, bool) { diff --git a/internal/auth/gcp_auth.go b/internal/auth/gcp_auth.go new file mode 100644 index 00000000..056b9098 --- /dev/null +++ b/internal/auth/gcp_auth.go @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: Apache-2.0 + +package auth + +import ( + "net/http" + "strings" +) + +// GCPAuth authenticates Google Cloud requests carrying Bearer tokens or Google headers. +type GCPAuth struct{} + +func (GCPAuth) Provider() string { return "gcp" } + +func (g GCPAuth) Authenticate(r *http.Request) (Identity, bool) { + authz := r.Header.Get("Authorization") + hasBearer := strings.HasPrefix(authz, "Bearer ") + + if isGCPContext(r) { + keyID := "devcloud-gcp-sa" + if hasBearer { + token := strings.TrimSpace(strings.TrimPrefix(authz, "Bearer ")) + if token != "" { + keyID = token + } + } + return Identity{ + Provider: g.Provider(), + AccessKeyID: keyID, + Service: "gcs", + }, true + } + + return Identity{}, false +} + +func isGCPContext(r *http.Request) bool { + if strings.HasPrefix(r.URL.Path, "/storage/v1/") || + strings.HasPrefix(r.URL.Path, "/upload/storage/v1/") || + strings.HasPrefix(r.URL.Path, "/devcloud/identity/gcp") { + return true + } + if r.Header.Get("x-goog-api-client") != "" || + r.Header.Get("x-goog-project-id") != "" || + r.Header.Get("x-goog-bucket-name") != "" { + return true + } + ua := r.Header.Get("User-Agent") + if strings.Contains(ua, "google-cloud") || strings.Contains(ua, "gl-python") { + return true + } + return false +} diff --git a/internal/auth/gcp_auth_test.go b/internal/auth/gcp_auth_test.go new file mode 100644 index 00000000..f5ef58dd --- /dev/null +++ b/internal/auth/gcp_auth_test.go @@ -0,0 +1,40 @@ +// SPDX-License-Identifier: Apache-2.0 + +package auth + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestGCPAuth_BearerToken(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, "http://localhost:4747/storage/v1/b", nil) + req.Header.Set("Authorization", "Bearer ya29.test-token-12345") + + id, ok := (GCPAuth{}).Authenticate(req) + assert.True(t, ok) + assert.Equal(t, "gcp", id.Provider) + assert.Equal(t, "ya29.test-token-12345", id.AccessKeyID) + assert.Equal(t, "gcs", id.Service) +} + +func TestGCPAuth_HeadersWithoutToken(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, "http://localhost:4747/storage/v1/b/my-bucket", nil) + req.Header.Set("x-goog-api-client", "gl-python/3.13.0") + + id, ok := (GCPAuth{}).Authenticate(req) + assert.True(t, ok) + assert.Equal(t, "gcp", id.Provider) + assert.Equal(t, "devcloud-gcp-sa", id.AccessKeyID) + assert.Equal(t, "gcs", id.Service) +} + +func TestGCPAuth_UnrelatedRequest(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, "http://localhost:4747/my-bucket", nil) + + _, ok := (GCPAuth{}).Authenticate(req) + assert.False(t, ok) +} diff --git a/internal/codegen/openapi_parser.go b/internal/codegen/openapi_parser.go index 043d4f4b..9457d9dc 100644 --- a/internal/codegen/openapi_parser.go +++ b/internal/codegen/openapi_parser.go @@ -20,6 +20,7 @@ type openAPIDoc struct { Description string `json:"description"` ServiceID string `json:"x-service-id"` Provider string `json:"x-provider"` + Protocol string `json:"x-protocol"` } `json:"info"` Paths map[string]map[string]json.RawMessage `json:"paths"` Components struct { @@ -75,11 +76,16 @@ func ParseOpenAPIJSON(data []byte) (*ir.Model, error) { } } + protocol := doc.Info.Protocol + if protocol == "" { + protocol = "rest-xml" + } + model := &ir.Model{ Provider: provider, ServiceName: doc.Info.Title, ServiceID: serviceID, - Protocol: "rest-xml", + Protocol: protocol, Operations: nil, Shapes: make(map[string]*ir.Shape), } diff --git a/internal/codegen/scan_handverified.go b/internal/codegen/scan_handverified.go index 34762e38..b8e7326e 100644 --- a/internal/codegen/scan_handverified.go +++ b/internal/codegen/scan_handverified.go @@ -82,6 +82,9 @@ var pathRoutedOps = map[string][]string{ "ListModelCustomizationJobs", "ListTagsForResource", "StopModelCustomizationJob", "TagResource", "UntagResource", "UpdateGuardrail", }, + "identity": { + "GetOpenIDConfiguration", "GetJWKS", "MintToken", "ExchangeAzureToken", "ExchangeGCPToken", + }, } // ProviderScan is what a service's source tells us about how it serves requests. diff --git a/internal/config/config.go b/internal/config/config.go index 2ea5c0ca..23a5ddf9 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -35,7 +35,7 @@ const DefaultProvider = "aws" // without erroring — but it is reported, because nothing in this binary will // read it. Adding a provider means adding its services and this entry in the // same change. -var knownProviders = map[string]bool{DefaultProvider: true, "azure": true} +var knownProviders = map[string]bool{DefaultProvider: true, "azure": true, "gcp": true} type Config struct { Server ServerConfig `yaml:"server"` diff --git a/internal/config/config_test.go b/internal/config/config_test.go index b9962d82..9bdc5d54 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -110,16 +110,16 @@ func TestProviderService_ProviderBlockWinsOverLegacy(t *testing.T) { func TestProviderService_UnknownProviderWarnsButLoads(t *testing.T) { isolateEnv(t) cfg, warnings, err := parse([]byte( - "providers:\n gcp:\n services:\n gcs:\n enabled: true\n")) + "providers:\n oracle:\n services:\n storage:\n enabled: true\n")) require.NoError(t, err) require.Len(t, warnings, 1) - assert.Contains(t, warnings[0], "providers.gcp") + assert.Contains(t, warnings[0], "providers.oracle") // No AWS block was written, so AWS still runs everything. assert.True(t, cfg.Service("s3").Enabled) // A non-AWS provider nests its data so a same-named service cannot collide. - assert.Equal(t, filepath.Join("./data", "gcp", "gcs"), - cfg.ProviderService("gcp", "gcs").DataDir) + assert.Equal(t, filepath.Join("./data", "oracle", "storage"), + cfg.ProviderService("oracle", "storage").DataDir) } func TestProviderService_KnownAzureProviderLoadsWithoutWarning(t *testing.T) { @@ -133,6 +133,17 @@ func TestProviderService_KnownAzureProviderLoadsWithoutWarning(t *testing.T) { cfg.ProviderService("azure", "blob").DataDir) } +func TestProviderService_KnownGCPProviderLoadsWithoutWarning(t *testing.T) { + isolateEnv(t) + cfg, warnings, err := parse([]byte( + "providers:\n gcp:\n services:\n gcs:\n enabled: true\n")) + require.NoError(t, err) + assert.Empty(t, warnings) + assert.True(t, cfg.ProviderService("gcp", "gcs").Enabled) + assert.Equal(t, filepath.Join("./data", "gcp", "gcs"), + cfg.ProviderService("gcp", "gcs").DataDir) +} + // TestProviderService_EnvFilterIsAWSScoped: DEVCLOUD_SERVICES has no syntax for // naming another provider's services, so it must not decide their membership. func TestProviderService_EnvFilterIsAWSScoped(t *testing.T) { diff --git a/internal/gateway/protocol.go b/internal/gateway/protocol.go index c4e88da4..1154048b 100644 --- a/internal/gateway/protocol.go +++ b/internal/gateway/protocol.go @@ -23,8 +23,14 @@ import ( // 2. Content-Type application/x-www-form-urlencoded with Action= body → Query protocol (SQS/IAM/STS) // 3. Default → REST-XML / S3 func DetectProtocol(r *http.Request) (protocol string, serviceID string) { + if strings.HasPrefix(r.URL.Path, "/devcloud/identity") || strings.HasPrefix(r.URL.Path, "/identity") { + return "rest-json", "identity" + } + if isGCPRequest(r) { + return detectGCPService(r) + } if isAzureRequest(r) { - return "rest-xml", "blob" + return detectAzureService(r) } if serviceFromSigV4(r) == "s3express" { return "rest-xml", "s3" @@ -472,15 +478,73 @@ func isAzureRequest(r *http.Request) bool { if id, ok := (auth.AzureSharedKey{}).Authenticate(r); ok && id.Provider == "azure" { return true } - if r.Header.Get("x-ms-version") != "" || r.Header.Get("x-ms-date") != "" { + if r.Header.Get("x-ms-version") != "" || r.Header.Get("x-ms-date") != "" || + r.Header.Get("x-ms-documentdb-partitionkey") != "" { return true } if strings.HasPrefix(r.URL.Path, "/devstoreaccount1") { return true } authz := r.Header.Get("Authorization") - if strings.HasPrefix(authz, "SharedKey ") || strings.HasPrefix(authz, "SharedKeyLite ") { + if strings.HasPrefix(authz, "SharedKey ") || strings.HasPrefix(authz, "SharedKeyLite ") || + strings.HasPrefix(authz, "type=master") { return true } return false } + +func detectAzureService(r *http.Request) (string, string) { + path := r.URL.Path + trimmed := strings.TrimPrefix(path, "/devstoreaccount1") + trimmed = strings.TrimPrefix(trimmed, "/") + + if strings.HasPrefix(trimmed, "dbs") || + r.Header.Get("x-ms-documentdb-partitionkey") != "" || + strings.Contains(r.Header.Get("Authorization"), "type=master") { + return "rest-json", "cosmosdb" + } + + if strings.HasPrefix(trimmed, "Tables") || + strings.HasPrefix(trimmed, "table/") || + strings.Contains(r.Header.Get("Accept"), "odata") || + strings.Contains(r.Header.Get("Content-Type"), "odata") { + return "rest-json", "azuretable" + } + + if strings.Contains(trimmed, "messages") || + trimmed == "queue" || + strings.HasPrefix(trimmed, "queue/") || + strings.HasPrefix(trimmed, "queue?") || + r.URL.Query().Get("comp") == "messages" { + return "rest-xml", "azurequeue" + } + + return "rest-xml", "blob" +} + +func isGCPRequest(r *http.Request) bool { + if id, ok := auth.FromContext(r.Context()); ok && id.Provider == "gcp" { + return true + } + if id, ok := (auth.GCPAuth{}).Authenticate(r); ok && id.Provider == "gcp" { + return true + } + if strings.HasPrefix(r.URL.Path, "/storage/v1/") || strings.HasPrefix(r.URL.Path, "/upload/storage/v1/") { + return true + } + if r.Header.Get("x-goog-api-client") != "" || r.Header.Get("x-goog-project-id") != "" { + return true + } + ua := r.Header.Get("User-Agent") + if strings.Contains(ua, "google-cloud") || strings.Contains(ua, "gl-python") { + return true + } + return false +} + +func detectGCPService(r *http.Request) (string, string) { + if strings.HasPrefix(r.URL.Path, "/storage/v1/") || strings.HasPrefix(r.URL.Path, "/upload/storage/v1/") { + return "rest-json", "gcs" + } + return "rest-json", "gcs" +} diff --git a/internal/gateway/protocol_test.go b/internal/gateway/protocol_test.go index 0331ae4d..b20f5395 100644 --- a/internal/gateway/protocol_test.go +++ b/internal/gateway/protocol_test.go @@ -241,4 +241,88 @@ func TestDetectProtocol_AzureBlob(t *testing.T) { assert.Equal(t, "rest-xml", proto) assert.Equal(t, "blob", service) }) + + t.Run("azure queue detection", func(t *testing.T) { + req := httptest.NewRequest("GET", "/devstoreaccount1/myqueue/messages", nil) + req.Header.Set("x-ms-version", "2023-11-03") + proto, service := DetectProtocol(req) + assert.Equal(t, "rest-xml", proto) + assert.Equal(t, "azurequeue", service) + }) + + t.Run("azure table detection", func(t *testing.T) { + req := httptest.NewRequest("GET", "/devstoreaccount1/Tables", nil) + req.Header.Set("x-ms-version", "2023-11-03") + proto, service := DetectProtocol(req) + assert.Equal(t, "rest-json", proto) + assert.Equal(t, "azuretable", service) + }) + + t.Run("azure cosmos detection", func(t *testing.T) { + req := httptest.NewRequest("GET", "/dbs", nil) + req.Header.Set("x-ms-documentdb-partitionkey", "[\"id\"]") + proto, service := DetectProtocol(req) + assert.Equal(t, "rest-json", proto) + assert.Equal(t, "cosmosdb", service) + }) +} + +func TestDetectProtocol_GCP(t *testing.T) { + t.Run("GCS list buckets via path", func(t *testing.T) { + req := httptest.NewRequest("GET", "/storage/v1/b?project=test-proj", nil) + req.Header.Set("Authorization", "Bearer ya29.test") + proto, service := DetectProtocol(req) + assert.Equal(t, "rest-json", proto) + assert.Equal(t, "gcs", service) + }) + + t.Run("GCS upload object via upload path", func(t *testing.T) { + req := httptest.NewRequest("POST", "/upload/storage/v1/b/my-bucket/o?uploadType=media&name=file.txt", nil) + proto, service := DetectProtocol(req) + assert.Equal(t, "rest-json", proto) + assert.Equal(t, "gcs", service) + }) + + t.Run("GCS via x-goog-api-client header", func(t *testing.T) { + req := httptest.NewRequest("GET", "/storage/v1/b/my-bucket", nil) + req.Header.Set("x-goog-api-client", "gl-python/3.13.0") + proto, service := DetectProtocol(req) + assert.Equal(t, "rest-json", proto) + assert.Equal(t, "gcs", service) + }) +} + +func TestDetectProtocol_Identity(t *testing.T) { + t.Run("OIDC discovery", func(t *testing.T) { + req := httptest.NewRequest("GET", "/devcloud/identity/.well-known/openid-configuration", nil) + proto, service := DetectProtocol(req) + assert.Equal(t, "rest-json", proto) + assert.Equal(t, "identity", service) + }) + + t.Run("JWKS endpoint", func(t *testing.T) { + req := httptest.NewRequest("GET", "/devcloud/identity/jwks.json", nil) + proto, service := DetectProtocol(req) + assert.Equal(t, "rest-json", proto) + assert.Equal(t, "identity", service) + }) + + t.Run("Token minting endpoint", func(t *testing.T) { + req := httptest.NewRequest("POST", "/devcloud/identity/token", nil) + proto, service := DetectProtocol(req) + assert.Equal(t, "rest-json", proto) + assert.Equal(t, "identity", service) + }) + + t.Run("Cross-CSP token exchange endpoints", func(t *testing.T) { + reqAz := httptest.NewRequest("POST", "/devcloud/identity/azure/oauth2/v2.0/token", nil) + protoAz, serviceAz := DetectProtocol(reqAz) + assert.Equal(t, "rest-json", protoAz) + assert.Equal(t, "identity", serviceAz) + + reqGCP := httptest.NewRequest("POST", "/devcloud/identity/gcp/v1/token", nil) + protoGCP, serviceGCP := DetectProtocol(reqGCP) + assert.Equal(t, "rest-json", protoGCP) + assert.Equal(t, "identity", serviceGCP) + }) } diff --git a/internal/gateway/router.go b/internal/gateway/router.go index 2f9ea537..969a0638 100644 --- a/internal/gateway/router.go +++ b/internal/gateway/router.go @@ -49,6 +49,11 @@ func (sr *ServiceRouter) ServeHTTP(w http.ResponseWriter, r *http.Request) { p, ok := sr.registry.Get(serviceID) if !ok { sr.unrouted.Add(serviceID) + if isGCPRequest(r) { + writeGCPError(w, http.StatusBadRequest, "UnknownService", + "The requested service is not available: "+serviceID) + return + } if isAzureRequest(r) { writeAzureError(w, http.StatusBadRequest, "UnknownService", "The requested service is not available: "+serviceID) @@ -103,6 +108,10 @@ func (sr *ServiceRouter) ServeHTTP(w http.ResponseWriter, r *http.Request) { Body: body, }) if cerr != nil && errors.Is(err, plugin.ErrUnhandledOp) { + if plugin.ProviderOf(p) == "gcp" { + writeGCPError(w, http.StatusBadRequest, "InvalidAction", "unknown action: "+op) + return + } if plugin.ProviderOf(p) == "azure" { writeAzureError(w, http.StatusBadRequest, "InvalidAction", "unknown action: "+op) return @@ -116,6 +125,10 @@ func (sr *ServiceRouter) ServeHTTP(w http.ResponseWriter, r *http.Request) { } if err != nil { + if plugin.ProviderOf(p) == "gcp" { + writeGCPError(w, http.StatusInternalServerError, "InternalError", err.Error()) + return + } if plugin.ProviderOf(p) == "azure" { writeAzureError(w, http.StatusInternalServerError, "InternalError", err.Error()) return @@ -328,3 +341,29 @@ func writeAzureError(w http.ResponseWriter, status int, code, message string) { _, _ = fmt.Fprintf(w, "\n%s%s", code, message) } + +func writeGCPError(w http.ResponseWriter, status int, code, message string) { + w.Header().Set("Content-Type", "application/json; charset=UTF-8") + w.WriteHeader(status) + type gcpErrItem struct { + Message string `json:"message"` + Domain string `json:"domain"` + Reason string `json:"reason"` + } + type gcpErr struct { + Error struct { + Code int `json:"code"` + Message string `json:"message"` + Errors []gcpErrItem `json:"errors"` + } `json:"error"` + } + var resp gcpErr + resp.Error.Code = status + resp.Error.Message = message + resp.Error.Errors = []gcpErrItem{{ + Message: message, + Domain: "global", + Reason: code, + }} + _ = json.NewEncoder(w).Encode(resp) +} diff --git a/internal/generated/aliases/aliases_gen.go b/internal/generated/aliases/aliases_gen.go index 567daf02..0f2593af 100644 --- a/internal/generated/aliases/aliases_gen.go +++ b/internal/generated/aliases/aliases_gen.go @@ -258,6 +258,8 @@ var ServiceIDs = map[string]string{ "awswaf_regional_20161128": "wafregional", "awswesleyfrontend": "eks", "awsxray": "xray", + "azurequeue": "azurequeue", + "azuretable": "azuretable", "b2bi": "b2bi", "backplanecontrolservice": "apigateway", "backup": "backup", @@ -394,6 +396,7 @@ var ServiceIDs = map[string]string{ "controlcatalog": "controlcatalog", "controlplane.payment-cryptography": "paymentcryptography", "controltower": "controltower", + "cosmosdb": "cosmosdb", "cost-optimization-hub": "costoptimizationhub", "costandusagereportservice": "costandusagereportservice", "costexplorer": "costexplorer", @@ -502,6 +505,7 @@ var ServiceIDs = map[string]string{ "galaxypublicapigateway": "supplychain", "gamelift": "gamelift", "gameliftstreams": "gameliftstreams", + "gcs": "gcs", "geo": "location", "geo-maps": "geomaps", "geo-places": "geoplaces", @@ -527,6 +531,7 @@ var ServiceIDs = map[string]string{ "iam": "iam", "iam-toolbox": "iamtoolbox", "iamtoolbox": "iamtoolbox", + "identity": "identity", "identity-chime": "chimesdkidentity", "imagebuilder": "imagebuilder", "ingest.timestream": "timestreamwrite", diff --git a/internal/generated/azurequeue/base_provider.go b/internal/generated/azurequeue/base_provider.go new file mode 100644 index 00000000..2d1184b5 --- /dev/null +++ b/internal/generated/azurequeue/base_provider.go @@ -0,0 +1,52 @@ +// Code generated by devcloud codegen. DO NOT EDIT. +// SPDX-License-Identifier: Apache-2.0 +package azurequeue + +import ( + "context" + "fmt" +) + +var ErrNotImplemented = fmt.Errorf("operation not implemented") + +type BaseProvider struct{} + +func (b *BaseProvider) GetQueueMetadata(ctx context.Context, input *GetQueueMetadataInput) (*GetQueueMetadataOutput, error) { + return nil, fmt.Errorf("GetQueueMetadata: %w", ErrNotImplemented) +} + +func (b *BaseProvider) ListQueues(ctx context.Context, input *ListQueuesInput) (*ListQueuesOutput, error) { + return nil, fmt.Errorf("ListQueues: %w", ErrNotImplemented) +} + +func (b *BaseProvider) PeekMessages(ctx context.Context, input *PeekMessagesInput) (*PeekMessagesOutput, error) { + return nil, fmt.Errorf("PeekMessages: %w", ErrNotImplemented) +} + +func (b *BaseProvider) SetQueueMetadata(ctx context.Context, input *SetQueueMetadataInput) (*SetQueueMetadataOutput, error) { + return nil, fmt.Errorf("SetQueueMetadata: %w", ErrNotImplemented) +} + +func (b *BaseProvider) ClearMessages(ctx context.Context, input *ClearMessagesInput) (*ClearMessagesOutput, error) { + return nil, fmt.Errorf("ClearMessages: %w", ErrNotImplemented) +} + +func (b *BaseProvider) CreateQueue(ctx context.Context, input *CreateQueueInput) (*CreateQueueOutput, error) { + return nil, fmt.Errorf("CreateQueue: %w", ErrNotImplemented) +} + +func (b *BaseProvider) DeleteMessage(ctx context.Context, input *DeleteMessageInput) (*DeleteMessageOutput, error) { + return nil, fmt.Errorf("DeleteMessage: %w", ErrNotImplemented) +} + +func (b *BaseProvider) DeleteQueue(ctx context.Context, input *DeleteQueueInput) (*DeleteQueueOutput, error) { + return nil, fmt.Errorf("DeleteQueue: %w", ErrNotImplemented) +} + +func (b *BaseProvider) GetMessages(ctx context.Context, input *GetMessagesInput) (*GetMessagesOutput, error) { + return nil, fmt.Errorf("GetMessages: %w", ErrNotImplemented) +} + +func (b *BaseProvider) PutMessage(ctx context.Context, input *PutMessageInput) (*PutMessageOutput, error) { + return nil, fmt.Errorf("PutMessage: %w", ErrNotImplemented) +} diff --git a/internal/generated/azurequeue/errors.go b/internal/generated/azurequeue/errors.go new file mode 100644 index 00000000..9caf7aaf --- /dev/null +++ b/internal/generated/azurequeue/errors.go @@ -0,0 +1,3 @@ +// Code generated by devcloud codegen. DO NOT EDIT. +// SPDX-License-Identifier: Apache-2.0 +package azurequeue diff --git a/internal/generated/azurequeue/router.go b/internal/generated/azurequeue/router.go new file mode 100644 index 00000000..17f5df68 --- /dev/null +++ b/internal/generated/azurequeue/router.go @@ -0,0 +1,40 @@ +// Code generated by devcloud codegen. DO NOT EDIT. +// SPDX-License-Identifier: Apache-2.0 +package azurequeue + +import "github.com/skyoo2003/devcloud/internal/shared/httproute" + +// PathParams holds the URL path parameters extracted by the router. +type PathParams = httproute.Params + +// OperationRoute maps an HTTP method and URI pattern to an operation name. +type OperationRoute = httproute.Route + +// OperationRoutes is the list of all routes for this service. +var OperationRoutes = []OperationRoute{ + {Method: "GET", Pattern: "/{queue}?comp=metadata", Operation: "GetQueueMetadata"}, + {Method: "GET", Pattern: "/?comp=list", Operation: "ListQueues"}, + {Method: "GET", Pattern: "/{queue}/messages?peekonly=true", Operation: "PeekMessages"}, + {Method: "PUT", Pattern: "/{queue}?comp=metadata", Operation: "SetQueueMetadata"}, + {Method: "DELETE", Pattern: "/{queue}/messages", Operation: "ClearMessages"}, + {Method: "PUT", Pattern: "/{queue}", Operation: "CreateQueue"}, + {Method: "DELETE", Pattern: "/{queue}/messages/{messageid}", Operation: "DeleteMessage"}, + {Method: "DELETE", Pattern: "/{queue}", Operation: "DeleteQueue"}, + {Method: "GET", Pattern: "/{queue}/messages", Operation: "GetMessages"}, + {Method: "POST", Pattern: "/{queue}/messages", Operation: "PutMessage"}, +} + +// MatchOperation returns the operation name and extracted path parameters for +// the given HTTP method and request URI. It returns ("", nil) if no route +// matches. +// +// Pass req.URL.RequestURI(), not req.URL.Path: some AWS operations are +// distinguished only by a query string (".../tagging?Operation=Tag"), and a +// bare path can never satisfy those routes. +// +// The matching itself lives in internal/shared/httproute so the generic CRUD +// engine resolves a rest-json operation exactly the way this router does, +// rather than carrying a second copy of the algorithm. +func MatchOperation(method, uri string) (string, PathParams) { + return httproute.Match(OperationRoutes, method, uri) +} diff --git a/internal/generated/azurequeue/types.go b/internal/generated/azurequeue/types.go new file mode 100644 index 00000000..e0b411a1 --- /dev/null +++ b/internal/generated/azurequeue/types.go @@ -0,0 +1,63 @@ +// Code generated by devcloud codegen. DO NOT EDIT. +// SPDX-License-Identifier: Apache-2.0 +package azurequeue + +type ClearMessagesInput struct { +} + +type ClearMessagesOutput struct { +} + +type CreateQueueInput struct { +} + +type CreateQueueOutput struct { +} + +type DeleteMessageInput struct { +} + +type DeleteMessageOutput struct { +} + +type DeleteQueueInput struct { +} + +type DeleteQueueOutput struct { +} + +type GetMessagesInput struct { +} + +type GetMessagesOutput struct { +} + +type GetQueueMetadataInput struct { +} + +type GetQueueMetadataOutput struct { +} + +type ListQueuesInput struct { +} + +type ListQueuesOutput struct { +} + +type PeekMessagesInput struct { +} + +type PeekMessagesOutput struct { +} + +type PutMessageInput struct { +} + +type PutMessageOutput struct { +} + +type SetQueueMetadataInput struct { +} + +type SetQueueMetadataOutput struct { +} diff --git a/internal/generated/azuretable/base_provider.go b/internal/generated/azuretable/base_provider.go new file mode 100644 index 00000000..0c4dcd26 --- /dev/null +++ b/internal/generated/azuretable/base_provider.go @@ -0,0 +1,44 @@ +// Code generated by devcloud codegen. DO NOT EDIT. +// SPDX-License-Identifier: Apache-2.0 +package azuretable + +import ( + "context" + "fmt" +) + +var ErrNotImplemented = fmt.Errorf("operation not implemented") + +type BaseProvider struct{} + +func (b *BaseProvider) CreateTable(ctx context.Context, input *CreateTableInput) (*CreateTableOutput, error) { + return nil, fmt.Errorf("CreateTable: %w", ErrNotImplemented) +} + +func (b *BaseProvider) DeleteEntity(ctx context.Context, input *DeleteEntityInput) (*DeleteEntityOutput, error) { + return nil, fmt.Errorf("DeleteEntity: %w", ErrNotImplemented) +} + +func (b *BaseProvider) DeleteTable(ctx context.Context, input *DeleteTableInput) (*DeleteTableOutput, error) { + return nil, fmt.Errorf("DeleteTable: %w", ErrNotImplemented) +} + +func (b *BaseProvider) GetEntity(ctx context.Context, input *GetEntityInput) (*GetEntityOutput, error) { + return nil, fmt.Errorf("GetEntity: %w", ErrNotImplemented) +} + +func (b *BaseProvider) InsertEntity(ctx context.Context, input *InsertEntityInput) (*InsertEntityOutput, error) { + return nil, fmt.Errorf("InsertEntity: %w", ErrNotImplemented) +} + +func (b *BaseProvider) QueryEntities(ctx context.Context, input *QueryEntitiesInput) (*QueryEntitiesOutput, error) { + return nil, fmt.Errorf("QueryEntities: %w", ErrNotImplemented) +} + +func (b *BaseProvider) QueryTables(ctx context.Context, input *QueryTablesInput) (*QueryTablesOutput, error) { + return nil, fmt.Errorf("QueryTables: %w", ErrNotImplemented) +} + +func (b *BaseProvider) UpdateEntity(ctx context.Context, input *UpdateEntityInput) (*UpdateEntityOutput, error) { + return nil, fmt.Errorf("UpdateEntity: %w", ErrNotImplemented) +} diff --git a/internal/generated/azuretable/errors.go b/internal/generated/azuretable/errors.go new file mode 100644 index 00000000..58c93ca9 --- /dev/null +++ b/internal/generated/azuretable/errors.go @@ -0,0 +1,3 @@ +// Code generated by devcloud codegen. DO NOT EDIT. +// SPDX-License-Identifier: Apache-2.0 +package azuretable diff --git a/internal/generated/azuretable/router.go b/internal/generated/azuretable/router.go new file mode 100644 index 00000000..78f51ae3 --- /dev/null +++ b/internal/generated/azuretable/router.go @@ -0,0 +1,38 @@ +// Code generated by devcloud codegen. DO NOT EDIT. +// SPDX-License-Identifier: Apache-2.0 +package azuretable + +import "github.com/skyoo2003/devcloud/internal/shared/httproute" + +// PathParams holds the URL path parameters extracted by the router. +type PathParams = httproute.Params + +// OperationRoute maps an HTTP method and URI pattern to an operation name. +type OperationRoute = httproute.Route + +// OperationRoutes is the list of all routes for this service. +var OperationRoutes = []OperationRoute{ + {Method: "POST", Pattern: "/Tables", Operation: "CreateTable"}, + {Method: "DELETE", Pattern: "/{table}(PartitionKey='{partitionKey}',RowKey='{rowKey}')", Operation: "DeleteEntity"}, + {Method: "DELETE", Pattern: "/Tables('{table}')", Operation: "DeleteTable"}, + {Method: "GET", Pattern: "/{table}(PartitionKey='{partitionKey}',RowKey='{rowKey}')", Operation: "GetEntity"}, + {Method: "POST", Pattern: "/{table}", Operation: "InsertEntity"}, + {Method: "GET", Pattern: "/{table}()", Operation: "QueryEntities"}, + {Method: "GET", Pattern: "/Tables", Operation: "QueryTables"}, + {Method: "PUT", Pattern: "/{table}(PartitionKey='{partitionKey}',RowKey='{rowKey}')", Operation: "UpdateEntity"}, +} + +// MatchOperation returns the operation name and extracted path parameters for +// the given HTTP method and request URI. It returns ("", nil) if no route +// matches. +// +// Pass req.URL.RequestURI(), not req.URL.Path: some AWS operations are +// distinguished only by a query string (".../tagging?Operation=Tag"), and a +// bare path can never satisfy those routes. +// +// The matching itself lives in internal/shared/httproute so the generic CRUD +// engine resolves a rest-json operation exactly the way this router does, +// rather than carrying a second copy of the algorithm. +func MatchOperation(method, uri string) (string, PathParams) { + return httproute.Match(OperationRoutes, method, uri) +} diff --git a/internal/generated/azuretable/types.go b/internal/generated/azuretable/types.go new file mode 100644 index 00000000..b0a84bb6 --- /dev/null +++ b/internal/generated/azuretable/types.go @@ -0,0 +1,51 @@ +// Code generated by devcloud codegen. DO NOT EDIT. +// SPDX-License-Identifier: Apache-2.0 +package azuretable + +type CreateTableInput struct { +} + +type CreateTableOutput struct { +} + +type DeleteEntityInput struct { +} + +type DeleteEntityOutput struct { +} + +type DeleteTableInput struct { +} + +type DeleteTableOutput struct { +} + +type GetEntityInput struct { +} + +type GetEntityOutput struct { +} + +type InsertEntityInput struct { +} + +type InsertEntityOutput struct { +} + +type QueryEntitiesInput struct { +} + +type QueryEntitiesOutput struct { +} + +type QueryTablesInput struct { +} + +type QueryTablesOutput struct { +} + +type UpdateEntityInput struct { +} + +type UpdateEntityOutput struct { +} diff --git a/internal/generated/compat/services.json b/internal/generated/compat/services.json index 6de86615..0b23ac18 100644 --- a/internal/generated/compat/services.json +++ b/internal/generated/compat/services.json @@ -1430,6 +1430,35 @@ "UpdateScalingPlan" ] }, + "azurequeue": { + "protocol": "rest-xml", + "servedOps": [ + "ClearMessages", + "CreateQueue", + "DeleteMessage", + "DeleteQueue", + "GetMessages", + "GetQueueMetadata", + "ListQueues", + "PeekMessages", + "PutMessage", + "SetQueueMetadata" + ] + }, + "azuretable": { + "protocol": "rest-json", + "servedOps": [ + "CreateTable", + "DeleteEntity", + "DeleteTable", + "GetEntity", + "InsertEntity", + "MergeEntity", + "QueryEntities", + "QueryTables", + "UpdateEntity" + ] + }, "b2bi": { "protocol": "json-1.0", "servedOps": [ @@ -5176,6 +5205,24 @@ "UpdateLandingZone" ] }, + "cosmosdb": { + "protocol": "rest-json", + "servedOps": [ + "CreateContainer", + "CreateDatabase", + "CreateDocument", + "DeleteContainer", + "DeleteDatabase", + "DeleteDocument", + "GetContainer", + "GetDatabase", + "GetDocument", + "ListContainers", + "ListDatabases", + "ListDocuments", + "ReplaceDocument" + ] + }, "costandusagereportservice": { "protocol": "json-1.1", "servedOps": [ @@ -8782,6 +8829,19 @@ "UpdateStreamGroup" ] }, + "gcs": { + "protocol": "rest-json", + "servedOps": [ + "DeleteBucket", + "DeleteObject", + "GetBucket", + "GetObject", + "InsertBucket", + "InsertObject", + "ListBuckets", + "ListObjects" + ] + }, "geomaps": { "protocol": "rest-json", "servedOps": [ @@ -9716,6 +9776,16 @@ "GetRequestAuthorizationDetails" ] }, + "identity": { + "protocol": "rest-json", + "servedOps": [ + "ExchangeAzureToken", + "ExchangeGCPToken", + "GetJWKS", + "GetOpenIDConfiguration", + "MintToken" + ] + }, "identitystore": { "protocol": "", "servedOps": [ diff --git a/internal/generated/cosmosdb/base_provider.go b/internal/generated/cosmosdb/base_provider.go new file mode 100644 index 00000000..5cdced63 --- /dev/null +++ b/internal/generated/cosmosdb/base_provider.go @@ -0,0 +1,64 @@ +// Code generated by devcloud codegen. DO NOT EDIT. +// SPDX-License-Identifier: Apache-2.0 +package cosmosdb + +import ( + "context" + "fmt" +) + +var ErrNotImplemented = fmt.Errorf("operation not implemented") + +type BaseProvider struct{} + +func (b *BaseProvider) CreateContainer(ctx context.Context, input *CreateContainerInput) (*CreateContainerOutput, error) { + return nil, fmt.Errorf("CreateContainer: %w", ErrNotImplemented) +} + +func (b *BaseProvider) CreateDatabase(ctx context.Context, input *CreateDatabaseInput) (*CreateDatabaseOutput, error) { + return nil, fmt.Errorf("CreateDatabase: %w", ErrNotImplemented) +} + +func (b *BaseProvider) CreateDocument(ctx context.Context, input *CreateDocumentInput) (*CreateDocumentOutput, error) { + return nil, fmt.Errorf("CreateDocument: %w", ErrNotImplemented) +} + +func (b *BaseProvider) DeleteContainer(ctx context.Context, input *DeleteContainerInput) (*DeleteContainerOutput, error) { + return nil, fmt.Errorf("DeleteContainer: %w", ErrNotImplemented) +} + +func (b *BaseProvider) DeleteDatabase(ctx context.Context, input *DeleteDatabaseInput) (*DeleteDatabaseOutput, error) { + return nil, fmt.Errorf("DeleteDatabase: %w", ErrNotImplemented) +} + +func (b *BaseProvider) DeleteDocument(ctx context.Context, input *DeleteDocumentInput) (*DeleteDocumentOutput, error) { + return nil, fmt.Errorf("DeleteDocument: %w", ErrNotImplemented) +} + +func (b *BaseProvider) GetContainer(ctx context.Context, input *GetContainerInput) (*GetContainerOutput, error) { + return nil, fmt.Errorf("GetContainer: %w", ErrNotImplemented) +} + +func (b *BaseProvider) GetDatabase(ctx context.Context, input *GetDatabaseInput) (*GetDatabaseOutput, error) { + return nil, fmt.Errorf("GetDatabase: %w", ErrNotImplemented) +} + +func (b *BaseProvider) GetDocument(ctx context.Context, input *GetDocumentInput) (*GetDocumentOutput, error) { + return nil, fmt.Errorf("GetDocument: %w", ErrNotImplemented) +} + +func (b *BaseProvider) ListContainers(ctx context.Context, input *ListContainersInput) (*ListContainersOutput, error) { + return nil, fmt.Errorf("ListContainers: %w", ErrNotImplemented) +} + +func (b *BaseProvider) ListDatabases(ctx context.Context, input *ListDatabasesInput) (*ListDatabasesOutput, error) { + return nil, fmt.Errorf("ListDatabases: %w", ErrNotImplemented) +} + +func (b *BaseProvider) ListDocuments(ctx context.Context, input *ListDocumentsInput) (*ListDocumentsOutput, error) { + return nil, fmt.Errorf("ListDocuments: %w", ErrNotImplemented) +} + +func (b *BaseProvider) ReplaceDocument(ctx context.Context, input *ReplaceDocumentInput) (*ReplaceDocumentOutput, error) { + return nil, fmt.Errorf("ReplaceDocument: %w", ErrNotImplemented) +} diff --git a/internal/generated/cosmosdb/errors.go b/internal/generated/cosmosdb/errors.go new file mode 100644 index 00000000..b292cb40 --- /dev/null +++ b/internal/generated/cosmosdb/errors.go @@ -0,0 +1,3 @@ +// Code generated by devcloud codegen. DO NOT EDIT. +// SPDX-License-Identifier: Apache-2.0 +package cosmosdb diff --git a/internal/generated/cosmosdb/router.go b/internal/generated/cosmosdb/router.go new file mode 100644 index 00000000..0c845c68 --- /dev/null +++ b/internal/generated/cosmosdb/router.go @@ -0,0 +1,43 @@ +// Code generated by devcloud codegen. DO NOT EDIT. +// SPDX-License-Identifier: Apache-2.0 +package cosmosdb + +import "github.com/skyoo2003/devcloud/internal/shared/httproute" + +// PathParams holds the URL path parameters extracted by the router. +type PathParams = httproute.Params + +// OperationRoute maps an HTTP method and URI pattern to an operation name. +type OperationRoute = httproute.Route + +// OperationRoutes is the list of all routes for this service. +var OperationRoutes = []OperationRoute{ + {Method: "POST", Pattern: "/dbs/{db}/colls", Operation: "CreateContainer"}, + {Method: "POST", Pattern: "/dbs", Operation: "CreateDatabase"}, + {Method: "POST", Pattern: "/dbs/{db}/colls/{coll}/docs", Operation: "CreateDocument"}, + {Method: "DELETE", Pattern: "/dbs/{db}/colls/{coll}", Operation: "DeleteContainer"}, + {Method: "DELETE", Pattern: "/dbs/{db}", Operation: "DeleteDatabase"}, + {Method: "DELETE", Pattern: "/dbs/{db}/colls/{coll}/docs/{doc}", Operation: "DeleteDocument"}, + {Method: "GET", Pattern: "/dbs/{db}/colls/{coll}", Operation: "GetContainer"}, + {Method: "GET", Pattern: "/dbs/{db}", Operation: "GetDatabase"}, + {Method: "GET", Pattern: "/dbs/{db}/colls/{coll}/docs/{doc}", Operation: "GetDocument"}, + {Method: "GET", Pattern: "/dbs/{db}/colls", Operation: "ListContainers"}, + {Method: "GET", Pattern: "/dbs", Operation: "ListDatabases"}, + {Method: "GET", Pattern: "/dbs/{db}/colls/{coll}/docs", Operation: "ListDocuments"}, + {Method: "PUT", Pattern: "/dbs/{db}/colls/{coll}/docs/{doc}", Operation: "ReplaceDocument"}, +} + +// MatchOperation returns the operation name and extracted path parameters for +// the given HTTP method and request URI. It returns ("", nil) if no route +// matches. +// +// Pass req.URL.RequestURI(), not req.URL.Path: some AWS operations are +// distinguished only by a query string (".../tagging?Operation=Tag"), and a +// bare path can never satisfy those routes. +// +// The matching itself lives in internal/shared/httproute so the generic CRUD +// engine resolves a rest-json operation exactly the way this router does, +// rather than carrying a second copy of the algorithm. +func MatchOperation(method, uri string) (string, PathParams) { + return httproute.Match(OperationRoutes, method, uri) +} diff --git a/internal/generated/cosmosdb/types.go b/internal/generated/cosmosdb/types.go new file mode 100644 index 00000000..79fd8ea5 --- /dev/null +++ b/internal/generated/cosmosdb/types.go @@ -0,0 +1,81 @@ +// Code generated by devcloud codegen. DO NOT EDIT. +// SPDX-License-Identifier: Apache-2.0 +package cosmosdb + +type CreateContainerInput struct { +} + +type CreateContainerOutput struct { +} + +type CreateDatabaseInput struct { +} + +type CreateDatabaseOutput struct { +} + +type CreateDocumentInput struct { +} + +type CreateDocumentOutput struct { +} + +type DeleteContainerInput struct { +} + +type DeleteContainerOutput struct { +} + +type DeleteDatabaseInput struct { +} + +type DeleteDatabaseOutput struct { +} + +type DeleteDocumentInput struct { +} + +type DeleteDocumentOutput struct { +} + +type GetContainerInput struct { +} + +type GetContainerOutput struct { +} + +type GetDatabaseInput struct { +} + +type GetDatabaseOutput struct { +} + +type GetDocumentInput struct { +} + +type GetDocumentOutput struct { +} + +type ListContainersInput struct { +} + +type ListContainersOutput struct { +} + +type ListDatabasesInput struct { +} + +type ListDatabasesOutput struct { +} + +type ListDocumentsInput struct { +} + +type ListDocumentsOutput struct { +} + +type ReplaceDocumentInput struct { +} + +type ReplaceDocumentOutput struct { +} diff --git a/internal/generated/crudregistry/registry_gen.go b/internal/generated/crudregistry/registry_gen.go index fa471ccb..c6f85ace 100644 --- a/internal/generated/crudregistry/registry_gen.go +++ b/internal/generated/crudregistry/registry_gen.go @@ -1277,6 +1277,28 @@ func init() { "GetScalingPlanResourceForecastData": {Verb: "Get", Resource: "ScalingPlanResourceForecastData", OutputListKey: "Datapoints", OutputItemKey: ""}, "UpdateScalingPlan": {Verb: "Update", Resource: "ScalingPlan", OutputListKey: "", OutputItemKey: ""}, }) + crud.Register("azurequeue", map[string]crud.OpMeta{ + "GetQueueMetadata": {Verb: "Get", Resource: "QueueMetadata", OutputListKey: "", OutputItemKey: "", Method: "GET", URI: "/{queue}?comp=metadata"}, + "ListQueues": {Verb: "List", Resource: "Queue", OutputListKey: "", OutputItemKey: "", Method: "GET", URI: "/?comp=list"}, + "PeekMessages": {Verb: "", Resource: "", OutputListKey: "", OutputItemKey: "", Method: "GET", URI: "/{queue}/messages?peekonly=true"}, + "SetQueueMetadata": {Verb: "Update", Resource: "QueueMetadata", OutputListKey: "", OutputItemKey: "", Method: "PUT", URI: "/{queue}?comp=metadata"}, + "ClearMessages": {Verb: "", Resource: "", OutputListKey: "", OutputItemKey: "", Method: "DELETE", URI: "/{queue}/messages"}, + "CreateQueue": {Verb: "Create", Resource: "Queue", OutputListKey: "", OutputItemKey: "", Method: "PUT", URI: "/{queue}"}, + "DeleteMessage": {Verb: "Delete", Resource: "Message", OutputListKey: "", OutputItemKey: "", Method: "DELETE", URI: "/{queue}/messages/{messageid}"}, + "DeleteQueue": {Verb: "Delete", Resource: "Queue", OutputListKey: "", OutputItemKey: "", Method: "DELETE", URI: "/{queue}"}, + "GetMessages": {Verb: "Get", Resource: "Message", OutputListKey: "", OutputItemKey: "", Method: "GET", URI: "/{queue}/messages"}, + "PutMessage": {Verb: "Update", Resource: "Message", OutputListKey: "", OutputItemKey: "", Method: "POST", URI: "/{queue}/messages"}, + }) + crud.Register("azuretable", map[string]crud.OpMeta{ + "CreateTable": {Verb: "Create", Resource: "Table", OutputListKey: "", OutputItemKey: "", Method: "POST", URI: "/Tables"}, + "DeleteEntity": {Verb: "Delete", Resource: "Entity", OutputListKey: "", OutputItemKey: "", Method: "DELETE", URI: "/{table}(PartitionKey='{partitionKey}',RowKey='{rowKey}')"}, + "DeleteTable": {Verb: "Delete", Resource: "Table", OutputListKey: "", OutputItemKey: "", Method: "DELETE", URI: "/Tables('{table}')"}, + "GetEntity": {Verb: "Get", Resource: "Entity", OutputListKey: "", OutputItemKey: "", Method: "GET", URI: "/{table}(PartitionKey='{partitionKey}',RowKey='{rowKey}')"}, + "InsertEntity": {Verb: "", Resource: "", OutputListKey: "", OutputItemKey: "", Method: "POST", URI: "/{table}"}, + "QueryEntities": {Verb: "List", Resource: "Entity", OutputListKey: "", OutputItemKey: "", Method: "GET", URI: "/{table}()"}, + "QueryTables": {Verb: "List", Resource: "Table", OutputListKey: "", OutputItemKey: "", Method: "GET", URI: "/Tables"}, + "UpdateEntity": {Verb: "Update", Resource: "Entity", OutputListKey: "", OutputItemKey: "", Method: "PUT", URI: "/{table}(PartitionKey='{partitionKey}',RowKey='{rowKey}')"}, + }) crud.Register("b2bi", map[string]crud.OpMeta{ "CreateCapability": {Verb: "Create", Resource: "Capability", OutputListKey: "instructionsDocuments", OutputItemKey: "", Method: "POST", URI: "/capabilities"}, "CreatePartnership": {Verb: "Create", Resource: "Partnership", OutputListKey: "capabilities", OutputItemKey: "capabilityOptions", Method: "POST", URI: "/partnerships"}, @@ -4728,6 +4750,21 @@ func init() { "UpdateEnabledControl": {Verb: "Update", Resource: "EnabledControl", OutputListKey: "", OutputItemKey: "", Method: "POST", URI: "/update-enabled-control"}, "UpdateLandingZone": {Verb: "Update", Resource: "LandingZone", OutputListKey: "", OutputItemKey: "", Method: "POST", URI: "/update-landingzone"}, }) + crud.Register("cosmosdb", map[string]crud.OpMeta{ + "CreateContainer": {Verb: "Create", Resource: "Container", OutputListKey: "", OutputItemKey: "", Method: "POST", URI: "/dbs/{db}/colls"}, + "CreateDatabase": {Verb: "Create", Resource: "Database", OutputListKey: "", OutputItemKey: "", Method: "POST", URI: "/dbs"}, + "CreateDocument": {Verb: "Create", Resource: "Document", OutputListKey: "", OutputItemKey: "", Method: "POST", URI: "/dbs/{db}/colls/{coll}/docs"}, + "DeleteContainer": {Verb: "Delete", Resource: "Container", OutputListKey: "", OutputItemKey: "", Method: "DELETE", URI: "/dbs/{db}/colls/{coll}"}, + "DeleteDatabase": {Verb: "Delete", Resource: "Database", OutputListKey: "", OutputItemKey: "", Method: "DELETE", URI: "/dbs/{db}"}, + "DeleteDocument": {Verb: "Delete", Resource: "Document", OutputListKey: "", OutputItemKey: "", Method: "DELETE", URI: "/dbs/{db}/colls/{coll}/docs/{doc}"}, + "GetContainer": {Verb: "Get", Resource: "Container", OutputListKey: "", OutputItemKey: "", Method: "GET", URI: "/dbs/{db}/colls/{coll}"}, + "GetDatabase": {Verb: "Get", Resource: "Database", OutputListKey: "", OutputItemKey: "", Method: "GET", URI: "/dbs/{db}"}, + "GetDocument": {Verb: "Get", Resource: "Document", OutputListKey: "", OutputItemKey: "", Method: "GET", URI: "/dbs/{db}/colls/{coll}/docs/{doc}"}, + "ListContainers": {Verb: "List", Resource: "Container", OutputListKey: "", OutputItemKey: "", Method: "GET", URI: "/dbs/{db}/colls"}, + "ListDatabases": {Verb: "List", Resource: "Database", OutputListKey: "", OutputItemKey: "", Method: "GET", URI: "/dbs"}, + "ListDocuments": {Verb: "List", Resource: "Document", OutputListKey: "", OutputItemKey: "", Method: "GET", URI: "/dbs/{db}/colls/{coll}/docs"}, + "ReplaceDocument": {Verb: "Update", Resource: "Document", OutputListKey: "", OutputItemKey: "", Method: "PUT", URI: "/dbs/{db}/colls/{coll}/docs/{doc}"}, + }) crud.Register("costandusagereportservice", map[string]crud.OpMeta{ "DeleteReportDefinition": {Verb: "Delete", Resource: "ReportDefinition", OutputListKey: "", OutputItemKey: ""}, "DescribeReportDefinitions": {Verb: "Get", Resource: "ReportDefinition", OutputListKey: "ReportDefinitions", OutputItemKey: ""}, @@ -8041,6 +8078,16 @@ func init() { "UpdateApplication": {Verb: "Update", Resource: "Application", OutputListKey: "ApplicationLogPaths", OutputItemKey: "RuntimeEnvironment", Method: "PATCH", URI: "/applications/{Identifier}"}, "UpdateStreamGroup": {Verb: "Update", Resource: "StreamGroup", OutputListKey: "AssociatedApplications", OutputItemKey: "DefaultApplication", Method: "PATCH", URI: "/streamgroups/{Identifier}"}, }) + crud.Register("gcs", map[string]crud.OpMeta{ + "DeleteBucket": {Verb: "Delete", Resource: "Bucket", OutputListKey: "", OutputItemKey: "", Method: "DELETE", URI: "/storage/v1/b/{bucket}"}, + "DeleteObject": {Verb: "Delete", Resource: "Object", OutputListKey: "", OutputItemKey: "", Method: "DELETE", URI: "/storage/v1/b/{bucket}/o/{object+}"}, + "GetBucket": {Verb: "Get", Resource: "Bucket", OutputListKey: "", OutputItemKey: "", Method: "GET", URI: "/storage/v1/b/{bucket}"}, + "GetObject": {Verb: "Get", Resource: "Object", OutputListKey: "", OutputItemKey: "", Method: "GET", URI: "/storage/v1/b/{bucket}/o/{object+}"}, + "InsertBucket": {Verb: "", Resource: "", OutputListKey: "", OutputItemKey: "", Method: "POST", URI: "/storage/v1/b"}, + "InsertObject": {Verb: "", Resource: "", OutputListKey: "", OutputItemKey: "", Method: "POST", URI: "/upload/storage/v1/b/{bucket}/o"}, + "ListBuckets": {Verb: "List", Resource: "Bucket", OutputListKey: "", OutputItemKey: "", Method: "GET", URI: "/storage/v1/b"}, + "ListObjects": {Verb: "List", Resource: "Object", OutputListKey: "", OutputItemKey: "", Method: "GET", URI: "/storage/v1/b/{bucket}/o"}, + }) crud.Register("geomaps", map[string]crud.OpMeta{ "GetGlyphs": {Verb: "Get", Resource: "Glyph", OutputListKey: "", OutputItemKey: "", Method: "GET", URI: "/v2/glyphs/{FontStack}/{FontUnicodeRange}"}, "GetSprites": {Verb: "Get", Resource: "Sprite", OutputListKey: "", OutputItemKey: "", Method: "GET", URI: "/v2/styles/{Style}/{ColorScheme}/{Variant}/sprites/{FileName}"}, @@ -8923,6 +8970,13 @@ func init() { crud.Register("iamtoolbox", map[string]crud.OpMeta{ "GetRequestAuthorizationDetails": {Verb: "Get", Resource: "RequestAuthorizationDetail", OutputListKey: "evaluations", OutputItemKey: "", Method: "GET", URI: "/authorization-details/{authorizationId}"}, }) + crud.Register("identity", map[string]crud.OpMeta{ + "ExchangeAzureToken": {Verb: "", Resource: "", OutputListKey: "", OutputItemKey: "", Method: "POST", URI: "/devcloud/identity/azure/oauth2/v2.0/token"}, + "ExchangeGCPToken": {Verb: "", Resource: "", OutputListKey: "", OutputItemKey: "", Method: "POST", URI: "/devcloud/identity/gcp/v1/token"}, + "GetJWKS": {Verb: "Get", Resource: "JWKS", OutputListKey: "", OutputItemKey: "", Method: "GET", URI: "/devcloud/identity/jwks.json"}, + "GetOpenIDConfiguration": {Verb: "Get", Resource: "OpenIDConfiguration", OutputListKey: "", OutputItemKey: "", Method: "GET", URI: "/devcloud/identity/.well-known/openid-configuration"}, + "MintToken": {Verb: "", Resource: "", OutputListKey: "", OutputItemKey: "", Method: "POST", URI: "/devcloud/identity/token"}, + }) crud.Register("imagebuilder", map[string]crud.OpMeta{ "CancelImageCreation": {Verb: "Toggle", Resource: "ImageCreation", OutputListKey: "", OutputItemKey: "", Method: "PUT", URI: "/CancelImageCreation"}, "CancelLifecycleExecution": {Verb: "Toggle", Resource: "LifecycleExecution", OutputListKey: "", OutputItemKey: "", Method: "PUT", URI: "/CancelLifecycleExecution"}, diff --git a/internal/generated/fidelity/manifest_gen.go b/internal/generated/fidelity/manifest_gen.go index 0dae5393..ce01a4d1 100644 --- a/internal/generated/fidelity/manifest_gen.go +++ b/internal/generated/fidelity/manifest_gen.go @@ -1415,6 +1415,29 @@ var Services = map[string]Service{ "GetScalingPlanResourceForecastData": TierAutoCRUD, "UpdateScalingPlan": TierAutoCRUD, }}, + "azurequeue": {Protocol: "rest-xml", ModelBacked: true, EngineWired: false, Operations: map[string]Tier{ + "ClearMessages": TierHandVerified, + "CreateQueue": TierHandVerified, + "DeleteMessage": TierHandVerified, + "DeleteQueue": TierHandVerified, + "GetMessages": TierHandVerified, + "GetQueueMetadata": TierHandVerified, + "ListQueues": TierHandVerified, + "PeekMessages": TierHandVerified, + "PutMessage": TierHandVerified, + "SetQueueMetadata": TierHandVerified, + }}, + "azuretable": {Protocol: "rest-json", ModelBacked: true, EngineWired: false, Operations: map[string]Tier{ + "CreateTable": TierHandVerified, + "DeleteEntity": TierHandVerified, + "DeleteTable": TierHandVerified, + "GetEntity": TierHandVerified, + "InsertEntity": TierHandVerified, + "MergeEntity": TierHandVerified, + "QueryEntities": TierHandVerified, + "QueryTables": TierHandVerified, + "UpdateEntity": TierHandVerified, + }}, "b2bi": {Protocol: "json-1.0", ModelBacked: true, EngineWired: true, Operations: map[string]Tier{ "CreateCapability": TierAutoCRUD, "CreatePartnership": TierAutoCRUD, @@ -5064,6 +5087,21 @@ var Services = map[string]Service{ "UpdateEnabledControl": TierAutoCRUD, "UpdateLandingZone": TierAutoCRUD, }}, + "cosmosdb": {Protocol: "rest-json", ModelBacked: true, EngineWired: false, Operations: map[string]Tier{ + "CreateContainer": TierHandVerified, + "CreateDatabase": TierHandVerified, + "CreateDocument": TierHandVerified, + "DeleteContainer": TierHandVerified, + "DeleteDatabase": TierHandVerified, + "DeleteDocument": TierHandVerified, + "GetContainer": TierHandVerified, + "GetDatabase": TierHandVerified, + "GetDocument": TierHandVerified, + "ListContainers": TierHandVerified, + "ListDatabases": TierHandVerified, + "ListDocuments": TierHandVerified, + "ReplaceDocument": TierHandVerified, + }}, "costandusagereportservice": {Protocol: "json-1.1", ModelBacked: true, EngineWired: true, Operations: map[string]Tier{ "DeleteReportDefinition": TierAutoCRUD, "DescribeReportDefinitions": TierAutoCRUD, @@ -8555,6 +8593,16 @@ var Services = map[string]Service{ "UpdateApplication": TierAutoCRUD, "UpdateStreamGroup": TierAutoCRUD, }}, + "gcs": {Protocol: "rest-json", ModelBacked: true, EngineWired: false, Operations: map[string]Tier{ + "DeleteBucket": TierHandVerified, + "DeleteObject": TierHandVerified, + "GetBucket": TierHandVerified, + "GetObject": TierHandVerified, + "InsertBucket": TierHandVerified, + "InsertObject": TierHandVerified, + "ListBuckets": TierHandVerified, + "ListObjects": TierHandVerified, + }}, "geomaps": {Protocol: "rest-json", ModelBacked: true, EngineWired: true, Operations: map[string]Tier{ "GetGlyphs": TierAutoCRUD, "GetSprites": TierAutoCRUD, @@ -9461,6 +9509,13 @@ var Services = map[string]Service{ "iamtoolbox": {Protocol: "rest-json", ModelBacked: true, EngineWired: true, Operations: map[string]Tier{ "GetRequestAuthorizationDetails": TierAutoCRUD, }}, + "identity": {Protocol: "rest-json", ModelBacked: true, EngineWired: false, Operations: map[string]Tier{ + "ExchangeAzureToken": TierHandVerified, + "ExchangeGCPToken": TierHandVerified, + "GetJWKS": TierHandVerified, + "GetOpenIDConfiguration": TierHandVerified, + "MintToken": TierHandVerified, + }}, "identitystore": {Protocol: "", ModelBacked: false, EngineWired: false, Operations: map[string]Tier{ "ActivateUser": TierHandVerified, "BatchCreateGroups": TierHandVerified, diff --git a/internal/generated/gcs/base_provider.go b/internal/generated/gcs/base_provider.go new file mode 100644 index 00000000..6b450ca8 --- /dev/null +++ b/internal/generated/gcs/base_provider.go @@ -0,0 +1,44 @@ +// Code generated by devcloud codegen. DO NOT EDIT. +// SPDX-License-Identifier: Apache-2.0 +package gcs + +import ( + "context" + "fmt" +) + +var ErrNotImplemented = fmt.Errorf("operation not implemented") + +type BaseProvider struct{} + +func (b *BaseProvider) DeleteBucket(ctx context.Context, input *DeleteBucketInput) (*DeleteBucketOutput, error) { + return nil, fmt.Errorf("DeleteBucket: %w", ErrNotImplemented) +} + +func (b *BaseProvider) DeleteObject(ctx context.Context, input *DeleteObjectInput) (*DeleteObjectOutput, error) { + return nil, fmt.Errorf("DeleteObject: %w", ErrNotImplemented) +} + +func (b *BaseProvider) GetBucket(ctx context.Context, input *GetBucketInput) (*GetBucketOutput, error) { + return nil, fmt.Errorf("GetBucket: %w", ErrNotImplemented) +} + +func (b *BaseProvider) GetObject(ctx context.Context, input *GetObjectInput) (*GetObjectOutput, error) { + return nil, fmt.Errorf("GetObject: %w", ErrNotImplemented) +} + +func (b *BaseProvider) InsertBucket(ctx context.Context, input *InsertBucketInput) (*InsertBucketOutput, error) { + return nil, fmt.Errorf("InsertBucket: %w", ErrNotImplemented) +} + +func (b *BaseProvider) InsertObject(ctx context.Context, input *InsertObjectInput) (*InsertObjectOutput, error) { + return nil, fmt.Errorf("InsertObject: %w", ErrNotImplemented) +} + +func (b *BaseProvider) ListBuckets(ctx context.Context, input *ListBucketsInput) (*ListBucketsOutput, error) { + return nil, fmt.Errorf("ListBuckets: %w", ErrNotImplemented) +} + +func (b *BaseProvider) ListObjects(ctx context.Context, input *ListObjectsInput) (*ListObjectsOutput, error) { + return nil, fmt.Errorf("ListObjects: %w", ErrNotImplemented) +} diff --git a/internal/generated/gcs/errors.go b/internal/generated/gcs/errors.go new file mode 100644 index 00000000..5d5791c9 --- /dev/null +++ b/internal/generated/gcs/errors.go @@ -0,0 +1,3 @@ +// Code generated by devcloud codegen. DO NOT EDIT. +// SPDX-License-Identifier: Apache-2.0 +package gcs diff --git a/internal/generated/gcs/router.go b/internal/generated/gcs/router.go new file mode 100644 index 00000000..05b6170d --- /dev/null +++ b/internal/generated/gcs/router.go @@ -0,0 +1,38 @@ +// Code generated by devcloud codegen. DO NOT EDIT. +// SPDX-License-Identifier: Apache-2.0 +package gcs + +import "github.com/skyoo2003/devcloud/internal/shared/httproute" + +// PathParams holds the URL path parameters extracted by the router. +type PathParams = httproute.Params + +// OperationRoute maps an HTTP method and URI pattern to an operation name. +type OperationRoute = httproute.Route + +// OperationRoutes is the list of all routes for this service. +var OperationRoutes = []OperationRoute{ + {Method: "DELETE", Pattern: "/storage/v1/b/{bucket}", Operation: "DeleteBucket"}, + {Method: "DELETE", Pattern: "/storage/v1/b/{bucket}/o/{object+}", Operation: "DeleteObject"}, + {Method: "GET", Pattern: "/storage/v1/b/{bucket}", Operation: "GetBucket"}, + {Method: "GET", Pattern: "/storage/v1/b/{bucket}/o/{object+}", Operation: "GetObject"}, + {Method: "POST", Pattern: "/storage/v1/b", Operation: "InsertBucket"}, + {Method: "POST", Pattern: "/upload/storage/v1/b/{bucket}/o", Operation: "InsertObject"}, + {Method: "GET", Pattern: "/storage/v1/b", Operation: "ListBuckets"}, + {Method: "GET", Pattern: "/storage/v1/b/{bucket}/o", Operation: "ListObjects"}, +} + +// MatchOperation returns the operation name and extracted path parameters for +// the given HTTP method and request URI. It returns ("", nil) if no route +// matches. +// +// Pass req.URL.RequestURI(), not req.URL.Path: some AWS operations are +// distinguished only by a query string (".../tagging?Operation=Tag"), and a +// bare path can never satisfy those routes. +// +// The matching itself lives in internal/shared/httproute so the generic CRUD +// engine resolves a rest-json operation exactly the way this router does, +// rather than carrying a second copy of the algorithm. +func MatchOperation(method, uri string) (string, PathParams) { + return httproute.Match(OperationRoutes, method, uri) +} diff --git a/internal/generated/gcs/types.go b/internal/generated/gcs/types.go new file mode 100644 index 00000000..d4c67a22 --- /dev/null +++ b/internal/generated/gcs/types.go @@ -0,0 +1,87 @@ +// Code generated by devcloud codegen. DO NOT EDIT. +// SPDX-License-Identifier: Apache-2.0 +package gcs + +type Bucket struct { + Etag string `json:"etag" xml:"etag"` + Id string `json:"id" xml:"id"` + Kind string `json:"kind" xml:"kind"` + Location string `json:"location" xml:"location"` + Name string `json:"name" xml:"name"` + ProjectNumber string `json:"projectNumber" xml:"projectNumber"` + StorageClass string `json:"storageClass" xml:"storageClass"` + TimeCreated string `json:"timeCreated" xml:"timeCreated"` + Updated string `json:"updated" xml:"updated"` +} + +type BucketList struct { + Items string `json:"items" xml:"items"` + Kind string `json:"kind" xml:"kind"` +} + +type DeleteBucketInput struct { +} + +type DeleteBucketOutput struct { +} + +type DeleteObjectInput struct { +} + +type DeleteObjectOutput struct { +} + +type GetBucketInput struct { +} + +type GetBucketOutput struct { +} + +type GetObjectInput struct { +} + +type GetObjectOutput struct { +} + +type InsertBucketInput struct { +} + +type InsertBucketOutput struct { +} + +type InsertObjectInput struct { +} + +type InsertObjectOutput struct { +} + +type ListBucketsInput struct { +} + +type ListBucketsOutput struct { +} + +type ListObjectsInput struct { +} + +type ListObjectsOutput struct { +} + +type Object struct { + Bucket string `json:"bucket" xml:"bucket"` + ContentType string `json:"contentType" xml:"contentType"` + Crc32c string `json:"crc32c" xml:"crc32c"` + Etag string `json:"etag" xml:"etag"` + Id string `json:"id" xml:"id"` + Kind string `json:"kind" xml:"kind"` + Md5Hash string `json:"md5Hash" xml:"md5Hash"` + Name string `json:"name" xml:"name"` + Size string `json:"size" xml:"size"` + TimeCreated string `json:"timeCreated" xml:"timeCreated"` + Updated string `json:"updated" xml:"updated"` +} + +type ObjectList struct { + Items string `json:"items" xml:"items"` + Kind string `json:"kind" xml:"kind"` +} diff --git a/internal/generated/identity/base_provider.go b/internal/generated/identity/base_provider.go new file mode 100644 index 00000000..0b9199e1 --- /dev/null +++ b/internal/generated/identity/base_provider.go @@ -0,0 +1,32 @@ +// Code generated by devcloud codegen. DO NOT EDIT. +// SPDX-License-Identifier: Apache-2.0 +package identity + +import ( + "context" + "fmt" +) + +var ErrNotImplemented = fmt.Errorf("operation not implemented") + +type BaseProvider struct{} + +func (b *BaseProvider) ExchangeAzureToken(ctx context.Context, input *ExchangeAzureTokenInput) (*ExchangeAzureTokenOutput, error) { + return nil, fmt.Errorf("ExchangeAzureToken: %w", ErrNotImplemented) +} + +func (b *BaseProvider) ExchangeGCPToken(ctx context.Context, input *ExchangeGCPTokenInput) (*ExchangeGCPTokenOutput, error) { + return nil, fmt.Errorf("ExchangeGCPToken: %w", ErrNotImplemented) +} + +func (b *BaseProvider) GetJWKS(ctx context.Context, input *GetJWKSInput) (*GetJWKSOutput, error) { + return nil, fmt.Errorf("GetJWKS: %w", ErrNotImplemented) +} + +func (b *BaseProvider) GetOpenIDConfiguration(ctx context.Context, input *GetOpenIDConfigurationInput) (*GetOpenIDConfigurationOutput, error) { + return nil, fmt.Errorf("GetOpenIDConfiguration: %w", ErrNotImplemented) +} + +func (b *BaseProvider) MintToken(ctx context.Context, input *MintTokenInput) (*MintTokenOutput, error) { + return nil, fmt.Errorf("MintToken: %w", ErrNotImplemented) +} diff --git a/internal/generated/identity/errors.go b/internal/generated/identity/errors.go new file mode 100644 index 00000000..18db35be --- /dev/null +++ b/internal/generated/identity/errors.go @@ -0,0 +1,3 @@ +// Code generated by devcloud codegen. DO NOT EDIT. +// SPDX-License-Identifier: Apache-2.0 +package identity diff --git a/internal/generated/identity/router.go b/internal/generated/identity/router.go new file mode 100644 index 00000000..18d429fc --- /dev/null +++ b/internal/generated/identity/router.go @@ -0,0 +1,35 @@ +// Code generated by devcloud codegen. DO NOT EDIT. +// SPDX-License-Identifier: Apache-2.0 +package identity + +import "github.com/skyoo2003/devcloud/internal/shared/httproute" + +// PathParams holds the URL path parameters extracted by the router. +type PathParams = httproute.Params + +// OperationRoute maps an HTTP method and URI pattern to an operation name. +type OperationRoute = httproute.Route + +// OperationRoutes is the list of all routes for this service. +var OperationRoutes = []OperationRoute{ + {Method: "POST", Pattern: "/devcloud/identity/azure/oauth2/v2.0/token", Operation: "ExchangeAzureToken"}, + {Method: "POST", Pattern: "/devcloud/identity/gcp/v1/token", Operation: "ExchangeGCPToken"}, + {Method: "GET", Pattern: "/devcloud/identity/jwks.json", Operation: "GetJWKS"}, + {Method: "GET", Pattern: "/devcloud/identity/.well-known/openid-configuration", Operation: "GetOpenIDConfiguration"}, + {Method: "POST", Pattern: "/devcloud/identity/token", Operation: "MintToken"}, +} + +// MatchOperation returns the operation name and extracted path parameters for +// the given HTTP method and request URI. It returns ("", nil) if no route +// matches. +// +// Pass req.URL.RequestURI(), not req.URL.Path: some AWS operations are +// distinguished only by a query string (".../tagging?Operation=Tag"), and a +// bare path can never satisfy those routes. +// +// The matching itself lives in internal/shared/httproute so the generic CRUD +// engine resolves a rest-json operation exactly the way this router does, +// rather than carrying a second copy of the algorithm. +func MatchOperation(method, uri string) (string, PathParams) { + return httproute.Match(OperationRoutes, method, uri) +} diff --git a/internal/generated/identity/types.go b/internal/generated/identity/types.go new file mode 100644 index 00000000..f93698dd --- /dev/null +++ b/internal/generated/identity/types.go @@ -0,0 +1,33 @@ +// Code generated by devcloud codegen. DO NOT EDIT. +// SPDX-License-Identifier: Apache-2.0 +package identity + +type ExchangeAzureTokenInput struct { +} + +type ExchangeAzureTokenOutput struct { +} + +type ExchangeGCPTokenInput struct { +} + +type ExchangeGCPTokenOutput struct { +} + +type GetJWKSInput struct { +} + +type GetJWKSOutput struct { +} + +type GetOpenIDConfigurationInput struct { +} + +type GetOpenIDConfigurationOutput struct { +} + +type MintTokenInput struct { +} + +type MintTokenOutput struct { +} diff --git a/internal/services/azurequeue/provider.go b/internal/services/azurequeue/provider.go new file mode 100644 index 00000000..e6178a89 --- /dev/null +++ b/internal/services/azurequeue/provider.go @@ -0,0 +1,405 @@ +// SPDX-License-Identifier: Apache-2.0 + +package azurequeue + +import ( + "context" + "crypto/rand" + "encoding/hex" + "encoding/xml" + "errors" + "fmt" + "io" + "net/http" + "strconv" + "strings" + "time" + + "github.com/skyoo2003/devcloud/internal/auth" + generated "github.com/skyoo2003/devcloud/internal/generated/azurequeue" + "github.com/skyoo2003/devcloud/internal/plugin" +) + +type Provider struct { + generated.BaseProvider + dataDir string + store *Store +} + +func (p *Provider) ServiceID() string { return "azurequeue" } +func (p *Provider) ServiceName() string { return "Azure Queue Storage" } +func (p *Provider) Protocol() plugin.ProtocolType { return plugin.ProtocolRESTXML } +func (p *Provider) Provider() string { return "azure" } + +func (p *Provider) Init(cfg plugin.PluginConfig) error { + p.dataDir = cfg.DataDir + if p.dataDir == "" { + p.dataDir = "./data/azure-queue" + } + p.store = NewStore(p.dataDir) + return nil +} + +func (p *Provider) Shutdown(ctx context.Context) error { + return nil +} + +func (p *Provider) ListResources(ctx context.Context) ([]plugin.Resource, error) { + queues, err := p.store.ListQueues("devstoreaccount1", "") + if err != nil { + return nil, err + } + var res []plugin.Resource + for _, q := range queues { + res = append(res, plugin.Resource{ + Type: "Queue", + ID: q.Name, + Name: q.Name, + }) + } + return res, nil +} + +func randomRequestID() string { + b := make([]byte, 16) + _, _ = rand.Read(b) + return hex.EncodeToString(b) +} + +func (p *Provider) reply(req *http.Request, status int, headers map[string]string, body []byte) *plugin.Response { + if headers == nil { + headers = make(map[string]string) + } + headers["x-ms-request-id"] = randomRequestID() + version := req.Header.Get("x-ms-version") + if version == "" { + version = "2023-11-03" + } + headers["x-ms-version"] = version + headers["Date"] = time.Now().UTC().Format(http.TimeFormat) + + ct := headers["Content-Type"] + if ct == "" { + ct = "application/xml" + } + return &plugin.Response{ + StatusCode: status, + Headers: headers, + Body: body, + ContentType: ct, + } +} + +func (p *Provider) errorReply(req *http.Request, status int, code, message string) *plugin.Response { + headers := map[string]string{ + "x-ms-error-code": code, + "Content-Type": "application/xml", + "x-ms-request-id": randomRequestID(), + "Date": time.Now().UTC().Format(http.TimeFormat), + } + version := req.Header.Get("x-ms-version") + if version == "" { + version = "2023-11-03" + } + headers["x-ms-version"] = version + + body := fmt.Sprintf("\n%s%s\nRequestId:%s\nTime:%s", + code, message, headers["x-ms-request-id"], time.Now().UTC().Format(time.RFC3339Nano)) + + return &plugin.Response{ + StatusCode: status, + Headers: headers, + Body: []byte(body), + ContentType: "application/xml", + } +} + +func (p *Provider) HandleRequest(ctx context.Context, _ string, req *http.Request) (*plugin.Response, error) { + account := "devstoreaccount1" + if id, ok := auth.FromContext(ctx); ok && id.AccessKeyID != "" { + account = id.AccessKeyID + } + + uri := req.URL.RequestURI() + for _, accPrefix := range []string{"/" + account + "/queue", "/devstoreaccount1/queue", "/" + account, "/devstoreaccount1", "/queue"} { + if strings.HasPrefix(uri, accPrefix+"/") { + uri = uri[len(accPrefix):] + break + } else if strings.HasPrefix(uri, accPrefix+"?") { + uri = "/" + uri[len(accPrefix):] + break + } else if uri == accPrefix { + uri = "/" + break + } + } + + op, params := generated.MatchOperation(req.Method, uri) + if op == "" { + op, params = generated.MatchOperation(req.Method, req.URL.Path) + } + + switch op { + case "ListQueues": + return p.handleListQueues(req, account) + case "CreateQueue": + return p.handleCreateQueue(req, account, params["queue"]) + case "DeleteQueue": + return p.handleDeleteQueue(req, account, params["queue"]) + case "GetQueueMetadata": + return p.handleGetQueueMetadata(req, account, params["queue"]) + case "SetQueueMetadata": + return p.handleSetQueueMetadata(req, account, params["queue"]) + case "PutMessage": + return p.handlePutMessage(req, account, params["queue"]) + case "GetMessages": + return p.handleGetMessages(req, account, params["queue"], false) + case "PeekMessages": + return p.handleGetMessages(req, account, params["queue"], true) + case "DeleteMessage": + return p.handleDeleteMessage(req, account, params["queue"], params["messageid"]) + case "ClearMessages": + return p.handleClearMessages(req, account, params["queue"]) + default: + return p.errorReply(req, http.StatusBadRequest, "InvalidAction", "operation not handled: "+req.Method+" "+req.URL.Path), nil + } +} + +type XMLQueueItem struct { + Name string `xml:"Name"` +} + +type XMLQueuesResult struct { + XMLName xml.Name `xml:"EnumerationResults"` + ServiceEndpoint string `xml:"ServiceEndpoint,attr"` + Queues []XMLQueueItem `xml:"Queues>Queue"` +} + +func (p *Provider) handleListQueues(req *http.Request, account string) (*plugin.Response, error) { + prefix := req.URL.Query().Get("prefix") + queues, err := p.store.ListQueues(account, prefix) + if err != nil { + return p.errorReply(req, http.StatusInternalServerError, "InternalError", err.Error()), nil + } + + var items []XMLQueueItem + for _, q := range queues { + items = append(items, XMLQueueItem{Name: q.Name}) + } + + result := XMLQueuesResult{ + ServiceEndpoint: fmt.Sprintf("http://127.0.0.1:4747/%s/", account), + Queues: items, + } + raw, _ := xml.MarshalIndent(result, "", " ") + body := []byte(xml.Header + string(raw)) + return p.reply(req, http.StatusOK, map[string]string{"Content-Type": "application/xml"}, body), nil +} + +func (p *Provider) handleCreateQueue(req *http.Request, account, queue string) (*plugin.Response, error) { + meta := extractMetadata(req.Header) + err := p.store.CreateQueue(account, queue, meta) + if err != nil { + if errors.Is(err, ErrQueueAlreadyExists) { + return p.reply(req, http.StatusNoContent, nil, nil), nil + } + if errors.Is(err, ErrInvalidQueueName) { + return p.errorReply(req, http.StatusBadRequest, "InvalidResourceName", "invalid queue name"), nil + } + return p.errorReply(req, http.StatusInternalServerError, "InternalError", err.Error()), nil + } + return p.reply(req, http.StatusCreated, nil, nil), nil +} + +func (p *Provider) handleDeleteQueue(req *http.Request, account, queue string) (*plugin.Response, error) { + err := p.store.DeleteQueue(account, queue) + if err != nil { + if errors.Is(err, ErrQueueNotFound) { + return p.errorReply(req, http.StatusNotFound, "QueueNotFound", "queue not found"), nil + } + return p.errorReply(req, http.StatusInternalServerError, "InternalError", err.Error()), nil + } + return p.reply(req, http.StatusNoContent, nil, nil), nil +} + +func (p *Provider) handleGetQueueMetadata(req *http.Request, account, queue string) (*plugin.Response, error) { + meta, count, err := p.store.GetQueueMetadata(account, queue) + if err != nil { + if errors.Is(err, ErrQueueNotFound) { + return p.errorReply(req, http.StatusNotFound, "QueueNotFound", "queue not found"), nil + } + return p.errorReply(req, http.StatusInternalServerError, "InternalError", err.Error()), nil + } + headers := map[string]string{ + "x-ms-approximate-messages-count": fmt.Sprintf("%d", count), + } + writeMetadataHeaders(headers, meta) + return p.reply(req, http.StatusOK, headers, nil), nil +} + +func (p *Provider) handleSetQueueMetadata(req *http.Request, account, queue string) (*plugin.Response, error) { + meta := extractMetadata(req.Header) + err := p.store.SetQueueMetadata(account, queue, meta) + if err != nil { + if errors.Is(err, ErrQueueNotFound) { + return p.errorReply(req, http.StatusNotFound, "QueueNotFound", "queue not found"), nil + } + return p.errorReply(req, http.StatusInternalServerError, "InternalError", err.Error()), nil + } + return p.reply(req, http.StatusNoContent, nil, nil), nil +} + +type XMLPutMessageRequest struct { + XMLName xml.Name `xml:"QueueMessage"` + Text string `xml:"MessageText"` +} + +type XMLMessageResponse struct { + MessageID string `xml:"MessageId"` + InsertionTime string `xml:"InsertionTime"` + ExpirationTime string `xml:"ExpirationTime"` + PopReceipt string `xml:"PopReceipt"` + TimeNextVisible string `xml:"TimeNextVisible"` + DequeueCount int `xml:"DequeueCount,omitempty"` + MessageText string `xml:"MessageText,omitempty"` +} + +type XMLQueueMessagesList struct { + XMLName xml.Name `xml:"QueueMessagesList"` + Messages []XMLMessageResponse `xml:"QueueMessage"` +} + +func (p *Provider) handlePutMessage(req *http.Request, account, queue string) (*plugin.Response, error) { + body, err := io.ReadAll(req.Body) + if err != nil { + return p.errorReply(req, http.StatusBadRequest, "InvalidXmlDocument", "failed to read body"), nil + } + var putReq XMLPutMessageRequest + if err := xml.Unmarshal(body, &putReq); err != nil { + return p.errorReply(req, http.StatusBadRequest, "InvalidXmlDocument", "invalid message XML"), nil + } + + visTimeoutSec, _ := strconv.Atoi(req.URL.Query().Get("visibilitytimeout")) + ttlSec, _ := strconv.Atoi(req.URL.Query().Get("messagettl")) + + msg, err := p.store.PutMessage(account, queue, putReq.Text, time.Duration(visTimeoutSec)*time.Second, time.Duration(ttlSec)*time.Second) + if err != nil { + if errors.Is(err, ErrQueueNotFound) { + return p.errorReply(req, http.StatusNotFound, "QueueNotFound", "queue not found"), nil + } + return p.errorReply(req, http.StatusInternalServerError, "InternalError", err.Error()), nil + } + + res := XMLQueueMessagesList{ + Messages: []XMLMessageResponse{{ + MessageID: msg.ID, + InsertionTime: msg.InsertionTime.Format(http.TimeFormat), + ExpirationTime: msg.ExpirationTime.Format(http.TimeFormat), + PopReceipt: msg.PopReceipt, + TimeNextVisible: msg.TimeNextVisible.Format(http.TimeFormat), + }}, + } + raw, _ := xml.MarshalIndent(res, "", " ") + out := []byte(xml.Header + string(raw)) + return p.reply(req, http.StatusCreated, map[string]string{"Content-Type": "application/xml"}, out), nil +} + +func (p *Provider) handleGetMessages(req *http.Request, account, queue string, peek bool) (*plugin.Response, error) { + numMessages := 1 + if n := req.URL.Query().Get("numofmessages"); n != "" { + if parsed, err := strconv.Atoi(n); err == nil && parsed > 0 { + numMessages = parsed + } + } + visTimeout := 30 * time.Second + if vt := req.URL.Query().Get("visibilitytimeout"); vt != "" { + if parsed, err := strconv.Atoi(vt); err == nil && parsed >= 0 { + visTimeout = time.Duration(parsed) * time.Second + } + } + + var msgs []*QueueMessage + var err error + if peek { + msgs, err = p.store.PeekMessages(account, queue, numMessages) + } else { + msgs, err = p.store.GetMessages(account, queue, numMessages, visTimeout) + } + if err != nil { + if errors.Is(err, ErrQueueNotFound) { + return p.errorReply(req, http.StatusNotFound, "QueueNotFound", "queue not found"), nil + } + return p.errorReply(req, http.StatusInternalServerError, "InternalError", err.Error()), nil + } + + var resMsgs []XMLMessageResponse + for _, m := range msgs { + resMsgs = append(resMsgs, XMLMessageResponse{ + MessageID: m.ID, + InsertionTime: m.InsertionTime.Format(http.TimeFormat), + ExpirationTime: m.ExpirationTime.Format(http.TimeFormat), + PopReceipt: m.PopReceipt, + TimeNextVisible: m.TimeNextVisible.Format(http.TimeFormat), + DequeueCount: m.DequeueCount, + MessageText: m.Text, + }) + } + + result := XMLQueueMessagesList{Messages: resMsgs} + raw, _ := xml.MarshalIndent(result, "", " ") + out := []byte(xml.Header + string(raw)) + return p.reply(req, http.StatusOK, map[string]string{"Content-Type": "application/xml"}, out), nil +} + +func (p *Provider) handleDeleteMessage(req *http.Request, account, queue, messageID string) (*plugin.Response, error) { + popReceipt := req.URL.Query().Get("popreceipt") + err := p.store.DeleteMessage(account, queue, messageID, popReceipt) + if err != nil { + if errors.Is(err, ErrQueueNotFound) { + return p.errorReply(req, http.StatusNotFound, "QueueNotFound", "queue not found"), nil + } + if errors.Is(err, ErrMessageNotFound) { + return p.errorReply(req, http.StatusNotFound, "MessageNotFound", "message not found"), nil + } + if errors.Is(err, ErrPopReceiptMismatch) { + return p.errorReply(req, http.StatusBadRequest, "PopReceiptMismatch", "pop receipt mismatch"), nil + } + return p.errorReply(req, http.StatusInternalServerError, "InternalError", err.Error()), nil + } + return p.reply(req, http.StatusNoContent, nil, nil), nil +} + +func (p *Provider) handleClearMessages(req *http.Request, account, queue string) (*plugin.Response, error) { + err := p.store.ClearMessages(account, queue) + if err != nil { + if errors.Is(err, ErrQueueNotFound) { + return p.errorReply(req, http.StatusNotFound, "QueueNotFound", "queue not found"), nil + } + return p.errorReply(req, http.StatusInternalServerError, "InternalError", err.Error()), nil + } + return p.reply(req, http.StatusNoContent, nil, nil), nil +} + +func extractMetadata(h http.Header) map[string]string { + meta := make(map[string]string) + for k, vv := range h { + lower := strings.ToLower(k) + if strings.HasPrefix(lower, "x-ms-meta-") && len(vv) > 0 { + metaKey := strings.TrimPrefix(lower, "x-ms-meta-") + meta[metaKey] = vv[0] + } + } + return meta +} + +func writeMetadataHeaders(headers map[string]string, meta map[string]string) { + for k, v := range meta { + headers["x-ms-meta-"+strings.ToLower(k)] = v + } +} + +func init() { + plugin.DefaultRegistry.Register("azurequeue", func() plugin.ServicePlugin { + return &Provider{} + }) +} diff --git a/internal/services/azurequeue/provider_test.go b/internal/services/azurequeue/provider_test.go new file mode 100644 index 00000000..479ad449 --- /dev/null +++ b/internal/services/azurequeue/provider_test.go @@ -0,0 +1,125 @@ +// SPDX-License-Identifier: Apache-2.0 + +package azurequeue + +import ( + "bytes" + "context" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/skyoo2003/devcloud/internal/plugin" +) + +func newTestProvider(t *testing.T) *Provider { + t.Helper() + dir, err := os.MkdirTemp("", "devcloud-queue-test-*") + require.NoError(t, err) + t.Cleanup(func() { _ = os.RemoveAll(dir) }) + + p := &Provider{} + err = p.Init(plugin.PluginConfig{DataDir: filepath.Join(dir, "queue")}) + require.NoError(t, err) + return p +} + +func TestQueueProvider_Metadata(t *testing.T) { + p := newTestProvider(t) + assert.Equal(t, "azurequeue", p.ServiceID()) + assert.Equal(t, "Azure Queue Storage", p.ServiceName()) + assert.Equal(t, plugin.ProtocolRESTXML, p.Protocol()) + assert.Equal(t, "azure", plugin.ProviderOf(p)) +} + +func TestQueueProvider_QueueLifecycle(t *testing.T) { + p := newTestProvider(t) + ctx := context.Background() + + // 1. Create Queue + reqCreate := httptest.NewRequest(http.MethodPut, "/devstoreaccount1/my-test-queue", nil) + resp, err := p.HandleRequest(ctx, "", reqCreate) + require.NoError(t, err) + assert.Equal(t, http.StatusCreated, resp.StatusCode) + + // 2. Duplicate create returns 204 No Content + respDup, err := p.HandleRequest(ctx, "", reqCreate) + require.NoError(t, err) + assert.Equal(t, http.StatusNoContent, respDup.StatusCode) + + // 3. Get Queue Metadata + reqMeta := httptest.NewRequest(http.MethodGet, "/devstoreaccount1/my-test-queue?comp=metadata", nil) + respMeta, err := p.HandleRequest(ctx, "", reqMeta) + require.NoError(t, err) + assert.Equal(t, http.StatusOK, respMeta.StatusCode) + assert.Equal(t, "0", respMeta.Headers["x-ms-approximate-messages-count"]) + + // 4. List Queues + reqList := httptest.NewRequest(http.MethodGet, "/devstoreaccount1?comp=list", nil) + respList, err := p.HandleRequest(ctx, "", reqList) + require.NoError(t, err) + assert.Equal(t, http.StatusOK, respList.StatusCode) + assert.Contains(t, string(respList.Body), "my-test-queue") + + // 5. Delete Queue + reqDel := httptest.NewRequest(http.MethodDelete, "/devstoreaccount1/my-test-queue", nil) + respDel, err := p.HandleRequest(ctx, "", reqDel) + require.NoError(t, err) + assert.Equal(t, http.StatusNoContent, respDel.StatusCode) + + // 6. Accessing deleted queue returns 404 + respGet404, err := p.HandleRequest(ctx, "", reqMeta) + require.NoError(t, err) + assert.Equal(t, http.StatusNotFound, respGet404.StatusCode) +} + +func TestQueueProvider_MessageLifecycle(t *testing.T) { + p := newTestProvider(t) + ctx := context.Background() + + // Create queue + reqCreate := httptest.NewRequest(http.MethodPut, "/devstoreaccount1/msg-queue", nil) + _, err := p.HandleRequest(ctx, "", reqCreate) + require.NoError(t, err) + + // 1. Put Message + msgXML := `hello world` + reqPut := httptest.NewRequest(http.MethodPost, "/devstoreaccount1/msg-queue/messages", bytes.NewReader([]byte(msgXML))) + respPut, err := p.HandleRequest(ctx, "", reqPut) + require.NoError(t, err) + assert.Equal(t, http.StatusCreated, respPut.StatusCode) + assert.Contains(t, string(respPut.Body), "") + assert.Contains(t, string(respPut.Body), "") + + // 2. Peek Message + reqPeek := httptest.NewRequest(http.MethodGet, "/devstoreaccount1/msg-queue/messages?peekonly=true", nil) + respPeek, err := p.HandleRequest(ctx, "", reqPeek) + require.NoError(t, err) + assert.Equal(t, http.StatusOK, respPeek.StatusCode) + assert.Contains(t, string(respPeek.Body), "hello world") + + // 3. Get Messages + reqGet := httptest.NewRequest(http.MethodGet, "/devstoreaccount1/msg-queue/messages", nil) + respGet, err := p.HandleRequest(ctx, "", reqGet) + require.NoError(t, err) + assert.Equal(t, http.StatusOK, respGet.StatusCode) + assert.Contains(t, string(respGet.Body), "hello world") + assert.Contains(t, string(respGet.Body), "1") + + // 4. Clear Messages + reqClear := httptest.NewRequest(http.MethodDelete, "/devstoreaccount1/msg-queue/messages", nil) + respClear, err := p.HandleRequest(ctx, "", reqClear) + require.NoError(t, err) + assert.Equal(t, http.StatusNoContent, respClear.StatusCode) + + // 5. Peek after clear returns empty list + respPeekEmpty, err := p.HandleRequest(ctx, "", reqPeek) + require.NoError(t, err) + assert.Equal(t, http.StatusOK, respPeekEmpty.StatusCode) + assert.NotContains(t, string(respPeekEmpty.Body), "hello world") +} diff --git a/internal/services/azurequeue/store.go b/internal/services/azurequeue/store.go new file mode 100644 index 00000000..3477e3a5 --- /dev/null +++ b/internal/services/azurequeue/store.go @@ -0,0 +1,294 @@ +// SPDX-License-Identifier: Apache-2.0 + +package azurequeue + +import ( + "crypto/rand" + "encoding/hex" + "errors" + "os" + "path/filepath" + "strings" + "sync" + "time" +) + +var ( + ErrQueueNotFound = errors.New("QueueNotFound") + ErrQueueAlreadyExists = errors.New("QueueAlreadyExists") + ErrMessageNotFound = errors.New("MessageNotFound") + ErrPopReceiptMismatch = errors.New("PopReceiptMismatch") + ErrInvalidQueueName = errors.New("InvalidQueueName") +) + +type QueueMessage struct { + ID string + Text string + InsertionTime time.Time + ExpirationTime time.Time + PopReceipt string + TimeNextVisible time.Time + DequeueCount int +} + +type Queue struct { + Name string + Account string + Metadata map[string]string + Messages []*QueueMessage +} + +type Store struct { + mu sync.RWMutex + baseDir string + queues map[string]*Queue // key: "account/queue" +} + +func NewStore(baseDir string) *Store { + abs, err := filepath.Abs(baseDir) + if err != nil { + abs = filepath.Clean(baseDir) + } + cleaned := filepath.Clean(abs) + _ = os.MkdirAll(cleaned, 0755) + return &Store{ + baseDir: cleaned, + queues: make(map[string]*Queue), + } +} + +func randomID() string { + b := make([]byte, 16) + _, _ = rand.Read(b) + return hex.EncodeToString(b) +} + +func (s *Store) queueKey(account, queue string) string { + if account == "" { + account = "devstoreaccount1" + } + return account + "/" + queue +} + +func (s *Store) CreateQueue(account, queue string, metadata map[string]string) error { + s.mu.Lock() + defer s.mu.Unlock() + + if queue == "" || strings.Contains(queue, "/") || strings.Contains(queue, "\\") || strings.Contains(queue, "..") { + return ErrInvalidQueueName + } + + key := s.queueKey(account, queue) + if _, exists := s.queues[key]; exists { + return ErrQueueAlreadyExists + } + + if metadata == nil { + metadata = make(map[string]string) + } + s.queues[key] = &Queue{ + Name: queue, + Account: account, + Metadata: metadata, + Messages: make([]*QueueMessage, 0), + } + return nil +} + +func (s *Store) DeleteQueue(account, queue string) error { + s.mu.Lock() + defer s.mu.Unlock() + + key := s.queueKey(account, queue) + if _, exists := s.queues[key]; !exists { + return ErrQueueNotFound + } + delete(s.queues, key) + return nil +} + +func (s *Store) ListQueues(account, prefix string) ([]*Queue, error) { + s.mu.RLock() + defer s.mu.RUnlock() + + if account == "" { + account = "devstoreaccount1" + } + var res []*Queue + accPrefix := account + "/" + for k, q := range s.queues { + if strings.HasPrefix(k, accPrefix) { + if prefix == "" || strings.HasPrefix(q.Name, prefix) { + res = append(res, q) + } + } + } + return res, nil +} + +func (s *Store) GetQueueMetadata(account, queue string) (map[string]string, int, error) { + s.mu.RLock() + defer s.mu.RUnlock() + + key := s.queueKey(account, queue) + q, exists := s.queues[key] + if !exists { + return nil, 0, ErrQueueNotFound + } + + now := time.Now().UTC() + count := 0 + for _, m := range q.Messages { + if m.ExpirationTime.After(now) { + count++ + } + } + return q.Metadata, count, nil +} + +func (s *Store) SetQueueMetadata(account, queue string, meta map[string]string) error { + s.mu.Lock() + defer s.mu.Unlock() + + key := s.queueKey(account, queue) + q, exists := s.queues[key] + if !exists { + return ErrQueueNotFound + } + q.Metadata = meta + return nil +} + +func (s *Store) PutMessage(account, queue, text string, visibilityTimeout, ttl time.Duration) (*QueueMessage, error) { + s.mu.Lock() + defer s.mu.Unlock() + + key := s.queueKey(account, queue) + q, exists := s.queues[key] + if !exists { + return nil, ErrQueueNotFound + } + + if ttl <= 0 { + ttl = 7 * 24 * time.Hour + } + now := time.Now().UTC() + msg := &QueueMessage{ + ID: randomID(), + Text: text, + InsertionTime: now, + ExpirationTime: now.Add(ttl), + PopReceipt: randomID(), + TimeNextVisible: now.Add(visibilityTimeout), + DequeueCount: 0, + } + q.Messages = append(q.Messages, msg) + return msg, nil +} + +func (s *Store) GetMessages(account, queue string, numMessages int, visibilityTimeout time.Duration) ([]*QueueMessage, error) { + s.mu.Lock() + defer s.mu.Unlock() + + key := s.queueKey(account, queue) + q, exists := s.queues[key] + if !exists { + return nil, ErrQueueNotFound + } + + if numMessages <= 0 { + numMessages = 1 + } + if numMessages > 32 { + numMessages = 32 + } + if visibilityTimeout <= 0 { + visibilityTimeout = 30 * time.Second + } + + now := time.Now().UTC() + var visible []*QueueMessage + for _, m := range q.Messages { + if m.ExpirationTime.After(now) && m.TimeNextVisible.Before(now) { + m.DequeueCount++ + m.PopReceipt = randomID() + m.TimeNextVisible = now.Add(visibilityTimeout) + visible = append(visible, m) + if len(visible) >= numMessages { + break + } + } + } + return visible, nil +} + +func (s *Store) PeekMessages(account, queue string, numMessages int) ([]*QueueMessage, error) { + s.mu.RLock() + defer s.mu.RUnlock() + + key := s.queueKey(account, queue) + q, exists := s.queues[key] + if !exists { + return nil, ErrQueueNotFound + } + + if numMessages <= 0 { + numMessages = 1 + } + if numMessages > 32 { + numMessages = 32 + } + + now := time.Now().UTC() + var visible []*QueueMessage + for _, m := range q.Messages { + if m.ExpirationTime.After(now) && m.TimeNextVisible.Before(now) { + visible = append(visible, m) + if len(visible) >= numMessages { + break + } + } + } + return visible, nil +} + +func (s *Store) DeleteMessage(account, queue, messageID, popReceipt string) error { + s.mu.Lock() + defer s.mu.Unlock() + + key := s.queueKey(account, queue) + q, exists := s.queues[key] + if !exists { + return ErrQueueNotFound + } + + idx := -1 + for i, m := range q.Messages { + if m.ID == messageID { + if popReceipt != "" && m.PopReceipt != popReceipt { + return ErrPopReceiptMismatch + } + idx = i + break + } + } + if idx == -1 { + return ErrMessageNotFound + } + + q.Messages = append(q.Messages[:idx], q.Messages[idx+1:]...) + return nil +} + +func (s *Store) ClearMessages(account, queue string) error { + s.mu.Lock() + defer s.mu.Unlock() + + key := s.queueKey(account, queue) + q, exists := s.queues[key] + if !exists { + return ErrQueueNotFound + } + q.Messages = make([]*QueueMessage, 0) + return nil +} diff --git a/internal/services/azuretable/provider.go b/internal/services/azuretable/provider.go new file mode 100644 index 00000000..99a52232 --- /dev/null +++ b/internal/services/azuretable/provider.go @@ -0,0 +1,395 @@ +// SPDX-License-Identifier: Apache-2.0 + +package azuretable + +import ( + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "strings" + "time" + + "github.com/skyoo2003/devcloud/internal/auth" + generated "github.com/skyoo2003/devcloud/internal/generated/azuretable" + "github.com/skyoo2003/devcloud/internal/plugin" +) + +type Provider struct { + generated.BaseProvider + dataDir string + store *Store +} + +func (p *Provider) ServiceID() string { return "azuretable" } +func (p *Provider) ServiceName() string { return "Azure Table Storage" } +func (p *Provider) Protocol() plugin.ProtocolType { return plugin.ProtocolRESTJSON } +func (p *Provider) Provider() string { return "azure" } + +func (p *Provider) Init(cfg plugin.PluginConfig) error { + p.dataDir = cfg.DataDir + if p.dataDir == "" { + p.dataDir = "./data/azure-table" + } + s, err := NewStore(p.dataDir) + if err != nil { + return err + } + p.store = s + return nil +} + +func (p *Provider) Shutdown(ctx context.Context) error { + if p.store != nil { + return p.store.Close() + } + return nil +} + +func (p *Provider) ListResources(ctx context.Context) ([]plugin.Resource, error) { + tables, err := p.store.ListTables("devstoreaccount1") + if err != nil { + return nil, err + } + var res []plugin.Resource + for _, t := range tables { + res = append(res, plugin.Resource{ + Type: "Table", + ID: t.TableName, + Name: t.TableName, + }) + } + return res, nil +} + +func randomRequestID() string { + b := make([]byte, 16) + _, _ = rand.Read(b) + return hex.EncodeToString(b) +} + +func (p *Provider) reply(req *http.Request, status int, headers map[string]string, body []byte) *plugin.Response { + if headers == nil { + headers = make(map[string]string) + } + headers["x-ms-request-id"] = randomRequestID() + version := req.Header.Get("x-ms-version") + if version == "" { + version = "2023-11-03" + } + headers["x-ms-version"] = version + headers["Date"] = time.Now().UTC().Format(http.TimeFormat) + + ct := headers["Content-Type"] + if ct == "" { + ct = "application/json;odata=nometadata" + } + return &plugin.Response{ + StatusCode: status, + Headers: headers, + Body: body, + ContentType: ct, + } +} + +func (p *Provider) errorReply(req *http.Request, status int, code, message string) *plugin.Response { + headers := map[string]string{ + "x-ms-error-code": code, + "Content-Type": "application/json;odata=minimalmetadata", + "x-ms-request-id": randomRequestID(), + "Date": time.Now().UTC().Format(http.TimeFormat), + } + version := req.Header.Get("x-ms-version") + if version == "" { + version = "2023-11-03" + } + headers["x-ms-version"] = version + + type odataErr struct { + Error struct { + Code string `json:"code"` + Message struct { + Lang string `json:"lang"` + Value string `json:"value"` + } `json:"message"` + } `json:"odata.error"` + } + var errObj odataErr + errObj.Error.Code = code + errObj.Error.Message.Lang = "en-US" + errObj.Error.Message.Value = message + raw, _ := json.Marshal(errObj) + + return &plugin.Response{ + StatusCode: status, + Headers: headers, + Body: raw, + ContentType: "application/json;odata=minimalmetadata", + } +} + +func (p *Provider) HandleRequest(ctx context.Context, _ string, req *http.Request) (*plugin.Response, error) { + account := "devstoreaccount1" + if id, ok := auth.FromContext(ctx); ok && id.AccessKeyID != "" { + account = id.AccessKeyID + } + + uri := req.URL.RequestURI() + for _, accPrefix := range []string{"/" + account, "/devstoreaccount1"} { + if strings.HasPrefix(uri, accPrefix+"/") { + uri = uri[len(accPrefix):] + break + } else if strings.HasPrefix(uri, accPrefix+"?") { + uri = "/" + uri[len(accPrefix):] + break + } else if uri == accPrefix { + uri = "/" + break + } + } + + op, params := matchTableOperation(req.Method, uri) + if op == "" { + op, params = generated.MatchOperation(req.Method, uri) + } + if op == "" { + op, params = generated.MatchOperation(req.Method, req.URL.Path) + } + + switch op { + case "QueryTables": + return p.handleQueryTables(req, account) + case "CreateTable": + return p.handleCreateTable(req, account) + case "DeleteTable": + return p.handleDeleteTable(req, account, params["table"]) + case "QueryEntities": + return p.handleQueryEntities(req, account, params["table"]) + case "InsertEntity": + return p.handleInsertEntity(req, account, params["table"]) + case "GetEntity": + return p.handleGetEntity(req, account, params["table"], params["partitionKey"], params["rowKey"]) + case "UpdateEntity": + return p.handleUpdateEntity(req, account, params["table"], params["partitionKey"], params["rowKey"], false) + case "MergeEntity": + return p.handleUpdateEntity(req, account, params["table"], params["partitionKey"], params["rowKey"], true) + case "DeleteEntity": + return p.handleDeleteEntity(req, account, params["table"], params["partitionKey"], params["rowKey"]) + default: + return p.errorReply(req, http.StatusBadRequest, "InvalidAction", "operation not handled: "+req.Method+" "+req.URL.Path), nil + } +} + +func (p *Provider) handleQueryTables(req *http.Request, account string) (*plugin.Response, error) { + tables, err := p.store.ListTables(account) + if err != nil { + return p.errorReply(req, http.StatusInternalServerError, "InternalError", err.Error()), nil + } + type queryTablesResponse struct { + Value []TableInfo `json:"value"` + } + raw, _ := json.Marshal(queryTablesResponse{Value: tables}) + return p.reply(req, http.StatusOK, nil, raw), nil +} + +func (p *Provider) handleCreateTable(req *http.Request, account string) (*plugin.Response, error) { + body, err := io.ReadAll(req.Body) + if err != nil { + return p.errorReply(req, http.StatusBadRequest, "InvalidInput", "failed to read body"), nil + } + var reqBody TableInfo + if err := json.Unmarshal(body, &reqBody); err != nil { + return p.errorReply(req, http.StatusBadRequest, "InvalidInput", "invalid json: "+err.Error()), nil + } + + err = p.store.CreateTable(account, reqBody.TableName) + if err != nil { + if errors.Is(err, ErrTableAlreadyExists) { + return p.errorReply(req, http.StatusConflict, "TableAlreadyExists", "table already exists"), nil + } + if errors.Is(err, ErrInvalidTableName) { + return p.errorReply(req, http.StatusBadRequest, "InvalidResourceName", "invalid table name"), nil + } + return p.errorReply(req, http.StatusInternalServerError, "InternalError", err.Error()), nil + } + + raw, _ := json.Marshal(reqBody) + return p.reply(req, http.StatusCreated, nil, raw), nil +} + +func (p *Provider) handleDeleteTable(req *http.Request, account, tableName string) (*plugin.Response, error) { + err := p.store.DeleteTable(account, tableName) + if err != nil { + if errors.Is(err, ErrTableNotFound) { + return p.errorReply(req, http.StatusNotFound, "TableNotFound", "table not found"), nil + } + return p.errorReply(req, http.StatusInternalServerError, "InternalError", err.Error()), nil + } + return p.reply(req, http.StatusNoContent, nil, nil), nil +} + +func (p *Provider) handleQueryEntities(req *http.Request, account, tableName string) (*plugin.Response, error) { + entities, err := p.store.QueryEntities(account, tableName) + if err != nil { + if errors.Is(err, ErrTableNotFound) { + return p.errorReply(req, http.StatusNotFound, "TableNotFound", "table not found"), nil + } + return p.errorReply(req, http.StatusInternalServerError, "InternalError", err.Error()), nil + } + type queryEntitiesResponse struct { + Value []Entity `json:"value"` + } + raw, _ := json.Marshal(queryEntitiesResponse{Value: entities}) + return p.reply(req, http.StatusOK, nil, raw), nil +} + +func (p *Provider) handleInsertEntity(req *http.Request, account, tableName string) (*plugin.Response, error) { + body, err := io.ReadAll(req.Body) + if err != nil { + return p.errorReply(req, http.StatusBadRequest, "InvalidInput", "failed to read body"), nil + } + var entity Entity + if err := json.Unmarshal(body, &entity); err != nil { + return p.errorReply(req, http.StatusBadRequest, "InvalidInput", "invalid entity JSON: "+err.Error()), nil + } + + err = p.store.InsertEntity(account, tableName, entity) + if err != nil { + if errors.Is(err, ErrTableNotFound) { + return p.errorReply(req, http.StatusNotFound, "TableNotFound", "table not found"), nil + } + if errors.Is(err, ErrEntityAlreadyExists) { + return p.errorReply(req, http.StatusConflict, "EntityAlreadyExists", "entity already exists"), nil + } + return p.errorReply(req, http.StatusInternalServerError, "InternalError", err.Error()), nil + } + + raw, _ := json.Marshal(entity) + headers := map[string]string{ + "ETag": fmt.Sprintf("%v", entity["odata.etag"]), + } + return p.reply(req, http.StatusCreated, headers, raw), nil +} + +func (p *Provider) handleGetEntity(req *http.Request, account, tableName, pk, rk string) (*plugin.Response, error) { + entity, err := p.store.GetEntity(account, tableName, pk, rk) + if err != nil { + if errors.Is(err, ErrEntityNotFound) || errors.Is(err, ErrTableNotFound) { + return p.errorReply(req, http.StatusNotFound, "ResourceNotFound", "entity not found"), nil + } + return p.errorReply(req, http.StatusInternalServerError, "InternalError", err.Error()), nil + } + raw, _ := json.Marshal(entity) + headers := map[string]string{ + "ETag": fmt.Sprintf("%v", entity["odata.etag"]), + } + return p.reply(req, http.StatusOK, headers, raw), nil +} + +func (p *Provider) handleUpdateEntity(req *http.Request, account, tableName, pk, rk string, merge bool) (*plugin.Response, error) { + body, err := io.ReadAll(req.Body) + if err != nil { + return p.errorReply(req, http.StatusBadRequest, "InvalidInput", "failed to read body"), nil + } + var entity Entity + if err := json.Unmarshal(body, &entity); err != nil { + return p.errorReply(req, http.StatusBadRequest, "InvalidInput", "invalid entity JSON: "+err.Error()), nil + } + + err = p.store.UpdateEntity(account, tableName, pk, rk, entity, merge) + if err != nil { + if errors.Is(err, ErrEntityNotFound) || errors.Is(err, ErrTableNotFound) { + return p.errorReply(req, http.StatusNotFound, "ResourceNotFound", "entity not found"), nil + } + return p.errorReply(req, http.StatusInternalServerError, "InternalError", err.Error()), nil + } + return p.reply(req, http.StatusNoContent, nil, nil), nil +} + +func (p *Provider) handleDeleteEntity(req *http.Request, account, tableName, pk, rk string) (*plugin.Response, error) { + err := p.store.DeleteEntity(account, tableName, pk, rk) + if err != nil { + if errors.Is(err, ErrEntityNotFound) || errors.Is(err, ErrTableNotFound) { + return p.errorReply(req, http.StatusNotFound, "ResourceNotFound", "entity not found"), nil + } + return p.errorReply(req, http.StatusInternalServerError, "InternalError", err.Error()), nil + } + return p.reply(req, http.StatusNoContent, nil, nil), nil +} + +func matchTableOperation(method, uri string) (string, map[string]string) { + cleanURI, _, _ := strings.Cut(uri, "?") + cleanURI = strings.TrimPrefix(cleanURI, "/") + + if cleanURI == "Tables" { + if method == http.MethodGet { + return "QueryTables", nil + } + if method == http.MethodPost { + return "CreateTable", nil + } + } + + if strings.HasPrefix(cleanURI, "Tables('") && strings.HasSuffix(cleanURI, "')") { + tableName := strings.TrimSuffix(strings.TrimPrefix(cleanURI, "Tables('"), "')") + if method == http.MethodDelete { + return "DeleteTable", map[string]string{"table": tableName} + } + } + + if idx := strings.Index(cleanURI, "("); idx != -1 && strings.HasSuffix(cleanURI, ")") { + tableName := cleanURI[:idx] + inner := cleanURI[idx+1 : len(cleanURI)-1] + if inner == "" { + if method == http.MethodGet { + return "QueryEntities", map[string]string{"table": tableName} + } + } + pk, rk := parseTableKeys(inner) + params := map[string]string{"table": tableName, "partitionKey": pk, "rowKey": rk} + switch method { + case http.MethodGet: + return "GetEntity", params + case http.MethodPut: + return "UpdateEntity", params + case "MERGE": + return "MergeEntity", params + case http.MethodDelete: + return "DeleteEntity", params + } + } + + if !strings.Contains(cleanURI, "(") && !strings.Contains(cleanURI, ")") && cleanURI != "" && cleanURI != "Tables" { + if method == http.MethodPost { + return "InsertEntity", map[string]string{"table": cleanURI} + } + if method == http.MethodGet { + return "QueryEntities", map[string]string{"table": cleanURI} + } + } + + return "", nil +} + +func parseTableKeys(s string) (pk string, rk string) { + parts := strings.Split(s, ",") + for _, part := range parts { + trimmed := strings.TrimSpace(part) + if strings.HasPrefix(trimmed, "PartitionKey='") && strings.HasSuffix(trimmed, "'") { + pk = strings.TrimSuffix(strings.TrimPrefix(trimmed, "PartitionKey='"), "'") + } else if strings.HasPrefix(trimmed, "RowKey='") && strings.HasSuffix(trimmed, "'") { + rk = strings.TrimSuffix(strings.TrimPrefix(trimmed, "RowKey='"), "'") + } + } + return pk, rk +} + +func init() { + plugin.DefaultRegistry.Register("azuretable", func() plugin.ServicePlugin { + return &Provider{} + }) +} diff --git a/internal/services/azuretable/provider_test.go b/internal/services/azuretable/provider_test.go new file mode 100644 index 00000000..983a5007 --- /dev/null +++ b/internal/services/azuretable/provider_test.go @@ -0,0 +1,145 @@ +// SPDX-License-Identifier: Apache-2.0 + +package azuretable + +import ( + "bytes" + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/skyoo2003/devcloud/internal/plugin" +) + +func newTestProvider(t *testing.T) *Provider { + t.Helper() + dir, err := os.MkdirTemp("", "devcloud-table-test-*") + require.NoError(t, err) + t.Cleanup(func() { _ = os.RemoveAll(dir) }) + + p := &Provider{} + err = p.Init(plugin.PluginConfig{DataDir: filepath.Join(dir, "table")}) + require.NoError(t, err) + return p +} + +func TestTableProvider_Metadata(t *testing.T) { + p := newTestProvider(t) + defer func() { _ = p.Shutdown(context.Background()) }() + assert.Equal(t, "azuretable", p.ServiceID()) + assert.Equal(t, "Azure Table Storage", p.ServiceName()) + assert.Equal(t, plugin.ProtocolRESTJSON, p.Protocol()) + assert.Equal(t, "azure", plugin.ProviderOf(p)) +} + +func TestTableProvider_TableLifecycle(t *testing.T) { + p := newTestProvider(t) + ctx := context.Background() + defer func() { _ = p.Shutdown(ctx) }() + + // 1. Create Table + reqCreate := httptest.NewRequest(http.MethodPost, "/devstoreaccount1/Tables", bytes.NewReader([]byte(`{"TableName": "Users"}`))) + reqCreate.Header.Set("Content-Type", "application/json") + resp, err := p.HandleRequest(ctx, "", reqCreate) + require.NoError(t, err) + assert.Equal(t, http.StatusCreated, resp.StatusCode) + + // 2. Duplicate Create returns 409 + reqDup := httptest.NewRequest(http.MethodPost, "/devstoreaccount1/Tables", bytes.NewReader([]byte(`{"TableName": "Users"}`))) + reqDup.Header.Set("Content-Type", "application/json") + respDup, err := p.HandleRequest(ctx, "", reqDup) + require.NoError(t, err) + assert.Equal(t, http.StatusConflict, respDup.StatusCode) + + // 3. Query Tables + reqQuery := httptest.NewRequest(http.MethodGet, "/devstoreaccount1/Tables", nil) + respQuery, err := p.HandleRequest(ctx, "", reqQuery) + require.NoError(t, err) + assert.Equal(t, http.StatusOK, respQuery.StatusCode) + assert.Contains(t, string(respQuery.Body), "Users") + + // 4. Delete Table + reqDel := httptest.NewRequest(http.MethodDelete, "/devstoreaccount1/Tables('Users')", nil) + respDel, err := p.HandleRequest(ctx, "", reqDel) + require.NoError(t, err) + assert.Equal(t, http.StatusNoContent, respDel.StatusCode) + + // 5. Delete again returns 404 + respDel404, err := p.HandleRequest(ctx, "", reqDel) + require.NoError(t, err) + assert.Equal(t, http.StatusNotFound, respDel404.StatusCode) +} + +func TestTableProvider_EntityCRUD(t *testing.T) { + p := newTestProvider(t) + ctx := context.Background() + defer func() { _ = p.Shutdown(ctx) }() + + // Create table + reqCreateT := httptest.NewRequest(http.MethodPost, "/devstoreaccount1/Tables", bytes.NewReader([]byte(`{"TableName": "Customers"}`))) + _, err := p.HandleRequest(ctx, "", reqCreateT) + require.NoError(t, err) + + // 1. Insert Entity + entity := map[string]any{ + "PartitionKey": "Seattle", + "RowKey": "user-001", + "Email": "alice@example.com", + "Age": 30, + } + body, _ := json.Marshal(entity) + reqInsert := httptest.NewRequest(http.MethodPost, "/devstoreaccount1/Customers", bytes.NewReader(body)) + respInsert, err := p.HandleRequest(ctx, "", reqInsert) + require.NoError(t, err) + assert.Equal(t, http.StatusCreated, respInsert.StatusCode) + assert.NotEmpty(t, respInsert.Headers["ETag"]) + + // 2. Get Entity + reqGet := httptest.NewRequest(http.MethodGet, "/devstoreaccount1/Customers(PartitionKey='Seattle',RowKey='user-001')", nil) + respGet, err := p.HandleRequest(ctx, "", reqGet) + require.NoError(t, err) + assert.Equal(t, http.StatusOK, respGet.StatusCode) + assert.Contains(t, string(respGet.Body), "alice@example.com") + + // 3. Update Entity + updateEntity := map[string]any{ + "PartitionKey": "Seattle", + "RowKey": "user-001", + "Email": "alice-updated@example.com", + } + upBody, _ := json.Marshal(updateEntity) + reqUpdate := httptest.NewRequest(http.MethodPut, "/devstoreaccount1/Customers(PartitionKey='Seattle',RowKey='user-001')", bytes.NewReader(upBody)) + respUpdate, err := p.HandleRequest(ctx, "", reqUpdate) + require.NoError(t, err) + assert.Equal(t, http.StatusNoContent, respUpdate.StatusCode) + + // Verify updated entity + respGet2, err := p.HandleRequest(ctx, "", reqGet) + require.NoError(t, err) + assert.Contains(t, string(respGet2.Body), "alice-updated@example.com") + + // 4. Query Entities + reqQuery := httptest.NewRequest(http.MethodGet, "/devstoreaccount1/Customers()", nil) + respQuery, err := p.HandleRequest(ctx, "", reqQuery) + require.NoError(t, err) + assert.Equal(t, http.StatusOK, respQuery.StatusCode) + assert.Contains(t, string(respQuery.Body), "alice-updated@example.com") + + // 5. Delete Entity + reqDel := httptest.NewRequest(http.MethodDelete, "/devstoreaccount1/Customers(PartitionKey='Seattle',RowKey='user-001')", nil) + respDel, err := p.HandleRequest(ctx, "", reqDel) + require.NoError(t, err) + assert.Equal(t, http.StatusNoContent, respDel.StatusCode) + + // 6. Get deleted entity returns 404 + respGet404, err := p.HandleRequest(ctx, "", reqGet) + require.NoError(t, err) + assert.Equal(t, http.StatusNotFound, respGet404.StatusCode) +} diff --git a/internal/services/azuretable/store.go b/internal/services/azuretable/store.go new file mode 100644 index 00000000..29d3896d --- /dev/null +++ b/internal/services/azuretable/store.go @@ -0,0 +1,308 @@ +// SPDX-License-Identifier: Apache-2.0 + +package azuretable + +import ( + "database/sql" + "encoding/json" + "errors" + "fmt" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/skyoo2003/devcloud/internal/storage/sqlite" +) + +var ( + ErrTableNotFound = errors.New("TableNotFound") + ErrTableAlreadyExists = errors.New("TableAlreadyExists") + ErrEntityNotFound = errors.New("EntityNotFound") + ErrEntityAlreadyExists = errors.New("EntityAlreadyExists") + ErrInvalidTableName = errors.New("InvalidTableName") +) + +type TableInfo struct { + TableName string `json:"TableName"` +} + +type Entity = map[string]any + +type Store struct { + mu sync.RWMutex + store *sqlite.Store +} + +var migrations = []sqlite.Migration{ + { + Version: 1, + SQL: ` +CREATE TABLE IF NOT EXISTS tables ( + account TEXT NOT NULL, + name TEXT NOT NULL, + created_at TIMESTAMP NOT NULL, + PRIMARY KEY (account, name) +); + +CREATE TABLE IF NOT EXISTS entities ( + account TEXT NOT NULL, + table_name TEXT NOT NULL, + partition_key TEXT NOT NULL, + row_key TEXT NOT NULL, + data TEXT NOT NULL, + timestamp TIMESTAMP NOT NULL, + etag TEXT NOT NULL, + PRIMARY KEY (account, table_name, partition_key, row_key) +); +CREATE INDEX IF NOT EXISTS idx_entities_table ON entities (account, table_name); +`, + }, +} + +func NewStore(dataDir string) (*Store, error) { + dbPath := filepath.Join(dataDir, "table.db") + s, err := sqlite.Open(dbPath, migrations) + if err != nil { + return nil, err + } + return &Store{store: s}, nil +} + +func (s *Store) Close() error { + return s.store.Close() +} + +func (s *Store) CreateTable(account, name string) error { + s.mu.Lock() + defer s.mu.Unlock() + + if name == "" || strings.Contains(name, "/") || strings.Contains(name, "\\") { + return ErrInvalidTableName + } + if account == "" { + account = "devstoreaccount1" + } + + var exists int + err := s.store.DB().QueryRow("SELECT COUNT(*) FROM tables WHERE account = ? AND name = ?", account, name).Scan(&exists) + if err != nil { + return err + } + if exists > 0 { + return ErrTableAlreadyExists + } + + _, err = s.store.DB().Exec("INSERT INTO tables (account, name, created_at) VALUES (?, ?, ?)", + account, name, time.Now().UTC()) + return err +} + +func (s *Store) DeleteTable(account, name string) error { + s.mu.Lock() + defer s.mu.Unlock() + + if account == "" { + account = "devstoreaccount1" + } + + res, err := s.store.DB().Exec("DELETE FROM tables WHERE account = ? AND name = ?", account, name) + if err != nil { + return err + } + rows, err := res.RowsAffected() + if err != nil || rows == 0 { + return ErrTableNotFound + } + + _, _ = s.store.DB().Exec("DELETE FROM entities WHERE account = ? AND table_name = ?", account, name) + return nil +} + +func (s *Store) ListTables(account string) ([]TableInfo, error) { + s.mu.RLock() + defer s.mu.RUnlock() + + if account == "" { + account = "devstoreaccount1" + } + + rows, err := s.store.DB().Query("SELECT name FROM tables WHERE account = ? ORDER BY name ASC", account) + if err != nil { + return nil, err + } + defer func() { _ = rows.Close() }() + + var list []TableInfo + for rows.Next() { + var name string + if err := rows.Scan(&name); err == nil { + list = append(list, TableInfo{TableName: name}) + } + } + return list, nil +} + +func (s *Store) InsertEntity(account, tableName string, entity Entity) error { + s.mu.Lock() + defer s.mu.Unlock() + + if account == "" { + account = "devstoreaccount1" + } + + // Verify table exists + var tableCount int + err := s.store.DB().QueryRow("SELECT COUNT(*) FROM tables WHERE account = ? AND name = ?", account, tableName).Scan(&tableCount) + if err != nil || tableCount == 0 { + return ErrTableNotFound + } + + pk, _ := entity["PartitionKey"].(string) + rk, _ := entity["RowKey"].(string) + if pk == "" || rk == "" { + return errors.New("PartitionKey and RowKey are required") + } + + now := time.Now().UTC() + etag := fmt.Sprintf("W/\"datetime'%s'\"", now.Format(time.RFC3339Nano)) + entity["Timestamp"] = now.Format(time.RFC3339) + entity["odata.etag"] = etag + + dataBytes, err := json.Marshal(entity) + if err != nil { + return err + } + + var exists int + _ = s.store.DB().QueryRow("SELECT COUNT(*) FROM entities WHERE account = ? AND table_name = ? AND partition_key = ? AND row_key = ?", + account, tableName, pk, rk).Scan(&exists) + if exists > 0 { + return ErrEntityAlreadyExists + } + + _, err = s.store.DB().Exec( + "INSERT INTO entities (account, table_name, partition_key, row_key, data, timestamp, etag) VALUES (?, ?, ?, ?, ?, ?, ?)", + account, tableName, pk, rk, string(dataBytes), now, etag) + return err +} + +func (s *Store) GetEntity(account, tableName, pk, rk string) (Entity, error) { + s.mu.RLock() + defer s.mu.RUnlock() + + if account == "" { + account = "devstoreaccount1" + } + + var dataStr string + err := s.store.DB().QueryRow( + "SELECT data FROM entities WHERE account = ? AND table_name = ? AND partition_key = ? AND row_key = ?", + account, tableName, pk, rk).Scan(&dataStr) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + return nil, ErrEntityNotFound + } + return nil, err + } + + var entity Entity + if err := json.Unmarshal([]byte(dataStr), &entity); err != nil { + return nil, err + } + return entity, nil +} + +func (s *Store) UpdateEntity(account, tableName, pk, rk string, entity Entity, merge bool) error { + s.mu.Lock() + defer s.mu.Unlock() + + if account == "" { + account = "devstoreaccount1" + } + + var existingDataStr string + err := s.store.DB().QueryRow( + "SELECT data FROM entities WHERE account = ? AND table_name = ? AND partition_key = ? AND row_key = ?", + account, tableName, pk, rk).Scan(&existingDataStr) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + return ErrEntityNotFound + } + return err + } + + targetEntity := make(Entity) + if merge { + _ = json.Unmarshal([]byte(existingDataStr), &targetEntity) + } + for k, v := range entity { + targetEntity[k] = v + } + targetEntity["PartitionKey"] = pk + targetEntity["RowKey"] = rk + now := time.Now().UTC() + etag := fmt.Sprintf("W/\"datetime'%s'\"", now.Format(time.RFC3339Nano)) + targetEntity["Timestamp"] = now.Format(time.RFC3339) + targetEntity["odata.etag"] = etag + + dataBytes, err := json.Marshal(targetEntity) + if err != nil { + return err + } + + _, err = s.store.DB().Exec( + "UPDATE entities SET data = ?, timestamp = ?, etag = ? WHERE account = ? AND table_name = ? AND partition_key = ? AND row_key = ?", + string(dataBytes), now, etag, account, tableName, pk, rk) + return err +} + +func (s *Store) DeleteEntity(account, tableName, pk, rk string) error { + s.mu.Lock() + defer s.mu.Unlock() + + if account == "" { + account = "devstoreaccount1" + } + + res, err := s.store.DB().Exec( + "DELETE FROM entities WHERE account = ? AND table_name = ? AND partition_key = ? AND row_key = ?", + account, tableName, pk, rk) + if err != nil { + return err + } + rows, err := res.RowsAffected() + if err != nil || rows == 0 { + return ErrEntityNotFound + } + return nil +} + +func (s *Store) QueryEntities(account, tableName string) ([]Entity, error) { + s.mu.RLock() + defer s.mu.RUnlock() + + if account == "" { + account = "devstoreaccount1" + } + + rows, err := s.store.DB().Query( + "SELECT data FROM entities WHERE account = ? AND table_name = ? ORDER BY partition_key, row_key ASC", + account, tableName) + if err != nil { + return nil, err + } + defer func() { _ = rows.Close() }() + + var list []Entity + for rows.Next() { + var dataStr string + if err := rows.Scan(&dataStr); err == nil { + var ent Entity + if json.Unmarshal([]byte(dataStr), &ent) == nil { + list = append(list, ent) + } + } + } + return list, nil +} diff --git a/internal/services/cosmosdb/provider.go b/internal/services/cosmosdb/provider.go new file mode 100644 index 00000000..195da892 --- /dev/null +++ b/internal/services/cosmosdb/provider.go @@ -0,0 +1,391 @@ +// SPDX-License-Identifier: Apache-2.0 + +package cosmosdb + +import ( + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "errors" + "io" + "net/http" + "strings" + "time" + + "github.com/skyoo2003/devcloud/internal/auth" + generated "github.com/skyoo2003/devcloud/internal/generated/cosmosdb" + "github.com/skyoo2003/devcloud/internal/plugin" +) + +type Provider struct { + generated.BaseProvider + dataDir string + store *Store +} + +func (p *Provider) ServiceID() string { return "cosmosdb" } +func (p *Provider) ServiceName() string { return "Azure Cosmos DB" } +func (p *Provider) Protocol() plugin.ProtocolType { return plugin.ProtocolRESTJSON } +func (p *Provider) Provider() string { return "azure" } + +func (p *Provider) Init(cfg plugin.PluginConfig) error { + p.dataDir = cfg.DataDir + if p.dataDir == "" { + p.dataDir = "./data/cosmos" + } + s, err := NewStore(p.dataDir) + if err != nil { + return err + } + p.store = s + return nil +} + +func (p *Provider) Shutdown(ctx context.Context) error { + if p.store != nil { + return p.store.Close() + } + return nil +} + +func (p *Provider) ListResources(ctx context.Context) ([]plugin.Resource, error) { + dbs, err := p.store.ListDatabases("devstoreaccount1") + if err != nil { + return nil, err + } + var res []plugin.Resource + for _, d := range dbs { + res = append(res, plugin.Resource{ + Type: "Database", + ID: d.ID, + Name: d.ID, + }) + } + return res, nil +} + +func randomActivityID() string { + b := make([]byte, 16) + _, _ = rand.Read(b) + return hex.EncodeToString(b) +} + +func (p *Provider) reply(req *http.Request, status int, headers map[string]string, body []byte) *plugin.Response { + if headers == nil { + headers = make(map[string]string) + } + headers["x-ms-activity-id"] = randomActivityID() + headers["x-ms-request-charge"] = "1" + headers["Date"] = time.Now().UTC().Format(http.TimeFormat) + + ct := headers["Content-Type"] + if ct == "" { + ct = "application/json" + } + return &plugin.Response{ + StatusCode: status, + Headers: headers, + Body: body, + ContentType: ct, + } +} + +func (p *Provider) errorReply(req *http.Request, status int, code, message string) *plugin.Response { + headers := map[string]string{ + "x-ms-activity-id": randomActivityID(), + "x-ms-error-code": code, + "Content-Type": "application/json", + "Date": time.Now().UTC().Format(http.TimeFormat), + } + type cosmosErr struct { + Code string `json:"code"` + Message string `json:"message"` + } + raw, _ := json.Marshal(cosmosErr{Code: code, Message: message}) + return &plugin.Response{ + StatusCode: status, + Headers: headers, + Body: raw, + ContentType: "application/json", + } +} + +func (p *Provider) HandleRequest(ctx context.Context, _ string, req *http.Request) (*plugin.Response, error) { + account := "devstoreaccount1" + if id, ok := auth.FromContext(ctx); ok && id.AccessKeyID != "" { + account = id.AccessKeyID + } + + uri := req.URL.RequestURI() + for _, accPrefix := range []string{"/" + account, "/devstoreaccount1"} { + if strings.HasPrefix(uri, accPrefix+"/") { + uri = uri[len(accPrefix):] + break + } else if strings.HasPrefix(uri, accPrefix+"?") { + uri = "/" + uri[len(accPrefix):] + break + } else if uri == accPrefix { + uri = "/" + break + } + } + + op, params := generated.MatchOperation(req.Method, uri) + if op == "" { + op, params = generated.MatchOperation(req.Method, req.URL.Path) + } + + switch op { + case "ListDatabases": + return p.handleListDatabases(req, account) + case "CreateDatabase": + return p.handleCreateDatabase(req, account) + case "GetDatabase": + return p.handleGetDatabase(req, account, params["db"]) + case "DeleteDatabase": + return p.handleDeleteDatabase(req, account, params["db"]) + case "ListContainers": + return p.handleListContainers(req, account, params["db"]) + case "CreateContainer": + return p.handleCreateContainer(req, account, params["db"]) + case "GetContainer": + return p.handleGetContainer(req, account, params["db"], params["coll"]) + case "DeleteContainer": + return p.handleDeleteContainer(req, account, params["db"], params["coll"]) + case "ListDocuments": + return p.handleListDocuments(req, account, params["db"], params["coll"]) + case "CreateDocument": + return p.handleCreateDocument(req, account, params["db"], params["coll"]) + case "GetDocument": + return p.handleGetDocument(req, account, params["db"], params["coll"], params["doc"]) + case "ReplaceDocument": + return p.handleReplaceDocument(req, account, params["db"], params["coll"], params["doc"]) + case "DeleteDocument": + return p.handleDeleteDocument(req, account, params["db"], params["coll"], params["doc"]) + default: + return p.errorReply(req, http.StatusBadRequest, "InvalidAction", "operation not handled: "+req.Method+" "+req.URL.Path), nil + } +} + +func (p *Provider) handleListDatabases(req *http.Request, account string) (*plugin.Response, error) { + dbs, err := p.store.ListDatabases(account) + if err != nil { + return p.errorReply(req, http.StatusInternalServerError, "InternalError", err.Error()), nil + } + type listDbsResponse struct { + RID string `json:"_rid"` + Databases []DatabaseInfo `json:"Databases"` + Count int `json:"_count"` + } + raw, _ := json.Marshal(listDbsResponse{ + Databases: dbs, + Count: len(dbs), + }) + return p.reply(req, http.StatusOK, nil, raw), nil +} + +func (p *Provider) handleCreateDatabase(req *http.Request, account string) (*plugin.Response, error) { + body, err := io.ReadAll(req.Body) + if err != nil { + return p.errorReply(req, http.StatusBadRequest, "BadRequest", "failed to read body"), nil + } + var reqBody struct { + ID string `json:"id"` + } + if err := json.Unmarshal(body, &reqBody); err != nil { + return p.errorReply(req, http.StatusBadRequest, "BadRequest", "invalid json: "+err.Error()), nil + } + + db, err := p.store.CreateDatabase(account, reqBody.ID) + if err != nil { + if errors.Is(err, ErrDatabaseAlreadyExists) { + return p.errorReply(req, http.StatusConflict, "Conflict", "database already exists"), nil + } + return p.errorReply(req, http.StatusInternalServerError, "InternalError", err.Error()), nil + } + + raw, _ := json.Marshal(db) + return p.reply(req, http.StatusCreated, nil, raw), nil +} + +func (p *Provider) handleGetDatabase(req *http.Request, account, dbID string) (*plugin.Response, error) { + db, err := p.store.GetDatabase(account, dbID) + if err != nil { + if errors.Is(err, ErrDatabaseNotFound) { + return p.errorReply(req, http.StatusNotFound, "NotFound", "database not found"), nil + } + return p.errorReply(req, http.StatusInternalServerError, "InternalError", err.Error()), nil + } + raw, _ := json.Marshal(db) + return p.reply(req, http.StatusOK, nil, raw), nil +} + +func (p *Provider) handleDeleteDatabase(req *http.Request, account, dbID string) (*plugin.Response, error) { + err := p.store.DeleteDatabase(account, dbID) + if err != nil { + if errors.Is(err, ErrDatabaseNotFound) { + return p.errorReply(req, http.StatusNotFound, "NotFound", "database not found"), nil + } + return p.errorReply(req, http.StatusInternalServerError, "InternalError", err.Error()), nil + } + return p.reply(req, http.StatusNoContent, nil, nil), nil +} + +func (p *Provider) handleListContainers(req *http.Request, account, dbID string) (*plugin.Response, error) { + colls, err := p.store.ListContainers(account, dbID) + if err != nil { + return p.errorReply(req, http.StatusInternalServerError, "InternalError", err.Error()), nil + } + type listCollsResponse struct { + RID string `json:"_rid"` + Containers []ContainerInfo `json:"DocumentCollections"` + Count int `json:"_count"` + } + raw, _ := json.Marshal(listCollsResponse{ + Containers: colls, + Count: len(colls), + }) + return p.reply(req, http.StatusOK, nil, raw), nil +} + +func (p *Provider) handleCreateContainer(req *http.Request, account, dbID string) (*plugin.Response, error) { + body, err := io.ReadAll(req.Body) + if err != nil { + return p.errorReply(req, http.StatusBadRequest, "BadRequest", "failed to read body"), nil + } + var reqBody struct { + ID string `json:"id"` + PartitionKey any `json:"partitionKey"` + } + if err := json.Unmarshal(body, &reqBody); err != nil { + return p.errorReply(req, http.StatusBadRequest, "BadRequest", "invalid json: "+err.Error()), nil + } + + coll, err := p.store.CreateContainer(account, dbID, reqBody.ID, reqBody.PartitionKey) + if err != nil { + if errors.Is(err, ErrDatabaseNotFound) { + return p.errorReply(req, http.StatusNotFound, "NotFound", "database not found"), nil + } + if errors.Is(err, ErrContainerAlreadyExists) { + return p.errorReply(req, http.StatusConflict, "Conflict", "container already exists"), nil + } + return p.errorReply(req, http.StatusInternalServerError, "InternalError", err.Error()), nil + } + + raw, _ := json.Marshal(coll) + return p.reply(req, http.StatusCreated, nil, raw), nil +} + +func (p *Provider) handleGetContainer(req *http.Request, account, dbID, collID string) (*plugin.Response, error) { + coll, err := p.store.GetContainer(account, dbID, collID) + if err != nil { + if errors.Is(err, ErrContainerNotFound) { + return p.errorReply(req, http.StatusNotFound, "NotFound", "container not found"), nil + } + return p.errorReply(req, http.StatusInternalServerError, "InternalError", err.Error()), nil + } + raw, _ := json.Marshal(coll) + return p.reply(req, http.StatusOK, nil, raw), nil +} + +func (p *Provider) handleDeleteContainer(req *http.Request, account, dbID, collID string) (*plugin.Response, error) { + err := p.store.DeleteContainer(account, dbID, collID) + if err != nil { + if errors.Is(err, ErrContainerNotFound) { + return p.errorReply(req, http.StatusNotFound, "NotFound", "container not found"), nil + } + return p.errorReply(req, http.StatusInternalServerError, "InternalError", err.Error()), nil + } + return p.reply(req, http.StatusNoContent, nil, nil), nil +} + +func (p *Provider) handleListDocuments(req *http.Request, account, dbID, collID string) (*plugin.Response, error) { + docs, err := p.store.ListDocuments(account, dbID, collID) + if err != nil { + return p.errorReply(req, http.StatusInternalServerError, "InternalError", err.Error()), nil + } + type listDocsResponse struct { + RID string `json:"_rid"` + Documents []Document `json:"Documents"` + Count int `json:"_count"` + } + raw, _ := json.Marshal(listDocsResponse{ + Documents: docs, + Count: len(docs), + }) + return p.reply(req, http.StatusOK, nil, raw), nil +} + +func (p *Provider) handleCreateDocument(req *http.Request, account, dbID, collID string) (*plugin.Response, error) { + body, err := io.ReadAll(req.Body) + if err != nil { + return p.errorReply(req, http.StatusBadRequest, "BadRequest", "failed to read body"), nil + } + var doc Document + if err := json.Unmarshal(body, &doc); err != nil { + return p.errorReply(req, http.StatusBadRequest, "BadRequest", "invalid json: "+err.Error()), nil + } + + created, err := p.store.CreateDocument(account, dbID, collID, doc) + if err != nil { + if errors.Is(err, ErrDocumentAlreadyExists) { + return p.errorReply(req, http.StatusConflict, "Conflict", "document already exists"), nil + } + return p.errorReply(req, http.StatusInternalServerError, "InternalError", err.Error()), nil + } + + raw, _ := json.Marshal(created) + return p.reply(req, http.StatusCreated, nil, raw), nil +} + +func (p *Provider) handleGetDocument(req *http.Request, account, dbID, collID, docID string) (*plugin.Response, error) { + doc, err := p.store.GetDocument(account, dbID, collID, docID) + if err != nil { + if errors.Is(err, ErrDocumentNotFound) { + return p.errorReply(req, http.StatusNotFound, "NotFound", "document not found"), nil + } + return p.errorReply(req, http.StatusInternalServerError, "InternalError", err.Error()), nil + } + raw, _ := json.Marshal(doc) + return p.reply(req, http.StatusOK, nil, raw), nil +} + +func (p *Provider) handleReplaceDocument(req *http.Request, account, dbID, collID, docID string) (*plugin.Response, error) { + body, err := io.ReadAll(req.Body) + if err != nil { + return p.errorReply(req, http.StatusBadRequest, "BadRequest", "failed to read body"), nil + } + var doc Document + if err := json.Unmarshal(body, &doc); err != nil { + return p.errorReply(req, http.StatusBadRequest, "BadRequest", "invalid json: "+err.Error()), nil + } + + replaced, err := p.store.ReplaceDocument(account, dbID, collID, docID, doc) + if err != nil { + if errors.Is(err, ErrDocumentNotFound) { + return p.errorReply(req, http.StatusNotFound, "NotFound", "document not found"), nil + } + return p.errorReply(req, http.StatusInternalServerError, "InternalError", err.Error()), nil + } + + raw, _ := json.Marshal(replaced) + return p.reply(req, http.StatusOK, nil, raw), nil +} + +func (p *Provider) handleDeleteDocument(req *http.Request, account, dbID, collID, docID string) (*plugin.Response, error) { + err := p.store.DeleteDocument(account, dbID, collID, docID) + if err != nil { + if errors.Is(err, ErrDocumentNotFound) { + return p.errorReply(req, http.StatusNotFound, "NotFound", "document not found"), nil + } + return p.errorReply(req, http.StatusInternalServerError, "InternalError", err.Error()), nil + } + return p.reply(req, http.StatusNoContent, nil, nil), nil +} + +func init() { + plugin.DefaultRegistry.Register("cosmosdb", func() plugin.ServicePlugin { + return &Provider{} + }) +} diff --git a/internal/services/cosmosdb/provider_test.go b/internal/services/cosmosdb/provider_test.go new file mode 100644 index 00000000..1ee3b830 --- /dev/null +++ b/internal/services/cosmosdb/provider_test.go @@ -0,0 +1,171 @@ +// SPDX-License-Identifier: Apache-2.0 + +package cosmosdb + +import ( + "bytes" + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/skyoo2003/devcloud/internal/plugin" +) + +func newTestProvider(t *testing.T) *Provider { + t.Helper() + dir, err := os.MkdirTemp("", "devcloud-cosmos-test-*") + require.NoError(t, err) + t.Cleanup(func() { _ = os.RemoveAll(dir) }) + + p := &Provider{} + err = p.Init(plugin.PluginConfig{DataDir: filepath.Join(dir, "cosmos")}) + require.NoError(t, err) + return p +} + +func TestCosmosProvider_Metadata(t *testing.T) { + p := newTestProvider(t) + defer func() { _ = p.Shutdown(context.Background()) }() + assert.Equal(t, "cosmosdb", p.ServiceID()) + assert.Equal(t, "Azure Cosmos DB", p.ServiceName()) + assert.Equal(t, plugin.ProtocolRESTJSON, p.Protocol()) + assert.Equal(t, "azure", plugin.ProviderOf(p)) +} + +func TestCosmosProvider_DatabaseLifecycle(t *testing.T) { + p := newTestProvider(t) + ctx := context.Background() + defer func() { _ = p.Shutdown(ctx) }() + + // 1. Create DB + reqCreate := httptest.NewRequest(http.MethodPost, "/devstoreaccount1/dbs", bytes.NewReader([]byte(`{"id": "mydb"}`))) + reqCreate.Header.Set("Content-Type", "application/json") + resp, err := p.HandleRequest(ctx, "", reqCreate) + require.NoError(t, err) + assert.Equal(t, http.StatusCreated, resp.StatusCode) + + // 2. Duplicate DB returns 409 + reqDup := httptest.NewRequest(http.MethodPost, "/devstoreaccount1/dbs", bytes.NewReader([]byte(`{"id": "mydb"}`))) + respDup, err := p.HandleRequest(ctx, "", reqDup) + require.NoError(t, err) + assert.Equal(t, http.StatusConflict, respDup.StatusCode) + + // 3. Get DB + reqGet := httptest.NewRequest(http.MethodGet, "/devstoreaccount1/dbs/mydb", nil) + respGet, err := p.HandleRequest(ctx, "", reqGet) + require.NoError(t, err) + assert.Equal(t, http.StatusOK, respGet.StatusCode) + assert.Contains(t, string(respGet.Body), `"id":"mydb"`) + + // 4. List DBs + reqList := httptest.NewRequest(http.MethodGet, "/devstoreaccount1/dbs", nil) + respList, err := p.HandleRequest(ctx, "", reqList) + require.NoError(t, err) + assert.Equal(t, http.StatusOK, respList.StatusCode) + assert.Contains(t, string(respList.Body), `"id":"mydb"`) + + // 5. Delete DB + reqDel := httptest.NewRequest(http.MethodDelete, "/devstoreaccount1/dbs/mydb", nil) + respDel, err := p.HandleRequest(ctx, "", reqDel) + require.NoError(t, err) + assert.Equal(t, http.StatusNoContent, respDel.StatusCode) + + // 6. Get deleted DB returns 404 + respGet404, err := p.HandleRequest(ctx, "", reqGet) + require.NoError(t, err) + assert.Equal(t, http.StatusNotFound, respGet404.StatusCode) +} + +func TestCosmosProvider_ContainerAndDocumentCRUD(t *testing.T) { + p := newTestProvider(t) + ctx := context.Background() + defer func() { _ = p.Shutdown(ctx) }() + + // 1. Setup DB + reqCreateDB := httptest.NewRequest(http.MethodPost, "/devstoreaccount1/dbs", bytes.NewReader([]byte(`{"id": "testdb"}`))) + _, err := p.HandleRequest(ctx, "", reqCreateDB) + require.NoError(t, err) + + // 2. Create Container + reqCreateColl := httptest.NewRequest(http.MethodPost, "/devstoreaccount1/dbs/testdb/colls", bytes.NewReader([]byte(`{"id": "users", "partitionKey": {"paths": ["/tenantId"]}}`))) + respColl, err := p.HandleRequest(ctx, "", reqCreateColl) + require.NoError(t, err) + assert.Equal(t, http.StatusCreated, respColl.StatusCode) + assert.Contains(t, string(respColl.Body), `"id":"users"`) + + // Duplicate container returns 409 + reqCollDup := httptest.NewRequest(http.MethodPost, "/devstoreaccount1/dbs/testdb/colls", bytes.NewReader([]byte(`{"id": "users", "partitionKey": {"paths": ["/tenantId"]}}`))) + respCollDup, err := p.HandleRequest(ctx, "", reqCollDup) + require.NoError(t, err) + assert.Equal(t, http.StatusConflict, respCollDup.StatusCode) + + // 3. Get Container + reqGetColl := httptest.NewRequest(http.MethodGet, "/devstoreaccount1/dbs/testdb/colls/users", nil) + respGetColl, err := p.HandleRequest(ctx, "", reqGetColl) + require.NoError(t, err) + assert.Equal(t, http.StatusOK, respGetColl.StatusCode) + + // 4. List Containers + reqListColls := httptest.NewRequest(http.MethodGet, "/devstoreaccount1/dbs/testdb/colls", nil) + respListColls, err := p.HandleRequest(ctx, "", reqListColls) + require.NoError(t, err) + assert.Equal(t, http.StatusOK, respListColls.StatusCode) + assert.Contains(t, string(respListColls.Body), `"users"`) + + // 5. Create Document + docData := map[string]any{ + "id": "doc-100", + "name": "Ada Lovelace", + "tenantId": "t-1", + "active": true, + } + body, _ := json.Marshal(docData) + reqCreateDoc := httptest.NewRequest(http.MethodPost, "/devstoreaccount1/dbs/testdb/colls/users/docs", bytes.NewReader(body)) + respDoc, err := p.HandleRequest(ctx, "", reqCreateDoc) + require.NoError(t, err) + assert.Equal(t, http.StatusCreated, respDoc.StatusCode) + assert.Contains(t, string(respDoc.Body), `"doc-100"`) + assert.Contains(t, string(respDoc.Body), `_rid`) + + // 6. Get Document + reqGetDoc := httptest.NewRequest(http.MethodGet, "/devstoreaccount1/dbs/testdb/colls/users/docs/doc-100", nil) + respGetDoc, err := p.HandleRequest(ctx, "", reqGetDoc) + require.NoError(t, err) + assert.Equal(t, http.StatusOK, respGetDoc.StatusCode) + assert.Contains(t, string(respGetDoc.Body), "Ada Lovelace") + + // 7. Replace Document + docData["name"] = "Countess Ada Lovelace" + bodyUp, _ := json.Marshal(docData) + reqReplaceDoc := httptest.NewRequest(http.MethodPut, "/devstoreaccount1/dbs/testdb/colls/users/docs/doc-100", bytes.NewReader(bodyUp)) + respReplaceDoc, err := p.HandleRequest(ctx, "", reqReplaceDoc) + require.NoError(t, err) + assert.Equal(t, http.StatusOK, respReplaceDoc.StatusCode) + assert.Contains(t, string(respReplaceDoc.Body), "Countess Ada Lovelace") + + // 8. List Documents + reqListDocs := httptest.NewRequest(http.MethodGet, "/devstoreaccount1/dbs/testdb/colls/users/docs", nil) + respListDocs, err := p.HandleRequest(ctx, "", reqListDocs) + require.NoError(t, err) + assert.Equal(t, http.StatusOK, respListDocs.StatusCode) + assert.Contains(t, string(respListDocs.Body), "Countess Ada Lovelace") + + // 9. Delete Document + reqDelDoc := httptest.NewRequest(http.MethodDelete, "/devstoreaccount1/dbs/testdb/colls/users/docs/doc-100", nil) + respDelDoc, err := p.HandleRequest(ctx, "", reqDelDoc) + require.NoError(t, err) + assert.Equal(t, http.StatusNoContent, respDelDoc.StatusCode) + + // 10. Delete Container + reqDelColl := httptest.NewRequest(http.MethodDelete, "/devstoreaccount1/dbs/testdb/colls/users", nil) + respDelColl, err := p.HandleRequest(ctx, "", reqDelColl) + require.NoError(t, err) + assert.Equal(t, http.StatusNoContent, respDelColl.StatusCode) +} diff --git a/internal/services/cosmosdb/store.go b/internal/services/cosmosdb/store.go new file mode 100644 index 00000000..3a8faf41 --- /dev/null +++ b/internal/services/cosmosdb/store.go @@ -0,0 +1,464 @@ +// SPDX-License-Identifier: Apache-2.0 + +package cosmosdb + +import ( + "crypto/rand" + "database/sql" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "path/filepath" + "sync" + "time" + + "github.com/skyoo2003/devcloud/internal/storage/sqlite" +) + +var ( + ErrDatabaseNotFound = errors.New("DatabaseNotFound") + ErrDatabaseAlreadyExists = errors.New("DatabaseAlreadyExists") + ErrContainerNotFound = errors.New("ContainerNotFound") + ErrContainerAlreadyExists = errors.New("ContainerAlreadyExists") + ErrDocumentNotFound = errors.New("DocumentNotFound") + ErrDocumentAlreadyExists = errors.New("DocumentAlreadyExists") +) + +type DatabaseInfo struct { + ID string `json:"id"` + RID string `json:"_rid"` + TS int64 `json:"_ts"` + ETag string `json:"_etag"` +} + +type ContainerInfo struct { + ID string `json:"id"` + RID string `json:"_rid"` + TS int64 `json:"_ts"` + ETag string `json:"_etag"` + PartitionKey any `json:"partitionKey,omitempty"` +} + +type Document = map[string]any + +type Store struct { + mu sync.RWMutex + store *sqlite.Store +} + +var migrations = []sqlite.Migration{ + { + Version: 1, + SQL: ` +CREATE TABLE IF NOT EXISTS databases ( + account TEXT NOT NULL, + id TEXT NOT NULL, + rid TEXT NOT NULL, + ts INTEGER NOT NULL, + etag TEXT NOT NULL, + PRIMARY KEY (account, id) +); + +CREATE TABLE IF NOT EXISTS containers ( + account TEXT NOT NULL, + database_id TEXT NOT NULL, + id TEXT NOT NULL, + rid TEXT NOT NULL, + partition_key TEXT NOT NULL, + ts INTEGER NOT NULL, + etag TEXT NOT NULL, + PRIMARY KEY (account, database_id, id) +); + +CREATE TABLE IF NOT EXISTS documents ( + account TEXT NOT NULL, + database_id TEXT NOT NULL, + container_id TEXT NOT NULL, + id TEXT NOT NULL, + rid TEXT NOT NULL, + data TEXT NOT NULL, + ts INTEGER NOT NULL, + etag TEXT NOT NULL, + PRIMARY KEY (account, database_id, container_id, id) +); +CREATE INDEX IF NOT EXISTS idx_docs_container ON documents (account, database_id, container_id); +`, + }, +} + +func NewStore(dataDir string) (*Store, error) { + dbPath := filepath.Join(dataDir, "cosmos.db") + s, err := sqlite.Open(dbPath, migrations) + if err != nil { + return nil, err + } + return &Store{store: s}, nil +} + +func (s *Store) Close() error { + return s.store.Close() +} + +func randomRID() string { + b := make([]byte, 8) + _, _ = rand.Read(b) + return hex.EncodeToString(b) +} + +func (s *Store) CreateDatabase(account, id string) (*DatabaseInfo, error) { + s.mu.Lock() + defer s.mu.Unlock() + + if account == "" { + account = "devstoreaccount1" + } + if id == "" { + return nil, errors.New("database id is required") + } + + var count int + _ = s.store.DB().QueryRow("SELECT COUNT(*) FROM databases WHERE account = ? AND id = ?", account, id).Scan(&count) + if count > 0 { + return nil, ErrDatabaseAlreadyExists + } + + now := time.Now().UTC() + rid := randomRID() + etag := fmt.Sprintf("\"%s\"", randomRID()) + ts := now.Unix() + + _, err := s.store.DB().Exec("INSERT INTO databases (account, id, rid, ts, etag) VALUES (?, ?, ?, ?, ?)", + account, id, rid, ts, etag) + if err != nil { + return nil, err + } + return &DatabaseInfo{ID: id, RID: rid, TS: ts, ETag: etag}, nil +} + +func (s *Store) GetDatabase(account, id string) (*DatabaseInfo, error) { + s.mu.RLock() + defer s.mu.RUnlock() + + if account == "" { + account = "devstoreaccount1" + } + + var rid, etag string + var ts int64 + err := s.store.DB().QueryRow("SELECT rid, ts, etag FROM databases WHERE account = ? AND id = ?", account, id).Scan(&rid, &ts, &etag) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + return nil, ErrDatabaseNotFound + } + return nil, err + } + return &DatabaseInfo{ID: id, RID: rid, TS: ts, ETag: etag}, nil +} + +func (s *Store) DeleteDatabase(account, id string) error { + s.mu.Lock() + defer s.mu.Unlock() + + if account == "" { + account = "devstoreaccount1" + } + + res, err := s.store.DB().Exec("DELETE FROM databases WHERE account = ? AND id = ?", account, id) + if err != nil { + return err + } + rows, err := res.RowsAffected() + if err != nil || rows == 0 { + return ErrDatabaseNotFound + } + + _, _ = s.store.DB().Exec("DELETE FROM containers WHERE account = ? AND database_id = ?", account, id) + _, _ = s.store.DB().Exec("DELETE FROM documents WHERE account = ? AND database_id = ?", account, id) + return nil +} + +func (s *Store) ListDatabases(account string) ([]DatabaseInfo, error) { + s.mu.RLock() + defer s.mu.RUnlock() + + if account == "" { + account = "devstoreaccount1" + } + + rows, err := s.store.DB().Query("SELECT id, rid, ts, etag FROM databases WHERE account = ? ORDER BY id ASC", account) + if err != nil { + return nil, err + } + defer func() { _ = rows.Close() }() + + var list []DatabaseInfo + for rows.Next() { + var id, rid, etag string + var ts int64 + if err := rows.Scan(&id, &rid, &ts, &etag); err == nil { + list = append(list, DatabaseInfo{ID: id, RID: rid, TS: ts, ETag: etag}) + } + } + return list, nil +} + +func (s *Store) CreateContainer(account, dbID, id string, partitionKey any) (*ContainerInfo, error) { + s.mu.Lock() + defer s.mu.Unlock() + + if account == "" { + account = "devstoreaccount1" + } + + var dbCount int + _ = s.store.DB().QueryRow("SELECT COUNT(*) FROM databases WHERE account = ? AND id = ?", account, dbID).Scan(&dbCount) + if dbCount == 0 { + return nil, ErrDatabaseNotFound + } + + var count int + _ = s.store.DB().QueryRow("SELECT COUNT(*) FROM containers WHERE account = ? AND database_id = ? AND id = ?", + account, dbID, id).Scan(&count) + if count > 0 { + return nil, ErrContainerAlreadyExists + } + + pkBytes, _ := json.Marshal(partitionKey) + now := time.Now().UTC() + rid := randomRID() + etag := fmt.Sprintf("\"%s\"", randomRID()) + ts := now.Unix() + + _, err := s.store.DB().Exec("INSERT INTO containers (account, database_id, id, rid, partition_key, ts, etag) VALUES (?, ?, ?, ?, ?, ?, ?)", + account, dbID, id, rid, string(pkBytes), ts, etag) + if err != nil { + return nil, err + } + return &ContainerInfo{ID: id, RID: rid, TS: ts, ETag: etag, PartitionKey: partitionKey}, nil +} + +func (s *Store) GetContainer(account, dbID, id string) (*ContainerInfo, error) { + s.mu.RLock() + defer s.mu.RUnlock() + + if account == "" { + account = "devstoreaccount1" + } + + var rid, pkStr, etag string + var ts int64 + err := s.store.DB().QueryRow("SELECT rid, partition_key, ts, etag FROM containers WHERE account = ? AND database_id = ? AND id = ?", + account, dbID, id).Scan(&rid, &pkStr, &ts, &etag) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + return nil, ErrContainerNotFound + } + return nil, err + } + + var pk any + _ = json.Unmarshal([]byte(pkStr), &pk) + return &ContainerInfo{ID: id, RID: rid, TS: ts, ETag: etag, PartitionKey: pk}, nil +} + +func (s *Store) DeleteContainer(account, dbID, id string) error { + s.mu.Lock() + defer s.mu.Unlock() + + if account == "" { + account = "devstoreaccount1" + } + + res, err := s.store.DB().Exec("DELETE FROM containers WHERE account = ? AND database_id = ? AND id = ?", account, dbID, id) + if err != nil { + return err + } + rows, err := res.RowsAffected() + if err != nil || rows == 0 { + return ErrContainerNotFound + } + + _, _ = s.store.DB().Exec("DELETE FROM documents WHERE account = ? AND database_id = ? AND container_id = ?", account, dbID, id) + return nil +} + +func (s *Store) ListContainers(account, dbID string) ([]ContainerInfo, error) { + s.mu.RLock() + defer s.mu.RUnlock() + + if account == "" { + account = "devstoreaccount1" + } + + rows, err := s.store.DB().Query("SELECT id, rid, partition_key, ts, etag FROM containers WHERE account = ? AND database_id = ? ORDER BY id ASC", account, dbID) + if err != nil { + return nil, err + } + defer func() { _ = rows.Close() }() + + var list []ContainerInfo + for rows.Next() { + var id, rid, pkStr, etag string + var ts int64 + if err := rows.Scan(&id, &rid, &pkStr, &ts, &etag); err == nil { + var pk any + _ = json.Unmarshal([]byte(pkStr), &pk) + list = append(list, ContainerInfo{ID: id, RID: rid, TS: ts, ETag: etag, PartitionKey: pk}) + } + } + return list, nil +} + +func (s *Store) CreateDocument(account, dbID, collID string, doc Document) (Document, error) { + s.mu.Lock() + defer s.mu.Unlock() + + if account == "" { + account = "devstoreaccount1" + } + + id, _ := doc["id"].(string) + if id == "" { + return nil, errors.New("document id is required") + } + + var count int + _ = s.store.DB().QueryRow("SELECT COUNT(*) FROM documents WHERE account = ? AND database_id = ? AND container_id = ? AND id = ?", + account, dbID, collID, id).Scan(&count) + if count > 0 { + return nil, ErrDocumentAlreadyExists + } + + now := time.Now().UTC() + rid := randomRID() + etag := fmt.Sprintf("\"%s\"", randomRID()) + ts := now.Unix() + + doc["_rid"] = rid + doc["_ts"] = ts + doc["_etag"] = etag + + dataBytes, err := json.Marshal(doc) + if err != nil { + return nil, err + } + + _, err = s.store.DB().Exec( + "INSERT INTO documents (account, database_id, container_id, id, rid, data, ts, etag) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + account, dbID, collID, id, rid, string(dataBytes), ts, etag) + if err != nil { + return nil, err + } + return doc, nil +} + +func (s *Store) GetDocument(account, dbID, collID, docID string) (Document, error) { + s.mu.RLock() + defer s.mu.RUnlock() + + if account == "" { + account = "devstoreaccount1" + } + + var dataStr string + err := s.store.DB().QueryRow( + "SELECT data FROM documents WHERE account = ? AND database_id = ? AND container_id = ? AND id = ?", + account, dbID, collID, docID).Scan(&dataStr) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + return nil, ErrDocumentNotFound + } + return nil, err + } + + var doc Document + if err := json.Unmarshal([]byte(dataStr), &doc); err != nil { + return nil, err + } + return doc, nil +} + +func (s *Store) ReplaceDocument(account, dbID, collID, docID string, doc Document) (Document, error) { + s.mu.Lock() + defer s.mu.Unlock() + + if account == "" { + account = "devstoreaccount1" + } + + doc["id"] = docID + now := time.Now().UTC() + etag := fmt.Sprintf("\"%s\"", randomRID()) + ts := now.Unix() + + doc["_ts"] = ts + doc["_etag"] = etag + + dataBytes, err := json.Marshal(doc) + if err != nil { + return nil, err + } + + res, err := s.store.DB().Exec( + "UPDATE documents SET data = ?, ts = ?, etag = ? WHERE account = ? AND database_id = ? AND container_id = ? AND id = ?", + string(dataBytes), ts, etag, account, dbID, collID, docID) + if err != nil { + return nil, err + } + rows, err := res.RowsAffected() + if err != nil || rows == 0 { + return nil, ErrDocumentNotFound + } + return doc, nil +} + +func (s *Store) DeleteDocument(account, dbID, collID, docID string) error { + s.mu.Lock() + defer s.mu.Unlock() + + if account == "" { + account = "devstoreaccount1" + } + + res, err := s.store.DB().Exec( + "DELETE FROM documents WHERE account = ? AND database_id = ? AND container_id = ? AND id = ?", + account, dbID, collID, docID) + if err != nil { + return err + } + rows, err := res.RowsAffected() + if err != nil || rows == 0 { + return ErrDocumentNotFound + } + return nil +} + +func (s *Store) ListDocuments(account, dbID, collID string) ([]Document, error) { + s.mu.RLock() + defer s.mu.RUnlock() + + if account == "" { + account = "devstoreaccount1" + } + + rows, err := s.store.DB().Query( + "SELECT data FROM documents WHERE account = ? AND database_id = ? AND container_id = ? ORDER BY id ASC", + account, dbID, collID) + if err != nil { + return nil, err + } + defer func() { _ = rows.Close() }() + + var list []Document + for rows.Next() { + var dataStr string + if err := rows.Scan(&dataStr); err == nil { + var doc Document + if json.Unmarshal([]byte(dataStr), &doc) == nil { + list = append(list, doc) + } + } + } + return list, nil +} diff --git a/internal/services/gcs/provider.go b/internal/services/gcs/provider.go new file mode 100644 index 00000000..647119cc --- /dev/null +++ b/internal/services/gcs/provider.go @@ -0,0 +1,370 @@ +// SPDX-License-Identifier: Apache-2.0 + +package gcs + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "mime" + "mime/multipart" + "net/http" + "strings" + + "github.com/skyoo2003/devcloud/internal/auth" + generated "github.com/skyoo2003/devcloud/internal/generated/gcs" + "github.com/skyoo2003/devcloud/internal/plugin" +) + +// Provider implements Google Cloud Storage. +type Provider struct { + generated.BaseProvider + dataDir string + store *FileStore +} + +func (p *Provider) ServiceID() string { return "gcs" } +func (p *Provider) ServiceName() string { return "Google Cloud Storage" } +func (p *Provider) Protocol() plugin.ProtocolType { return plugin.ProtocolRESTJSON } +func (p *Provider) Provider() string { return "gcp" } + +func (p *Provider) Init(cfg plugin.PluginConfig) error { + p.dataDir = cfg.DataDir + if p.dataDir == "" { + p.dataDir = "./data/gcp-storage" + } + p.store = NewFileStore(p.dataDir) + return nil +} + +func (p *Provider) Shutdown(ctx context.Context) error { + return nil +} + +func (p *Provider) ListResources(ctx context.Context) ([]plugin.Resource, error) { + buckets, err := p.store.ListBuckets("default") + if err != nil { + return nil, err + } + var res []plugin.Resource + for _, b := range buckets { + res = append(res, plugin.Resource{ + Type: "storage#bucket", + ID: b.ID, + Name: b.Name, + }) + } + return res, nil +} + +func (p *Provider) jsonReply(status int, data any) *plugin.Response { + raw, err := json.MarshalIndent(data, "", " ") + if err != nil { + return p.errorReply(http.StatusInternalServerError, "internalError", err.Error()) + } + return &plugin.Response{ + StatusCode: status, + Headers: map[string]string{"Content-Type": "application/json; charset=UTF-8"}, + Body: raw, + ContentType: "application/json; charset=UTF-8", + } +} + +func (p *Provider) errorReply(status int, reason, message string) *plugin.Response { + type gcpErrItem struct { + Message string `json:"message"` + Domain string `json:"domain"` + Reason string `json:"reason"` + } + type gcpErr struct { + Error struct { + Code int `json:"code"` + Message string `json:"message"` + Errors []gcpErrItem `json:"errors"` + } `json:"error"` + } + var resp gcpErr + resp.Error.Code = status + resp.Error.Message = message + resp.Error.Errors = []gcpErrItem{{ + Message: message, + Domain: "global", + Reason: reason, + }} + raw, _ := json.MarshalIndent(resp, "", " ") + return &plugin.Response{ + StatusCode: status, + Headers: map[string]string{"Content-Type": "application/json; charset=UTF-8"}, + Body: raw, + ContentType: "application/json; charset=UTF-8", + } +} + +func (p *Provider) HandleRequest(ctx context.Context, _ string, req *http.Request) (*plugin.Response, error) { + project := req.URL.Query().Get("project") + if project == "" { + if id, ok := auth.FromContext(ctx); ok && id.AccessKeyID != "" { + project = id.AccessKeyID + } else { + project = "default" + } + } + + op, params := generated.MatchOperation(req.Method, req.URL.RequestURI()) + if op == "" { + // Fallback match using path without query + op, params = generated.MatchOperation(req.Method, req.URL.Path) + } + + switch op { + case "ListBuckets": + return p.handleListBuckets(project) + case "InsertBucket": + return p.handleInsertBucket(project, req) + case "GetBucket": + return p.handleGetBucket(project, params["bucket"]) + case "DeleteBucket": + return p.handleDeleteBucket(project, params["bucket"]) + case "ListObjects": + return p.handleListObjects(project, params["bucket"], req.URL.Query().Get("prefix")) + case "GetObject": + return p.handleGetObject(project, params["bucket"], params["object"], req) + case "InsertObject": + return p.handleInsertObject(project, params["bucket"], req) + case "DeleteObject": + return p.handleDeleteObject(project, params["bucket"], params["object"]) + default: + return p.errorReply(http.StatusBadRequest, "unknownAction", "operation not handled: "+req.Method+" "+req.URL.Path), nil + } +} + +func (p *Provider) handleListBuckets(project string) (*plugin.Response, error) { + buckets, err := p.store.ListBuckets(project) + if err != nil { + return p.errorReply(http.StatusInternalServerError, "internalError", err.Error()), nil + } + type listBucketsResponse struct { + Kind string `json:"kind"` + Items []BucketInfo `json:"items"` + } + return p.jsonReply(http.StatusOK, listBucketsResponse{ + Kind: "storage#buckets", + Items: buckets, + }), nil +} + +func (p *Provider) handleInsertBucket(project string, req *http.Request) (*plugin.Response, error) { + body, err := io.ReadAll(req.Body) + if err != nil { + return p.errorReply(http.StatusBadRequest, "badRequest", "failed to read body"), nil + } + var reqBody struct { + Name string `json:"name"` + Location string `json:"location"` + StorageClass string `json:"storageClass"` + } + if len(body) > 0 { + _ = json.Unmarshal(body, &reqBody) + } + bucketName := reqBody.Name + if bucketName == "" { + bucketName = req.URL.Query().Get("name") + } + if bucketName == "" { + return p.errorReply(http.StatusBadRequest, "required", "bucket name is required"), nil + } + + bucket, err := p.store.CreateBucket(project, bucketName, reqBody.Location, reqBody.StorageClass) + if err != nil { + if errors.Is(err, ErrBucketAlreadyExists) { + return p.errorReply(http.StatusConflict, "conflict", "bucket already exists"), nil + } + if errors.Is(err, ErrInvalidName) { + return p.errorReply(http.StatusBadRequest, "invalid", "invalid bucket name"), nil + } + return p.errorReply(http.StatusInternalServerError, "internalError", err.Error()), nil + } + + return p.jsonReply(http.StatusOK, bucket), nil +} + +func (p *Provider) handleGetBucket(project, bucket string) (*plugin.Response, error) { + b, err := p.store.GetBucket(project, bucket) + if err != nil { + if errors.Is(err, ErrBucketNotFound) { + return p.errorReply(http.StatusNotFound, "notFound", "bucket not found"), nil + } + return p.errorReply(http.StatusInternalServerError, "internalError", err.Error()), nil + } + return p.jsonReply(http.StatusOK, b), nil +} + +func (p *Provider) handleDeleteBucket(project, bucket string) (*plugin.Response, error) { + err := p.store.DeleteBucket(project, bucket) + if err != nil { + if errors.Is(err, ErrBucketNotFound) { + return p.errorReply(http.StatusNotFound, "notFound", "bucket not found"), nil + } + if errors.Is(err, ErrBucketNotEmpty) { + return p.errorReply(http.StatusConflict, "conflict", "bucket is not empty"), nil + } + return p.errorReply(http.StatusInternalServerError, "internalError", err.Error()), nil + } + return &plugin.Response{ + StatusCode: http.StatusNoContent, + Headers: map[string]string{}, + }, nil +} + +func (p *Provider) handleListObjects(project, bucket, prefix string) (*plugin.Response, error) { + objects, err := p.store.ListObjects(project, bucket, prefix) + if err != nil { + if errors.Is(err, ErrBucketNotFound) { + return p.errorReply(http.StatusNotFound, "notFound", "bucket not found"), nil + } + return p.errorReply(http.StatusInternalServerError, "internalError", err.Error()), nil + } + type listObjectsResponse struct { + Kind string `json:"kind"` + Items []ObjectInfo `json:"items"` + } + return p.jsonReply(http.StatusOK, listObjectsResponse{ + Kind: "storage#objects", + Items: objects, + }), nil +} + +func (p *Provider) handleGetObject(project, bucket, object string, req *http.Request) (*plugin.Response, error) { + alt := req.URL.Query().Get("alt") + if alt == "media" { + info, data, err := p.store.GetObject(project, bucket, object) + if err != nil { + if errors.Is(err, ErrObjectNotFound) || errors.Is(err, ErrBucketNotFound) { + return p.errorReply(http.StatusNotFound, "notFound", "object not found"), nil + } + return p.errorReply(http.StatusInternalServerError, "internalError", err.Error()), nil + } + headers := map[string]string{ + "Content-Type": info.ContentType, + "ETag": info.ETag, + "Content-Length": fmt.Sprintf("%d", len(data)), + "x-goog-hash": fmt.Sprintf("crc32c=%s,md5=%s", info.CRC32c, info.MD5Hash), + } + return &plugin.Response{ + StatusCode: http.StatusOK, + Headers: headers, + Body: data, + ContentType: info.ContentType, + }, nil + } + + info, err := p.store.GetObjectMetadata(project, bucket, object) + if err != nil { + if errors.Is(err, ErrObjectNotFound) || errors.Is(err, ErrBucketNotFound) { + return p.errorReply(http.StatusNotFound, "notFound", "object not found"), nil + } + return p.errorReply(http.StatusInternalServerError, "internalError", err.Error()), nil + } + return p.jsonReply(http.StatusOK, info), nil +} + +func (p *Provider) handleInsertObject(project, bucket string, req *http.Request) (*plugin.Response, error) { + contentType := req.Header.Get("Content-Type") + mediaType, params, _ := mime.ParseMediaType(contentType) + + objectName := req.URL.Query().Get("name") + var payload []byte + var meta map[string]string + objContentType := "application/octet-stream" + + if strings.HasPrefix(mediaType, "multipart/") { + mr := multipart.NewReader(req.Body, params["boundary"]) + for { + part, err := mr.NextPart() + if err == io.EOF { + break + } + if err != nil { + return p.errorReply(http.StatusBadRequest, "badRequest", "invalid multipart payload: "+err.Error()), nil + } + partType := part.Header.Get("Content-Type") + if strings.Contains(partType, "application/json") { + var objMetadata struct { + Name string `json:"name"` + ContentType string `json:"contentType"` + Metadata map[string]string `json:"metadata"` + } + _ = json.NewDecoder(part).Decode(&objMetadata) + if objMetadata.Name != "" { + objectName = objMetadata.Name + } + if objMetadata.ContentType != "" { + objContentType = objMetadata.ContentType + } + if objMetadata.Metadata != nil { + meta = objMetadata.Metadata + } + } else { + if partType != "" { + objContentType = partType + } + data, rerr := io.ReadAll(part) + if rerr != nil { + return p.errorReply(http.StatusBadRequest, "badRequest", "failed to read part data"), nil + } + payload = data + } + } + } else { + // Raw media or direct upload + data, err := io.ReadAll(req.Body) + if err != nil { + return p.errorReply(http.StatusBadRequest, "badRequest", "failed to read body"), nil + } + payload = data + if ct := req.Header.Get("X-Upload-Content-Type"); ct != "" { + objContentType = ct + } else if contentType != "" { + objContentType = contentType + } + } + + if objectName == "" { + return p.errorReply(http.StatusBadRequest, "required", "object name is required"), nil + } + + info, err := p.store.PutObject(project, bucket, objectName, objContentType, payload, meta) + if err != nil { + if errors.Is(err, ErrBucketNotFound) { + return p.errorReply(http.StatusNotFound, "notFound", "bucket not found"), nil + } + if errors.Is(err, ErrInvalidName) { + return p.errorReply(http.StatusBadRequest, "invalid", "invalid object name"), nil + } + return p.errorReply(http.StatusInternalServerError, "internalError", err.Error()), nil + } + + return p.jsonReply(http.StatusOK, info), nil +} + +func (p *Provider) handleDeleteObject(project, bucket, object string) (*plugin.Response, error) { + err := p.store.DeleteObject(project, bucket, object) + if err != nil { + if errors.Is(err, ErrObjectNotFound) || errors.Is(err, ErrBucketNotFound) { + return p.errorReply(http.StatusNotFound, "notFound", "object not found"), nil + } + return p.errorReply(http.StatusInternalServerError, "internalError", err.Error()), nil + } + return &plugin.Response{ + StatusCode: http.StatusNoContent, + Headers: map[string]string{}, + }, nil +} + +func init() { + plugin.DefaultRegistry.Register("gcs", func() plugin.ServicePlugin { + return &Provider{} + }) +} diff --git a/internal/services/gcs/provider_test.go b/internal/services/gcs/provider_test.go new file mode 100644 index 00000000..bdfc94a9 --- /dev/null +++ b/internal/services/gcs/provider_test.go @@ -0,0 +1,205 @@ +// SPDX-License-Identifier: Apache-2.0 + +package gcs + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "mime/multipart" + "net/http" + "net/http/httptest" + "net/textproto" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/skyoo2003/devcloud/internal/plugin" +) + +func newTestProvider(t *testing.T) *Provider { + t.Helper() + dir, err := os.MkdirTemp("", "devcloud-gcs-test-*") + require.NoError(t, err) + t.Cleanup(func() { _ = os.RemoveAll(dir) }) + + p := &Provider{} + err = p.Init(plugin.PluginConfig{DataDir: filepath.Join(dir, "gcs")}) + require.NoError(t, err) + return p +} + +func TestGCSProvider_Metadata(t *testing.T) { + p := newTestProvider(t) + assert.Equal(t, "gcs", p.ServiceID()) + assert.Equal(t, "Google Cloud Storage", p.ServiceName()) + assert.Equal(t, plugin.ProtocolRESTJSON, p.Protocol()) + assert.Equal(t, "gcp", plugin.ProviderOf(p)) +} + +func TestGCSProvider_BucketLifecycle(t *testing.T) { + p := newTestProvider(t) + ctx := context.Background() + + // 1. Create Bucket + createBody := `{"name": "my-bucket", "location": "US", "storageClass": "STANDARD"}` + reqCreate := httptest.NewRequest(http.MethodPost, "/storage/v1/b?project=test-proj", stringsReader(createBody)) + reqCreate.Header.Set("Content-Type", "application/json") + resp, err := p.HandleRequest(ctx, "", reqCreate) + require.NoError(t, err) + assert.Equal(t, http.StatusOK, resp.StatusCode) + + var b BucketInfo + err = json.Unmarshal(resp.Body, &b) + require.NoError(t, err) + assert.Equal(t, "my-bucket", b.Name) + assert.Equal(t, "US", b.Location) + + // 2. Duplicate Create returns 409 Conflict + reqDup := httptest.NewRequest(http.MethodPost, "/storage/v1/b?project=test-proj", stringsReader(createBody)) + reqDup.Header.Set("Content-Type", "application/json") + respDup, err := p.HandleRequest(ctx, "", reqDup) + require.NoError(t, err) + assert.Equal(t, http.StatusConflict, respDup.StatusCode) + + // 3. Get Bucket + reqGet := httptest.NewRequest(http.MethodGet, "/storage/v1/b/my-bucket?project=test-proj", nil) + respGet, err := p.HandleRequest(ctx, "", reqGet) + require.NoError(t, err) + assert.Equal(t, http.StatusOK, respGet.StatusCode) + + // 4. List Buckets + reqList := httptest.NewRequest(http.MethodGet, "/storage/v1/b?project=test-proj", nil) + respList, err := p.HandleRequest(ctx, "", reqList) + require.NoError(t, err) + assert.Equal(t, http.StatusOK, respList.StatusCode) + assert.Contains(t, string(respList.Body), "my-bucket") + + // 5. Delete Bucket + reqDel := httptest.NewRequest(http.MethodDelete, "/storage/v1/b/my-bucket?project=test-proj", nil) + respDel, err := p.HandleRequest(ctx, "", reqDel) + require.NoError(t, err) + assert.Equal(t, http.StatusNoContent, respDel.StatusCode) + + // 6. Get Deleted Bucket returns 404 + respGet404, err := p.HandleRequest(ctx, "", reqGet) + require.NoError(t, err) + assert.Equal(t, http.StatusNotFound, respGet404.StatusCode) +} + +func TestGCSProvider_ObjectLifecycle(t *testing.T) { + p := newTestProvider(t) + ctx := context.Background() + + // Setup bucket + createBody := `{"name": "test-bucket"}` + reqCreateB := httptest.NewRequest(http.MethodPost, "/storage/v1/b?project=test-proj", stringsReader(createBody)) + _, err := p.HandleRequest(ctx, "", reqCreateB) + require.NoError(t, err) + + // 1. Upload Object (Media upload) + payload := []byte("Hello Google Cloud Storage from DevCloud!") + reqUpload := httptest.NewRequest(http.MethodPost, "/upload/storage/v1/b/test-bucket/o?uploadType=media&name=notes/sample.txt&project=test-proj", bytes.NewReader(payload)) + reqUpload.Header.Set("Content-Type", "text/plain") + respUpload, err := p.HandleRequest(ctx, "", reqUpload) + require.NoError(t, err) + assert.Equal(t, http.StatusOK, respUpload.StatusCode) + + var obj ObjectInfo + err = json.Unmarshal(respUpload.Body, &obj) + require.NoError(t, err) + assert.Equal(t, "notes/sample.txt", obj.Name) + assert.NotEmpty(t, obj.MD5Hash) + assert.NotEmpty(t, obj.CRC32c) + + // 2. Get Object Metadata + reqMeta := httptest.NewRequest(http.MethodGet, "/storage/v1/b/test-bucket/o/notes/sample.txt?project=test-proj", nil) + respMeta, err := p.HandleRequest(ctx, "", reqMeta) + require.NoError(t, err) + assert.Equal(t, http.StatusOK, respMeta.StatusCode) + assert.Contains(t, string(respMeta.Body), "text/plain") + + // 3. Download Object Media (alt=media) + reqDown := httptest.NewRequest(http.MethodGet, "/storage/v1/b/test-bucket/o/notes/sample.txt?alt=media&project=test-proj", nil) + respDown, err := p.HandleRequest(ctx, "", reqDown) + require.NoError(t, err) + assert.Equal(t, http.StatusOK, respDown.StatusCode) + assert.Equal(t, payload, respDown.Body) + assert.NotEmpty(t, respDown.Headers["x-goog-hash"]) + + // 4. List Objects with prefix + reqList := httptest.NewRequest(http.MethodGet, "/storage/v1/b/test-bucket/o?prefix=notes/&project=test-proj", nil) + respList, err := p.HandleRequest(ctx, "", reqList) + require.NoError(t, err) + assert.Equal(t, http.StatusOK, respList.StatusCode) + assert.Contains(t, string(respList.Body), "notes/sample.txt") + + // 5. Delete Object + reqDel := httptest.NewRequest(http.MethodDelete, "/storage/v1/b/test-bucket/o/notes/sample.txt?project=test-proj", nil) + respDel, err := p.HandleRequest(ctx, "", reqDel) + require.NoError(t, err) + assert.Equal(t, http.StatusNoContent, respDel.StatusCode) + + // 6. Download Deleted Object returns 404 + respDown404, err := p.HandleRequest(ctx, "", reqDown) + require.NoError(t, err) + assert.Equal(t, http.StatusNotFound, respDown404.StatusCode) +} + +func TestGCSProvider_MultipartUpload(t *testing.T) { + p := newTestProvider(t) + ctx := context.Background() + + // Setup bucket + createBody := `{"name": "mp-bucket"}` + reqCreateB := httptest.NewRequest(http.MethodPost, "/storage/v1/b?project=test-proj", stringsReader(createBody)) + _, err := p.HandleRequest(ctx, "", reqCreateB) + require.NoError(t, err) + + // Build multipart body + var body bytes.Buffer + writer := multipart.NewWriter(&body) + + // Part 1: JSON metadata + hJSON := make(textproto.MIMEHeader) + hJSON.Set("Content-Type", "application/json; charset=UTF-8") + part1, err := writer.CreatePart(hJSON) + require.NoError(t, err) + _, _ = part1.Write([]byte(`{"name": "data/large.bin", "contentType": "application/octet-stream"}`)) + + // Part 2: Payload + hMedia := make(textproto.MIMEHeader) + hMedia.Set("Content-Type", "application/octet-stream") + part2, err := writer.CreatePart(hMedia) + require.NoError(t, err) + payload := []byte("Multipart upload bytes in DevCloud GCS") + _, _ = part2.Write(payload) + _ = writer.Close() + + reqUpload := httptest.NewRequest(http.MethodPost, "/upload/storage/v1/b/mp-bucket/o?uploadType=multipart&project=test-proj", &body) + reqUpload.Header.Set("Content-Type", fmt.Sprintf("multipart/related; boundary=%s", writer.Boundary())) + + respUpload, err := p.HandleRequest(ctx, "", reqUpload) + require.NoError(t, err) + assert.Equal(t, http.StatusOK, respUpload.StatusCode) + + var obj ObjectInfo + err = json.Unmarshal(respUpload.Body, &obj) + require.NoError(t, err) + assert.Equal(t, "data/large.bin", obj.Name) + assert.Equal(t, int64(len(payload)), obj.Size) + + // Verify download + reqDown := httptest.NewRequest(http.MethodGet, "/storage/v1/b/mp-bucket/o/data/large.bin?alt=media&project=test-proj", nil) + respDown, err := p.HandleRequest(ctx, "", reqDown) + require.NoError(t, err) + assert.Equal(t, payload, respDown.Body) +} + +func stringsReader(s string) *bytes.Reader { + return bytes.NewReader([]byte(s)) +} diff --git a/internal/services/gcs/store.go b/internal/services/gcs/store.go new file mode 100644 index 00000000..381d890a --- /dev/null +++ b/internal/services/gcs/store.go @@ -0,0 +1,484 @@ +// SPDX-License-Identifier: Apache-2.0 + +package gcs + +import ( + "crypto/md5" + "encoding/base64" + "encoding/binary" + "encoding/json" + "errors" + "fmt" + "hash/crc32" + "os" + "path/filepath" + "sort" + "strings" + "sync" + "time" +) + +var ( + ErrBucketNotFound = errors.New("notFound: bucket not found") + ErrBucketAlreadyExists = errors.New("conflict: bucket already exists") + ErrBucketNotEmpty = errors.New("conflict: bucket is not empty") + ErrObjectNotFound = errors.New("notFound: object not found") + ErrConditionNotMet = errors.New("conditionNotMet: precondition failed") + ErrInvalidName = errors.New("invalid: invalid bucket or object name") +) + +type BucketInfo struct { + Kind string `json:"kind"` + ID string `json:"id"` + Name string `json:"name"` + ProjectNumber string `json:"projectNumber,omitempty"` + Location string `json:"location"` + StorageClass string `json:"storageClass"` + TimeCreated time.Time `json:"timeCreated"` + Updated time.Time `json:"updated"` + ETag string `json:"etag"` +} + +type ObjectInfo struct { + Kind string `json:"kind"` + ID string `json:"id"` + Name string `json:"name"` + Bucket string `json:"bucket"` + Size int64 `json:"size,string"` + ContentType string `json:"contentType"` + MD5Hash string `json:"md5Hash"` + CRC32c string `json:"crc32c"` + ETag string `json:"etag"` + TimeCreated time.Time `json:"timeCreated"` + Updated time.Time `json:"updated"` + Metadata map[string]string `json:"metadata,omitempty"` +} + +type FileStore struct { + mu sync.RWMutex + baseDir string +} + +func NewFileStore(baseDir string) *FileStore { + abs, err := filepath.Abs(baseDir) + if err != nil { + abs = filepath.Clean(baseDir) + } + cleaned := filepath.Clean(abs) + _ = os.MkdirAll(cleaned, 0755) + return &FileStore{baseDir: cleaned} +} + +func (fs *FileStore) safeBucketDir(project, bucket string) (string, error) { + if project == "" { + project = "default" + } + if bucket == "" || bucket == "." || strings.Contains(bucket, "..") || strings.Contains(bucket, "/") || strings.Contains(bucket, "\\") { + return "", ErrInvalidName + } + if !filepath.IsLocal(bucket) || !filepath.IsLocal(project) { + return "", ErrInvalidName + } + + dir := filepath.Clean(filepath.Join(fs.baseDir, project, bucket)) + basePrefix := fs.baseDir + string(filepath.Separator) + if !strings.HasPrefix(dir, basePrefix) { + return "", ErrInvalidName + } + rel, err := filepath.Rel(fs.baseDir, dir) + if err != nil || !filepath.IsLocal(rel) || rel == "." { + return "", ErrInvalidName + } + return dir, nil +} + +func (fs *FileStore) safeObjectPaths(project, bucket, object string) (dataPath, metaPath string, err error) { + bucketDir, err := fs.safeBucketDir(project, bucket) + if err != nil { + return "", "", err + } + if object == "" || strings.Contains(object, "..") || strings.Contains(object, "\\") || strings.HasPrefix(object, "/") { + return "", "", ErrInvalidName + } + if !filepath.IsLocal(object) || object == "." { + return "", "", ErrInvalidName + } + + cleanObject := filepath.Clean(object) + if strings.Contains(cleanObject, "..") || strings.HasPrefix(cleanObject, "/") || cleanObject == "." { + return "", "", ErrInvalidName + } + if !filepath.IsLocal(cleanObject) { + return "", "", ErrInvalidName + } + + dataDir := filepath.Clean(filepath.Join(bucketDir, "data")) + dataPath = filepath.Clean(filepath.Join(dataDir, cleanObject)) + dataPrefix := dataDir + string(filepath.Separator) + if !strings.HasPrefix(dataPath, dataPrefix) { + return "", "", ErrInvalidName + } + relData, err := filepath.Rel(dataDir, dataPath) + if err != nil || !filepath.IsLocal(relData) || relData == "." { + return "", "", ErrInvalidName + } + + metaDir := filepath.Clean(filepath.Join(bucketDir, "meta")) + metaPath = filepath.Clean(filepath.Join(metaDir, cleanObject+".json")) + metaPrefix := metaDir + string(filepath.Separator) + if !strings.HasPrefix(metaPath, metaPrefix) { + return "", "", ErrInvalidName + } + relMeta, err := filepath.Rel(metaDir, metaPath) + if err != nil || !filepath.IsLocal(relMeta) || relMeta == "." { + return "", "", ErrInvalidName + } + + return dataPath, metaPath, nil +} + +func (fs *FileStore) CreateBucket(project, bucket, location, storageClass string) (*BucketInfo, error) { + fs.mu.Lock() + defer fs.mu.Unlock() + + dir, err := fs.safeBucketDir(project, bucket) + if err != nil { + return nil, err + } + + metaFile := filepath.Join(dir, "bucket.json") + if _, err := os.Stat(metaFile); err == nil { + return nil, ErrBucketAlreadyExists + } + + if err := os.MkdirAll(filepath.Join(dir, "data"), 0755); err != nil { + return nil, err + } + if err := os.MkdirAll(filepath.Join(dir, "meta"), 0755); err != nil { + return nil, err + } + + if location == "" { + location = "US" + } + if storageClass == "" { + storageClass = "STANDARD" + } + now := time.Now().UTC() + info := &BucketInfo{ + Kind: "storage#bucket", + ID: bucket, + Name: bucket, + Location: location, + StorageClass: storageClass, + TimeCreated: now, + Updated: now, + ETag: fmt.Sprintf("\"%d\"", now.UnixNano()), + } + + raw, err := json.MarshalIndent(info, "", " ") + if err != nil { + return nil, err + } + if err := os.WriteFile(metaFile, raw, 0644); err != nil { + return nil, err + } + return info, nil +} + +func (fs *FileStore) GetBucket(project, bucket string) (*BucketInfo, error) { + fs.mu.RLock() + defer fs.mu.RUnlock() + + dir, err := fs.safeBucketDir(project, bucket) + if err != nil { + return nil, err + } + + metaFile := filepath.Join(dir, "bucket.json") + raw, err := os.ReadFile(metaFile) + if err != nil { + if os.IsNotExist(err) { + return nil, ErrBucketNotFound + } + return nil, err + } + + var info BucketInfo + if err := json.Unmarshal(raw, &info); err != nil { + return nil, err + } + return &info, nil +} + +func (fs *FileStore) DeleteBucket(project, bucket string) error { + fs.mu.Lock() + defer fs.mu.Unlock() + + dir, err := fs.safeBucketDir(project, bucket) + if err != nil { + return err + } + + metaFile := filepath.Join(dir, "bucket.json") + if _, err := os.Stat(metaFile); os.IsNotExist(err) { + return ErrBucketNotFound + } + + // Check if bucket has objects + dataDir := filepath.Join(dir, "data") + entries, err := os.ReadDir(dataDir) + if err == nil && len(entries) > 0 { + return ErrBucketNotEmpty + } + + return os.RemoveAll(dir) +} + +func (fs *FileStore) ListBuckets(project string) ([]BucketInfo, error) { + fs.mu.RLock() + defer fs.mu.RUnlock() + + if project == "" { + project = "default" + } + if project == "." || strings.Contains(project, "..") || strings.Contains(project, "/") || strings.Contains(project, "\\") { + return nil, ErrInvalidName + } + if !filepath.IsLocal(project) { + return nil, ErrInvalidName + } + + projDir := filepath.Clean(filepath.Join(fs.baseDir, project)) + basePrefix := fs.baseDir + string(filepath.Separator) + if !strings.HasPrefix(projDir, basePrefix) { + return nil, ErrInvalidName + } + rel, err := filepath.Rel(fs.baseDir, projDir) + if err != nil || !filepath.IsLocal(rel) || rel == "." { + return nil, ErrInvalidName + } + + entries, err := os.ReadDir(projDir) + if err != nil { + if os.IsNotExist(err) { + return []BucketInfo{}, nil + } + return nil, err + } + + var list []BucketInfo + for _, entry := range entries { + if !entry.IsDir() { + continue + } + bucketName := entry.Name() + dir, err := fs.safeBucketDir(project, bucketName) + if err != nil { + continue + } + metaFile := filepath.Clean(filepath.Join(dir, "bucket.json")) + if !strings.HasPrefix(metaFile, dir+string(filepath.Separator)) { + continue + } + relMeta, err := filepath.Rel(dir, metaFile) + if err != nil || !filepath.IsLocal(relMeta) { + continue + } + raw, err := os.ReadFile(metaFile) + if err == nil { + var info BucketInfo + if json.Unmarshal(raw, &info) == nil { + list = append(list, info) + } + } + } + sort.Slice(list, func(i, j int) bool { + return list[i].Name < list[j].Name + }) + return list, nil +} + +func computeMD5Base64(data []byte) string { + sum := md5.Sum(data) + return base64.StdEncoding.EncodeToString(sum[:]) +} + +func computeCRC32cBase64(data []byte) string { + table := crc32.MakeTable(crc32.Castagnoli) + checksum := crc32.Checksum(data, table) + b := make([]byte, 4) + binary.BigEndian.PutUint32(b, checksum) + return base64.StdEncoding.EncodeToString(b) +} + +func (fs *FileStore) PutObject(project, bucket, object, contentType string, data []byte, metadata map[string]string) (*ObjectInfo, error) { + fs.mu.Lock() + defer fs.mu.Unlock() + + dataPath, metaPath, err := fs.safeObjectPaths(project, bucket, object) + if err != nil { + return nil, err + } + + // Verify bucket exists + bucketDir, err := fs.safeBucketDir(project, bucket) + if err != nil { + return nil, err + } + if _, err := os.Stat(filepath.Join(bucketDir, "bucket.json")); os.IsNotExist(err) { + return nil, ErrBucketNotFound + } + + if err := os.MkdirAll(filepath.Dir(dataPath), 0755); err != nil { + return nil, err + } + if err := os.MkdirAll(filepath.Dir(metaPath), 0755); err != nil { + return nil, err + } + + if err := os.WriteFile(dataPath, data, 0644); err != nil { + return nil, err + } + + if contentType == "" { + contentType = "application/octet-stream" + } + now := time.Now().UTC() + info := &ObjectInfo{ + Kind: "storage#object", + ID: fmt.Sprintf("%s/%s/%d", bucket, object, now.UnixNano()), + Name: object, + Bucket: bucket, + Size: int64(len(data)), + ContentType: contentType, + MD5Hash: computeMD5Base64(data), + CRC32c: computeCRC32cBase64(data), + ETag: fmt.Sprintf("\"%d\"", now.UnixNano()), + TimeCreated: now, + Updated: now, + Metadata: metadata, + } + + raw, err := json.MarshalIndent(info, "", " ") + if err != nil { + return nil, err + } + if err := os.WriteFile(metaPath, raw, 0644); err != nil { + return nil, err + } + return info, nil +} + +func (fs *FileStore) GetObject(project, bucket, object string) (*ObjectInfo, []byte, error) { + fs.mu.RLock() + defer fs.mu.RUnlock() + + dataPath, metaPath, err := fs.safeObjectPaths(project, bucket, object) + if err != nil { + return nil, nil, err + } + + rawMeta, err := os.ReadFile(metaPath) + if err != nil { + if os.IsNotExist(err) { + return nil, nil, ErrObjectNotFound + } + return nil, nil, err + } + + var info ObjectInfo + if err := json.Unmarshal(rawMeta, &info); err != nil { + return nil, nil, err + } + + data, err := os.ReadFile(dataPath) + if err != nil { + if os.IsNotExist(err) { + return nil, nil, ErrObjectNotFound + } + return nil, nil, err + } + + return &info, data, nil +} + +func (fs *FileStore) GetObjectMetadata(project, bucket, object string) (*ObjectInfo, error) { + fs.mu.RLock() + defer fs.mu.RUnlock() + + _, metaPath, err := fs.safeObjectPaths(project, bucket, object) + if err != nil { + return nil, err + } + + rawMeta, err := os.ReadFile(metaPath) + if err != nil { + if os.IsNotExist(err) { + return nil, ErrObjectNotFound + } + return nil, err + } + + var info ObjectInfo + if err := json.Unmarshal(rawMeta, &info); err != nil { + return nil, err + } + return &info, nil +} + +func (fs *FileStore) DeleteObject(project, bucket, object string) error { + fs.mu.Lock() + defer fs.mu.Unlock() + + dataPath, metaPath, err := fs.safeObjectPaths(project, bucket, object) + if err != nil { + return err + } + + if _, err := os.Stat(metaPath); os.IsNotExist(err) { + return ErrObjectNotFound + } + + _ = os.Remove(dataPath) + _ = os.Remove(metaPath) + return nil +} + +func (fs *FileStore) ListObjects(project, bucket, prefix string) ([]ObjectInfo, error) { + fs.mu.RLock() + defer fs.mu.RUnlock() + + bucketDir, err := fs.safeBucketDir(project, bucket) + if err != nil { + return nil, err + } + + if _, err := os.Stat(filepath.Join(bucketDir, "bucket.json")); os.IsNotExist(err) { + return nil, ErrBucketNotFound + } + + metaDir := filepath.Join(bucketDir, "meta") + var objects []ObjectInfo + + _ = filepath.Walk(metaDir, func(path string, info os.FileInfo, err error) error { + if err != nil || info.IsDir() || !strings.HasSuffix(info.Name(), ".json") { + return nil + } + raw, rerr := os.ReadFile(path) + if rerr == nil { + var obj ObjectInfo + if json.Unmarshal(raw, &obj) == nil { + if prefix == "" || strings.HasPrefix(obj.Name, prefix) { + objects = append(objects, obj) + } + } + } + return nil + }) + + sort.Slice(objects, func(i, j int) bool { + return objects[i].Name < objects[j].Name + }) + return objects, nil +} diff --git a/internal/services/iam/sts_federation.go b/internal/services/iam/sts_federation.go index 694ae01c..4b8e12c9 100644 --- a/internal/services/iam/sts_federation.go +++ b/internal/services/iam/sts_federation.go @@ -12,6 +12,7 @@ import ( "time" "github.com/skyoo2003/devcloud/internal/plugin" + "github.com/skyoo2003/devcloud/internal/services/identity" ) // --- Response structs --- @@ -130,11 +131,10 @@ func (p *STSProvider) handleAssumeRoleWithWebIdentity(_ context.Context, form ur expiration := time.Now().UTC().Add(time.Hour).Format(time.RFC3339) assumedArn := deriveAssumedRoleArn(roleArn, sessionName) - subject := "sub-devcloud-user" - if len(webToken) > 0 { - end := min(len(webToken), 16) - subject = "sub-" + webToken[:end] - } + parsed := identity.ParseWebIdentityToken(webToken) + subject := parsed.Subject + provider := parsed.Issuer + audience := parsed.Audience return stsXMLResponse(http.StatusOK, assumeRoleWithWebIdentityResponse{ AssumeRoleWithWebIdentityResult: assumeRoleWithWebIdentityResult{ @@ -150,8 +150,8 @@ func (p *STSProvider) handleAssumeRoleWithWebIdentity(_ context.Context, form ur AssumedRoleID: fmt.Sprintf("%s:%s", roaID, sessionName), }, PackedPolicySize: 1, - Provider: "https://sts.amazonaws.com", - Audience: "sts.devcloud", + Provider: provider, + Audience: audience, }, }) } diff --git a/internal/services/identity/cross_csp_exchange.go b/internal/services/identity/cross_csp_exchange.go new file mode 100644 index 00000000..3a5da8e5 --- /dev/null +++ b/internal/services/identity/cross_csp_exchange.go @@ -0,0 +1,155 @@ +// SPDX-License-Identifier: Apache-2.0 + +package identity + +import ( + "crypto/rand" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "strings" + + "github.com/skyoo2003/devcloud/internal/plugin" +) + +type CrossCSPExchange struct { + issuer *OIDCIssuer +} + +func NewCrossCSPExchange(issuer *OIDCIssuer) *CrossCSPExchange { + return &CrossCSPExchange{issuer: issuer} +} + +func parseRequestFormOrJSON(req *http.Request) (url.Values, map[string]any, error) { + bodyBytes, err := io.ReadAll(req.Body) + if err != nil { + return nil, nil, err + } + contentType := req.Header.Get("Content-Type") + if strings.Contains(contentType, "application/json") { + var jsonBody map[string]any + if err := json.Unmarshal(bodyBytes, &jsonBody); err != nil { + return nil, nil, err + } + values := url.Values{} + for k, v := range jsonBody { + if s, ok := v.(string); ok { + values.Set(k, s) + } + } + return values, jsonBody, nil + } + + form, err := url.ParseQuery(string(bodyBytes)) + if err != nil { + return nil, nil, err + } + return form, nil, nil +} + +// HandleAzureTokenExchange handles Microsoft Entra / Azure AD token exchange requests +// (e.g. POST /devcloud/identity/azure/oauth2/v2.0/token or POST /devcloud/identity/azure/token). +func (c *CrossCSPExchange) HandleAzureTokenExchange(req *http.Request) (*plugin.Response, error) { + form, _, err := parseRequestFormOrJSON(req) + if err != nil { + return &plugin.Response{ + StatusCode: http.StatusBadRequest, + ContentType: "application/json", + Body: []byte(`{"error":"invalid_request","error_description":"failed to parse body"}`), + }, nil + } + + grantType := form.Get("grant_type") + clientAssertion := form.Get("client_assertion") + assertionType := form.Get("client_assertion_type") + + sub := "azure-workload-identity" + if clientAssertion != "" { + claims := ParseWebIdentityToken(clientAssertion) + if claims.Subject != "" { + sub = claims.Subject + } + } + + scope := form.Get("scope") + if scope == "" { + scope = "https://management.azure.com/.default" + } + + bytes := make([]byte, 16) + _, _ = rand.Read(bytes) + randSuffix := hex.EncodeToString(bytes) + + azureToken := fmt.Sprintf("azure-bearer-%s-%s", sub, randSuffix) + + resp := map[string]any{ + "token_type": "Bearer", + "expires_in": 3600, + "ext_expires_in": 3600, + "access_token": azureToken, + "scope": scope, + "grant_type": grantType, + "assertion_type": assertionType, + } + + raw, _ := json.Marshal(resp) + return &plugin.Response{ + StatusCode: http.StatusOK, + ContentType: "application/json", + Body: raw, + }, nil +} + +// HandleGCPTokenExchange handles Google Cloud STS token exchange requests +// (e.g. POST /devcloud/identity/gcp/v1/token or POST /devcloud/identity/gcp/token). +func (c *CrossCSPExchange) HandleGCPTokenExchange(req *http.Request) (*plugin.Response, error) { + form, _, err := parseRequestFormOrJSON(req) + if err != nil { + return &plugin.Response{ + StatusCode: http.StatusBadRequest, + ContentType: "application/json", + Body: []byte(`{"error":"invalid_request","error_description":"failed to parse body"}`), + }, nil + } + + grantType := form.Get("grant_type") + subjectToken := form.Get("subject_token") + subjectTokenType := form.Get("subject_token_type") + requestedTokenType := form.Get("requested_token_type") + audience := form.Get("audience") + + sub := "gcp-federated-identity" + if subjectToken != "" { + claims := ParseWebIdentityToken(subjectToken) + if claims.Subject != "" { + sub = claims.Subject + } + } + + bytes := make([]byte, 16) + _, _ = rand.Read(bytes) + randSuffix := hex.EncodeToString(bytes) + + gcpToken := fmt.Sprintf("ya29.devcloud-%s-%s", sub, randSuffix) + + resp := map[string]any{ + "access_token": gcpToken, + "issued_token_type": "urn:ietf:params:oauth:token-type:access_token", + "token_type": "Bearer", + "expires_in": 3600, + "grant_type": grantType, + "subject_token_type": subjectTokenType, + "requested_token_type": requestedTokenType, + "audience": audience, + } + + raw, _ := json.Marshal(resp) + return &plugin.Response{ + StatusCode: http.StatusOK, + ContentType: "application/json", + Body: raw, + }, nil +} diff --git a/internal/services/identity/oidc_issuer.go b/internal/services/identity/oidc_issuer.go new file mode 100644 index 00000000..40fd4cf0 --- /dev/null +++ b/internal/services/identity/oidc_issuer.go @@ -0,0 +1,228 @@ +// SPDX-License-Identifier: Apache-2.0 + +package identity + +import ( + "crypto" + "crypto/rand" + "crypto/rsa" + "crypto/sha256" + "encoding/base64" + "encoding/json" + "fmt" + "math/big" + "strings" + "sync" + "time" +) + +type OIDCIssuer struct { + mu sync.RWMutex + privateKey *rsa.PrivateKey + keyID string +} + +func NewOIDCIssuer() (*OIDCIssuer, error) { + key, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + return nil, fmt.Errorf("failed to generate rsa key: %w", err) + } + return &OIDCIssuer{ + privateKey: key, + keyID: "devcloud-key-1", + }, nil +} + +type JWKKey struct { + Kty string `json:"kty"` + Use string `json:"use"` + Alg string `json:"alg"` + Kid string `json:"kid"` + N string `json:"n"` + E string `json:"e"` +} + +type JWKSResponse struct { + Keys []JWKKey `json:"keys"` +} + +func (iss *OIDCIssuer) JWKS() JWKSResponse { + iss.mu.RLock() + defer iss.mu.RUnlock() + + pub := &iss.privateKey.PublicKey + nBytes := pub.N.Bytes() + eBytes := big.NewInt(int64(pub.E)).Bytes() + + return JWKSResponse{ + Keys: []JWKKey{ + { + Kty: "RSA", + Use: "sig", + Alg: "RS256", + Kid: iss.keyID, + N: base64.RawURLEncoding.EncodeToString(nBytes), + E: base64.RawURLEncoding.EncodeToString(eBytes), + }, + }, + } +} + +func (iss *OIDCIssuer) OpenIDConfiguration(issuerURL string) map[string]any { + if issuerURL == "" { + issuerURL = "http://localhost:4747/devcloud/identity" + } + issuerURL = strings.TrimRight(issuerURL, "/") + return map[string]any{ + "issuer": issuerURL, + "jwks_uri": issuerURL + "/jwks.json", + "token_endpoint": issuerURL + "/token", + "response_types_supported": []string{"id_token", "token"}, + "subject_types_supported": []string{"public"}, + "id_token_signing_alg_values_supported": []string{"RS256"}, + "scopes_supported": []string{"openid", "email", "profile"}, + "claims_supported": []string{"sub", "iss", "aud", "exp", "iat"}, + } +} + +type MintTokenOptions struct { + Subject string + Issuer string + Audience any // string or []string + ExpiresIn time.Duration + Claims map[string]any +} + +func (iss *OIDCIssuer) MintToken(opts MintTokenOptions) (string, error) { + iss.mu.RLock() + defer iss.mu.RUnlock() + + now := time.Now().UTC() + expDur := opts.ExpiresIn + if expDur <= 0 { + expDur = time.Hour + } + if opts.Subject == "" { + opts.Subject = "devcloud-user" + } + if opts.Issuer == "" { + opts.Issuer = "http://localhost:4747/devcloud/identity" + } + if opts.Audience == nil { + opts.Audience = "sts.devcloud" + } + + header := map[string]string{ + "alg": "RS256", + "typ": "JWT", + "kid": iss.keyID, + } + headerJSON, err := json.Marshal(header) + if err != nil { + return "", err + } + + claims := map[string]any{ + "sub": opts.Subject, + "iss": opts.Issuer, + "aud": opts.Audience, + "iat": now.Unix(), + "exp": now.Add(expDur).Unix(), + } + for k, v := range opts.Claims { + claims[k] = v + } + claimsJSON, err := json.Marshal(claims) + if err != nil { + return "", err + } + + headerB64 := base64.RawURLEncoding.EncodeToString(headerJSON) + payloadB64 := base64.RawURLEncoding.EncodeToString(claimsJSON) + sigInput := headerB64 + "." + payloadB64 + + hashed := sha256.Sum256([]byte(sigInput)) + sig, err := rsa.SignPKCS1v15(rand.Reader, iss.privateKey, crypto.SHA256, hashed[:]) + if err != nil { + return "", err + } + sigB64 := base64.RawURLEncoding.EncodeToString(sig) + + return sigInput + "." + sigB64, nil +} + +type ParsedIdentity struct { + Subject string + Issuer string + Audience string + Claims map[string]any +} + +// ParseWebIdentityToken parses an identity token (OIDC JWT, Google Access Token, Azure token, or raw string) +// and extracts the subject, issuer, and audience claims. +func ParseWebIdentityToken(token string) ParsedIdentity { + result := ParsedIdentity{ + Subject: "sub-devcloud-user", + Issuer: "https://sts.amazonaws.com", + Audience: "sts.devcloud", + Claims: make(map[string]any), + } + if token == "" { + return result + } + + // 1. Google OAuth2 / STS Access Token (ya29...) + if strings.HasPrefix(token, "ya29.") { + sub := strings.TrimPrefix(token, "ya29.") + if len(sub) > 32 { + sub = sub[:32] + } + result.Subject = "gcp:" + sub + result.Issuer = "https://accounts.google.com" + result.Audience = "sts.amazonaws.com" + return result + } + + // 2. Azure AD token simulation + if strings.HasPrefix(token, "azure-bearer-") { + sub := strings.TrimPrefix(token, "azure-bearer-") + result.Subject = "azure:" + sub + result.Issuer = "https://sts.windows.net/" + result.Audience = "https://management.azure.com" + return result + } + + // 3. Standard 3-part JWT + parts := strings.Split(token, ".") + if len(parts) == 3 { + payloadBytes, err := base64.RawURLEncoding.DecodeString(parts[1]) + if err != nil { + payloadBytes, err = base64.StdEncoding.DecodeString(parts[1]) + } + if err == nil { + var claims map[string]any + if json.Unmarshal(payloadBytes, &claims) == nil { + result.Claims = claims + if sub, ok := claims["sub"].(string); ok && sub != "" { + result.Subject = sub + } + if iss, ok := claims["iss"].(string); ok && iss != "" { + result.Issuer = iss + } + if aud, ok := claims["aud"].(string); ok && aud != "" { + result.Audience = aud + } else if auds, ok := claims["aud"].([]any); ok && len(auds) > 0 { + if firstAud, ok := auds[0].(string); ok { + result.Audience = firstAud + } + } + return result + } + } + } + + // 4. Fallback for non-JWT arbitrary strings + end := min(len(token), 16) + result.Subject = "sub-" + token[:end] + return result +} diff --git a/internal/services/identity/provider.go b/internal/services/identity/provider.go new file mode 100644 index 00000000..c3ab763e --- /dev/null +++ b/internal/services/identity/provider.go @@ -0,0 +1,199 @@ +// SPDX-License-Identifier: Apache-2.0 + +package identity + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "strconv" + "strings" + "time" + + generated "github.com/skyoo2003/devcloud/internal/generated/identity" + "github.com/skyoo2003/devcloud/internal/plugin" +) + +// Provider implements the DevCloud Federated Identity service. +type Provider struct { + generated.BaseProvider + issuer *OIDCIssuer + exchange *CrossCSPExchange +} + +// NewProvider creates a new Federated Identity Provider instance. +func NewProvider() *Provider { + iss, err := NewOIDCIssuer() + if err != nil { + panic(fmt.Sprintf("failed to initialize OIDC issuer: %v", err)) + } + return &Provider{ + issuer: iss, + exchange: NewCrossCSPExchange(iss), + } +} + +func (p *Provider) ServiceID() string { return "identity" } +func (p *Provider) ServiceName() string { return "DevCloud Federated Identity" } +func (p *Provider) Protocol() plugin.ProtocolType { return plugin.ProtocolRESTJSON } +func (p *Provider) Provider() string { return "devcloud" } + +func (p *Provider) Init(_ plugin.PluginConfig) error { + return nil +} + +func (p *Provider) Shutdown(_ context.Context) error { + return nil +} + +func (p *Provider) ListResources(_ context.Context) ([]plugin.Resource, error) { + return []plugin.Resource{ + {Type: "OIDCProvider", ID: "devcloud-local-oidc", Name: "DevCloud Local OIDC Issuer"}, + }, nil +} + +func (p *Provider) HandleRequest(_ context.Context, _ string, req *http.Request) (*plugin.Response, error) { + path := req.URL.Path + + // 1. OpenID Discovery + if req.Method == http.MethodGet && (strings.HasSuffix(path, "/.well-known/openid-configuration") || strings.Contains(path, "openid-configuration")) { + issuerURL := fmt.Sprintf("http://%s/devcloud/identity", req.Host) + if req.Host == "" { + issuerURL = "http://localhost:4747/devcloud/identity" + } + cfg := p.issuer.OpenIDConfiguration(issuerURL) + body, _ := json.Marshal(cfg) + return &plugin.Response{ + StatusCode: http.StatusOK, + ContentType: "application/json", + Body: body, + }, nil + } + + // 2. JWKS + if req.Method == http.MethodGet && strings.HasSuffix(path, "/jwks.json") { + jwks := p.issuer.JWKS() + body, _ := json.Marshal(jwks) + return &plugin.Response{ + StatusCode: http.StatusOK, + ContentType: "application/json", + Body: body, + }, nil + } + + // 3. Token Minting + if req.Method == http.MethodPost && (strings.HasSuffix(path, "/identity/token") || path == "/token") { + return p.handleMintToken(req) + } + + // 4. Azure AD Token Exchange + if req.Method == http.MethodPost && (strings.Contains(path, "/azure/") || strings.HasSuffix(path, "/azure/token")) { + return p.exchange.HandleAzureTokenExchange(req) + } + + // 5. GCP Token Exchange + if req.Method == http.MethodPost && (strings.Contains(path, "/gcp/") || strings.HasSuffix(path, "/gcp/token")) { + return p.exchange.HandleGCPTokenExchange(req) + } + + return &plugin.Response{ + StatusCode: http.StatusNotFound, + ContentType: "application/json", + Body: []byte(`{"error":"not_found","error_description":"endpoint not found"}`), + }, nil +} + +func (p *Provider) handleMintToken(req *http.Request) (*plugin.Response, error) { + form, jsonBody, err := parseRequestFormOrJSON(req) + if err != nil { + return &plugin.Response{ + StatusCode: http.StatusBadRequest, + ContentType: "application/json", + Body: []byte(`{"error":"invalid_request","error_description":"failed to parse body"}`), + }, nil + } + + sub := form.Get("sub") + if sub == "" { + sub = form.Get("subject") + } + if sub == "" { + sub = "devcloud-workload-user" + } + + iss := form.Get("iss") + if iss == "" { + iss = form.Get("issuer") + } + if iss == "" { + host := req.Host + if host == "" { + host = "localhost:4747" + } + iss = fmt.Sprintf("http://%s/devcloud/identity", host) + } + + aud := form.Get("aud") + if aud == "" { + aud = form.Get("audience") + } + if aud == "" { + aud = "sts.devcloud" + } + + expSec := 3600 + if expStr := form.Get("expires_in"); expStr != "" { + if n, err := strconv.Atoi(expStr); err == nil && n > 0 { + expSec = n + } + } else if jsonBody != nil { + if expFloat, ok := jsonBody["expires_in"].(float64); ok && expFloat > 0 { + expSec = int(expFloat) + } + } + + var customClaims map[string]any + if jsonBody != nil { + if c, ok := jsonBody["claims"].(map[string]any); ok { + customClaims = c + } + } + + tokenStr, err := p.issuer.MintToken(MintTokenOptions{ + Subject: sub, + Issuer: iss, + Audience: aud, + ExpiresIn: time.Duration(expSec) * time.Second, + Claims: customClaims, + }) + if err != nil { + return &plugin.Response{ + StatusCode: http.StatusInternalServerError, + ContentType: "application/json", + Body: []byte(fmt.Sprintf(`{"error":"server_error","error_description":%q}`, err.Error())), + }, nil + } + + resp := map[string]any{ + "access_token": tokenStr, + "id_token": tokenStr, + "token_type": "Bearer", + "expires_in": expSec, + "sub": sub, + "iss": iss, + "aud": aud, + } + raw, _ := json.Marshal(resp) + return &plugin.Response{ + StatusCode: http.StatusOK, + ContentType: "application/json", + Body: raw, + }, nil +} + +func init() { + plugin.DefaultRegistry.Register("identity", func() plugin.ServicePlugin { + return NewProvider() + }) +} diff --git a/internal/services/identity/provider_test.go b/internal/services/identity/provider_test.go new file mode 100644 index 00000000..381884aa --- /dev/null +++ b/internal/services/identity/provider_test.go @@ -0,0 +1,138 @@ +// SPDX-License-Identifier: Apache-2.0 + +package identity + +import ( + "bytes" + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "net/url" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/skyoo2003/devcloud/internal/plugin" +) + +func TestProvider_Metadata(t *testing.T) { + p := NewProvider() + assert.Equal(t, "identity", p.ServiceID()) + assert.Equal(t, "DevCloud Federated Identity", p.ServiceName()) + assert.Equal(t, plugin.ProtocolRESTJSON, p.Protocol()) + assert.Equal(t, "devcloud", plugin.ProviderOf(p)) +} + +func TestProvider_OIDCDiscoveryAndJWKS(t *testing.T) { + p := NewProvider() + ctx := context.Background() + + // 1. Discovery + reqDisc := httptest.NewRequest(http.MethodGet, "/devcloud/identity/.well-known/openid-configuration", nil) + respDisc, err := p.HandleRequest(ctx, "", reqDisc) + require.NoError(t, err) + assert.Equal(t, http.StatusOK, respDisc.StatusCode) + + var disc map[string]any + err = json.Unmarshal(respDisc.Body, &disc) + require.NoError(t, err) + assert.Contains(t, disc["issuer"], "/devcloud/identity") + assert.Contains(t, disc["jwks_uri"], "/jwks.json") + + // 2. JWKS + reqJWKS := httptest.NewRequest(http.MethodGet, "/devcloud/identity/jwks.json", nil) + respJWKS, err := p.HandleRequest(ctx, "", reqJWKS) + require.NoError(t, err) + assert.Equal(t, http.StatusOK, respJWKS.StatusCode) + + var jwks JWKSResponse + err = json.Unmarshal(respJWKS.Body, &jwks) + require.NoError(t, err) + require.Len(t, jwks.Keys, 1) + assert.Equal(t, "RSA", jwks.Keys[0].Kty) + assert.Equal(t, "RS256", jwks.Keys[0].Alg) + assert.NotEmpty(t, jwks.Keys[0].N) + assert.NotEmpty(t, jwks.Keys[0].E) +} + +func TestProvider_MintToken(t *testing.T) { + p := NewProvider() + ctx := context.Background() + + mintPayload := map[string]any{ + "sub": "user@corp.example.com", + "aud": "sts.amazonaws.com", + "claims": map[string]any{ + "team": "infrastructure", + }, + } + body, _ := json.Marshal(mintPayload) + reqMint := httptest.NewRequest(http.MethodPost, "/devcloud/identity/token", bytes.NewReader(body)) + reqMint.Header.Set("Content-Type", "application/json") + respMint, err := p.HandleRequest(ctx, "", reqMint) + require.NoError(t, err) + assert.Equal(t, http.StatusOK, respMint.StatusCode) + + var tokenResp map[string]any + err = json.Unmarshal(respMint.Body, &tokenResp) + require.NoError(t, err) + assert.Equal(t, "Bearer", tokenResp["token_type"]) + tokenStr, ok := tokenResp["access_token"].(string) + require.True(t, ok) + assert.NotEmpty(t, tokenStr) + + // Verify token claims parsing + parsed := ParseWebIdentityToken(tokenStr) + assert.Equal(t, "user@corp.example.com", parsed.Subject) + assert.Equal(t, "sts.amazonaws.com", parsed.Audience) + assert.Equal(t, "infrastructure", parsed.Claims["team"]) +} + +func TestProvider_CrossCSPExchange(t *testing.T) { + p := NewProvider() + ctx := context.Background() + + // First mint a token to use as subject assertion + tokenStr, err := p.issuer.MintToken(MintTokenOptions{ + Subject: "workload-gha-runner", + }) + require.NoError(t, err) + + // 1. Azure Token Exchange + azureReqBody := url.Values{ + "grant_type": {"urn:ietf:params:oauth:grant-type:token-exchange"}, + "client_assertion": {tokenStr}, + "client_id": {"my-azure-app-client"}, + } + reqAzure := httptest.NewRequest(http.MethodPost, "/devcloud/identity/azure/oauth2/v2.0/token", bytes.NewReader([]byte(azureReqBody.Encode()))) + reqAzure.Header.Set("Content-Type", "application/x-www-form-urlencoded") + respAzure, err := p.HandleRequest(ctx, "", reqAzure) + require.NoError(t, err) + assert.Equal(t, http.StatusOK, respAzure.StatusCode) + + var azResp map[string]any + err = json.Unmarshal(respAzure.Body, &azResp) + require.NoError(t, err) + assert.Equal(t, "Bearer", azResp["token_type"]) + assert.NotEmpty(t, azResp["access_token"]) + + // 2. GCP Token Exchange + gcpReqBody := url.Values{ + "grant_type": {"urn:ietf:params:oauth:grant-type:token-exchange"}, + "subject_token": {tokenStr}, + } + reqGCP := httptest.NewRequest(http.MethodPost, "/devcloud/identity/gcp/v1/token", bytes.NewReader([]byte(gcpReqBody.Encode()))) + reqGCP.Header.Set("Content-Type", "application/x-www-form-urlencoded") + respGCP, err := p.HandleRequest(ctx, "", reqGCP) + require.NoError(t, err) + assert.Equal(t, http.StatusOK, respGCP.StatusCode) + + var gcpResp map[string]any + err = json.Unmarshal(respGCP.Body, &gcpResp) + require.NoError(t, err) + assert.Equal(t, "Bearer", gcpResp["token_type"]) + gcpToken, _ := gcpResp["access_token"].(string) + assert.Contains(t, gcpToken, "ya29.") +} diff --git a/test/compatibility/exclusions.json b/test/compatibility/exclusions.json index 42191269..b235853c 100644 --- a/test/compatibility/exclusions.json +++ b/test/compatibility/exclusions.json @@ -1,7 +1,12 @@ { "_comment": "The registered services no boto3 test can exercise, and why. Read by test/compatibility/_coverage.py and by cmd/devcloud/coverage_test.go, which derives the Compatibility-tested figure docs/coverage.md publishes. Adding an entry lowers that figure; it is a deliberate edit, and two gates fail until the doc moves with it.", "noBoto3Client": { + "azurequeue": "Azure Queue Storage uses the azure-storage-queue SDK, not boto3", + "azuretable": "Azure Table Storage uses the azure-data-tables SDK, not boto3", "blob": "Azure Blob Storage uses the azure-storage-blob SDK, not boto3", + "cosmosdb": "Azure Cosmos DB uses the azure-cosmos SDK, not boto3", + "gcs": "Google Cloud Storage uses the google-cloud-storage SDK, not boto3", + "identity": "DevCloud Federated Identity Playground uses direct OIDC/OAuth2 endpoints, not boto3", "sagemakerruntimehttp2": "botocore has no HTTP/2 SageMaker Runtime client", "transcribestreaming": "botocore has no streaming Transcribe client" }, diff --git a/test/compatibility/test_azure_cosmos.py b/test/compatibility/test_azure_cosmos.py new file mode 100644 index 00000000..e8896ff9 --- /dev/null +++ b/test/compatibility/test_azure_cosmos.py @@ -0,0 +1,145 @@ +import pytest +import requests +from conftest import DEVCLOUD_URL + + +@pytest.fixture +def cosmos_endpoint(devcloud_server): + return f"{DEVCLOUD_URL}/devstoreaccount1" + + +def test_azure_cosmos_lifecycle_and_documents(cosmos_endpoint): + db_id = "test-db" + coll_id = "users" + headers = { + "x-ms-version": "2018-12-31", + "Content-Type": "application/json", + "Authorization": "type=master&ver=1.0&sig=c29tZXNpZ25hdHVyZQ==", + } + + # 1. Create Database + resp_create_db = requests.post( + f"{cosmos_endpoint}/dbs", headers=headers, json={"id": db_id} + ) + assert resp_create_db.status_code == 201 + assert resp_create_db.json()["id"] == db_id + + # 2. Duplicate Create Database returns 409 + resp_dup_db = requests.post( + f"{cosmos_endpoint}/dbs", headers=headers, json={"id": db_id} + ) + assert resp_dup_db.status_code == 409 + + # 3. List Databases + resp_list_dbs = requests.get(f"{cosmos_endpoint}/dbs", headers=headers) + assert resp_list_dbs.status_code == 200 + dbs = resp_list_dbs.json().get("Databases", []) + assert any(d["id"] == db_id for d in dbs) + + # 4. Get Database + resp_get_db = requests.get(f"{cosmos_endpoint}/dbs/{db_id}", headers=headers) + assert resp_get_db.status_code == 200 + assert resp_get_db.json()["id"] == db_id + + # 5. Create Container + resp_create_coll = requests.post( + f"{cosmos_endpoint}/dbs/{db_id}/colls", + headers=headers, + json={"id": coll_id, "partitionKey": {"paths": ["/pk"]}}, + ) + assert resp_create_coll.status_code == 201 + assert resp_create_coll.json()["id"] == coll_id + + # 6. Duplicate Container returns 409 + resp_dup_coll = requests.post( + f"{cosmos_endpoint}/dbs/{db_id}/colls", + headers=headers, + json={"id": coll_id, "partitionKey": {"paths": ["/pk"]}}, + ) + assert resp_dup_coll.status_code == 409 + + # 7. List Containers + resp_list_colls = requests.get( + f"{cosmos_endpoint}/dbs/{db_id}/colls", headers=headers + ) + assert resp_list_colls.status_code == 200 + colls = resp_list_colls.json().get("DocumentCollections", []) + assert any(c["id"] == coll_id for c in colls) + + # 8. Get Container + resp_get_coll = requests.get( + f"{cosmos_endpoint}/dbs/{db_id}/colls/{coll_id}", headers=headers + ) + assert resp_get_coll.status_code == 200 + assert resp_get_coll.json()["id"] == coll_id + + # 9. Create Document + doc = { + "id": "doc-42", + "pk": "tenant-1", + "name": "Grace Hopper", + "compiler": "A-0", + } + resp_create_doc = requests.post( + f"{cosmos_endpoint}/dbs/{db_id}/colls/{coll_id}/docs", + headers=headers, + json=doc, + ) + assert resp_create_doc.status_code == 201 + created_doc = resp_create_doc.json() + assert created_doc["id"] == "doc-42" + assert "_rid" in created_doc + + # 10. Duplicate Document returns 409 + resp_dup_doc = requests.post( + f"{cosmos_endpoint}/dbs/{db_id}/colls/{coll_id}/docs", + headers=headers, + json=doc, + ) + assert resp_dup_doc.status_code == 409 + + # 11. Get Document + resp_get_doc = requests.get( + f"{cosmos_endpoint}/dbs/{db_id}/colls/{coll_id}/docs/doc-42", + headers=headers, + ) + assert resp_get_doc.status_code == 200 + assert resp_get_doc.json()["name"] == "Grace Hopper" + + # 12. Replace Document + doc["name"] = "Rear Admiral Grace Hopper" + resp_put_doc = requests.put( + f"{cosmos_endpoint}/dbs/{db_id}/colls/{coll_id}/docs/doc-42", + headers=headers, + json=doc, + ) + assert resp_put_doc.status_code == 200 + assert resp_put_doc.json()["name"] == "Rear Admiral Grace Hopper" + + # 13. List Documents + resp_list_docs = requests.get( + f"{cosmos_endpoint}/dbs/{db_id}/colls/{coll_id}/docs", + headers=headers, + ) + assert resp_list_docs.status_code == 200 + docs = resp_list_docs.json().get("Documents", []) + assert len(docs) == 1 + assert docs[0]["name"] == "Rear Admiral Grace Hopper" + + # 14. Delete Document + resp_del_doc = requests.delete( + f"{cosmos_endpoint}/dbs/{db_id}/colls/{coll_id}/docs/doc-42", + headers=headers, + ) + assert resp_del_doc.status_code == 204 + + # 15. Delete Container + resp_del_coll = requests.delete( + f"{cosmos_endpoint}/dbs/{db_id}/colls/{coll_id}", + headers=headers, + ) + assert resp_del_coll.status_code == 204 + + # 16. Delete Database + resp_del_db = requests.delete(f"{cosmos_endpoint}/dbs/{db_id}", headers=headers) + assert resp_del_db.status_code == 204 diff --git a/test/compatibility/test_azure_queue.py b/test/compatibility/test_azure_queue.py new file mode 100644 index 00000000..fc5e5a6c --- /dev/null +++ b/test/compatibility/test_azure_queue.py @@ -0,0 +1,93 @@ +import pytest +import requests +import xml.etree.ElementTree as ET +from conftest import DEVCLOUD_URL + + +@pytest.fixture +def queue_endpoint(devcloud_server): + return f"{DEVCLOUD_URL}/devstoreaccount1/queue" + + +def test_azure_queue_lifecycle(queue_endpoint): + queue_name = "test-compat-queue" + headers = { + "x-ms-version": "2023-11-03", + "Authorization": "SharedKey devstoreaccount1:Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==", + } + + # 1. Create Queue + resp = requests.put(f"{queue_endpoint}/{queue_name}", headers=headers) + assert resp.status_code in (201, 204) + + # 2. Duplicate create returns 204 + resp_dup = requests.put(f"{queue_endpoint}/{queue_name}", headers=headers) + assert resp_dup.status_code == 204 + + # 3. Get Queue Metadata + resp_meta = requests.get( + f"{queue_endpoint}/{queue_name}?comp=metadata", headers=headers + ) + assert resp_meta.status_code == 200 + assert "x-ms-approximate-messages-count" in resp_meta.headers + + # 4. List Queues + resp_list = requests.get(f"{queue_endpoint}?comp=list", headers=headers) + assert resp_list.status_code == 200 + assert queue_name in resp_list.text + + # 5. Put Message + msg_xml = ( + "DevCloud Queue Message" + ) + resp_put = requests.post( + f"{queue_endpoint}/{queue_name}/messages", headers=headers, data=msg_xml + ) + assert resp_put.status_code == 201 + assert "" in resp_put.text + + # Parse message ID and pop receipt + root = ET.fromstring(resp_put.text) + msg_elem = root.find("QueueMessage") + assert msg_elem is not None + msg_id = msg_elem.find("MessageId").text + pop_receipt = msg_elem.find("PopReceipt").text + assert pop_receipt is not None + + # 6. Peek Messages + resp_peek = requests.get( + f"{queue_endpoint}/{queue_name}/messages?peekonly=true", headers=headers + ) + assert resp_peek.status_code == 200 + assert "DevCloud Queue Message" in resp_peek.text + + # 7. Get Messages (Dequeues message) + resp_get = requests.get(f"{queue_endpoint}/{queue_name}/messages", headers=headers) + assert resp_get.status_code == 200 + assert "DevCloud Queue Message" in resp_get.text + + root_get = ET.fromstring(resp_get.text) + new_pop_receipt = root_get.find("QueueMessage/PopReceipt").text + + # 8. Delete Message + resp_del_msg = requests.delete( + f"{queue_endpoint}/{queue_name}/messages/{msg_id}?popreceipt={new_pop_receipt}", + headers=headers, + ) + assert resp_del_msg.status_code == 204 + + # 9. Clear Messages + resp_clear = requests.delete( + f"{queue_endpoint}/{queue_name}/messages", headers=headers + ) + assert resp_clear.status_code == 204 + + # 10. Delete Queue + resp_del = requests.delete(f"{queue_endpoint}/{queue_name}", headers=headers) + assert resp_del.status_code == 204 + + # 11. Access deleted queue returns 404 + resp_deleted = requests.get( + f"{queue_endpoint}/{queue_name}?comp=metadata", headers=headers + ) + assert resp_deleted.status_code == 404 diff --git a/test/compatibility/test_azure_table.py b/test/compatibility/test_azure_table.py new file mode 100644 index 00000000..3985dc71 --- /dev/null +++ b/test/compatibility/test_azure_table.py @@ -0,0 +1,101 @@ +import pytest +import requests +from conftest import DEVCLOUD_URL + + +@pytest.fixture +def table_endpoint(devcloud_server): + return f"{DEVCLOUD_URL}/devstoreaccount1" + + +def test_azure_table_lifecycle_and_entities(table_endpoint): + table_name = "DevCloudUsers" + headers = { + "x-ms-version": "2023-11-03", + "Accept": "application/json;odata=nometadata", + "Content-Type": "application/json", + "Authorization": "SharedKey devstoreaccount1:Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==", + } + + # 1. Create Table + resp_create = requests.post( + f"{table_endpoint}/Tables", headers=headers, json={"TableName": table_name} + ) + assert resp_create.status_code in (201, 204) + + # 2. Duplicate Create returns 409 + resp_dup = requests.post( + f"{table_endpoint}/Tables", headers=headers, json={"TableName": table_name} + ) + assert resp_dup.status_code == 409 + + # 3. Query Tables + resp_tables = requests.get(f"{table_endpoint}/Tables", headers=headers) + assert resp_tables.status_code == 200 + tables_list = [t["TableName"] for t in resp_tables.json().get("value", [])] + assert table_name in tables_list + + # 4. Insert Entity + entity = { + "PartitionKey": "Developers", + "RowKey": "dev-001", + "Name": "Gopher", + "Role": "Cloud Engineer", + "Level": 5, + } + resp_insert = requests.post( + f"{table_endpoint}/{table_name}", headers=headers, json=entity + ) + assert resp_insert.status_code in (201, 204) + + # 5. Get Entity + resp_get = requests.get( + f"{table_endpoint}/{table_name}(PartitionKey='Developers',RowKey='dev-001')", + headers=headers, + ) + assert resp_get.status_code == 200 + retrieved = resp_get.json() + assert retrieved["Name"] == "Gopher" + assert retrieved["Role"] == "Cloud Engineer" + + # 6. Update Entity + entity["Role"] = "Principal Engineer" + resp_update = requests.put( + f"{table_endpoint}/{table_name}(PartitionKey='Developers',RowKey='dev-001')", + headers=headers, + json=entity, + ) + assert resp_update.status_code == 204 + + # 7. Query Entities + resp_query = requests.get(f"{table_endpoint}/{table_name}()", headers=headers) + assert resp_query.status_code == 200 + entities = resp_query.json().get("value", []) + assert len(entities) >= 1 + assert entities[0]["Role"] == "Principal Engineer" + + # 8. Delete Entity + resp_del_ent = requests.delete( + f"{table_endpoint}/{table_name}(PartitionKey='Developers',RowKey='dev-001')", + headers=headers, + ) + assert resp_del_ent.status_code == 204 + + # 9. Get Deleted Entity returns 404 + resp_get_404 = requests.get( + f"{table_endpoint}/{table_name}(PartitionKey='Developers',RowKey='dev-001')", + headers=headers, + ) + assert resp_get_404.status_code == 404 + + # 10. Delete Table + resp_del_tbl = requests.delete( + f"{table_endpoint}/Tables('{table_name}')", headers=headers + ) + assert resp_del_tbl.status_code == 204 + + # 11. Delete Table again returns 404 + resp_del_tbl_404 = requests.delete( + f"{table_endpoint}/Tables('{table_name}')", headers=headers + ) + assert resp_del_tbl_404.status_code == 404 diff --git a/test/compatibility/test_federated_identity.py b/test/compatibility/test_federated_identity.py new file mode 100644 index 00000000..92e998a9 --- /dev/null +++ b/test/compatibility/test_federated_identity.py @@ -0,0 +1,118 @@ +import pytest +import requests +from conftest import DEVCLOUD_URL + + +@pytest.fixture +def identity_url(devcloud_server): + return f"{DEVCLOUD_URL}/devcloud/identity" + + +def test_federated_identity_oidc_discovery_and_jwks(identity_url): + # 1. Discovery document + resp_disc = requests.get(f"{identity_url}/.well-known/openid-configuration") + assert resp_disc.status_code == 200 + disc = resp_disc.json() + assert disc["issuer"].endswith("/devcloud/identity") + assert disc["jwks_uri"].endswith("/jwks.json") + assert "RS256" in disc["id_token_signing_alg_values_supported"] + + # 2. JWKS + resp_jwks = requests.get(f"{identity_url}/jwks.json") + assert resp_jwks.status_code == 200 + jwks = resp_jwks.json() + assert len(jwks.get("keys", [])) >= 1 + key = jwks["keys"][0] + assert key["kty"] == "RSA" + assert key["alg"] == "RS256" + assert key["use"] == "sig" + assert len(key["n"]) > 0 + assert len(key["e"]) > 0 + + +def test_federated_identity_mint_and_sts_web_identity_exchange( + identity_url, sts_client +): + # 1. Mint an OIDC token from local provider + custom_sub = "repo:octocat/microservice-api:ref:refs/heads/main" + mint_payload = { + "sub": custom_sub, + "aud": "sts.amazonaws.com", + "claims": { + "repository_owner": "octocat", + "environment": "production", + }, + } + resp_mint = requests.post(f"{identity_url}/token", json=mint_payload) + assert resp_mint.status_code == 200 + token_data = resp_mint.json() + token = token_data.get("access_token") + assert token is not None + assert token_data.get("token_type") == "Bearer" + + # 2. Exchange OIDC token with AWS STS AssumeRoleWithWebIdentity via boto3 + sts = sts_client + role_arn = "arn:aws:iam::123456789012:role/DeployProductionRole" + assumed = sts.assume_role_with_web_identity( + RoleArn=role_arn, + RoleSessionName="github-actions-session", + WebIdentityToken=token, + ) + + assert "Credentials" in assumed + creds = assumed["Credentials"] + assert creds["AccessKeyId"].startswith("ASIA") + assert len(creds["SecretAccessKey"]) > 0 + assert len(creds["SessionToken"]) > 0 + assert assumed["SubjectFromWebIdentityToken"] == custom_sub + assert assumed["Audience"] == "sts.amazonaws.com" + assert "AssumedRoleUser" in assumed + assert "github-actions-session" in assumed["AssumedRoleUser"]["Arn"] + + +def test_cross_csp_token_exchange(identity_url, sts_client): + # 1. Mint token from DevCloud OIDC + resp_mint = requests.post( + f"{identity_url}/token", json={"sub": "service-mesh-controller"} + ) + assert resp_mint.status_code == 200 + token = resp_mint.json()["access_token"] + + # 2. Exchange for Azure AD Bearer token + resp_az = requests.post( + f"{identity_url}/azure/oauth2/v2.0/token", + data={ + "grant_type": "urn:ietf:params:oauth:grant-type:token-exchange", + "client_assertion": token, + "scope": "https://management.azure.com/.default", + }, + ) + assert resp_az.status_code == 200 + az_data = resp_az.json() + assert az_data["token_type"] == "Bearer" + assert len(az_data["access_token"]) > 0 + + # 3. Exchange for GCP OAuth2 access token + resp_gcp = requests.post( + f"{identity_url}/gcp/v1/token", + data={ + "grant_type": "urn:ietf:params:oauth:grant-type:token-exchange", + "subject_token": token, + }, + ) + assert resp_gcp.status_code == 200 + gcp_data = resp_gcp.json() + assert gcp_data["token_type"] == "Bearer" + gcp_token = gcp_data["access_token"] + assert gcp_token.startswith("ya29.") + + # 4. Exchange GCP token for AWS STS temporary credentials + sts = sts_client + assumed_gcp = sts.assume_role_with_web_identity( + RoleArn="arn:aws:iam::123456789012:role/CrossCloudGCPRole", + RoleSessionName="gcp-federated-session", + WebIdentityToken=gcp_token, + ) + assert assumed_gcp["Credentials"]["AccessKeyId"].startswith("ASIA") + assert "gcp:" in assumed_gcp["SubjectFromWebIdentityToken"] + assert assumed_gcp["Provider"] == "https://accounts.google.com" diff --git a/test/compatibility/test_gcp_storage.py b/test/compatibility/test_gcp_storage.py new file mode 100644 index 00000000..ea6254d0 --- /dev/null +++ b/test/compatibility/test_gcp_storage.py @@ -0,0 +1,147 @@ +import pytest +import requests +from conftest import DEVCLOUD_URL + + +@pytest.fixture +def gcs_base_url(devcloud_server): + return DEVCLOUD_URL + + +def test_gcs_bucket_lifecycle(gcs_base_url): + project = "test-proj" + bucket_name = "test-lifecycle-bucket" + headers = { + "Authorization": "Bearer ya29.devcloud-pilot-token", + "Content-Type": "application/json", + } + + # 1. Create bucket + resp = requests.post( + f"{gcs_base_url}/storage/v1/b?project={project}", + headers=headers, + json={"name": bucket_name, "location": "US", "storageClass": "STANDARD"}, + ) + assert resp.status_code == 200 + data = resp.json() + assert data["name"] == bucket_name + assert data["kind"] == "storage#bucket" + + # 2. Duplicate create raises 409 + resp_dup = requests.post( + f"{gcs_base_url}/storage/v1/b?project={project}", + headers=headers, + json={"name": bucket_name}, + ) + assert resp_dup.status_code == 409 + + # 3. Get bucket + resp_get = requests.get( + f"{gcs_base_url}/storage/v1/b/{bucket_name}?project={project}", + headers=headers, + ) + assert resp_get.status_code == 200 + assert resp_get.json()["name"] == bucket_name + + # 4. List buckets + resp_list = requests.get( + f"{gcs_base_url}/storage/v1/b?project={project}", + headers=headers, + ) + assert resp_list.status_code == 200 + bucket_names = [b["name"] for b in resp_list.json().get("items", [])] + assert bucket_name in bucket_names + + # 5. Delete bucket + resp_del = requests.delete( + f"{gcs_base_url}/storage/v1/b/{bucket_name}?project={project}", + headers=headers, + ) + assert resp_del.status_code == 204 + + # 6. Accessing deleted bucket returns 404 + resp_deleted = requests.get( + f"{gcs_base_url}/storage/v1/b/{bucket_name}?project={project}", + headers=headers, + ) + assert resp_deleted.status_code == 404 + + +def test_gcs_object_crud_lifecycle(gcs_base_url): + project = "test-proj" + bucket_name = "test-object-crud-bucket" + headers = {"Authorization": "Bearer ya29.devcloud-pilot-token"} + + # Setup bucket + requests.post( + f"{gcs_base_url}/storage/v1/b?project={project}", + headers=headers, + json={"name": bucket_name}, + ) + + try: + object_name = "docs/welcome.txt" + payload = b"Hello Google Cloud Storage from local DevCloud runtime!" + + # 1. Upload object via media upload + resp_upload = requests.post( + f"{gcs_base_url}/upload/storage/v1/b/{bucket_name}/o?uploadType=media&name={object_name}&project={project}", + headers={ + "Authorization": "Bearer ya29.devcloud-pilot-token", + "Content-Type": "text/plain", + }, + data=payload, + ) + assert resp_upload.status_code == 200 + obj_data = resp_upload.json() + assert obj_data["name"] == object_name + assert obj_data["bucket"] == bucket_name + assert obj_data["md5Hash"] != "" + assert obj_data["crc32c"] != "" + + # 2. Get object metadata + resp_meta = requests.get( + f"{gcs_base_url}/storage/v1/b/{bucket_name}/o/{object_name}?project={project}", + headers=headers, + ) + assert resp_meta.status_code == 200 + assert resp_meta.json()["contentType"] == "text/plain" + + # 3. Download object payload (alt=media) + resp_down = requests.get( + f"{gcs_base_url}/storage/v1/b/{bucket_name}/o/{object_name}?alt=media&project={project}", + headers=headers, + ) + assert resp_down.status_code == 200 + assert resp_down.content == payload + assert "x-goog-hash" in resp_down.headers + + # 4. List objects + resp_list = requests.get( + f"{gcs_base_url}/storage/v1/b/{bucket_name}/o?prefix=docs/&project={project}", + headers=headers, + ) + assert resp_list.status_code == 200 + items = resp_list.json().get("items", []) + assert any(item["name"] == object_name for item in items) + + # 5. Delete object + resp_del = requests.delete( + f"{gcs_base_url}/storage/v1/b/{bucket_name}/o/{object_name}?project={project}", + headers=headers, + ) + assert resp_del.status_code == 204 + + # 6. Download deleted object returns 404 + resp_down_del = requests.get( + f"{gcs_base_url}/storage/v1/b/{bucket_name}/o/{object_name}?alt=media&project={project}", + headers=headers, + ) + assert resp_down_del.status_code == 404 + + finally: + # Cleanup bucket + requests.delete( + f"{gcs_base_url}/storage/v1/b/{bucket_name}?project={project}", + headers=headers, + )