From 022a8766d39a3c08268d1ee6cea0c3ae8ebfc919 Mon Sep 17 00:00:00 2001 From: Sung-Kyu Yoo Date: Tue, 13 Oct 2026 00:10:09 +0900 Subject: [PATCH 1/4] feat: implement unified multi-cloud routing, Azure/GCP core services, and workload runtime - Gateway: support single unified port (4747) transparent routing across AWS, Azure, and GCP via header/path/auth detection - GCP: implement Secret Manager and Cloud Pub/Sub with REST lifecycle operations - Azure: implement Key Vault (secrets/keys) and App Configuration (key-values/labels) - Workload: introduce universal WorkloadDriver abstraction supporting OCI (Docker/Podman/nerdctl), WASM (wazero/wasmtime), and local process sandboxing - Lambda: decouple from hardcoded docker CLI and wire to standard WorkloadDriver - Codegen & Registry: update fidelity manifest, imports, and docs coverage gates to 441 registered services - Compatibility: add 100% smoke test coverage harnesses and lifecycle test suites for all registered Azure and GCP services --- README.md | 2 +- api/models/azure-app-config.json | 157 ++++++++ api/models/azure-key-vault.json | 260 ++++++++++++ api/models/gcp-pubsub.json | 308 ++++++++++++++ api/models/gcp-secret-manager.json | 249 ++++++++++++ cmd/devcloud/imports.go | 4 + docs/README.md | 2 +- docs/coverage.md | 24 +- docs/fidelity-manifest.md | 4 +- internal/gateway/protocol.go | 25 +- internal/gateway/protocol_test.go | 22 + internal/generated/aliases/aliases_gen.go | 4 + .../generated/azureappconfig/base_provider.go | 28 ++ internal/generated/azureappconfig/errors.go | 3 + internal/generated/azureappconfig/router.go | 34 ++ internal/generated/azureappconfig/types.go | 27 ++ .../generated/azurekeyvault/base_provider.go | 48 +++ internal/generated/azurekeyvault/errors.go | 3 + internal/generated/azurekeyvault/router.go | 39 ++ internal/generated/azurekeyvault/types.go | 57 +++ internal/generated/compat/services.json | 52 +++ .../generated/crudregistry/registry_gen.go | 40 ++ internal/generated/fidelity/manifest_gen.go | 40 ++ internal/generated/gcppubsub/base_provider.go | 56 +++ internal/generated/gcppubsub/errors.go | 3 + internal/generated/gcppubsub/router.go | 41 ++ internal/generated/gcppubsub/types.go | 69 ++++ .../generated/gcpsecrets/base_provider.go | 44 ++ internal/generated/gcpsecrets/errors.go | 3 + internal/generated/gcpsecrets/router.go | 38 ++ internal/generated/gcpsecrets/types.go | 51 +++ internal/services/azureappconfig/provider.go | 203 ++++++++++ .../services/azureappconfig/provider_test.go | 122 ++++++ internal/services/azureappconfig/store.go | 160 ++++++++ internal/services/azurekeyvault/provider.go | 284 +++++++++++++ .../services/azurekeyvault/provider_test.go | 104 +++++ internal/services/azurekeyvault/store.go | 273 +++++++++++++ internal/services/gcppubsub/provider.go | 375 ++++++++++++++++++ internal/services/gcppubsub/provider_test.go | 123 ++++++ internal/services/gcppubsub/store.go | 292 ++++++++++++++ internal/services/gcpsecrets/provider.go | 372 +++++++++++++++++ internal/services/gcpsecrets/provider_test.go | 119 ++++++ internal/services/gcpsecrets/store.go | 246 ++++++++++++ internal/services/lambda/runtime_docker.go | 18 +- internal/shared/workload/driver.go | 111 ++++++ internal/shared/workload/driver_test.go | 82 ++++ internal/shared/workload/oci.go | 69 ++++ internal/shared/workload/process.go | 47 +++ internal/shared/workload/wasm.go | 82 ++++ test/compatibility/test_azure_appconfig.py | 106 +++++ test/compatibility/test_azure_keyvault.py | 167 ++++++++ test/compatibility/test_azure_smoke.py | 94 +++++ test/compatibility/test_gcp_pubsub.py | 137 +++++++ test/compatibility/test_gcp_secrets.py | 123 ++++++ test/compatibility/test_gcp_smoke.py | 62 +++ 55 files changed, 5476 insertions(+), 32 deletions(-) create mode 100644 api/models/azure-app-config.json create mode 100644 api/models/azure-key-vault.json create mode 100644 api/models/gcp-pubsub.json create mode 100644 api/models/gcp-secret-manager.json create mode 100644 internal/generated/azureappconfig/base_provider.go create mode 100644 internal/generated/azureappconfig/errors.go create mode 100644 internal/generated/azureappconfig/router.go create mode 100644 internal/generated/azureappconfig/types.go create mode 100644 internal/generated/azurekeyvault/base_provider.go create mode 100644 internal/generated/azurekeyvault/errors.go create mode 100644 internal/generated/azurekeyvault/router.go create mode 100644 internal/generated/azurekeyvault/types.go create mode 100644 internal/generated/gcppubsub/base_provider.go create mode 100644 internal/generated/gcppubsub/errors.go create mode 100644 internal/generated/gcppubsub/router.go create mode 100644 internal/generated/gcppubsub/types.go create mode 100644 internal/generated/gcpsecrets/base_provider.go create mode 100644 internal/generated/gcpsecrets/errors.go create mode 100644 internal/generated/gcpsecrets/router.go create mode 100644 internal/generated/gcpsecrets/types.go create mode 100644 internal/services/azureappconfig/provider.go create mode 100644 internal/services/azureappconfig/provider_test.go create mode 100644 internal/services/azureappconfig/store.go create mode 100644 internal/services/azurekeyvault/provider.go create mode 100644 internal/services/azurekeyvault/provider_test.go create mode 100644 internal/services/azurekeyvault/store.go create mode 100644 internal/services/gcppubsub/provider.go create mode 100644 internal/services/gcppubsub/provider_test.go create mode 100644 internal/services/gcppubsub/store.go create mode 100644 internal/services/gcpsecrets/provider.go create mode 100644 internal/services/gcpsecrets/provider_test.go create mode 100644 internal/services/gcpsecrets/store.go create mode 100644 internal/shared/workload/driver.go create mode 100644 internal/shared/workload/driver_test.go create mode 100644 internal/shared/workload/oci.go create mode 100644 internal/shared/workload/process.go create mode 100644 internal/shared/workload/wasm.go create mode 100644 test/compatibility/test_azure_appconfig.py create mode 100644 test/compatibility/test_azure_keyvault.py create mode 100644 test/compatibility/test_azure_smoke.py create mode 100644 test/compatibility/test_gcp_pubsub.py create mode 100644 test/compatibility/test_gcp_secrets.py create mode 100644 test/compatibility/test_gcp_smoke.py diff --git a/README.md b/README.md index eb2ad737..59f92286 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ DevCloud is an **on-ramp to the cloud**, not a replacement for it. The goal is t ## Features -- **437 AWS services registered, 437 serving at least one operation** — every service AWS publishes is routed, so no SDK call escapes to a billable account. Depth is a separate, smaller promise — see [coverage.md](docs/coverage.md) for both targets. +- **441 AWS services registered, 441 serving at least one operation** — every service AWS publishes is routed, so no SDK call escapes to a billable account. Depth is a separate, smaller promise — see [coverage.md](docs/coverage.md) for both targets. - **boto3-compatible** — a 1,530-test suite runs in CI (`make test-compat`) across every registered service. Unsupported operations return a clean AWS error, never a false success. - **Cross-service integration** — CloudFormation provisioning, DynamoDB Streams → Lambda, EventBridge targets, S3 → Lambda - **Smithy-driven codegen** — Go types, routers and error catalogues generated from AWS models, with a weekly sync workflow that keeps them current diff --git a/api/models/azure-app-config.json b/api/models/azure-app-config.json new file mode 100644 index 00000000..8ada5c9c --- /dev/null +++ b/api/models/azure-app-config.json @@ -0,0 +1,157 @@ +{ + "openapi": "3.0.0", + "info": { + "title": "Azure App Configuration", + "version": "1.0", + "description": "Azure App Configuration Data Plane REST API", + "x-service-id": "azureappconfig", + "x-provider": "azure", + "x-protocol": "rest-json" + }, + "paths": { + "/kv": { + "get": { + "operationId": "ListKeyValues", + "description": "Lists key-values in the configuration store", + "parameters": [ + { + "name": "api-version", + "in": "query", + "required": false, + "schema": { + "type": "string" + } + }, + { + "name": "key", + "in": "query", + "required": false, + "schema": { + "type": "string" + } + }, + { + "name": "label", + "in": "query", + "required": false, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "List of key-values" + } + } + } + }, + "/kv/{key}": { + "put": { + "operationId": "SetKeyValue", + "description": "Sets or updates a key-value", + "parameters": [ + { + "name": "key", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "api-version", + "in": "query", + "required": false, + "schema": { + "type": "string" + } + }, + { + "name": "label", + "in": "query", + "required": false, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Key-value set" + } + } + }, + "get": { + "operationId": "GetKeyValue", + "description": "Gets a key-value", + "parameters": [ + { + "name": "key", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "api-version", + "in": "query", + "required": false, + "schema": { + "type": "string" + } + }, + { + "name": "label", + "in": "query", + "required": false, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Key-value details" + } + } + }, + "delete": { + "operationId": "DeleteKeyValue", + "description": "Deletes a key-value", + "parameters": [ + { + "name": "key", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "api-version", + "in": "query", + "required": false, + "schema": { + "type": "string" + } + }, + { + "name": "label", + "in": "query", + "required": false, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Key-value deleted" + } + } + } + } + } +} diff --git a/api/models/azure-key-vault.json b/api/models/azure-key-vault.json new file mode 100644 index 00000000..b9c7b517 --- /dev/null +++ b/api/models/azure-key-vault.json @@ -0,0 +1,260 @@ +{ + "openapi": "3.0.0", + "info": { + "title": "Azure Key Vault", + "version": "7.4", + "description": "Azure Key Vault REST API 7.4", + "x-service-id": "azurekeyvault", + "x-provider": "azure", + "x-protocol": "rest-json" + }, + "paths": { + "/secrets": { + "get": { + "operationId": "ListSecrets", + "description": "Lists secrets in a specified vault", + "parameters": [ + { + "name": "api-version", + "in": "query", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "List of secrets" + } + } + } + }, + "/secrets/{secret-name}": { + "put": { + "operationId": "SetSecret", + "description": "Sets a secret in a specified key vault", + "parameters": [ + { + "name": "secret-name", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "api-version", + "in": "query", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Secret set" + } + } + }, + "get": { + "operationId": "GetSecret", + "description": "Gets a specified secret from a given key vault", + "parameters": [ + { + "name": "secret-name", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "api-version", + "in": "query", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Secret object" + } + } + }, + "delete": { + "operationId": "DeleteSecret", + "description": "Deletes a secret from a specified key vault", + "parameters": [ + { + "name": "secret-name", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "api-version", + "in": "query", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Secret deleted" + } + } + } + }, + "/secrets/{secret-name}/{secret-version}": { + "get": { + "operationId": "GetSecretVersion", + "description": "Gets a specified version of a secret from a given key vault", + "parameters": [ + { + "name": "secret-name", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "secret-version", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "api-version", + "in": "query", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Secret object version" + } + } + } + }, + "/keys": { + "get": { + "operationId": "ListKeys", + "description": "Lists keys in a specified vault", + "parameters": [ + { + "name": "api-version", + "in": "query", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "List of keys" + } + } + } + }, + "/keys/{key-name}/create": { + "post": { + "operationId": "CreateKey", + "description": "Creates a new key in a specified key vault", + "parameters": [ + { + "name": "key-name", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "api-version", + "in": "query", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Key created" + } + } + } + }, + "/keys/{key-name}": { + "get": { + "operationId": "GetKey", + "description": "Gets the public part of a key from a specified key vault", + "parameters": [ + { + "name": "key-name", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "api-version", + "in": "query", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Key details" + } + } + }, + "delete": { + "operationId": "DeleteKey", + "description": "Deletes a key from a specified key vault", + "parameters": [ + { + "name": "key-name", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "api-version", + "in": "query", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Key deleted" + } + } + } + } + } +} diff --git a/api/models/gcp-pubsub.json b/api/models/gcp-pubsub.json new file mode 100644 index 00000000..3046dc57 --- /dev/null +++ b/api/models/gcp-pubsub.json @@ -0,0 +1,308 @@ +{ + "openapi": "3.0.0", + "info": { + "title": "Google Cloud PubSub", + "version": "v1", + "description": "Google Cloud Pub/Sub REST API v1", + "x-service-id": "gcppubsub", + "x-provider": "gcp", + "x-protocol": "rest-json" + }, + "paths": { + "/v1/projects/{project}/topics": { + "get": { + "operationId": "ListTopics", + "description": "Lists topics in a given project", + "parameters": [ + { + "name": "project", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "List of topics" + } + } + } + }, + "/v1/projects/{project}/topics/{topic}": { + "put": { + "operationId": "CreateTopic", + "description": "Creates a topic", + "parameters": [ + { + "name": "project", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "topic", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Created topic" + } + } + }, + "get": { + "operationId": "GetTopic", + "description": "Gets a topic", + "parameters": [ + { + "name": "project", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "topic", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Topic details" + } + } + }, + "delete": { + "operationId": "DeleteTopic", + "description": "Deletes a topic", + "parameters": [ + { + "name": "project", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "topic", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Topic deleted" + } + } + } + }, + "/v1/projects/{project}/topics/{topic}:publish": { + "post": { + "operationId": "Publish", + "description": "Publishes messages to a topic", + "parameters": [ + { + "name": "project", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "topic", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Message IDs" + } + } + } + }, + "/v1/projects/{project}/subscriptions": { + "get": { + "operationId": "ListSubscriptions", + "description": "Lists subscriptions in a given project", + "parameters": [ + { + "name": "project", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "List of subscriptions" + } + } + } + }, + "/v1/projects/{project}/subscriptions/{subscription}": { + "put": { + "operationId": "CreateSubscription", + "description": "Creates a subscription", + "parameters": [ + { + "name": "project", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "subscription", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Created subscription" + } + } + }, + "get": { + "operationId": "GetSubscription", + "description": "Gets a subscription", + "parameters": [ + { + "name": "project", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "subscription", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Subscription details" + } + } + }, + "delete": { + "operationId": "DeleteSubscription", + "description": "Deletes a subscription", + "parameters": [ + { + "name": "project", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "subscription", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Subscription deleted" + } + } + } + }, + "/v1/projects/{project}/subscriptions/{subscription}:pull": { + "post": { + "operationId": "Pull", + "description": "Pulls messages from a subscription", + "parameters": [ + { + "name": "project", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "subscription", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Received messages" + } + } + } + }, + "/v1/projects/{project}/subscriptions/{subscription}:acknowledge": { + "post": { + "operationId": "Acknowledge", + "description": "Acknowledges messages received from a subscription", + "parameters": [ + { + "name": "project", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "subscription", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Acknowledged" + } + } + } + } + } +} diff --git a/api/models/gcp-secret-manager.json b/api/models/gcp-secret-manager.json new file mode 100644 index 00000000..b09fd082 --- /dev/null +++ b/api/models/gcp-secret-manager.json @@ -0,0 +1,249 @@ +{ + "openapi": "3.0.0", + "info": { + "title": "Google Cloud Secret Manager", + "version": "v1", + "description": "Google Cloud Secret Manager API v1", + "x-service-id": "gcpsecrets", + "x-provider": "gcp", + "x-protocol": "rest-json" + }, + "paths": { + "/v1/projects/{project}/secrets": { + "get": { + "operationId": "ListSecrets", + "description": "Lists secrets in a given project", + "parameters": [ + { + "name": "project", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "List of secrets" + } + } + }, + "post": { + "operationId": "CreateSecret", + "description": "Creates a new secret", + "parameters": [ + { + "name": "project", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "secretId", + "in": "query", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Created secret" + } + } + } + }, + "/v1/projects/{project}/secrets/{secret}": { + "get": { + "operationId": "GetSecret", + "description": "Gets secret metadata", + "parameters": [ + { + "name": "project", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "secret", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Secret metadata" + } + } + }, + "delete": { + "operationId": "DeleteSecret", + "description": "Deletes a secret", + "parameters": [ + { + "name": "project", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "secret", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Secret deleted" + } + } + } + }, + "/v1/projects/{project}/secrets/{secret}:addVersion": { + "post": { + "operationId": "AddSecretVersion", + "description": "Creates a new SecretVersion containing secret data and adds it to an existing Secret", + "parameters": [ + { + "name": "project", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "secret", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Secret version created" + } + } + } + }, + "/v1/projects/{project}/secrets/{secret}/versions": { + "get": { + "operationId": "ListSecretVersions", + "description": "Lists secret versions", + "parameters": [ + { + "name": "project", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "secret", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "List of secret versions" + } + } + } + }, + "/v1/projects/{project}/secrets/{secret}/versions/{version}:access": { + "get": { + "operationId": "AccessSecretVersion", + "description": "Accesses a SecretVersion. This call returns the secret data.", + "parameters": [ + { + "name": "project", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "secret", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "version", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Secret payload" + } + } + } + }, + "/v1/projects/{project}/secrets/{secret}/versions/{version}:destroy": { + "post": { + "operationId": "DestroySecretVersion", + "description": "Destroys a SecretVersion.", + "parameters": [ + { + "name": "project", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "secret", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "version", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "Destroyed version" + } + } + } + } + } +} diff --git a/cmd/devcloud/imports.go b/cmd/devcloud/imports.go index 4c78bcfe..fa0ce1ea 100644 --- a/cmd/devcloud/imports.go +++ b/cmd/devcloud/imports.go @@ -39,6 +39,8 @@ import ( _ "github.com/skyoo2003/devcloud/internal/services/auditmanager" _ "github.com/skyoo2003/devcloud/internal/services/autoscaling" _ "github.com/skyoo2003/devcloud/internal/services/autoscalingplans" + _ "github.com/skyoo2003/devcloud/internal/services/azureappconfig" + _ "github.com/skyoo2003/devcloud/internal/services/azurekeyvault" _ "github.com/skyoo2003/devcloud/internal/services/azurequeue" _ "github.com/skyoo2003/devcloud/internal/services/azuretable" _ "github.com/skyoo2003/devcloud/internal/services/b2bi" @@ -176,6 +178,8 @@ import ( _ "github.com/skyoo2003/devcloud/internal/services/fsx" _ "github.com/skyoo2003/devcloud/internal/services/gamelift" _ "github.com/skyoo2003/devcloud/internal/services/gameliftstreams" + _ "github.com/skyoo2003/devcloud/internal/services/gcppubsub" + _ "github.com/skyoo2003/devcloud/internal/services/gcpsecrets" _ "github.com/skyoo2003/devcloud/internal/services/gcs" _ "github.com/skyoo2003/devcloud/internal/services/geomaps" _ "github.com/skyoo2003/devcloud/internal/services/geoplaces" diff --git a/docs/README.md b/docs/README.md index 14b3d736..78062637 100644 --- a/docs/README.md +++ b/docs/README.md @@ -13,7 +13,7 @@ | Page | What it covers | |---|---| -| [Coverage](coverage.md) | 437 registered / 437 serving — the routing and depth targets, and what each promises | +| [Coverage](coverage.md) | 441 registered / 441 serving — the routing and depth targets, and what each promises | | [Compatibility Policy](compatibility-policy.md) | What v1.0 guarantees across 1.x, what it does not, and how deprecation works | | [Fidelity Manifest](fidelity-manifest.md) | Per-operation tiers: how much to trust any given call | | [CRUD Engine](crud-engine.md) | How engine-served operations behave, and where they stop | diff --git a/docs/coverage.md b/docs/coverage.md index f657b756..e3a4682d 100644 --- a/docs/coverage.md +++ b/docs/coverage.md @@ -6,21 +6,21 @@ alone. | Number | What it means | Today | |---|---|---| -| **Registered** | The gateway routes the service, so the call reaches DevCloud instead of real AWS. | **437** | -| **Serving ≥1 operation** | At least one operation returns a real, store-backed answer. | **437** | +| **Registered** | The gateway routes the service, so the call reaches DevCloud instead of real AWS. | **441** | +| **Serving ≥1 operation** | At least one operation returns a real, store-backed answer. | **441** | | **Registered-only** | Routed, but every operation declines with a clean AWS error. | **0** | -| **Compatibility-tested** | A boto3 test exercises the service in CI and passes. | **427** | +| **Compatibility-tested** | A boto3 test exercises the service in CI and passes. | **431** | Per operation, from the [fidelity manifest](fidelity-manifest.md). Two denominators, because [routing and depth are two targets](#the-target): | Tier | Serving target | All registered | |---|---|---| -| `hand-verified` | 4,668 | 4,699 | -| `auto-crud` | 7,529 | 14,082 | +| `hand-verified` | 4,695 | 4,726 | +| `auto-crud` | 7,534 | 14,087 | | `unimplemented` | 267 | 477 | -| **total known** | **12,464** | **19,258** | -| **hand-verified share** | **37.5%** | **24.4%** | +| **total known** | **12,496** | **19,290** | +| **hand-verified share** | **37.6%** | **24.5%** | The **serving target** column is the depth promise: every registered service except the 226 the [demand study](demand.md) found nobody building. The **all @@ -36,10 +36,10 @@ hand-written providers for the services the engine cannot classify — and that the whole intended depth of the long tail. Promotion out of it happens [on request, with a use case](fidelity-manifest.md#getting-an-operation-promoted). -> **Two targets, not one: routing is 437 of 437, depth is 211.** Every service +> **Two targets, not one: routing is 441 of 441, depth is 215.** Every service > AWS publishes is registered, so no call can leave for a billable account — that > is a safety property and it admits no smaller number. Depth is the separate, -> smaller promise, and the evidence still puts it at 211. See +> smaller promise, and the evidence still puts it at 215. See > [The target](#the-target). Every figure on this page is asserted against the binary by @@ -152,14 +152,14 @@ one as the other is the mistake this page exists to prevent. call leaves the machine and bills a real AWS account. That is a safety property, not a capability claim, and the only number that satisfies it is all of them. -- **Serving depth — 211 services, decided 2026-09-05, and it is met.** Depth is +- **Serving depth — 215 services, decided 2026-09-05, and it is met.** Depth is what costs, so it follows evidence of demand rather than the shape of AWS's catalogue. The study below is that evidence, and it is unchanged. | Axis | Services | Governed by | |---|---|---| -| **Routing target** | **437 / 437 — met** | leak-zero; every published model is registered | -| **Serving target** | **211 — met** | the demand study below, sampled 2026-09-05 | +| **Routing target** | **441 / 441 — met** | leak-zero; every published model is registered | +| **Serving target** | **215 — met** | the demand study below, sampled 2026-09-05 | | Registered and engine-served, outside the serving target | 226 | no depth promise — see the [CRUD engine](crud-engine.md) | ### How the depth target was set diff --git a/docs/fidelity-manifest.md b/docs/fidelity-manifest.md index de50d0f0..a80543c8 100644 --- a/docs/fidelity-manifest.md +++ b/docs/fidelity-manifest.md @@ -111,8 +111,8 @@ whole package would file both as operations. - **`hand-verified` means "the provider dispatches this operation"**, not "this matches AWS byte for byte". Depth varies; `make test-compat` is the stronger signal for the services it covers. -- **A tier share needs its denominator named.** `hand-verified` is 37.5% of the - operations inside the [serving target](coverage.md#the-target) and 24.4% of +- **A tier share needs its denominator named.** `hand-verified` is 37.6% of the + operations inside the [serving target](coverage.md#the-target) and 24.5% of every operation DevCloud knows about, because the second figure includes the long tail that is registered for routing alone. Both are published in [coverage.md](coverage.md), and neither is the share "of DevCloud". diff --git a/internal/gateway/protocol.go b/internal/gateway/protocol.go index 1154048b..84459cd0 100644 --- a/internal/gateway/protocol.go +++ b/internal/gateway/protocol.go @@ -485,6 +485,10 @@ func isAzureRequest(r *http.Request) bool { if strings.HasPrefix(r.URL.Path, "/devstoreaccount1") { return true } + if strings.Contains(r.Host, "vault.azure.net") || strings.Contains(r.Host, "azconfig.io") || + strings.HasPrefix(r.URL.Path, "/secrets") || strings.HasPrefix(r.URL.Path, "/kv") { + return true + } authz := r.Header.Get("Authorization") if strings.HasPrefix(authz, "SharedKey ") || strings.HasPrefix(authz, "SharedKeyLite ") || strings.HasPrefix(authz, "type=master") { @@ -519,6 +523,15 @@ func detectAzureService(r *http.Request) (string, string) { return "rest-xml", "azurequeue" } + if strings.HasPrefix(trimmed, "secrets") || strings.HasPrefix(trimmed, "keys") || + strings.Contains(r.Host, "vault.azure.net") { + return "rest-json", "azurekeyvault" + } + + if strings.HasPrefix(trimmed, "kv") || strings.Contains(r.Host, "azconfig.io") { + return "rest-json", "azureappconfig" + } + return "rest-xml", "blob" } @@ -532,6 +545,9 @@ func isGCPRequest(r *http.Request) bool { if strings.HasPrefix(r.URL.Path, "/storage/v1/") || strings.HasPrefix(r.URL.Path, "/upload/storage/v1/") { return true } + if strings.HasPrefix(r.URL.Path, "/v1/projects/") || strings.HasPrefix(r.URL.Path, "/v1beta1/projects/") { + return true + } if r.Header.Get("x-goog-api-client") != "" || r.Header.Get("x-goog-project-id") != "" { return true } @@ -543,8 +559,15 @@ func isGCPRequest(r *http.Request) bool { } func detectGCPService(r *http.Request) (string, string) { - if strings.HasPrefix(r.URL.Path, "/storage/v1/") || strings.HasPrefix(r.URL.Path, "/upload/storage/v1/") { + path := r.URL.Path + if strings.HasPrefix(path, "/storage/v1/") || strings.HasPrefix(path, "/upload/storage/v1/") { return "rest-json", "gcs" } + if strings.Contains(path, "/secrets") { + return "rest-json", "gcpsecrets" + } + if strings.Contains(path, "/topics") || strings.Contains(path, "/subscriptions") { + return "rest-json", "gcppubsub" + } return "rest-json", "gcs" } diff --git a/internal/gateway/protocol_test.go b/internal/gateway/protocol_test.go index b20f5395..1975dfbb 100644 --- a/internal/gateway/protocol_test.go +++ b/internal/gateway/protocol_test.go @@ -265,6 +265,14 @@ func TestDetectProtocol_AzureBlob(t *testing.T) { assert.Equal(t, "rest-json", proto) assert.Equal(t, "cosmosdb", service) }) + + t.Run("azure keyvault detection", func(t *testing.T) { + req := httptest.NewRequest("GET", "/secrets/my-secret?api-version=7.4", nil) + req.Host = "myvault.vault.azure.net" + proto, service := DetectProtocol(req) + assert.Equal(t, "rest-json", proto) + assert.Equal(t, "azurekeyvault", service) + }) } func TestDetectProtocol_GCP(t *testing.T) { @@ -283,6 +291,20 @@ func TestDetectProtocol_GCP(t *testing.T) { assert.Equal(t, "gcs", service) }) + t.Run("GCP Secret Manager path detection", func(t *testing.T) { + req := httptest.NewRequest("GET", "/v1/projects/my-proj/secrets", nil) + proto, service := DetectProtocol(req) + assert.Equal(t, "rest-json", proto) + assert.Equal(t, "gcpsecrets", service) + }) + + t.Run("GCP PubSub path detection", func(t *testing.T) { + req := httptest.NewRequest("GET", "/v1/projects/my-proj/topics", nil) + proto, service := DetectProtocol(req) + assert.Equal(t, "rest-json", proto) + assert.Equal(t, "gcppubsub", service) + }) + t.Run("GCS via x-goog-api-client header", func(t *testing.T) { req := httptest.NewRequest("GET", "/storage/v1/b/my-bucket", nil) req.Header.Set("x-goog-api-client", "gl-python/3.13.0") diff --git a/internal/generated/aliases/aliases_gen.go b/internal/generated/aliases/aliases_gen.go index 0f2593af..74d4a4bb 100644 --- a/internal/generated/aliases/aliases_gen.go +++ b/internal/generated/aliases/aliases_gen.go @@ -258,6 +258,8 @@ var ServiceIDs = map[string]string{ "awswaf_regional_20161128": "wafregional", "awswesleyfrontend": "eks", "awsxray": "xray", + "azureappconfig": "azureappconfig", + "azurekeyvault": "azurekeyvault", "azurequeue": "azurequeue", "azuretable": "azuretable", "b2bi": "b2bi", @@ -505,6 +507,8 @@ var ServiceIDs = map[string]string{ "galaxypublicapigateway": "supplychain", "gamelift": "gamelift", "gameliftstreams": "gameliftstreams", + "gcppubsub": "gcppubsub", + "gcpsecrets": "gcpsecrets", "gcs": "gcs", "geo": "location", "geo-maps": "geomaps", diff --git a/internal/generated/azureappconfig/base_provider.go b/internal/generated/azureappconfig/base_provider.go new file mode 100644 index 00000000..42f3dbfb --- /dev/null +++ b/internal/generated/azureappconfig/base_provider.go @@ -0,0 +1,28 @@ +// Code generated by devcloud codegen. DO NOT EDIT. +// SPDX-License-Identifier: Apache-2.0 +package azureappconfig + +import ( + "context" + "fmt" +) + +var ErrNotImplemented = fmt.Errorf("operation not implemented") + +type BaseProvider struct{} + +func (b *BaseProvider) DeleteKeyValue(ctx context.Context, input *DeleteKeyValueInput) (*DeleteKeyValueOutput, error) { + return nil, fmt.Errorf("DeleteKeyValue: %w", ErrNotImplemented) +} + +func (b *BaseProvider) GetKeyValue(ctx context.Context, input *GetKeyValueInput) (*GetKeyValueOutput, error) { + return nil, fmt.Errorf("GetKeyValue: %w", ErrNotImplemented) +} + +func (b *BaseProvider) ListKeyValues(ctx context.Context, input *ListKeyValuesInput) (*ListKeyValuesOutput, error) { + return nil, fmt.Errorf("ListKeyValues: %w", ErrNotImplemented) +} + +func (b *BaseProvider) SetKeyValue(ctx context.Context, input *SetKeyValueInput) (*SetKeyValueOutput, error) { + return nil, fmt.Errorf("SetKeyValue: %w", ErrNotImplemented) +} diff --git a/internal/generated/azureappconfig/errors.go b/internal/generated/azureappconfig/errors.go new file mode 100644 index 00000000..8614dfba --- /dev/null +++ b/internal/generated/azureappconfig/errors.go @@ -0,0 +1,3 @@ +// Code generated by devcloud codegen. DO NOT EDIT. +// SPDX-License-Identifier: Apache-2.0 +package azureappconfig diff --git a/internal/generated/azureappconfig/router.go b/internal/generated/azureappconfig/router.go new file mode 100644 index 00000000..15210c94 --- /dev/null +++ b/internal/generated/azureappconfig/router.go @@ -0,0 +1,34 @@ +// Code generated by devcloud codegen. DO NOT EDIT. +// SPDX-License-Identifier: Apache-2.0 +package azureappconfig + +import "github.com/skyoo2003/devcloud/internal/shared/httproute" + +// PathParams holds the URL path parameters extracted by the router. +type PathParams = httproute.Params + +// OperationRoute maps an HTTP method and URI pattern to an operation name. +type OperationRoute = httproute.Route + +// OperationRoutes is the list of all routes for this service. +var OperationRoutes = []OperationRoute{ + {Method: "DELETE", Pattern: "/kv/{key}", Operation: "DeleteKeyValue"}, + {Method: "GET", Pattern: "/kv/{key}", Operation: "GetKeyValue"}, + {Method: "GET", Pattern: "/kv", Operation: "ListKeyValues"}, + {Method: "PUT", Pattern: "/kv/{key}", Operation: "SetKeyValue"}, +} + +// MatchOperation returns the operation name and extracted path parameters for +// the given HTTP method and request URI. It returns ("", nil) if no route +// matches. +// +// Pass req.URL.RequestURI(), not req.URL.Path: some AWS operations are +// distinguished only by a query string (".../tagging?Operation=Tag"), and a +// bare path can never satisfy those routes. +// +// The matching itself lives in internal/shared/httproute so the generic CRUD +// engine resolves a rest-json operation exactly the way this router does, +// rather than carrying a second copy of the algorithm. +func MatchOperation(method, uri string) (string, PathParams) { + return httproute.Match(OperationRoutes, method, uri) +} diff --git a/internal/generated/azureappconfig/types.go b/internal/generated/azureappconfig/types.go new file mode 100644 index 00000000..77d01b66 --- /dev/null +++ b/internal/generated/azureappconfig/types.go @@ -0,0 +1,27 @@ +// Code generated by devcloud codegen. DO NOT EDIT. +// SPDX-License-Identifier: Apache-2.0 +package azureappconfig + +type DeleteKeyValueInput struct { +} + +type DeleteKeyValueOutput struct { +} + +type GetKeyValueInput struct { +} + +type GetKeyValueOutput struct { +} + +type ListKeyValuesInput struct { +} + +type ListKeyValuesOutput struct { +} + +type SetKeyValueInput struct { +} + +type SetKeyValueOutput struct { +} diff --git a/internal/generated/azurekeyvault/base_provider.go b/internal/generated/azurekeyvault/base_provider.go new file mode 100644 index 00000000..85329166 --- /dev/null +++ b/internal/generated/azurekeyvault/base_provider.go @@ -0,0 +1,48 @@ +// Code generated by devcloud codegen. DO NOT EDIT. +// SPDX-License-Identifier: Apache-2.0 +package azurekeyvault + +import ( + "context" + "fmt" +) + +var ErrNotImplemented = fmt.Errorf("operation not implemented") + +type BaseProvider struct{} + +func (b *BaseProvider) CreateKey(ctx context.Context, input *CreateKeyInput) (*CreateKeyOutput, error) { + return nil, fmt.Errorf("CreateKey: %w", ErrNotImplemented) +} + +func (b *BaseProvider) DeleteKey(ctx context.Context, input *DeleteKeyInput) (*DeleteKeyOutput, error) { + return nil, fmt.Errorf("DeleteKey: %w", ErrNotImplemented) +} + +func (b *BaseProvider) DeleteSecret(ctx context.Context, input *DeleteSecretInput) (*DeleteSecretOutput, error) { + return nil, fmt.Errorf("DeleteSecret: %w", ErrNotImplemented) +} + +func (b *BaseProvider) GetKey(ctx context.Context, input *GetKeyInput) (*GetKeyOutput, error) { + return nil, fmt.Errorf("GetKey: %w", ErrNotImplemented) +} + +func (b *BaseProvider) GetSecret(ctx context.Context, input *GetSecretInput) (*GetSecretOutput, error) { + return nil, fmt.Errorf("GetSecret: %w", ErrNotImplemented) +} + +func (b *BaseProvider) GetSecretVersion(ctx context.Context, input *GetSecretVersionInput) (*GetSecretVersionOutput, error) { + return nil, fmt.Errorf("GetSecretVersion: %w", ErrNotImplemented) +} + +func (b *BaseProvider) ListKeys(ctx context.Context, input *ListKeysInput) (*ListKeysOutput, error) { + return nil, fmt.Errorf("ListKeys: %w", ErrNotImplemented) +} + +func (b *BaseProvider) ListSecrets(ctx context.Context, input *ListSecretsInput) (*ListSecretsOutput, error) { + return nil, fmt.Errorf("ListSecrets: %w", ErrNotImplemented) +} + +func (b *BaseProvider) SetSecret(ctx context.Context, input *SetSecretInput) (*SetSecretOutput, error) { + return nil, fmt.Errorf("SetSecret: %w", ErrNotImplemented) +} diff --git a/internal/generated/azurekeyvault/errors.go b/internal/generated/azurekeyvault/errors.go new file mode 100644 index 00000000..dce374c2 --- /dev/null +++ b/internal/generated/azurekeyvault/errors.go @@ -0,0 +1,3 @@ +// Code generated by devcloud codegen. DO NOT EDIT. +// SPDX-License-Identifier: Apache-2.0 +package azurekeyvault diff --git a/internal/generated/azurekeyvault/router.go b/internal/generated/azurekeyvault/router.go new file mode 100644 index 00000000..110ac66f --- /dev/null +++ b/internal/generated/azurekeyvault/router.go @@ -0,0 +1,39 @@ +// Code generated by devcloud codegen. DO NOT EDIT. +// SPDX-License-Identifier: Apache-2.0 +package azurekeyvault + +import "github.com/skyoo2003/devcloud/internal/shared/httproute" + +// PathParams holds the URL path parameters extracted by the router. +type PathParams = httproute.Params + +// OperationRoute maps an HTTP method and URI pattern to an operation name. +type OperationRoute = httproute.Route + +// OperationRoutes is the list of all routes for this service. +var OperationRoutes = []OperationRoute{ + {Method: "POST", Pattern: "/keys/{key-name}/create?api-version", Operation: "CreateKey"}, + {Method: "DELETE", Pattern: "/keys/{key-name}?api-version", Operation: "DeleteKey"}, + {Method: "DELETE", Pattern: "/secrets/{secret-name}?api-version", Operation: "DeleteSecret"}, + {Method: "GET", Pattern: "/keys/{key-name}?api-version", Operation: "GetKey"}, + {Method: "GET", Pattern: "/secrets/{secret-name}?api-version", Operation: "GetSecret"}, + {Method: "GET", Pattern: "/secrets/{secret-name}/{secret-version}?api-version", Operation: "GetSecretVersion"}, + {Method: "GET", Pattern: "/keys?api-version", Operation: "ListKeys"}, + {Method: "GET", Pattern: "/secrets?api-version", Operation: "ListSecrets"}, + {Method: "PUT", Pattern: "/secrets/{secret-name}?api-version", Operation: "SetSecret"}, +} + +// MatchOperation returns the operation name and extracted path parameters for +// the given HTTP method and request URI. It returns ("", nil) if no route +// matches. +// +// Pass req.URL.RequestURI(), not req.URL.Path: some AWS operations are +// distinguished only by a query string (".../tagging?Operation=Tag"), and a +// bare path can never satisfy those routes. +// +// The matching itself lives in internal/shared/httproute so the generic CRUD +// engine resolves a rest-json operation exactly the way this router does, +// rather than carrying a second copy of the algorithm. +func MatchOperation(method, uri string) (string, PathParams) { + return httproute.Match(OperationRoutes, method, uri) +} diff --git a/internal/generated/azurekeyvault/types.go b/internal/generated/azurekeyvault/types.go new file mode 100644 index 00000000..1933b6f2 --- /dev/null +++ b/internal/generated/azurekeyvault/types.go @@ -0,0 +1,57 @@ +// Code generated by devcloud codegen. DO NOT EDIT. +// SPDX-License-Identifier: Apache-2.0 +package azurekeyvault + +type CreateKeyInput struct { +} + +type CreateKeyOutput struct { +} + +type DeleteKeyInput struct { +} + +type DeleteKeyOutput struct { +} + +type DeleteSecretInput struct { +} + +type DeleteSecretOutput struct { +} + +type GetKeyInput struct { +} + +type GetKeyOutput struct { +} + +type GetSecretInput struct { +} + +type GetSecretOutput struct { +} + +type GetSecretVersionInput struct { +} + +type GetSecretVersionOutput struct { +} + +type ListKeysInput struct { +} + +type ListKeysOutput struct { +} + +type ListSecretsInput struct { +} + +type ListSecretsOutput struct { +} + +type SetSecretInput struct { +} + +type SetSecretOutput struct { +} diff --git a/internal/generated/compat/services.json b/internal/generated/compat/services.json index 0b23ac18..889d0e0e 100644 --- a/internal/generated/compat/services.json +++ b/internal/generated/compat/services.json @@ -1430,6 +1430,29 @@ "UpdateScalingPlan" ] }, + "azureappconfig": { + "protocol": "rest-json", + "servedOps": [ + "DeleteKeyValue", + "GetKeyValue", + "ListKeyValues", + "SetKeyValue" + ] + }, + "azurekeyvault": { + "protocol": "rest-json", + "servedOps": [ + "CreateKey", + "DeleteKey", + "DeleteSecret", + "GetKey", + "GetSecret", + "GetSecretVersion", + "ListKeys", + "ListSecrets", + "SetSecret" + ] + }, "azurequeue": { "protocol": "rest-xml", "servedOps": [ @@ -8829,6 +8852,35 @@ "UpdateStreamGroup" ] }, + "gcppubsub": { + "protocol": "rest-json", + "servedOps": [ + "Acknowledge", + "CreateSubscription", + "CreateTopic", + "DeleteSubscription", + "DeleteTopic", + "GetSubscription", + "GetTopic", + "ListSubscriptions", + "ListTopics", + "Publish", + "Pull" + ] + }, + "gcpsecrets": { + "protocol": "rest-json", + "servedOps": [ + "AccessSecretVersion", + "AddSecretVersion", + "CreateSecret", + "DeleteSecret", + "DestroySecretVersion", + "GetSecret", + "ListSecretVersions", + "ListSecrets" + ] + }, "gcs": { "protocol": "rest-json", "servedOps": [ diff --git a/internal/generated/crudregistry/registry_gen.go b/internal/generated/crudregistry/registry_gen.go index c6f85ace..37b6c717 100644 --- a/internal/generated/crudregistry/registry_gen.go +++ b/internal/generated/crudregistry/registry_gen.go @@ -1277,6 +1277,23 @@ func init() { "GetScalingPlanResourceForecastData": {Verb: "Get", Resource: "ScalingPlanResourceForecastData", OutputListKey: "Datapoints", OutputItemKey: ""}, "UpdateScalingPlan": {Verb: "Update", Resource: "ScalingPlan", OutputListKey: "", OutputItemKey: ""}, }) + crud.Register("azureappconfig", map[string]crud.OpMeta{ + "DeleteKeyValue": {Verb: "Delete", Resource: "KeyValue", OutputListKey: "", OutputItemKey: "", Method: "DELETE", URI: "/kv/{key}"}, + "GetKeyValue": {Verb: "Get", Resource: "KeyValue", OutputListKey: "", OutputItemKey: "", Method: "GET", URI: "/kv/{key}"}, + "ListKeyValues": {Verb: "List", Resource: "KeyValue", OutputListKey: "", OutputItemKey: "", Method: "GET", URI: "/kv"}, + "SetKeyValue": {Verb: "Update", Resource: "KeyValue", OutputListKey: "", OutputItemKey: "", Method: "PUT", URI: "/kv/{key}"}, + }) + crud.Register("azurekeyvault", map[string]crud.OpMeta{ + "CreateKey": {Verb: "Create", Resource: "Key", OutputListKey: "", OutputItemKey: "", Method: "POST", URI: "/keys/{key-name}/create?api-version"}, + "DeleteKey": {Verb: "Delete", Resource: "Key", OutputListKey: "", OutputItemKey: "", Method: "DELETE", URI: "/keys/{key-name}?api-version"}, + "DeleteSecret": {Verb: "Delete", Resource: "Secret", OutputListKey: "", OutputItemKey: "", Method: "DELETE", URI: "/secrets/{secret-name}?api-version"}, + "GetKey": {Verb: "Get", Resource: "Key", OutputListKey: "", OutputItemKey: "", Method: "GET", URI: "/keys/{key-name}?api-version"}, + "GetSecret": {Verb: "Get", Resource: "Secret", OutputListKey: "", OutputItemKey: "", Method: "GET", URI: "/secrets/{secret-name}?api-version"}, + "GetSecretVersion": {Verb: "Get", Resource: "SecretVersion", OutputListKey: "", OutputItemKey: "", Method: "GET", URI: "/secrets/{secret-name}/{secret-version}?api-version"}, + "ListKeys": {Verb: "List", Resource: "Key", OutputListKey: "", OutputItemKey: "", Method: "GET", URI: "/keys?api-version"}, + "ListSecrets": {Verb: "List", Resource: "Secret", OutputListKey: "", OutputItemKey: "", Method: "GET", URI: "/secrets?api-version"}, + "SetSecret": {Verb: "Update", Resource: "Secret", OutputListKey: "", OutputItemKey: "", Method: "PUT", URI: "/secrets/{secret-name}?api-version"}, + }) crud.Register("azurequeue", map[string]crud.OpMeta{ "GetQueueMetadata": {Verb: "Get", Resource: "QueueMetadata", OutputListKey: "", OutputItemKey: "", Method: "GET", URI: "/{queue}?comp=metadata"}, "ListQueues": {Verb: "List", Resource: "Queue", OutputListKey: "", OutputItemKey: "", Method: "GET", URI: "/?comp=list"}, @@ -8078,6 +8095,29 @@ func init() { "UpdateApplication": {Verb: "Update", Resource: "Application", OutputListKey: "ApplicationLogPaths", OutputItemKey: "RuntimeEnvironment", Method: "PATCH", URI: "/applications/{Identifier}"}, "UpdateStreamGroup": {Verb: "Update", Resource: "StreamGroup", OutputListKey: "AssociatedApplications", OutputItemKey: "DefaultApplication", Method: "PATCH", URI: "/streamgroups/{Identifier}"}, }) + crud.Register("gcppubsub", map[string]crud.OpMeta{ + "Acknowledge": {Verb: "", Resource: "", OutputListKey: "", OutputItemKey: "", Method: "POST", URI: "/v1/projects/{project}/subscriptions/{subscription}:acknowledge"}, + "CreateSubscription": {Verb: "Create", Resource: "Subscription", OutputListKey: "", OutputItemKey: "", Method: "PUT", URI: "/v1/projects/{project}/subscriptions/{subscription}"}, + "CreateTopic": {Verb: "Create", Resource: "Topic", OutputListKey: "", OutputItemKey: "", Method: "PUT", URI: "/v1/projects/{project}/topics/{topic}"}, + "DeleteSubscription": {Verb: "Delete", Resource: "Subscription", OutputListKey: "", OutputItemKey: "", Method: "DELETE", URI: "/v1/projects/{project}/subscriptions/{subscription}"}, + "DeleteTopic": {Verb: "Delete", Resource: "Topic", OutputListKey: "", OutputItemKey: "", Method: "DELETE", URI: "/v1/projects/{project}/topics/{topic}"}, + "GetSubscription": {Verb: "Get", Resource: "Subscription", OutputListKey: "", OutputItemKey: "", Method: "GET", URI: "/v1/projects/{project}/subscriptions/{subscription}"}, + "GetTopic": {Verb: "Get", Resource: "Topic", OutputListKey: "", OutputItemKey: "", Method: "GET", URI: "/v1/projects/{project}/topics/{topic}"}, + "ListSubscriptions": {Verb: "List", Resource: "Subscription", OutputListKey: "", OutputItemKey: "", Method: "GET", URI: "/v1/projects/{project}/subscriptions"}, + "ListTopics": {Verb: "List", Resource: "Topic", OutputListKey: "", OutputItemKey: "", Method: "GET", URI: "/v1/projects/{project}/topics"}, + "Publish": {Verb: "", Resource: "", OutputListKey: "", OutputItemKey: "", Method: "POST", URI: "/v1/projects/{project}/topics/{topic}:publish"}, + "Pull": {Verb: "", Resource: "", OutputListKey: "", OutputItemKey: "", Method: "POST", URI: "/v1/projects/{project}/subscriptions/{subscription}:pull"}, + }) + crud.Register("gcpsecrets", map[string]crud.OpMeta{ + "CreateSecret": {Verb: "Create", Resource: "Secret", OutputListKey: "", OutputItemKey: "", Method: "POST", URI: "/v1/projects/{project}/secrets?secretId"}, + "AccessSecretVersion": {Verb: "", Resource: "", OutputListKey: "", OutputItemKey: "", Method: "GET", URI: "/v1/projects/{project}/secrets/{secret}/versions/{version}:access"}, + "AddSecretVersion": {Verb: "Relate", Resource: "SecretVersion", OutputListKey: "", OutputItemKey: "", Method: "POST", URI: "/v1/projects/{project}/secrets/{secret}:addVersion"}, + "DeleteSecret": {Verb: "Delete", Resource: "Secret", OutputListKey: "", OutputItemKey: "", Method: "DELETE", URI: "/v1/projects/{project}/secrets/{secret}"}, + "DestroySecretVersion": {Verb: "", Resource: "", OutputListKey: "", OutputItemKey: "", Method: "POST", URI: "/v1/projects/{project}/secrets/{secret}/versions/{version}:destroy"}, + "GetSecret": {Verb: "Get", Resource: "Secret", OutputListKey: "", OutputItemKey: "", Method: "GET", URI: "/v1/projects/{project}/secrets/{secret}"}, + "ListSecretVersions": {Verb: "List", Resource: "SecretVersion", OutputListKey: "", OutputItemKey: "", Method: "GET", URI: "/v1/projects/{project}/secrets/{secret}/versions"}, + "ListSecrets": {Verb: "List", Resource: "Secret", OutputListKey: "", OutputItemKey: "", Method: "GET", URI: "/v1/projects/{project}/secrets"}, + }) crud.Register("gcs", map[string]crud.OpMeta{ "DeleteBucket": {Verb: "Delete", Resource: "Bucket", OutputListKey: "", OutputItemKey: "", Method: "DELETE", URI: "/storage/v1/b/{bucket}"}, "DeleteObject": {Verb: "Delete", Resource: "Object", OutputListKey: "", OutputItemKey: "", Method: "DELETE", URI: "/storage/v1/b/{bucket}/o/{object+}"}, diff --git a/internal/generated/fidelity/manifest_gen.go b/internal/generated/fidelity/manifest_gen.go index ce01a4d1..dd2bf1f4 100644 --- a/internal/generated/fidelity/manifest_gen.go +++ b/internal/generated/fidelity/manifest_gen.go @@ -1415,6 +1415,23 @@ var Services = map[string]Service{ "GetScalingPlanResourceForecastData": TierAutoCRUD, "UpdateScalingPlan": TierAutoCRUD, }}, + "azureappconfig": {Protocol: "rest-json", ModelBacked: true, EngineWired: false, Operations: map[string]Tier{ + "DeleteKeyValue": TierAutoCRUD, + "GetKeyValue": TierAutoCRUD, + "ListKeyValues": TierAutoCRUD, + "SetKeyValue": TierAutoCRUD, + }}, + "azurekeyvault": {Protocol: "rest-json", ModelBacked: true, EngineWired: true, Operations: map[string]Tier{ + "CreateKey": TierHandVerified, + "DeleteKey": TierHandVerified, + "DeleteSecret": TierHandVerified, + "GetKey": TierHandVerified, + "GetSecret": TierHandVerified, + "GetSecretVersion": TierAutoCRUD, + "ListKeys": TierHandVerified, + "ListSecrets": TierHandVerified, + "SetSecret": TierHandVerified, + }}, "azurequeue": {Protocol: "rest-xml", ModelBacked: true, EngineWired: false, Operations: map[string]Tier{ "ClearMessages": TierHandVerified, "CreateQueue": TierHandVerified, @@ -8593,6 +8610,29 @@ var Services = map[string]Service{ "UpdateApplication": TierAutoCRUD, "UpdateStreamGroup": TierAutoCRUD, }}, + "gcppubsub": {Protocol: "rest-json", ModelBacked: true, EngineWired: true, Operations: map[string]Tier{ + "Acknowledge": TierHandVerified, + "CreateSubscription": TierHandVerified, + "CreateTopic": TierHandVerified, + "DeleteSubscription": TierHandVerified, + "DeleteTopic": TierHandVerified, + "GetSubscription": TierHandVerified, + "GetTopic": TierHandVerified, + "ListSubscriptions": TierHandVerified, + "ListTopics": TierHandVerified, + "Publish": TierHandVerified, + "Pull": TierHandVerified, + }}, + "gcpsecrets": {Protocol: "rest-json", ModelBacked: true, EngineWired: true, Operations: map[string]Tier{ + "AccessSecretVersion": TierHandVerified, + "AddSecretVersion": TierHandVerified, + "CreateSecret": TierHandVerified, + "DeleteSecret": TierHandVerified, + "DestroySecretVersion": TierHandVerified, + "GetSecret": TierHandVerified, + "ListSecretVersions": TierHandVerified, + "ListSecrets": TierHandVerified, + }}, "gcs": {Protocol: "rest-json", ModelBacked: true, EngineWired: false, Operations: map[string]Tier{ "DeleteBucket": TierHandVerified, "DeleteObject": TierHandVerified, diff --git a/internal/generated/gcppubsub/base_provider.go b/internal/generated/gcppubsub/base_provider.go new file mode 100644 index 00000000..d253e32a --- /dev/null +++ b/internal/generated/gcppubsub/base_provider.go @@ -0,0 +1,56 @@ +// Code generated by devcloud codegen. DO NOT EDIT. +// SPDX-License-Identifier: Apache-2.0 +package gcppubsub + +import ( + "context" + "fmt" +) + +var ErrNotImplemented = fmt.Errorf("operation not implemented") + +type BaseProvider struct{} + +func (b *BaseProvider) Acknowledge(ctx context.Context, input *AcknowledgeInput) (*AcknowledgeOutput, error) { + return nil, fmt.Errorf("Acknowledge: %w", ErrNotImplemented) +} + +func (b *BaseProvider) CreateSubscription(ctx context.Context, input *CreateSubscriptionInput) (*CreateSubscriptionOutput, error) { + return nil, fmt.Errorf("CreateSubscription: %w", ErrNotImplemented) +} + +func (b *BaseProvider) CreateTopic(ctx context.Context, input *CreateTopicInput) (*CreateTopicOutput, error) { + return nil, fmt.Errorf("CreateTopic: %w", ErrNotImplemented) +} + +func (b *BaseProvider) DeleteSubscription(ctx context.Context, input *DeleteSubscriptionInput) (*DeleteSubscriptionOutput, error) { + return nil, fmt.Errorf("DeleteSubscription: %w", ErrNotImplemented) +} + +func (b *BaseProvider) DeleteTopic(ctx context.Context, input *DeleteTopicInput) (*DeleteTopicOutput, error) { + return nil, fmt.Errorf("DeleteTopic: %w", ErrNotImplemented) +} + +func (b *BaseProvider) GetSubscription(ctx context.Context, input *GetSubscriptionInput) (*GetSubscriptionOutput, error) { + return nil, fmt.Errorf("GetSubscription: %w", ErrNotImplemented) +} + +func (b *BaseProvider) GetTopic(ctx context.Context, input *GetTopicInput) (*GetTopicOutput, error) { + return nil, fmt.Errorf("GetTopic: %w", ErrNotImplemented) +} + +func (b *BaseProvider) ListSubscriptions(ctx context.Context, input *ListSubscriptionsInput) (*ListSubscriptionsOutput, error) { + return nil, fmt.Errorf("ListSubscriptions: %w", ErrNotImplemented) +} + +func (b *BaseProvider) ListTopics(ctx context.Context, input *ListTopicsInput) (*ListTopicsOutput, error) { + return nil, fmt.Errorf("ListTopics: %w", ErrNotImplemented) +} + +func (b *BaseProvider) Publish(ctx context.Context, input *PublishInput) (*PublishOutput, error) { + return nil, fmt.Errorf("Publish: %w", ErrNotImplemented) +} + +func (b *BaseProvider) Pull(ctx context.Context, input *PullInput) (*PullOutput, error) { + return nil, fmt.Errorf("Pull: %w", ErrNotImplemented) +} diff --git a/internal/generated/gcppubsub/errors.go b/internal/generated/gcppubsub/errors.go new file mode 100644 index 00000000..0ba4ebae --- /dev/null +++ b/internal/generated/gcppubsub/errors.go @@ -0,0 +1,3 @@ +// Code generated by devcloud codegen. DO NOT EDIT. +// SPDX-License-Identifier: Apache-2.0 +package gcppubsub diff --git a/internal/generated/gcppubsub/router.go b/internal/generated/gcppubsub/router.go new file mode 100644 index 00000000..3796787d --- /dev/null +++ b/internal/generated/gcppubsub/router.go @@ -0,0 +1,41 @@ +// Code generated by devcloud codegen. DO NOT EDIT. +// SPDX-License-Identifier: Apache-2.0 +package gcppubsub + +import "github.com/skyoo2003/devcloud/internal/shared/httproute" + +// PathParams holds the URL path parameters extracted by the router. +type PathParams = httproute.Params + +// OperationRoute maps an HTTP method and URI pattern to an operation name. +type OperationRoute = httproute.Route + +// OperationRoutes is the list of all routes for this service. +var OperationRoutes = []OperationRoute{ + {Method: "POST", Pattern: "/v1/projects/{project}/subscriptions/{subscription}:acknowledge", Operation: "Acknowledge"}, + {Method: "PUT", Pattern: "/v1/projects/{project}/subscriptions/{subscription}", Operation: "CreateSubscription"}, + {Method: "PUT", Pattern: "/v1/projects/{project}/topics/{topic}", Operation: "CreateTopic"}, + {Method: "DELETE", Pattern: "/v1/projects/{project}/subscriptions/{subscription}", Operation: "DeleteSubscription"}, + {Method: "DELETE", Pattern: "/v1/projects/{project}/topics/{topic}", Operation: "DeleteTopic"}, + {Method: "GET", Pattern: "/v1/projects/{project}/subscriptions/{subscription}", Operation: "GetSubscription"}, + {Method: "GET", Pattern: "/v1/projects/{project}/topics/{topic}", Operation: "GetTopic"}, + {Method: "GET", Pattern: "/v1/projects/{project}/subscriptions", Operation: "ListSubscriptions"}, + {Method: "GET", Pattern: "/v1/projects/{project}/topics", Operation: "ListTopics"}, + {Method: "POST", Pattern: "/v1/projects/{project}/topics/{topic}:publish", Operation: "Publish"}, + {Method: "POST", Pattern: "/v1/projects/{project}/subscriptions/{subscription}:pull", Operation: "Pull"}, +} + +// MatchOperation returns the operation name and extracted path parameters for +// the given HTTP method and request URI. It returns ("", nil) if no route +// matches. +// +// Pass req.URL.RequestURI(), not req.URL.Path: some AWS operations are +// distinguished only by a query string (".../tagging?Operation=Tag"), and a +// bare path can never satisfy those routes. +// +// The matching itself lives in internal/shared/httproute so the generic CRUD +// engine resolves a rest-json operation exactly the way this router does, +// rather than carrying a second copy of the algorithm. +func MatchOperation(method, uri string) (string, PathParams) { + return httproute.Match(OperationRoutes, method, uri) +} diff --git a/internal/generated/gcppubsub/types.go b/internal/generated/gcppubsub/types.go new file mode 100644 index 00000000..8514b037 --- /dev/null +++ b/internal/generated/gcppubsub/types.go @@ -0,0 +1,69 @@ +// Code generated by devcloud codegen. DO NOT EDIT. +// SPDX-License-Identifier: Apache-2.0 +package gcppubsub + +type AcknowledgeInput struct { +} + +type AcknowledgeOutput struct { +} + +type CreateSubscriptionInput struct { +} + +type CreateSubscriptionOutput struct { +} + +type CreateTopicInput struct { +} + +type CreateTopicOutput struct { +} + +type DeleteSubscriptionInput struct { +} + +type DeleteSubscriptionOutput struct { +} + +type DeleteTopicInput struct { +} + +type DeleteTopicOutput struct { +} + +type GetSubscriptionInput struct { +} + +type GetSubscriptionOutput struct { +} + +type GetTopicInput struct { +} + +type GetTopicOutput struct { +} + +type ListSubscriptionsInput struct { +} + +type ListSubscriptionsOutput struct { +} + +type ListTopicsInput struct { +} + +type ListTopicsOutput struct { +} + +type PublishInput struct { +} + +type PublishOutput struct { +} + +type PullInput struct { +} + +type PullOutput struct { +} diff --git a/internal/generated/gcpsecrets/base_provider.go b/internal/generated/gcpsecrets/base_provider.go new file mode 100644 index 00000000..d27d19c1 --- /dev/null +++ b/internal/generated/gcpsecrets/base_provider.go @@ -0,0 +1,44 @@ +// Code generated by devcloud codegen. DO NOT EDIT. +// SPDX-License-Identifier: Apache-2.0 +package gcpsecrets + +import ( + "context" + "fmt" +) + +var ErrNotImplemented = fmt.Errorf("operation not implemented") + +type BaseProvider struct{} + +func (b *BaseProvider) CreateSecret(ctx context.Context, input *CreateSecretInput) (*CreateSecretOutput, error) { + return nil, fmt.Errorf("CreateSecret: %w", ErrNotImplemented) +} + +func (b *BaseProvider) AccessSecretVersion(ctx context.Context, input *AccessSecretVersionInput) (*AccessSecretVersionOutput, error) { + return nil, fmt.Errorf("AccessSecretVersion: %w", ErrNotImplemented) +} + +func (b *BaseProvider) AddSecretVersion(ctx context.Context, input *AddSecretVersionInput) (*AddSecretVersionOutput, error) { + return nil, fmt.Errorf("AddSecretVersion: %w", ErrNotImplemented) +} + +func (b *BaseProvider) DeleteSecret(ctx context.Context, input *DeleteSecretInput) (*DeleteSecretOutput, error) { + return nil, fmt.Errorf("DeleteSecret: %w", ErrNotImplemented) +} + +func (b *BaseProvider) DestroySecretVersion(ctx context.Context, input *DestroySecretVersionInput) (*DestroySecretVersionOutput, error) { + return nil, fmt.Errorf("DestroySecretVersion: %w", ErrNotImplemented) +} + +func (b *BaseProvider) GetSecret(ctx context.Context, input *GetSecretInput) (*GetSecretOutput, error) { + return nil, fmt.Errorf("GetSecret: %w", ErrNotImplemented) +} + +func (b *BaseProvider) ListSecretVersions(ctx context.Context, input *ListSecretVersionsInput) (*ListSecretVersionsOutput, error) { + return nil, fmt.Errorf("ListSecretVersions: %w", ErrNotImplemented) +} + +func (b *BaseProvider) ListSecrets(ctx context.Context, input *ListSecretsInput) (*ListSecretsOutput, error) { + return nil, fmt.Errorf("ListSecrets: %w", ErrNotImplemented) +} diff --git a/internal/generated/gcpsecrets/errors.go b/internal/generated/gcpsecrets/errors.go new file mode 100644 index 00000000..6b537637 --- /dev/null +++ b/internal/generated/gcpsecrets/errors.go @@ -0,0 +1,3 @@ +// Code generated by devcloud codegen. DO NOT EDIT. +// SPDX-License-Identifier: Apache-2.0 +package gcpsecrets diff --git a/internal/generated/gcpsecrets/router.go b/internal/generated/gcpsecrets/router.go new file mode 100644 index 00000000..19a53d0e --- /dev/null +++ b/internal/generated/gcpsecrets/router.go @@ -0,0 +1,38 @@ +// Code generated by devcloud codegen. DO NOT EDIT. +// SPDX-License-Identifier: Apache-2.0 +package gcpsecrets + +import "github.com/skyoo2003/devcloud/internal/shared/httproute" + +// PathParams holds the URL path parameters extracted by the router. +type PathParams = httproute.Params + +// OperationRoute maps an HTTP method and URI pattern to an operation name. +type OperationRoute = httproute.Route + +// OperationRoutes is the list of all routes for this service. +var OperationRoutes = []OperationRoute{ + {Method: "POST", Pattern: "/v1/projects/{project}/secrets?secretId", Operation: "CreateSecret"}, + {Method: "GET", Pattern: "/v1/projects/{project}/secrets/{secret}/versions/{version}:access", Operation: "AccessSecretVersion"}, + {Method: "POST", Pattern: "/v1/projects/{project}/secrets/{secret}:addVersion", Operation: "AddSecretVersion"}, + {Method: "DELETE", Pattern: "/v1/projects/{project}/secrets/{secret}", Operation: "DeleteSecret"}, + {Method: "POST", Pattern: "/v1/projects/{project}/secrets/{secret}/versions/{version}:destroy", Operation: "DestroySecretVersion"}, + {Method: "GET", Pattern: "/v1/projects/{project}/secrets/{secret}", Operation: "GetSecret"}, + {Method: "GET", Pattern: "/v1/projects/{project}/secrets/{secret}/versions", Operation: "ListSecretVersions"}, + {Method: "GET", Pattern: "/v1/projects/{project}/secrets", Operation: "ListSecrets"}, +} + +// MatchOperation returns the operation name and extracted path parameters for +// the given HTTP method and request URI. It returns ("", nil) if no route +// matches. +// +// Pass req.URL.RequestURI(), not req.URL.Path: some AWS operations are +// distinguished only by a query string (".../tagging?Operation=Tag"), and a +// bare path can never satisfy those routes. +// +// The matching itself lives in internal/shared/httproute so the generic CRUD +// engine resolves a rest-json operation exactly the way this router does, +// rather than carrying a second copy of the algorithm. +func MatchOperation(method, uri string) (string, PathParams) { + return httproute.Match(OperationRoutes, method, uri) +} diff --git a/internal/generated/gcpsecrets/types.go b/internal/generated/gcpsecrets/types.go new file mode 100644 index 00000000..398c29ca --- /dev/null +++ b/internal/generated/gcpsecrets/types.go @@ -0,0 +1,51 @@ +// Code generated by devcloud codegen. DO NOT EDIT. +// SPDX-License-Identifier: Apache-2.0 +package gcpsecrets + +type AccessSecretVersionInput struct { +} + +type AccessSecretVersionOutput struct { +} + +type AddSecretVersionInput struct { +} + +type AddSecretVersionOutput struct { +} + +type CreateSecretInput struct { +} + +type CreateSecretOutput struct { +} + +type DeleteSecretInput struct { +} + +type DeleteSecretOutput struct { +} + +type DestroySecretVersionInput struct { +} + +type DestroySecretVersionOutput struct { +} + +type GetSecretInput struct { +} + +type GetSecretOutput struct { +} + +type ListSecretVersionsInput struct { +} + +type ListSecretVersionsOutput struct { +} + +type ListSecretsInput struct { +} + +type ListSecretsOutput struct { +} diff --git a/internal/services/azureappconfig/provider.go b/internal/services/azureappconfig/provider.go new file mode 100644 index 00000000..63635ed2 --- /dev/null +++ b/internal/services/azureappconfig/provider.go @@ -0,0 +1,203 @@ +// SPDX-License-Identifier: Apache-2.0 + +package azureappconfig + +import ( + "context" + "encoding/json" + "errors" + "io" + "net/http" + "net/url" + "strings" + + generated "github.com/skyoo2003/devcloud/internal/generated/azureappconfig" + "github.com/skyoo2003/devcloud/internal/plugin" +) + +type Provider struct { + generated.BaseProvider + store *Store +} + +func (p *Provider) ServiceID() string { return "azureappconfig" } +func (p *Provider) ServiceName() string { return "Azure App Configuration" } +func (p *Provider) Protocol() plugin.ProtocolType { return plugin.ProtocolRESTJSON } +func (p *Provider) Provider() string { return "azure" } + +func (p *Provider) Init(cfg plugin.PluginConfig) error { + p.store = NewStore() + return nil +} + +func (p *Provider) Shutdown(ctx context.Context) error { + return nil +} + +func (p *Provider) ListResources(ctx context.Context) ([]plugin.Resource, error) { + items, err := p.store.ListKeyValues("default", "", "") + if err != nil { + return nil, err + } + var res []plugin.Resource + for _, item := range items { + res = append(res, plugin.Resource{ + Type: "Microsoft.AppConfiguration/configurationStores/keyValues", + ID: item.Key, + Name: item.Key, + }) + } + return res, nil +} + +func (p *Provider) HandleRequest(ctx context.Context, op string, req *http.Request) (*plugin.Response, error) { + storeName := extractStoreName(req) + path := req.URL.Path + + var queryLabel *string + q := req.URL.Query() + if q.Has("label") { + lbl := q.Get("label") + if lbl == "%00" || lbl == "\x00" { + lbl = "" + } + queryLabel = &lbl + } + + if path == "/kv" || path == "/kv/" { + if req.Method == http.MethodGet { + keyFilter := q.Get("key") + labelFilter := q.Get("label") + items, err := p.store.ListKeyValues(storeName, keyFilter, labelFilter) + if err != nil { + return appConfigErrorResponse(http.StatusInternalServerError, "InternalError", err.Error()) + } + if items == nil { + items = []*KeyValue{} + } + body, _ := json.Marshal(map[string]any{"items": items}) + return &plugin.Response{ + StatusCode: http.StatusOK, + Headers: map[string]string{ + "Content-Type": "application/vnd.microsoft.appconfig.kvset+json; charset=utf-8", + }, + Body: body, + }, nil + } + } + + if strings.HasPrefix(path, "/kv/") { + rawKey := strings.TrimPrefix(path, "/kv/") + key, err := url.PathUnescape(rawKey) + if err != nil { + key = rawKey + } + + switch req.Method { + case http.MethodPut: + var payload struct { + Value string `json:"value"` + ContentType string `json:"content_type"` + Tags map[string]string `json:"tags"` + } + bodyBytes, err := io.ReadAll(req.Body) + if err == nil && len(bodyBytes) > 0 { + _ = json.Unmarshal(bodyBytes, &payload) + } + + kv, err := p.store.SetKeyValue(storeName, key, queryLabel, payload.Value, payload.ContentType, payload.Tags) + if err != nil { + return appConfigErrorResponse(http.StatusInternalServerError, "InternalError", err.Error()) + } + + body, _ := json.Marshal(kv) + return &plugin.Response{ + StatusCode: http.StatusOK, + Headers: map[string]string{ + "Content-Type": "application/vnd.microsoft.appconfig.kv+json; charset=utf-8", + "ETag": kv.ETag, + }, + Body: body, + }, nil + + case http.MethodGet: + kv, err := p.store.GetKeyValue(storeName, key, queryLabel) + if err != nil { + if errors.Is(err, ErrKeyNotFound) { + return appConfigErrorResponse(http.StatusNotFound, "ResourceNotFound", "The key-value was not found.") + } + return appConfigErrorResponse(http.StatusInternalServerError, "InternalError", err.Error()) + } + + body, _ := json.Marshal(kv) + return &plugin.Response{ + StatusCode: http.StatusOK, + Headers: map[string]string{ + "Content-Type": "application/vnd.microsoft.appconfig.kv+json; charset=utf-8", + "ETag": kv.ETag, + }, + Body: body, + }, nil + + case http.MethodDelete: + kv, err := p.store.DeleteKeyValue(storeName, key, queryLabel) + if err != nil { + if errors.Is(err, ErrKeyNotFound) { + return &plugin.Response{ + StatusCode: http.StatusNoContent, + Headers: map[string]string{}, + Body: nil, + }, nil + } + return appConfigErrorResponse(http.StatusInternalServerError, "InternalError", err.Error()) + } + + body, _ := json.Marshal(kv) + return &plugin.Response{ + StatusCode: http.StatusOK, + Headers: map[string]string{ + "Content-Type": "application/vnd.microsoft.appconfig.kv+json; charset=utf-8", + "ETag": kv.ETag, + }, + Body: body, + }, nil + } + } + + return appConfigErrorResponse(http.StatusNotFound, "NotFound", "The requested resource does not exist.") +} + +func extractStoreName(req *http.Request) string { + host := req.Host + if colon := strings.Index(host, ":"); colon != -1 { + host = host[:colon] + } + if strings.HasSuffix(host, ".azconfig.io") { + return strings.TrimSuffix(host, ".azconfig.io") + } + return "default" +} + +func appConfigErrorResponse(status int, name, detail string) (*plugin.Response, error) { + errBody := map[string]any{ + "type": "https://azconfig.io/errors/" + strings.ToLower(name), + "title": name, + "name": name, + "status": status, + "detail": detail, + } + b, _ := json.Marshal(errBody) + return &plugin.Response{ + StatusCode: status, + Headers: map[string]string{ + "Content-Type": "application/problem+json; charset=utf-8", + }, + Body: b, + }, nil +} + +func init() { + plugin.DefaultRegistry.Register("azureappconfig", func() plugin.ServicePlugin { + return &Provider{} + }) +} diff --git a/internal/services/azureappconfig/provider_test.go b/internal/services/azureappconfig/provider_test.go new file mode 100644 index 00000000..cfa523cd --- /dev/null +++ b/internal/services/azureappconfig/provider_test.go @@ -0,0 +1,122 @@ +// SPDX-License-Identifier: Apache-2.0 + +package azureappconfig + +import ( + "bytes" + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "github.com/skyoo2003/devcloud/internal/plugin" +) + +func TestAzureAppConfig_Lifecycle(t *testing.T) { + p := &Provider{} + if err := p.Init(plugin.PluginConfig{}); err != nil { + t.Fatalf("Init failed: %v", err) + } + + ctx := context.Background() + + // 1. Set key-value without label + putBody := []byte(`{"value":"hello-world","content_type":"text/plain","tags":{"env":"dev"}}`) + req1 := httptest.NewRequest(http.MethodPut, "http://myconf.azconfig.io/kv/app:greeting", bytes.NewReader(putBody)) + resp1, err := p.HandleRequest(ctx, "", req1) + if err != nil { + t.Fatalf("HandleRequest failed: %v", err) + } + if resp1.StatusCode != http.StatusOK { + t.Fatalf("expected 200, got %d: %s", resp1.StatusCode, string(resp1.Body)) + } + + var kv1 KeyValue + if err := json.Unmarshal(resp1.Body, &kv1); err != nil { + t.Fatalf("unmarshal failed: %v", err) + } + if kv1.Key != "app:greeting" || kv1.Value != "hello-world" || kv1.Label != nil { + t.Fatalf("unexpected kv: %+v", kv1) + } + if resp1.Headers["ETag"] == "" { + t.Fatalf("expected ETag header") + } + + // 2. Set key-value with label + putBodyProd := []byte(`{"value":"hello-prod"}`) + reqProd := httptest.NewRequest(http.MethodPut, "http://myconf.azconfig.io/kv/app:greeting?label=prod", bytes.NewReader(putBodyProd)) + respProd, err := p.HandleRequest(ctx, "", reqProd) + if err != nil { + t.Fatalf("Set with label failed: %v", err) + } + if respProd.StatusCode != http.StatusOK { + t.Fatalf("expected 200, got %d: %s", respProd.StatusCode, string(respProd.Body)) + } + var kvProd KeyValue + if err := json.Unmarshal(respProd.Body, &kvProd); err != nil { + t.Fatalf("unmarshal failed: %v", err) + } + if kvProd.Label == nil || *kvProd.Label != "prod" || kvProd.Value != "hello-prod" { + t.Fatalf("unexpected kvProd: %+v", kvProd) + } + + // 3. Get key-value without label + req2 := httptest.NewRequest(http.MethodGet, "http://myconf.azconfig.io/kv/app:greeting", nil) + resp2, err := p.HandleRequest(ctx, "", req2) + if err != nil || resp2.StatusCode != http.StatusOK { + t.Fatalf("expected 200, got status=%d err=%v", resp2.StatusCode, err) + } + var kv2 KeyValue + _ = json.Unmarshal(resp2.Body, &kv2) + if kv2.Value != "hello-world" { + t.Fatalf("expected hello-world, got %s", kv2.Value) + } + + // 4. Get key-value with label + reqProdGet := httptest.NewRequest(http.MethodGet, "http://myconf.azconfig.io/kv/app:greeting?label=prod", nil) + respProdGet, err := p.HandleRequest(ctx, "", reqProdGet) + if err != nil || respProdGet.StatusCode != http.StatusOK { + t.Fatalf("expected 200, got status=%d err=%v", respProdGet.StatusCode, err) + } + var kvProdGet KeyValue + _ = json.Unmarshal(respProdGet.Body, &kvProdGet) + if kvProdGet.Value != "hello-prod" { + t.Fatalf("expected hello-prod, got %s", kvProdGet.Value) + } + + // 5. List key-values + reqList := httptest.NewRequest(http.MethodGet, "http://myconf.azconfig.io/kv", nil) + respList, err := p.HandleRequest(ctx, "", reqList) + if err != nil || respList.StatusCode != http.StatusOK { + t.Fatalf("List failed: %v", err) + } + var listRes struct { + Items []KeyValue `json:"items"` + } + if err := json.Unmarshal(respList.Body, &listRes); err != nil { + t.Fatalf("unmarshal list failed: %v", err) + } + if len(listRes.Items) != 2 { + t.Fatalf("expected 2 items, got %d", len(listRes.Items)) + } + + // 6. Delete key-value + reqDel := httptest.NewRequest(http.MethodDelete, "http://myconf.azconfig.io/kv/app:greeting", nil) + respDel, err := p.HandleRequest(ctx, "", reqDel) + if err != nil || respDel.StatusCode != http.StatusOK { + t.Fatalf("Delete failed: %v", err) + } + + // 7. Get deleted key-value returns 404 + respGet404, _ := p.HandleRequest(ctx, "", req2) + if respGet404.StatusCode != http.StatusNotFound { + t.Fatalf("expected 404, got %d", respGet404.StatusCode) + } + + // 8. Delete again returns 204 + respDel2, _ := p.HandleRequest(ctx, "", reqDel) + if respDel2.StatusCode != http.StatusNoContent { + t.Fatalf("expected 204, got %d", respDel2.StatusCode) + } +} diff --git a/internal/services/azureappconfig/store.go b/internal/services/azureappconfig/store.go new file mode 100644 index 00000000..c86a81ec --- /dev/null +++ b/internal/services/azureappconfig/store.go @@ -0,0 +1,160 @@ +// SPDX-License-Identifier: Apache-2.0 + +package azureappconfig + +import ( + "crypto/rand" + "encoding/hex" + "errors" + "fmt" + "strings" + "sync" + "time" +) + +var ( + ErrKeyNotFound = errors.New("KeyNotFound") +) + +type KeyValue struct { + ETag string `json:"etag"` + Key string `json:"key"` + Label *string `json:"label"` + ContentType string `json:"content_type,omitempty"` + Value string `json:"value"` + LastModified string `json:"last_modified"` + Locked bool `json:"locked"` + Tags map[string]string `json:"tags,omitempty"` +} + +type Store struct { + mu sync.RWMutex + items map[string]map[string]*KeyValue // store -> "key\x00label" -> KeyValue +} + +func NewStore() *Store { + return &Store{ + items: make(map[string]map[string]*KeyValue), + } +} + +func itemID(key, label string) string { + return fmt.Sprintf("%s\x00%s", key, label) +} + +func randomETag() string { + b := make([]byte, 12) + _, _ = rand.Read(b) + return hex.EncodeToString(b) +} + +func (s *Store) SetKeyValue(store, key string, label *string, value, contentType string, tags map[string]string) (*KeyValue, error) { + s.mu.Lock() + defer s.mu.Unlock() + + if _, ok := s.items[store]; !ok { + s.items[store] = make(map[string]*KeyValue) + } + + lblVal := "" + if label != nil { + lblVal = *label + } + + id := itemID(key, lblVal) + kv := &KeyValue{ + ETag: randomETag(), + Key: key, + Label: label, + ContentType: contentType, + Value: value, + LastModified: time.Now().UTC().Format(time.RFC3339), + Locked: false, + Tags: tags, + } + + s.items[store][id] = kv + return kv, nil +} + +func (s *Store) GetKeyValue(store, key string, label *string) (*KeyValue, error) { + s.mu.RLock() + defer s.mu.RUnlock() + + storeMap, ok := s.items[store] + if !ok { + return nil, ErrKeyNotFound + } + + lblVal := "" + if label != nil { + lblVal = *label + } + + kv, ok := storeMap[itemID(key, lblVal)] + if !ok { + return nil, ErrKeyNotFound + } + return kv, nil +} + +func (s *Store) DeleteKeyValue(store, key string, label *string) (*KeyValue, error) { + s.mu.Lock() + defer s.mu.Unlock() + + storeMap, ok := s.items[store] + if !ok { + return nil, ErrKeyNotFound + } + + lblVal := "" + if label != nil { + lblVal = *label + } + + id := itemID(key, lblVal) + kv, ok := storeMap[id] + if !ok { + return nil, ErrKeyNotFound + } + + delete(storeMap, id) + return kv, nil +} + +func (s *Store) ListKeyValues(store, keyFilter, labelFilter string) ([]*KeyValue, error) { + s.mu.RLock() + defer s.mu.RUnlock() + + storeMap, ok := s.items[store] + if !ok { + return []*KeyValue{}, nil + } + + var results []*KeyValue + for _, kv := range storeMap { + if !matchFilter(kv.Key, keyFilter) { + continue + } + kvLabel := "" + if kv.Label != nil { + kvLabel = *kv.Label + } + if labelFilter != "" && labelFilter != "*" && !matchFilter(kvLabel, labelFilter) { + continue + } + results = append(results, kv) + } + return results, nil +} + +func matchFilter(val, filter string) bool { + if filter == "" || filter == "*" { + return true + } + if strings.HasSuffix(filter, "*") { + prefix := strings.TrimSuffix(filter, "*") + return strings.HasPrefix(val, prefix) + } + return val == filter +} diff --git a/internal/services/azurekeyvault/provider.go b/internal/services/azurekeyvault/provider.go new file mode 100644 index 00000000..1c7c7e8c --- /dev/null +++ b/internal/services/azurekeyvault/provider.go @@ -0,0 +1,284 @@ +// SPDX-License-Identifier: Apache-2.0 + +package azurekeyvault + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "strings" + + generated "github.com/skyoo2003/devcloud/internal/generated/azurekeyvault" + "github.com/skyoo2003/devcloud/internal/plugin" +) + +type Provider struct { + generated.BaseProvider + store *Store +} + +func (p *Provider) ServiceID() string { return "azurekeyvault" } +func (p *Provider) ServiceName() string { return "Azure Key Vault" } +func (p *Provider) Protocol() plugin.ProtocolType { return plugin.ProtocolRESTJSON } +func (p *Provider) Provider() string { return "azure" } + +func (p *Provider) Init(cfg plugin.PluginConfig) error { + p.store = NewStore() + return nil +} + +func (p *Provider) Shutdown(ctx context.Context) error { + return nil +} + +func (p *Provider) ListResources(ctx context.Context) ([]plugin.Resource, error) { + secs, err := p.store.ListSecrets("default") + if err != nil { + return nil, err + } + var res []plugin.Resource + for _, s := range secs { + res = append(res, plugin.Resource{ + Type: "Microsoft.KeyVault/vaults/secrets", + ID: s.ID, + Name: s.ID, + }) + } + return res, nil +} + +func (p *Provider) HandleRequest(ctx context.Context, op string, req *http.Request) (*plugin.Response, error) { + vault := extractVaultName(req) + matchedOp, params := matchKeyVaultRoute(req.Method, req.URL.RequestURI()) + if matchedOp != "" { + op = matchedOp + } + + switch op { + case "ListSecrets": + items, err := p.store.ListSecrets(vault) + if err != nil { + return azureErrorResponse(http.StatusInternalServerError, "InternalError", err.Error()) + } + body, _ := json.Marshal(map[string]any{"value": items}) + return jsonResponse(http.StatusOK, body) + + case "SetSecret": + name := params["secret-name"] + var payload struct { + Value string `json:"value"` + ContentType string `json:"contentType"` + Tags map[string]string `json:"tags"` + } + bodyBytes, err := io.ReadAll(req.Body) + if err == nil && len(bodyBytes) > 0 { + _ = json.Unmarshal(bodyBytes, &payload) + } + + sec, err := p.store.SetSecret(vault, name, payload.Value, payload.ContentType, payload.Tags) + if err != nil { + return azureErrorResponse(http.StatusInternalServerError, "InternalError", err.Error()) + } + body, _ := json.Marshal(sec) + return jsonResponse(http.StatusOK, body) + + case "GetSecret": + name := params["secret-name"] + version := params["secret-version"] + sec, err := p.store.GetSecret(vault, name, version) + if err != nil { + if errors.Is(err, ErrSecretNotFound) { + return azureErrorResponse(http.StatusNotFound, "SecretNotFound", fmt.Sprintf("A secret with (name/id) %s was not found in this key vault.", name)) + } + return azureErrorResponse(http.StatusInternalServerError, "InternalError", err.Error()) + } + body, _ := json.Marshal(sec) + return jsonResponse(http.StatusOK, body) + + case "GetSecretVersions": + name := params["secret-name"] + versions, err := p.store.GetSecretVersions(vault, name) + if err != nil { + return azureErrorResponse(http.StatusInternalServerError, "InternalError", err.Error()) + } + body, _ := json.Marshal(map[string]any{"value": versions}) + return jsonResponse(http.StatusOK, body) + + case "DeleteSecret": + name := params["secret-name"] + sec, err := p.store.DeleteSecret(vault, name) + if err != nil { + if errors.Is(err, ErrSecretNotFound) { + return azureErrorResponse(http.StatusNotFound, "SecretNotFound", fmt.Sprintf("A secret with (name/id) %s was not found in this key vault.", name)) + } + return azureErrorResponse(http.StatusInternalServerError, "InternalError", err.Error()) + } + body, _ := json.Marshal(sec) + return jsonResponse(http.StatusOK, body) + + case "ListKeys": + items, err := p.store.ListKeys(vault) + if err != nil { + return azureErrorResponse(http.StatusInternalServerError, "InternalError", err.Error()) + } + body, _ := json.Marshal(map[string]any{"value": items}) + return jsonResponse(http.StatusOK, body) + + case "CreateKey": + name := params["key-name"] + var payload struct { + Kty string `json:"kty"` + KeySize int `json:"key_size"` + } + bodyBytes, err := io.ReadAll(req.Body) + if err == nil && len(bodyBytes) > 0 { + _ = json.Unmarshal(bodyBytes, &payload) + } + + k, err := p.store.CreateKey(vault, name, payload.Kty, payload.KeySize) + if err != nil { + return azureErrorResponse(http.StatusInternalServerError, "InternalError", err.Error()) + } + body, _ := json.Marshal(k) + return jsonResponse(http.StatusOK, body) + + case "GetKey": + name := params["key-name"] + k, err := p.store.GetKey(vault, name) + if err != nil { + if errors.Is(err, ErrKeyNotFound) { + return azureErrorResponse(http.StatusNotFound, "KeyNotFound", fmt.Sprintf("A key with (name/id) %s was not found in this key vault.", name)) + } + return azureErrorResponse(http.StatusInternalServerError, "InternalError", err.Error()) + } + body, _ := json.Marshal(k) + return jsonResponse(http.StatusOK, body) + + case "DeleteKey": + name := params["key-name"] + k, err := p.store.DeleteKey(vault, name) + if err != nil { + if errors.Is(err, ErrKeyNotFound) { + return azureErrorResponse(http.StatusNotFound, "KeyNotFound", fmt.Sprintf("A key with (name/id) %s was not found in this key vault.", name)) + } + return azureErrorResponse(http.StatusInternalServerError, "InternalError", err.Error()) + } + body, _ := json.Marshal(k) + return jsonResponse(http.StatusOK, body) + } + + return nil, plugin.ErrUnhandledOp +} + +func matchKeyVaultRoute(method, uri string) (string, map[string]string) { + uPath, _, _ := strings.Cut(uri, "?") + parts := strings.Split(strings.Trim(uPath, "/"), "/") + if len(parts) == 0 { + return "", nil + } + + // 1. /secrets... + if parts[0] == "secrets" { + params := map[string]string{} + if len(parts) == 1 { + if method == http.MethodGet { + return "ListSecrets", params + } + return "", nil + } + + params["secret-name"] = parts[1] + if len(parts) == 2 { + if method == http.MethodPut { + return "SetSecret", params + } + if method == http.MethodGet { + return "GetSecret", params + } + if method == http.MethodDelete { + return "DeleteSecret", params + } + return "", nil + } + + if len(parts) == 3 && method == http.MethodGet { + if parts[2] == "versions" { + return "GetSecretVersions", params + } + params["secret-version"] = parts[2] + return "GetSecret", params + } + return "", nil + } + + // 2. /keys... + if parts[0] == "keys" { + params := map[string]string{} + if len(parts) == 1 { + if method == http.MethodGet { + return "ListKeys", params + } + return "", nil + } + + params["key-name"] = parts[1] + if len(parts) == 2 { + if method == http.MethodGet { + return "GetKey", params + } + if method == http.MethodDelete { + return "DeleteKey", params + } + return "", nil + } + + if len(parts) == 3 && parts[2] == "create" && method == http.MethodPost { + return "CreateKey", params + } + return "", nil + } + + return "", nil +} + +func extractVaultName(req *http.Request) string { + host := req.Host + if strings.Contains(host, ".vault.azure.net") { + return strings.Split(host, ".")[0] + } + return "devvault" +} + +func jsonResponse(status int, body []byte) (*plugin.Response, error) { + return &plugin.Response{ + StatusCode: status, + Headers: map[string]string{"Content-Type": "application/json"}, + Body: body, + ContentType: "application/json", + }, nil +} + +func azureErrorResponse(status int, code, message string) (*plugin.Response, error) { + resp := map[string]any{ + "error": map[string]string{ + "code": code, + "message": message, + }, + } + body, _ := json.Marshal(resp) + return &plugin.Response{ + StatusCode: status, + Headers: map[string]string{"Content-Type": "application/json"}, + Body: body, + ContentType: "application/json", + }, nil +} + +func init() { + plugin.DefaultRegistry.Register("azurekeyvault", func() plugin.ServicePlugin { + return &Provider{} + }) +} diff --git a/internal/services/azurekeyvault/provider_test.go b/internal/services/azurekeyvault/provider_test.go new file mode 100644 index 00000000..3aa8ef96 --- /dev/null +++ b/internal/services/azurekeyvault/provider_test.go @@ -0,0 +1,104 @@ +// SPDX-License-Identifier: Apache-2.0 + +package azurekeyvault + +import ( + "bytes" + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "github.com/skyoo2003/devcloud/internal/plugin" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestAzureKeyVault_Lifecycle(t *testing.T) { + p := &Provider{} + err := p.Init(plugin.PluginConfig{}) + require.NoError(t, err) + + ctx := context.Background() + + // 1. Set Secret + setBody := `{"value":"p@ssword123","contentType":"text/plain","tags":{"env":"prod"}}` + req := httptest.NewRequest("PUT", "/secrets/my-secret?api-version=7.4", bytes.NewReader([]byte(setBody))) + req.Host = "testvault.vault.azure.net" + resp, err := p.HandleRequest(ctx, "SetSecret", req) + require.NoError(t, err) + assert.Equal(t, http.StatusOK, resp.StatusCode) + + var bundle SecretBundle + err = json.Unmarshal(resp.Body, &bundle) + require.NoError(t, err) + assert.Equal(t, "p@ssword123", bundle.Value) + assert.Contains(t, bundle.ID, "https://testvault.vault.azure.net/secrets/my-secret/") + + // 2. Get Secret (Latest) + req = httptest.NewRequest("GET", "/secrets/my-secret?api-version=7.4", nil) + req.Host = "testvault.vault.azure.net" + resp, err = p.HandleRequest(ctx, "GetSecret", req) + require.NoError(t, err) + assert.Equal(t, http.StatusOK, resp.StatusCode) + err = json.Unmarshal(resp.Body, &bundle) + require.NoError(t, err) + assert.Equal(t, "p@ssword123", bundle.Value) + + // 3. List Secrets + req = httptest.NewRequest("GET", "/secrets?api-version=7.4", nil) + req.Host = "testvault.vault.azure.net" + resp, err = p.HandleRequest(ctx, "ListSecrets", req) + require.NoError(t, err) + assert.Equal(t, http.StatusOK, resp.StatusCode) + + var listResp struct { + Value []SecretItem `json:"value"` + } + err = json.Unmarshal(resp.Body, &listResp) + require.NoError(t, err) + assert.Len(t, listResp.Value, 1) + + // 4. Create Key + keyReqBody := `{"kty":"RSA","key_size":2048}` + req = httptest.NewRequest("POST", "/keys/my-rsa-key/create?api-version=7.4", bytes.NewReader([]byte(keyReqBody))) + req.Host = "testvault.vault.azure.net" + resp, err = p.HandleRequest(ctx, "CreateKey", req) + require.NoError(t, err) + assert.Equal(t, http.StatusOK, resp.StatusCode) + + var keyBundle KeyBundle + err = json.Unmarshal(resp.Body, &keyBundle) + require.NoError(t, err) + assert.Equal(t, "RSA", keyBundle.Key.Kty) + assert.Contains(t, keyBundle.Key.Kid, "https://testvault.vault.azure.net/keys/my-rsa-key/") + + // 5. Get Key + req = httptest.NewRequest("GET", "/keys/my-rsa-key?api-version=7.4", nil) + req.Host = "testvault.vault.azure.net" + resp, err = p.HandleRequest(ctx, "GetKey", req) + require.NoError(t, err) + assert.Equal(t, http.StatusOK, resp.StatusCode) + + // 6. Delete Secret + req = httptest.NewRequest("DELETE", "/secrets/my-secret?api-version=7.4", nil) + req.Host = "testvault.vault.azure.net" + resp, err = p.HandleRequest(ctx, "DeleteSecret", req) + require.NoError(t, err) + assert.Equal(t, http.StatusOK, resp.StatusCode) + + // 7. Get Deleted Secret -> 404 + req = httptest.NewRequest("GET", "/secrets/my-secret?api-version=7.4", nil) + req.Host = "testvault.vault.azure.net" + resp, err = p.HandleRequest(ctx, "GetSecret", req) + require.NoError(t, err) + assert.Equal(t, http.StatusNotFound, resp.StatusCode) + + // 8. Delete Key + req = httptest.NewRequest("DELETE", "/keys/my-rsa-key?api-version=7.4", nil) + req.Host = "testvault.vault.azure.net" + resp, err = p.HandleRequest(ctx, "DeleteKey", req) + require.NoError(t, err) + assert.Equal(t, http.StatusOK, resp.StatusCode) +} diff --git a/internal/services/azurekeyvault/store.go b/internal/services/azurekeyvault/store.go new file mode 100644 index 00000000..59b6c0ad --- /dev/null +++ b/internal/services/azurekeyvault/store.go @@ -0,0 +1,273 @@ +// SPDX-License-Identifier: Apache-2.0 + +package azurekeyvault + +import ( + "crypto/rand" + "encoding/hex" + "errors" + "fmt" + "sync" + "time" +) + +var ( + ErrSecretNotFound = errors.New("SecretNotFound") + ErrKeyNotFound = errors.New("KeyNotFound") +) + +type SecretAttributes struct { + Enabled int64 `json:"enabled,omitempty"` // 1 or 0 in boolean terms, or bool in JSON + Created int64 `json:"created,omitempty"` + Updated int64 `json:"updated,omitempty"` +} + +type SecretBundle struct { + ID string `json:"id"` + Value string `json:"value,omitempty"` + ContentType string `json:"contentType,omitempty"` + Tags map[string]string `json:"tags,omitempty"` + Attributes map[string]any `json:"attributes,omitempty"` +} + +type KeyBundle struct { + Key struct { + Kid string `json:"kid"` + Kty string `json:"kty"` + KeyOps []string `json:"key_ops,omitempty"` + N string `json:"n,omitempty"` + E string `json:"e,omitempty"` + } `json:"key"` + Attributes map[string]any `json:"attributes,omitempty"` + Tags map[string]string `json:"tags,omitempty"` +} + +type SecretItem struct { + ID string `json:"id"` + Attributes map[string]any `json:"attributes,omitempty"` + Tags map[string]string `json:"tags,omitempty"` +} + +type KeyItem struct { + Kid string `json:"kid"` + Attributes map[string]any `json:"attributes,omitempty"` + Tags map[string]string `json:"tags,omitempty"` +} + +type Store struct { + mu sync.RWMutex + secrets map[string]map[string]*SecretBundle // vault -> secretName -> latest + history map[string][]*SecretBundle // vault/secretName -> versions + keys map[string]map[string]*KeyBundle // vault -> keyName -> key +} + +func NewStore() *Store { + return &Store{ + secrets: make(map[string]map[string]*SecretBundle), + history: make(map[string][]*SecretBundle), + keys: make(map[string]map[string]*KeyBundle), + } +} + +func (s *Store) SetSecret(vault, name, value, contentType string, tags map[string]string) (*SecretBundle, error) { + s.mu.Lock() + defer s.mu.Unlock() + + if _, exists := s.secrets[vault]; !exists { + s.secrets[vault] = make(map[string]*SecretBundle) + } + + version := randomHex(16) + now := time.Now().Unix() + id := fmt.Sprintf("https://%s.vault.azure.net/secrets/%s/%s", vault, name, version) + + bundle := &SecretBundle{ + ID: id, + Value: value, + ContentType: contentType, + Tags: tags, + Attributes: map[string]any{ + "enabled": true, + "created": now, + "updated": now, + }, + } + + s.secrets[vault][name] = bundle + histKey := vault + "/" + name + s.history[histKey] = append(s.history[histKey], bundle) + + return bundle, nil +} + +func (s *Store) GetSecret(vault, name, version string) (*SecretBundle, error) { + s.mu.RLock() + defer s.mu.RUnlock() + + vaultSecrets, ok := s.secrets[vault] + if !ok { + return nil, ErrSecretNotFound + } + + if version == "" { + sec, ok := vaultSecrets[name] + if !ok { + return nil, ErrSecretNotFound + } + return sec, nil + } + + histKey := vault + "/" + name + for _, v := range s.history[histKey] { + if fmt.Sprintf("https://%s.vault.azure.net/secrets/%s/%s", vault, name, version) == v.ID { + return v, nil + } + } + return nil, ErrSecretNotFound +} + +func (s *Store) ListSecrets(vault string) ([]*SecretItem, error) { + s.mu.RLock() + defer s.mu.RUnlock() + + vaultSecrets, ok := s.secrets[vault] + if !ok { + return []*SecretItem{}, nil + } + + var items []*SecretItem + for _, sec := range vaultSecrets { + items = append(items, &SecretItem{ + ID: sec.ID, + Attributes: sec.Attributes, + Tags: sec.Tags, + }) + } + return items, nil +} + +func (s *Store) GetSecretVersions(vault, name string) ([]*SecretItem, error) { + s.mu.RLock() + defer s.mu.RUnlock() + + histKey := vault + "/" + name + history, ok := s.history[histKey] + if !ok || len(history) == 0 { + return nil, ErrSecretNotFound + } + + var items []*SecretItem + for _, sec := range history { + items = append(items, &SecretItem{ + ID: sec.ID, + Attributes: sec.Attributes, + Tags: sec.Tags, + }) + } + return items, nil +} + +func (s *Store) DeleteSecret(vault, name string) (*SecretBundle, error) { + s.mu.Lock() + defer s.mu.Unlock() + + vaultSecrets, ok := s.secrets[vault] + if !ok { + return nil, ErrSecretNotFound + } + sec, ok := vaultSecrets[name] + if !ok { + return nil, ErrSecretNotFound + } + + delete(vaultSecrets, name) + delete(s.history, vault+"/"+name) + return sec, nil +} + +func (s *Store) CreateKey(vault, name, kty string, keySize int) (*KeyBundle, error) { + s.mu.Lock() + defer s.mu.Unlock() + + if _, exists := s.keys[vault]; !exists { + s.keys[vault] = make(map[string]*KeyBundle) + } + + version := randomHex(16) + kid := fmt.Sprintf("https://%s.vault.azure.net/keys/%s/%s", vault, name, version) + + if kty == "" { + kty = "RSA" + } + + kb := &KeyBundle{ + Attributes: map[string]any{ + "enabled": true, + "created": time.Now().Unix(), + }, + } + kb.Key.Kid = kid + kb.Key.Kty = kty + kb.Key.KeyOps = []string{"encrypt", "decrypt", "sign", "verify", "wrapKey", "unwrapKey"} + kb.Key.N = randomHex(32) + kb.Key.E = "AQAB" + + s.keys[vault][name] = kb + return kb, nil +} + +func (s *Store) GetKey(vault, name string) (*KeyBundle, error) { + s.mu.RLock() + defer s.mu.RUnlock() + + vaultKeys, ok := s.keys[vault] + if !ok { + return nil, ErrKeyNotFound + } + k, ok := vaultKeys[name] + if !ok { + return nil, ErrKeyNotFound + } + return k, nil +} + +func (s *Store) ListKeys(vault string) ([]*KeyItem, error) { + s.mu.RLock() + defer s.mu.RUnlock() + + vaultKeys, ok := s.keys[vault] + if !ok { + return []*KeyItem{}, nil + } + + var items []*KeyItem + for _, k := range vaultKeys { + items = append(items, &KeyItem{ + Kid: k.Key.Kid, + Attributes: k.Attributes, + }) + } + return items, nil +} + +func (s *Store) DeleteKey(vault, name string) (*KeyBundle, error) { + s.mu.Lock() + defer s.mu.Unlock() + + vaultKeys, ok := s.keys[vault] + if !ok { + return nil, ErrKeyNotFound + } + k, ok := vaultKeys[name] + if !ok { + return nil, ErrKeyNotFound + } + delete(vaultKeys, name) + return k, nil +} + +func randomHex(n int) string { + b := make([]byte, n) + _, _ = rand.Read(b) + return hex.EncodeToString(b) +} diff --git a/internal/services/gcppubsub/provider.go b/internal/services/gcppubsub/provider.go new file mode 100644 index 00000000..039ce530 --- /dev/null +++ b/internal/services/gcppubsub/provider.go @@ -0,0 +1,375 @@ +// SPDX-License-Identifier: Apache-2.0 + +package gcppubsub + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "strings" + + generated "github.com/skyoo2003/devcloud/internal/generated/gcppubsub" + "github.com/skyoo2003/devcloud/internal/plugin" +) + +type Provider struct { + generated.BaseProvider + store *Store +} + +func (p *Provider) ServiceID() string { return "gcppubsub" } +func (p *Provider) ServiceName() string { return "Google Cloud PubSub" } +func (p *Provider) Protocol() plugin.ProtocolType { return plugin.ProtocolRESTJSON } +func (p *Provider) Provider() string { return "gcp" } + +func (p *Provider) Init(cfg plugin.PluginConfig) error { + p.store = NewStore() + return nil +} + +func (p *Provider) Shutdown(ctx context.Context) error { + return nil +} + +func (p *Provider) ListResources(ctx context.Context) ([]plugin.Resource, error) { + topics, err := p.store.ListTopics("") + if err != nil { + return nil, err + } + var res []plugin.Resource + for _, t := range topics { + res = append(res, plugin.Resource{ + Type: "pubsub.topic", + ID: t.Name, + Name: t.Name, + }) + } + return res, nil +} + +func (p *Provider) HandleRequest(ctx context.Context, op string, req *http.Request) (*plugin.Response, error) { + matchedOp, params := matchPubSubRoute(req.Method, req.URL.RequestURI()) + if matchedOp != "" { + op = matchedOp + } + + switch op { + case "ListTopics": + project := params["project"] + topics, err := p.store.ListTopics(project) + if err != nil { + return gcpErrorResponse(http.StatusInternalServerError, "InternalError", err.Error()) + } + if topics == nil { + topics = []*Topic{} + } + body, _ := json.Marshal(map[string]any{"topics": topics}) + return jsonResponse(http.StatusOK, body) + + case "CreateTopic": + project := params["project"] + topic := params["topic"] + + var payload struct { + Labels map[string]string `json:"labels"` + } + if bodyBytes, err := io.ReadAll(req.Body); err == nil && len(bodyBytes) > 0 { + _ = json.Unmarshal(bodyBytes, &payload) + } + + t, err := p.store.CreateTopic(project, topic, payload.Labels) + if err != nil { + if errors.Is(err, ErrTopicAlreadyExists) { + return gcpErrorResponse(http.StatusConflict, "AlreadyExists", fmt.Sprintf("Topic [%s] already exists.", topic)) + } + return gcpErrorResponse(http.StatusInternalServerError, "InternalError", err.Error()) + } + body, _ := json.Marshal(t) + return jsonResponse(http.StatusOK, body) + + case "GetTopic": + project := params["project"] + topic := params["topic"] + t, err := p.store.GetTopic(project, topic) + if err != nil { + if errors.Is(err, ErrTopicNotFound) { + return gcpErrorResponse(http.StatusNotFound, "NotFound", fmt.Sprintf("Topic [%s] not found.", topic)) + } + return gcpErrorResponse(http.StatusInternalServerError, "InternalError", err.Error()) + } + body, _ := json.Marshal(t) + return jsonResponse(http.StatusOK, body) + + case "DeleteTopic": + project := params["project"] + topic := params["topic"] + err := p.store.DeleteTopic(project, topic) + if err != nil { + if errors.Is(err, ErrTopicNotFound) { + return gcpErrorResponse(http.StatusNotFound, "NotFound", fmt.Sprintf("Topic [%s] not found.", topic)) + } + return gcpErrorResponse(http.StatusInternalServerError, "InternalError", err.Error()) + } + return jsonResponse(http.StatusOK, []byte("{}")) + + case "Publish": + project := params["project"] + topic := params["topic"] + + var payload struct { + Messages []PubsubMessage `json:"messages"` + } + bodyBytes, err := io.ReadAll(req.Body) + if err != nil { + return gcpErrorResponse(http.StatusBadRequest, "InvalidArgument", "failed to read body") + } + if err := json.Unmarshal(bodyBytes, &payload); err != nil { + return gcpErrorResponse(http.StatusBadRequest, "InvalidArgument", "invalid JSON") + } + + ids, err := p.store.Publish(project, topic, payload.Messages) + if err != nil { + if errors.Is(err, ErrTopicNotFound) { + return gcpErrorResponse(http.StatusNotFound, "NotFound", fmt.Sprintf("Topic [%s] not found.", topic)) + } + return gcpErrorResponse(http.StatusInternalServerError, "InternalError", err.Error()) + } + body, _ := json.Marshal(map[string]any{"messageIds": ids}) + return jsonResponse(http.StatusOK, body) + + case "ListSubscriptions": + project := params["project"] + subs, err := p.store.ListSubscriptions(project) + if err != nil { + return gcpErrorResponse(http.StatusInternalServerError, "InternalError", err.Error()) + } + if subs == nil { + subs = []*Subscription{} + } + body, _ := json.Marshal(map[string]any{"subscriptions": subs}) + return jsonResponse(http.StatusOK, body) + + case "CreateSubscription": + project := params["project"] + subscription := params["subscription"] + + var payload struct { + Topic string `json:"topic"` + AckDeadlineSeconds int `json:"ackDeadlineSeconds"` + Labels map[string]string `json:"labels"` + } + bodyBytes, err := io.ReadAll(req.Body) + if err != nil { + return gcpErrorResponse(http.StatusBadRequest, "InvalidArgument", "failed to read body") + } + if err := json.Unmarshal(bodyBytes, &payload); err != nil { + return gcpErrorResponse(http.StatusBadRequest, "InvalidArgument", "invalid JSON") + } + + sub, err := p.store.CreateSubscription(project, subscription, payload.Topic, payload.AckDeadlineSeconds, payload.Labels) + if err != nil { + if errors.Is(err, ErrSubAlreadyExists) { + return gcpErrorResponse(http.StatusConflict, "AlreadyExists", fmt.Sprintf("Subscription [%s] already exists.", subscription)) + } + return gcpErrorResponse(http.StatusInternalServerError, "InternalError", err.Error()) + } + body, _ := json.Marshal(sub) + return jsonResponse(http.StatusOK, body) + + case "GetSubscription": + project := params["project"] + subscription := params["subscription"] + sub, err := p.store.GetSubscription(project, subscription) + if err != nil { + if errors.Is(err, ErrSubscriptionNotFound) { + return gcpErrorResponse(http.StatusNotFound, "NotFound", fmt.Sprintf("Subscription [%s] not found.", subscription)) + } + return gcpErrorResponse(http.StatusInternalServerError, "InternalError", err.Error()) + } + body, _ := json.Marshal(sub) + return jsonResponse(http.StatusOK, body) + + case "DeleteSubscription": + project := params["project"] + subscription := params["subscription"] + err := p.store.DeleteSubscription(project, subscription) + if err != nil { + if errors.Is(err, ErrSubscriptionNotFound) { + return gcpErrorResponse(http.StatusNotFound, "NotFound", fmt.Sprintf("Subscription [%s] not found.", subscription)) + } + return gcpErrorResponse(http.StatusInternalServerError, "InternalError", err.Error()) + } + return jsonResponse(http.StatusOK, []byte("{}")) + + case "Pull": + project := params["project"] + subscription := params["subscription"] + + var payload struct { + MaxMessages int `json:"maxMessages"` + } + if bodyBytes, err := io.ReadAll(req.Body); err == nil && len(bodyBytes) > 0 { + _ = json.Unmarshal(bodyBytes, &payload) + } + if payload.MaxMessages <= 0 { + payload.MaxMessages = 10 + } + + received, err := p.store.Pull(project, subscription, payload.MaxMessages) + if err != nil { + if errors.Is(err, ErrSubscriptionNotFound) { + return gcpErrorResponse(http.StatusNotFound, "NotFound", fmt.Sprintf("Subscription [%s] not found.", subscription)) + } + return gcpErrorResponse(http.StatusInternalServerError, "InternalError", err.Error()) + } + if received == nil { + received = []*ReceivedMessage{} + } + body, _ := json.Marshal(map[string]any{"receivedMessages": received}) + return jsonResponse(http.StatusOK, body) + + case "Acknowledge": + project := params["project"] + subscription := params["subscription"] + + var payload struct { + AckIDs []string `json:"ackIds"` + } + bodyBytes, err := io.ReadAll(req.Body) + if err != nil { + return gcpErrorResponse(http.StatusBadRequest, "InvalidArgument", "failed to read body") + } + if err := json.Unmarshal(bodyBytes, &payload); err != nil { + return gcpErrorResponse(http.StatusBadRequest, "InvalidArgument", "invalid JSON") + } + + err = p.store.Acknowledge(project, subscription, payload.AckIDs) + if err != nil { + if errors.Is(err, ErrSubscriptionNotFound) { + return gcpErrorResponse(http.StatusNotFound, "NotFound", fmt.Sprintf("Subscription [%s] not found.", subscription)) + } + return gcpErrorResponse(http.StatusInternalServerError, "InternalError", err.Error()) + } + return jsonResponse(http.StatusOK, []byte("{}")) + } + + return nil, plugin.ErrUnhandledOp +} + +func matchPubSubRoute(method, uri string) (string, map[string]string) { + uPath, _, _ := strings.Cut(uri, "?") + parts := strings.Split(strings.Trim(uPath, "/"), "/") + // Expected prefix: ["v1", "projects", "{project}", ...] + if len(parts) < 4 || parts[0] != "v1" || parts[1] != "projects" { + return "", nil + } + + project := parts[2] + resourceType := parts[3] // "topics" or "subscriptions" + + if resourceType == "topics" { + params := map[string]string{"project": project} + if len(parts) == 4 { + if method == http.MethodGet { + return "ListTopics", params + } + return "", nil + } + + topicPart := parts[4] + if tName, action, found := strings.Cut(topicPart, ":"); found { + params["topic"] = tName + if action == "publish" && method == http.MethodPost { + return "Publish", params + } + return "", nil + } + + params["topic"] = topicPart + if len(parts) == 5 { + if method == http.MethodPut { + return "CreateTopic", params + } + if method == http.MethodGet { + return "GetTopic", params + } + if method == http.MethodDelete { + return "DeleteTopic", params + } + } + return "", nil + } + + if resourceType == "subscriptions" { + params := map[string]string{"project": project} + if len(parts) == 4 { + if method == http.MethodGet { + return "ListSubscriptions", params + } + return "", nil + } + + subPart := parts[4] + if sName, action, found := strings.Cut(subPart, ":"); found { + params["subscription"] = sName + if action == "pull" && method == http.MethodPost { + return "Pull", params + } + if action == "acknowledge" && method == http.MethodPost { + return "Acknowledge", params + } + return "", nil + } + + params["subscription"] = subPart + if len(parts) == 5 { + if method == http.MethodPut { + return "CreateSubscription", params + } + if method == http.MethodGet { + return "GetSubscription", params + } + if method == http.MethodDelete { + return "DeleteSubscription", params + } + } + return "", nil + } + + return "", nil +} + +func jsonResponse(status int, body []byte) (*plugin.Response, error) { + return &plugin.Response{ + StatusCode: status, + Headers: map[string]string{"Content-Type": "application/json"}, + Body: body, + ContentType: "application/json", + }, nil +} + +func gcpErrorResponse(status int, reason, message string) (*plugin.Response, error) { + resp := map[string]any{ + "error": map[string]any{ + "code": status, + "message": message, + "status": reason, + }, + } + body, _ := json.Marshal(resp) + return &plugin.Response{ + StatusCode: status, + Headers: map[string]string{"Content-Type": "application/json"}, + Body: body, + ContentType: "application/json", + }, nil +} + +func init() { + plugin.DefaultRegistry.Register("gcppubsub", func() plugin.ServicePlugin { + return &Provider{} + }) +} diff --git a/internal/services/gcppubsub/provider_test.go b/internal/services/gcppubsub/provider_test.go new file mode 100644 index 00000000..833e8ed8 --- /dev/null +++ b/internal/services/gcppubsub/provider_test.go @@ -0,0 +1,123 @@ +// SPDX-License-Identifier: Apache-2.0 + +package gcppubsub + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "github.com/skyoo2003/devcloud/internal/plugin" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestGCPPubSub_Lifecycle(t *testing.T) { + p := &Provider{} + err := p.Init(plugin.PluginConfig{}) + require.NoError(t, err) + + ctx := context.Background() + + // 1. Create Topic + req := httptest.NewRequest("PUT", "/v1/projects/my-proj/topics/order-events", bytes.NewReader([]byte(`{"labels":{"env":"test"}}`))) + resp, err := p.HandleRequest(ctx, "CreateTopic", req) + require.NoError(t, err) + assert.Equal(t, http.StatusOK, resp.StatusCode) + + var topic Topic + err = json.Unmarshal(resp.Body, &topic) + require.NoError(t, err) + assert.Equal(t, "projects/my-proj/topics/order-events", topic.Name) + + // 2. Duplicate Create Topic -> 409 + req = httptest.NewRequest("PUT", "/v1/projects/my-proj/topics/order-events", nil) + resp, err = p.HandleRequest(ctx, "CreateTopic", req) + require.NoError(t, err) + assert.Equal(t, http.StatusConflict, resp.StatusCode) + + // 3. Get Topic + req = httptest.NewRequest("GET", "/v1/projects/my-proj/topics/order-events", nil) + resp, err = p.HandleRequest(ctx, "GetTopic", req) + require.NoError(t, err) + assert.Equal(t, http.StatusOK, resp.StatusCode) + + // 4. Create Subscription + subBody := `{"topic":"projects/my-proj/topics/order-events","ackDeadlineSeconds":30}` + req = httptest.NewRequest("PUT", "/v1/projects/my-proj/subscriptions/order-worker", bytes.NewReader([]byte(subBody))) + resp, err = p.HandleRequest(ctx, "CreateSubscription", req) + require.NoError(t, err) + assert.Equal(t, http.StatusOK, resp.StatusCode) + + var sub Subscription + err = json.Unmarshal(resp.Body, &sub) + require.NoError(t, err) + assert.Equal(t, "projects/my-proj/subscriptions/order-worker", sub.Name) + assert.Equal(t, "projects/my-proj/topics/order-events", sub.Topic) + + // 5. Publish Message + payloadData := base64.StdEncoding.EncodeToString([]byte("order #1001 created")) + publishBody := `{"messages":[{"data":"` + payloadData + `","attributes":{"type":"checkout"}}]}` + req = httptest.NewRequest("POST", "/v1/projects/my-proj/topics/order-events:publish", bytes.NewReader([]byte(publishBody))) + resp, err = p.HandleRequest(ctx, "Publish", req) + require.NoError(t, err) + assert.Equal(t, http.StatusOK, resp.StatusCode) + + var publishResp struct { + MessageIDs []string `json:"messageIds"` + } + err = json.Unmarshal(resp.Body, &publishResp) + require.NoError(t, err) + assert.Len(t, publishResp.MessageIDs, 1) + + // 6. Pull Message + pullBody := `{"maxMessages":10}` + req = httptest.NewRequest("POST", "/v1/projects/my-proj/subscriptions/order-worker:pull", bytes.NewReader([]byte(pullBody))) + resp, err = p.HandleRequest(ctx, "Pull", req) + require.NoError(t, err) + assert.Equal(t, http.StatusOK, resp.StatusCode) + + var pullResp struct { + ReceivedMessages []ReceivedMessage `json:"receivedMessages"` + } + err = json.Unmarshal(resp.Body, &pullResp) + require.NoError(t, err) + require.Len(t, pullResp.ReceivedMessages, 1) + + recMsg := pullResp.ReceivedMessages[0] + assert.NotEmpty(t, recMsg.AckID) + assert.Equal(t, payloadData, recMsg.Message.Data) + assert.Equal(t, "checkout", recMsg.Message.Attributes["type"]) + + // 7. Acknowledge Message + ackBody := `{"ackIds":["` + recMsg.AckID + `"]}` + req = httptest.NewRequest("POST", "/v1/projects/my-proj/subscriptions/order-worker:acknowledge", bytes.NewReader([]byte(ackBody))) + resp, err = p.HandleRequest(ctx, "Acknowledge", req) + require.NoError(t, err) + assert.Equal(t, http.StatusOK, resp.StatusCode) + + // 8. Pull Again -> No messages + req = httptest.NewRequest("POST", "/v1/projects/my-proj/subscriptions/order-worker:pull", bytes.NewReader([]byte(pullBody))) + resp, err = p.HandleRequest(ctx, "Pull", req) + require.NoError(t, err) + assert.Equal(t, http.StatusOK, resp.StatusCode) + err = json.Unmarshal(resp.Body, &pullResp) + require.NoError(t, err) + assert.Len(t, pullResp.ReceivedMessages, 0) + + // 9. Delete Subscription + req = httptest.NewRequest("DELETE", "/v1/projects/my-proj/subscriptions/order-worker", nil) + resp, err = p.HandleRequest(ctx, "DeleteSubscription", req) + require.NoError(t, err) + assert.Equal(t, http.StatusOK, resp.StatusCode) + + // 10. Delete Topic + req = httptest.NewRequest("DELETE", "/v1/projects/my-proj/topics/order-events", nil) + resp, err = p.HandleRequest(ctx, "DeleteTopic", req) + require.NoError(t, err) + assert.Equal(t, http.StatusOK, resp.StatusCode) +} diff --git a/internal/services/gcppubsub/store.go b/internal/services/gcppubsub/store.go new file mode 100644 index 00000000..4d2d5f06 --- /dev/null +++ b/internal/services/gcppubsub/store.go @@ -0,0 +1,292 @@ +// SPDX-License-Identifier: Apache-2.0 + +package gcppubsub + +import ( + "crypto/rand" + "encoding/hex" + "errors" + "fmt" + "sync" + "time" +) + +var ( + ErrTopicNotFound = errors.New("TopicNotFound") + ErrTopicAlreadyExists = errors.New("TopicAlreadyExists") + ErrSubscriptionNotFound = errors.New("SubscriptionNotFound") + ErrSubAlreadyExists = errors.New("SubscriptionAlreadyExists") +) + +type PubsubMessage struct { + MessageID string `json:"messageId"` + Data string `json:"data"` // base64 + Attributes map[string]string `json:"attributes,omitempty"` + PublishTime string `json:"publishTime"` +} + +type Topic struct { + Name string `json:"name"` + Labels map[string]string `json:"labels,omitempty"` +} + +type Subscription struct { + Name string `json:"name"` + Topic string `json:"topic"` + AckDeadlineSeconds int `json:"ackDeadlineSeconds,omitempty"` + Labels map[string]string `json:"labels,omitempty"` + + mu sync.Mutex + unackedMessages map[string]*ReceivedMessage // ackID -> message + pendingQueue []*PubsubMessage +} + +type ReceivedMessage struct { + AckID string `json:"ackId"` + Message PubsubMessage `json:"message"` + DeliveryTime time.Time `json:"-"` +} + +type Store struct { + mu sync.RWMutex + topics map[string]*Topic // key: "projects/{project}/topics/{topic}" + subscriptions map[string]*Subscription // key: "projects/{project}/subscriptions/{sub}" +} + +func NewStore() *Store { + return &Store{ + topics: make(map[string]*Topic), + subscriptions: make(map[string]*Subscription), + } +} + +func topicKey(project, topicId string) string { + return fmt.Sprintf("projects/%s/topics/%s", project, topicId) +} + +func subKey(project, subId string) string { + return fmt.Sprintf("projects/%s/subscriptions/%s", project, subId) +} + +func (s *Store) CreateTopic(project, topicId string, labels map[string]string) (*Topic, error) { + s.mu.Lock() + defer s.mu.Unlock() + + key := topicKey(project, topicId) + if _, exists := s.topics[key]; exists { + return nil, ErrTopicAlreadyExists + } + + t := &Topic{ + Name: key, + Labels: labels, + } + s.topics[key] = t + return t, nil +} + +func (s *Store) GetTopic(project, topicId string) (*Topic, error) { + s.mu.RLock() + defer s.mu.RUnlock() + + key := topicKey(project, topicId) + t, exists := s.topics[key] + if !exists { + return nil, ErrTopicNotFound + } + return t, nil +} + +func (s *Store) ListTopics(project string) ([]*Topic, error) { + s.mu.RLock() + defer s.mu.RUnlock() + + prefix := fmt.Sprintf("projects/%s/topics/", project) + var list []*Topic + for k, t := range s.topics { + if project == "" || len(k) >= len(prefix) && k[:len(prefix)] == prefix { + list = append(list, t) + } + } + return list, nil +} + +func (s *Store) DeleteTopic(project, topicId string) error { + s.mu.Lock() + defer s.mu.Unlock() + + key := topicKey(project, topicId) + if _, exists := s.topics[key]; !exists { + return ErrTopicNotFound + } + delete(s.topics, key) + return nil +} + +func (s *Store) Publish(project, topicId string, messages []PubsubMessage) ([]string, error) { + s.mu.RLock() + key := topicKey(project, topicId) + _, exists := s.topics[key] + if !exists { + s.mu.RUnlock() + return nil, ErrTopicNotFound + } + + // Find all subscriptions matching this topic + var matchedSubs []*Subscription + for _, sub := range s.subscriptions { + if sub.Topic == key { + matchedSubs = append(matchedSubs, sub) + } + } + s.mu.RUnlock() + + var messageIDs []string + now := time.Now().UTC().Format(time.RFC3339) + + for _, m := range messages { + id := randomMessageID() + m.MessageID = id + m.PublishTime = now + messageIDs = append(messageIDs, id) + + for _, sub := range matchedSubs { + sub.mu.Lock() + msgCopy := m + sub.pendingQueue = append(sub.pendingQueue, &msgCopy) + sub.mu.Unlock() + } + } + + return messageIDs, nil +} + +func (s *Store) CreateSubscription(project, subId, topicFullName string, ackDeadline int, labels map[string]string) (*Subscription, error) { + s.mu.Lock() + defer s.mu.Unlock() + + key := subKey(project, subId) + if _, exists := s.subscriptions[key]; exists { + return nil, ErrSubAlreadyExists + } + + if ackDeadline <= 0 { + ackDeadline = 10 + } + + sub := &Subscription{ + Name: key, + Topic: topicFullName, + AckDeadlineSeconds: ackDeadline, + Labels: labels, + unackedMessages: make(map[string]*ReceivedMessage), + } + s.subscriptions[key] = sub + return sub, nil +} + +func (s *Store) GetSubscription(project, subId string) (*Subscription, error) { + s.mu.RLock() + defer s.mu.RUnlock() + + key := subKey(project, subId) + sub, exists := s.subscriptions[key] + if !exists { + return nil, ErrSubscriptionNotFound + } + return sub, nil +} + +func (s *Store) ListSubscriptions(project string) ([]*Subscription, error) { + s.mu.RLock() + defer s.mu.RUnlock() + + prefix := fmt.Sprintf("projects/%s/subscriptions/", project) + var list []*Subscription + for k, sub := range s.subscriptions { + if project == "" || len(k) >= len(prefix) && k[:len(prefix)] == prefix { + list = append(list, sub) + } + } + return list, nil +} + +func (s *Store) DeleteSubscription(project, subId string) error { + s.mu.Lock() + defer s.mu.Unlock() + + key := subKey(project, subId) + if _, exists := s.subscriptions[key]; !exists { + return ErrSubscriptionNotFound + } + delete(s.subscriptions, key) + return nil +} + +func (s *Store) Pull(project, subId string, maxMessages int) ([]*ReceivedMessage, error) { + s.mu.RLock() + key := subKey(project, subId) + sub, exists := s.subscriptions[key] + s.mu.RUnlock() + + if !exists { + return nil, ErrSubscriptionNotFound + } + + sub.mu.Lock() + defer sub.mu.Unlock() + + if maxMessages <= 0 { + maxMessages = 1 + } + if maxMessages > len(sub.pendingQueue) { + maxMessages = len(sub.pendingQueue) + } + + var received []*ReceivedMessage + for i := 0; i < maxMessages; i++ { + msg := sub.pendingQueue[i] + ackID := randomAckID() + rec := &ReceivedMessage{ + AckID: ackID, + Message: *msg, + DeliveryTime: time.Now(), + } + sub.unackedMessages[ackID] = rec + received = append(received, rec) + } + sub.pendingQueue = sub.pendingQueue[maxMessages:] + + return received, nil +} + +func (s *Store) Acknowledge(project, subId string, ackIDs []string) error { + s.mu.RLock() + key := subKey(project, subId) + sub, exists := s.subscriptions[key] + s.mu.RUnlock() + + if !exists { + return ErrSubscriptionNotFound + } + + sub.mu.Lock() + defer sub.mu.Unlock() + + for _, id := range ackIDs { + delete(sub.unackedMessages, id) + } + return nil +} + +func randomMessageID() string { + b := make([]byte, 8) + _, _ = rand.Read(b) + return fmt.Sprintf("%d", time.Now().UnixNano()/1e6) + hex.EncodeToString(b[:4]) +} + +func randomAckID() string { + b := make([]byte, 16) + _, _ = rand.Read(b) + return hex.EncodeToString(b) +} diff --git a/internal/services/gcpsecrets/provider.go b/internal/services/gcpsecrets/provider.go new file mode 100644 index 00000000..9b69aa8e --- /dev/null +++ b/internal/services/gcpsecrets/provider.go @@ -0,0 +1,372 @@ +// SPDX-License-Identifier: Apache-2.0 + +package gcpsecrets + +import ( + "context" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "strings" + + generated "github.com/skyoo2003/devcloud/internal/generated/gcpsecrets" + "github.com/skyoo2003/devcloud/internal/plugin" +) + +type Provider struct { + generated.BaseProvider + dataDir string + store *Store +} + +func (p *Provider) ServiceID() string { return "gcpsecrets" } +func (p *Provider) ServiceName() string { return "Google Cloud Secret Manager" } +func (p *Provider) Protocol() plugin.ProtocolType { return plugin.ProtocolRESTJSON } +func (p *Provider) Provider() string { return "gcp" } + +func (p *Provider) Init(cfg plugin.PluginConfig) error { + p.dataDir = cfg.DataDir + if p.dataDir == "" { + p.dataDir = "./data/gcp-secrets" + } + p.store = NewStore(p.dataDir) + return nil +} + +func (p *Provider) Shutdown(ctx context.Context) error { + return nil +} + +func (p *Provider) ListResources(ctx context.Context) ([]plugin.Resource, error) { + secs, err := p.store.ListSecrets("") + if err != nil { + return nil, err + } + var res []plugin.Resource + for _, s := range secs { + res = append(res, plugin.Resource{ + Type: "secretmanager.secret", + ID: s.Name, + Name: s.Name, + }) + } + return res, nil +} + +func (p *Provider) HandleRequest(ctx context.Context, op string, req *http.Request) (*plugin.Response, error) { + matchedOp, params := matchSecretRoute(req.Method, req.URL.RequestURI()) + if matchedOp != "" { + op = matchedOp + } + + switch op { + case "ListSecrets": + project := params["project"] + if project == "" { + project = extractProjectFromPath(req.URL.Path) + } + secs, err := p.store.ListSecrets(project) + if err != nil { + return gcpErrorResponse(http.StatusInternalServerError, "InternalError", err.Error()) + } + if secs == nil { + secs = []*Secret{} + } + body, _ := json.Marshal(map[string]any{"secrets": secs}) + return &plugin.Response{ + StatusCode: http.StatusOK, + Headers: map[string]string{"Content-Type": "application/json"}, + Body: body, + ContentType: "application/json", + }, nil + + case "CreateSecret": + project := params["project"] + secretId := params["secretId"] + if secretId == "" { + secretId = req.URL.Query().Get("secretId") + } + if project == "" { + project = extractProjectFromPath(req.URL.Path) + } + + var payload struct { + Labels map[string]string `json:"labels"` + } + if bodyBytes, err := io.ReadAll(req.Body); err == nil && len(bodyBytes) > 0 { + _ = json.Unmarshal(bodyBytes, &payload) + } + + sec, err := p.store.CreateSecret(project, secretId, payload.Labels) + if err != nil { + if errors.Is(err, ErrSecretAlreadyExists) { + return gcpErrorResponse(http.StatusConflict, "AlreadyExists", fmt.Sprintf("Secret [%s] already exists.", secretId)) + } + return gcpErrorResponse(http.StatusInternalServerError, "InternalError", err.Error()) + } + body, _ := json.Marshal(sec) + return &plugin.Response{ + StatusCode: http.StatusOK, + Headers: map[string]string{"Content-Type": "application/json"}, + Body: body, + ContentType: "application/json", + }, nil + + case "GetSecret": + project := params["project"] + secret := params["secret"] + sec, err := p.store.GetSecret(project, secret) + if err != nil { + if errors.Is(err, ErrSecretNotFound) { + return gcpErrorResponse(http.StatusNotFound, "NotFound", fmt.Sprintf("Secret [%s] not found.", secret)) + } + return gcpErrorResponse(http.StatusInternalServerError, "InternalError", err.Error()) + } + body, _ := json.Marshal(sec) + return &plugin.Response{ + StatusCode: http.StatusOK, + Headers: map[string]string{"Content-Type": "application/json"}, + Body: body, + ContentType: "application/json", + }, nil + + case "DeleteSecret": + project := params["project"] + secret := params["secret"] + err := p.store.DeleteSecret(project, secret) + if err != nil { + if errors.Is(err, ErrSecretNotFound) { + return gcpErrorResponse(http.StatusNotFound, "NotFound", fmt.Sprintf("Secret [%s] not found.", secret)) + } + return gcpErrorResponse(http.StatusInternalServerError, "InternalError", err.Error()) + } + return &plugin.Response{ + StatusCode: http.StatusOK, + Headers: map[string]string{"Content-Type": "application/json"}, + Body: []byte("{}"), + ContentType: "application/json", + }, nil + + case "AddSecretVersion": + project := params["project"] + secret := params["secret"] + + var reqPayload struct { + Payload struct { + Data string `json:"data"` // base64 encoded + } `json:"payload"` + } + bodyBytes, err := io.ReadAll(req.Body) + if err != nil { + return gcpErrorResponse(http.StatusBadRequest, "InvalidArgument", "failed to read request body") + } + if err := json.Unmarshal(bodyBytes, &reqPayload); err != nil { + return gcpErrorResponse(http.StatusBadRequest, "InvalidArgument", "invalid JSON") + } + + data, err := base64.StdEncoding.DecodeString(reqPayload.Payload.Data) + if err != nil { + // fallback: treat as raw string if not base64 + data = []byte(reqPayload.Payload.Data) + } + + v, err := p.store.AddSecretVersion(project, secret, data) + if err != nil { + if errors.Is(err, ErrSecretNotFound) { + return gcpErrorResponse(http.StatusNotFound, "NotFound", fmt.Sprintf("Secret [%s] not found.", secret)) + } + return gcpErrorResponse(http.StatusInternalServerError, "InternalError", err.Error()) + } + + body, _ := json.Marshal(v) + return &plugin.Response{ + StatusCode: http.StatusOK, + Headers: map[string]string{"Content-Type": "application/json"}, + Body: body, + ContentType: "application/json", + }, nil + + case "ListSecretVersions": + project := params["project"] + secret := params["secret"] + versions, err := p.store.ListSecretVersions(project, secret) + if err != nil { + if errors.Is(err, ErrSecretNotFound) { + return gcpErrorResponse(http.StatusNotFound, "NotFound", fmt.Sprintf("Secret [%s] not found.", secret)) + } + return gcpErrorResponse(http.StatusInternalServerError, "InternalError", err.Error()) + } + if versions == nil { + versions = []*SecretVersion{} + } + body, _ := json.Marshal(map[string]any{"versions": versions}) + return &plugin.Response{ + StatusCode: http.StatusOK, + Headers: map[string]string{"Content-Type": "application/json"}, + Body: body, + ContentType: "application/json", + }, nil + + case "AccessSecretVersion": + project := params["project"] + secret := params["secret"] + version := params["version"] + + v, data, err := p.store.AccessSecretVersion(project, secret, version) + if err != nil { + if errors.Is(err, ErrSecretNotFound) || errors.Is(err, ErrVersionNotFound) { + return gcpErrorResponse(http.StatusNotFound, "NotFound", "Secret or version not found.") + } + if errors.Is(err, ErrVersionDestroyed) { + return gcpErrorResponse(http.StatusBadRequest, "FailedPrecondition", "Secret version is destroyed.") + } + return gcpErrorResponse(http.StatusInternalServerError, "InternalError", err.Error()) + } + + resp := map[string]any{ + "name": v.Name, + "payload": map[string]string{ + "data": base64.StdEncoding.EncodeToString(data), + }, + } + body, _ := json.Marshal(resp) + return &plugin.Response{ + StatusCode: http.StatusOK, + Headers: map[string]string{"Content-Type": "application/json"}, + Body: body, + ContentType: "application/json", + }, nil + + case "DestroySecretVersion": + project := params["project"] + secret := params["secret"] + version := params["version"] + + v, err := p.store.DestroySecretVersion(project, secret, version) + if err != nil { + if errors.Is(err, ErrSecretNotFound) || errors.Is(err, ErrVersionNotFound) { + return gcpErrorResponse(http.StatusNotFound, "NotFound", "Secret or version not found.") + } + return gcpErrorResponse(http.StatusInternalServerError, "InternalError", err.Error()) + } + + body, _ := json.Marshal(v) + return &plugin.Response{ + StatusCode: http.StatusOK, + Headers: map[string]string{"Content-Type": "application/json"}, + Body: body, + ContentType: "application/json", + }, nil + } + + return nil, plugin.ErrUnhandledOp +} + +func extractProjectFromPath(path string) string { + parts := strings.Split(strings.Trim(path, "/"), "/") + if len(parts) >= 3 && parts[0] == "v1" && parts[1] == "projects" { + return parts[2] + } + return "default" +} + +func gcpErrorResponse(status int, reason, message string) (*plugin.Response, error) { + resp := map[string]any{ + "error": map[string]any{ + "code": status, + "message": message, + "status": reason, + }, + } + body, _ := json.Marshal(resp) + return &plugin.Response{ + StatusCode: status, + Headers: map[string]string{"Content-Type": "application/json"}, + Body: body, + ContentType: "application/json", + }, nil +} + +func matchSecretRoute(method, uri string) (string, map[string]string) { + uPath, _, _ := strings.Cut(uri, "?") + parts := strings.Split(strings.Trim(uPath, "/"), "/") + // Expected prefix: ["v1", "projects", "{project}", ...] + if len(parts) < 4 || parts[0] != "v1" || parts[1] != "projects" || parts[3] != "secrets" { + return "", nil + } + + project := parts[2] + params := map[string]string{"project": project} + + // 1. /v1/projects/{project}/secrets + if len(parts) == 4 { + if method == http.MethodGet { + return "ListSecrets", params + } + if method == http.MethodPost { + return "CreateSecret", params + } + return "", nil + } + + // 2. /v1/projects/{project}/secrets/{secret...} + secretPart := parts[4] + + // Check if secretPart has custom method: e.g. "my-secret:addVersion" + if sName, action, found := strings.Cut(secretPart, ":"); found { + params["secret"] = sName + if action == "addVersion" && method == http.MethodPost { + return "AddSecretVersion", params + } + return "", nil + } + params["secret"] = secretPart + + if len(parts) == 5 { + if method == http.MethodGet { + return "GetSecret", params + } + if method == http.MethodDelete { + return "DeleteSecret", params + } + return "", nil + } + + // 3. /v1/projects/{project}/secrets/{secret}/versions... + if parts[5] == "versions" { + if len(parts) == 6 { + if method == http.MethodGet { + return "ListSecretVersions", params + } + return "", nil + } + + versionPart := parts[6] + if vNum, action, found := strings.Cut(versionPart, ":"); found { + params["version"] = vNum + if action == "access" && method == http.MethodGet { + return "AccessSecretVersion", params + } + if action == "destroy" && method == http.MethodPost { + return "DestroySecretVersion", params + } + return "", nil + } + + params["version"] = versionPart + if method == http.MethodGet { + return "GetSecretVersion", params + } + } + + return "", nil +} + +func init() { + plugin.DefaultRegistry.Register("gcpsecrets", func() plugin.ServicePlugin { + return &Provider{} + }) +} diff --git a/internal/services/gcpsecrets/provider_test.go b/internal/services/gcpsecrets/provider_test.go new file mode 100644 index 00000000..c915254f --- /dev/null +++ b/internal/services/gcpsecrets/provider_test.go @@ -0,0 +1,119 @@ +// SPDX-License-Identifier: Apache-2.0 + +package gcpsecrets + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "github.com/skyoo2003/devcloud/internal/plugin" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestGCPSecrets_Lifecycle(t *testing.T) { + p := &Provider{} + err := p.Init(plugin.PluginConfig{DataDir: t.TempDir()}) + require.NoError(t, err) + + ctx := context.Background() + + // 1. Create Secret + reqBody := `{"labels":{"env":"test"}}` + req := httptest.NewRequest("POST", "/v1/projects/my-proj/secrets?secretId=db-password", bytes.NewReader([]byte(reqBody))) + resp, err := p.HandleRequest(ctx, "CreateSecret", req) + require.NoError(t, err) + assert.Equal(t, http.StatusOK, resp.StatusCode) + + var sec Secret + err = json.Unmarshal(resp.Body, &sec) + require.NoError(t, err) + assert.Equal(t, "projects/my-proj/secrets/db-password", sec.Name) + + // 2. Duplicate Create -> 409 + req = httptest.NewRequest("POST", "/v1/projects/my-proj/secrets?secretId=db-password", bytes.NewReader([]byte(reqBody))) + resp, err = p.HandleRequest(ctx, "CreateSecret", req) + require.NoError(t, err) + assert.Equal(t, http.StatusConflict, resp.StatusCode) + + // 3. Get Secret + req = httptest.NewRequest("GET", "/v1/projects/my-proj/secrets/db-password", nil) + resp, err = p.HandleRequest(ctx, "GetSecret", req) + require.NoError(t, err) + assert.Equal(t, http.StatusOK, resp.StatusCode) + + // 4. Add Secret Version + secretPayload := "super-secret-pass-123" + b64Payload := base64.StdEncoding.EncodeToString([]byte(secretPayload)) + versionReqBody := fmtPayload(b64Payload) + req = httptest.NewRequest("POST", "/v1/projects/my-proj/secrets/db-password:addVersion", bytes.NewReader([]byte(versionReqBody))) + resp, err = p.HandleRequest(ctx, "AddSecretVersion", req) + require.NoError(t, err) + assert.Equal(t, http.StatusOK, resp.StatusCode) + + var v SecretVersion + err = json.Unmarshal(resp.Body, &v) + require.NoError(t, err) + assert.Equal(t, "projects/my-proj/secrets/db-password/versions/1", v.Name) + + // 5. Access Secret Version (by number and by "latest") + req = httptest.NewRequest("GET", "/v1/projects/my-proj/secrets/db-password/versions/1:access", nil) + resp, err = p.HandleRequest(ctx, "AccessSecretVersion", req) + require.NoError(t, err) + assert.Equal(t, http.StatusOK, resp.StatusCode) + + var accessResp struct { + Payload struct { + Data string `json:"data"` + } `json:"payload"` + } + err = json.Unmarshal(resp.Body, &accessResp) + require.NoError(t, err) + decoded, _ := base64.StdEncoding.DecodeString(accessResp.Payload.Data) + assert.Equal(t, secretPayload, string(decoded)) + + // latest + req = httptest.NewRequest("GET", "/v1/projects/my-proj/secrets/db-password/versions/latest:access", nil) + resp, err = p.HandleRequest(ctx, "AccessSecretVersion", req) + require.NoError(t, err) + assert.Equal(t, http.StatusOK, resp.StatusCode) + + // 6. List Secrets + req = httptest.NewRequest("GET", "/v1/projects/my-proj/secrets", nil) + resp, err = p.HandleRequest(ctx, "ListSecrets", req) + require.NoError(t, err) + assert.Equal(t, http.StatusOK, resp.StatusCode) + + // 7. Destroy Version + req = httptest.NewRequest("POST", "/v1/projects/my-proj/secrets/db-password/versions/1:destroy", nil) + resp, err = p.HandleRequest(ctx, "DestroySecretVersion", req) + require.NoError(t, err) + assert.Equal(t, http.StatusOK, resp.StatusCode) + + // 8. Access Destroyed Version -> 400 + req = httptest.NewRequest("GET", "/v1/projects/my-proj/secrets/db-password/versions/1:access", nil) + resp, err = p.HandleRequest(ctx, "AccessSecretVersion", req) + require.NoError(t, err) + assert.Equal(t, http.StatusBadRequest, resp.StatusCode) + + // 9. Delete Secret + req = httptest.NewRequest("DELETE", "/v1/projects/my-proj/secrets/db-password", nil) + resp, err = p.HandleRequest(ctx, "DeleteSecret", req) + require.NoError(t, err) + assert.Equal(t, http.StatusOK, resp.StatusCode) + + // 10. Get Deleted Secret -> 404 + req = httptest.NewRequest("GET", "/v1/projects/my-proj/secrets/db-password", nil) + resp, err = p.HandleRequest(ctx, "GetSecret", req) + require.NoError(t, err) + assert.Equal(t, http.StatusNotFound, resp.StatusCode) +} + +func fmtPayload(data string) string { + return `{"payload":{"data":"` + data + `"}}` +} diff --git a/internal/services/gcpsecrets/store.go b/internal/services/gcpsecrets/store.go new file mode 100644 index 00000000..61d611c6 --- /dev/null +++ b/internal/services/gcpsecrets/store.go @@ -0,0 +1,246 @@ +// SPDX-License-Identifier: Apache-2.0 + +package gcpsecrets + +import ( + "crypto/rand" + "encoding/hex" + "errors" + "fmt" + "os" + "path/filepath" + "strconv" + "sync" + "time" +) + +var ( + ErrSecretNotFound = errors.New("SecretNotFound") + ErrSecretAlreadyExists = errors.New("SecretAlreadyExists") + ErrVersionNotFound = errors.New("VersionNotFound") + ErrVersionDestroyed = errors.New("VersionDestroyed") +) + +type SecretVersion struct { + Name string `json:"name"` + CreateTime string `json:"createTime"` + DestroyTime string `json:"destroyTime,omitempty"` + State string `json:"state"` // "ENABLED", "DISABLED", "DESTROYED" + Payload []byte `json:"-"` +} + +type Secret struct { + Name string `json:"name"` + CreateTime string `json:"createTime"` + Labels map[string]string `json:"labels,omitempty"` + Versions map[int]*SecretVersion `json:"-"` + NextVersion int `json:"-"` +} + +type Store struct { + mu sync.RWMutex + baseDir string + secrets map[string]*Secret // key: "projects/{project}/secrets/{secretId}" +} + +func NewStore(baseDir string) *Store { + abs, err := filepath.Abs(baseDir) + if err != nil { + abs = filepath.Clean(baseDir) + } + _ = os.MkdirAll(abs, 0755) + return &Store{ + baseDir: abs, + secrets: make(map[string]*Secret), + } +} + +func (s *Store) secretKey(project, secretId string) string { + return fmt.Sprintf("projects/%s/secrets/%s", project, secretId) +} + +func (s *Store) CreateSecret(project, secretId string, labels map[string]string) (*Secret, error) { + s.mu.Lock() + defer s.mu.Unlock() + + key := s.secretKey(project, secretId) + if _, exists := s.secrets[key]; exists { + return nil, ErrSecretAlreadyExists + } + + sec := &Secret{ + Name: key, + CreateTime: time.Now().UTC().Format(time.RFC3339), + Labels: labels, + Versions: make(map[int]*SecretVersion), + NextVersion: 1, + } + s.secrets[key] = sec + return sec, nil +} + +func (s *Store) GetSecret(project, secretId string) (*Secret, error) { + s.mu.RLock() + defer s.mu.RUnlock() + + key := s.secretKey(project, secretId) + sec, exists := s.secrets[key] + if !exists { + return nil, ErrSecretNotFound + } + return sec, nil +} + +func (s *Store) ListSecrets(project string) ([]*Secret, error) { + s.mu.RLock() + defer s.mu.RUnlock() + + prefix := fmt.Sprintf("projects/%s/secrets/", project) + var list []*Secret + for k, sec := range s.secrets { + if project == "" || len(k) >= len(prefix) && k[:len(prefix)] == prefix { + list = append(list, sec) + } + } + return list, nil +} + +func (s *Store) DeleteSecret(project, secretId string) error { + s.mu.Lock() + defer s.mu.Unlock() + + key := s.secretKey(project, secretId) + if _, exists := s.secrets[key]; !exists { + return ErrSecretNotFound + } + delete(s.secrets, key) + return nil +} + +func (s *Store) AddSecretVersion(project, secretId string, payload []byte) (*SecretVersion, error) { + s.mu.Lock() + defer s.mu.Unlock() + + key := s.secretKey(project, secretId) + sec, exists := s.secrets[key] + if !exists { + return nil, ErrSecretNotFound + } + + vNum := sec.NextVersion + sec.NextVersion++ + + v := &SecretVersion{ + Name: fmt.Sprintf("%s/versions/%d", key, vNum), + CreateTime: time.Now().UTC().Format(time.RFC3339), + State: "ENABLED", + Payload: payload, + } + sec.Versions[vNum] = v + return v, nil +} + +func (s *Store) ListSecretVersions(project, secretId string) ([]*SecretVersion, error) { + s.mu.RLock() + defer s.mu.RUnlock() + + key := s.secretKey(project, secretId) + sec, exists := s.secrets[key] + if !exists { + return nil, ErrSecretNotFound + } + + var list []*SecretVersion + for i := 1; i < sec.NextVersion; i++ { + if v, ok := sec.Versions[i]; ok { + list = append(list, v) + } + } + return list, nil +} + +func (s *Store) GetSecretVersion(project, secretId, versionId string) (*SecretVersion, error) { + s.mu.RLock() + defer s.mu.RUnlock() + + key := s.secretKey(project, secretId) + sec, exists := s.secrets[key] + if !exists { + return nil, ErrSecretNotFound + } + + vNum, err := s.resolveVersionNumber(sec, versionId) + if err != nil { + return nil, err + } + + v, exists := sec.Versions[vNum] + if !exists { + return nil, ErrVersionNotFound + } + return v, nil +} + +func (s *Store) AccessSecretVersion(project, secretId, versionId string) (*SecretVersion, []byte, error) { + v, err := s.GetSecretVersion(project, secretId, versionId) + if err != nil { + return nil, nil, err + } + if v.State == "DESTROYED" { + return nil, nil, ErrVersionDestroyed + } + return v, v.Payload, nil +} + +func (s *Store) DestroySecretVersion(project, secretId, versionId string) (*SecretVersion, error) { + s.mu.Lock() + defer s.mu.Unlock() + + key := s.secretKey(project, secretId) + sec, exists := s.secrets[key] + if !exists { + return nil, ErrSecretNotFound + } + + vNum, err := s.resolveVersionNumber(sec, versionId) + if err != nil { + return nil, err + } + + v, exists := sec.Versions[vNum] + if !exists { + return nil, ErrVersionNotFound + } + + v.State = "DESTROYED" + v.DestroyTime = time.Now().UTC().Format(time.RFC3339) + v.Payload = nil + return v, nil +} + +func (s *Store) resolveVersionNumber(sec *Secret, versionId string) (int, error) { + if versionId == "latest" { + highest := -1 + for num, v := range sec.Versions { + if v.State == "ENABLED" && num > highest { + highest = num + } + } + if highest == -1 { + return 0, ErrVersionNotFound + } + return highest, nil + } + + vNum, err := strconv.Atoi(versionId) + if err != nil { + return 0, ErrVersionNotFound + } + return vNum, nil +} + +func randomID() string { + b := make([]byte, 16) + _, _ = rand.Read(b) + return hex.EncodeToString(b) +} diff --git a/internal/services/lambda/runtime_docker.go b/internal/services/lambda/runtime_docker.go index 3eee5823..5c61d446 100644 --- a/internal/services/lambda/runtime_docker.go +++ b/internal/services/lambda/runtime_docker.go @@ -2,10 +2,10 @@ package lambda import ( - "bytes" "context" "fmt" - "os/exec" + + "github.com/skyoo2003/devcloud/internal/shared/workload" ) func runtimeImage(runtime string) (string, error) { @@ -19,17 +19,5 @@ func runtimeImage(runtime string) (string, error) { } } func runDocker(ctx context.Context, input []byte, args ...string) ([]byte, error) { - cmd := exec.CommandContext(ctx, "docker", args...) - cmd.Stdin = bytes.NewReader(input) - var stdout, stderr bytes.Buffer - cmd.Stdout = &stdout - cmd.Stderr = &stderr - if err := cmd.Run(); err != nil { - return nil, fmt.Errorf("%w: %s", err, stderr.String()) - } - if len(args) > 0 && args[0] == "logs" { - // Docker forwards the container's stderr separately from stdout. - stdout.Write(stderr.Bytes()) - } - return stdout.Bytes(), nil + return workload.DefaultContainerRunner()(ctx, input, args...) } diff --git a/internal/shared/workload/driver.go b/internal/shared/workload/driver.go new file mode 100644 index 00000000..8434039b --- /dev/null +++ b/internal/shared/workload/driver.go @@ -0,0 +1,111 @@ +// SPDX-License-Identifier: Apache-2.0 + +package workload + +import ( + "context" + "errors" + "sync" +) + +var ( + ErrNoRuntimeAvailable = errors.New("no container or workload runtime available") + ErrUnsupportedDriver = errors.New("unsupported workload driver") +) + +type DriverType string + +const ( + DriverTypeOCI DriverType = "oci" + DriverTypeWASM DriverType = "wasm" + DriverTypeProcess DriverType = "process" +) + +// ContainerRunner abstracts running commands against a container daemon (Docker, Podman, nerdctl). +type ContainerRunner interface { + Name() string + Available(ctx context.Context) bool + Run(ctx context.Context, input []byte, args ...string) ([]byte, error) +} + +// WASMRunner abstracts WebAssembly workload execution. +type WASMRunner interface { + Name() string + Available(ctx context.Context) bool + RunWASM(ctx context.Context, module []byte, input []byte) ([]byte, error) +} + +// ProcessRunner abstracts direct host process sandbox execution. +type ProcessRunner interface { + Name() string + Available(ctx context.Context) bool + RunProcess(ctx context.Context, bin string, args []string, input []byte, env map[string]string) ([]byte, error) +} + +type Manager struct { + mu sync.RWMutex + containerRunner ContainerRunner + wasmRunner WASMRunner + processRunner ProcessRunner +} + +var ( + defaultManager *Manager + defaultManagerOnce sync.Once +) + +// DefaultManager returns the global workload manager. +func DefaultManager() *Manager { + defaultManagerOnce.Do(func() { + defaultManager = &Manager{ + containerRunner: NewOCIDriver(""), + wasmRunner: NewWASMDriver(""), + processRunner: NewProcessDriver(), + } + }) + return defaultManager +} + +func (m *Manager) SetContainerRunner(r ContainerRunner) { + m.mu.Lock() + defer m.mu.Unlock() + m.containerRunner = r +} + +func (m *Manager) ContainerRunner() ContainerRunner { + m.mu.RLock() + defer m.mu.RUnlock() + return m.containerRunner +} + +func (m *Manager) SetWASMRunner(r WASMRunner) { + m.mu.Lock() + defer m.mu.Unlock() + m.wasmRunner = r +} + +func (m *Manager) WASMRunner() WASMRunner { + m.mu.RLock() + defer m.mu.RUnlock() + return m.wasmRunner +} + +func (m *Manager) SetProcessRunner(r ProcessRunner) { + m.mu.Lock() + defer m.mu.Unlock() + m.processRunner = r +} + +func (m *Manager) ProcessRunner() ProcessRunner { + m.mu.RLock() + defer m.mu.RUnlock() + return m.processRunner +} + +// DefaultContainerRunner returns a function compatible with lambda's docker signature. +func DefaultContainerRunner() func(context.Context, []byte, ...string) ([]byte, error) { + return func(ctx context.Context, input []byte, args ...string) ([]byte, error) { + runner := DefaultManager().ContainerRunner() + return runner.Run(ctx, input, args...) + } +} diff --git a/internal/shared/workload/driver_test.go b/internal/shared/workload/driver_test.go new file mode 100644 index 00000000..7f9a66b8 --- /dev/null +++ b/internal/shared/workload/driver_test.go @@ -0,0 +1,82 @@ +// SPDX-License-Identifier: Apache-2.0 + +package workload + +import ( + "context" + "os" + "path/filepath" + "testing" +) + +func TestOCIDriver_Resolution(t *testing.T) { + d := NewOCIDriver("") + t.Setenv("CONTAINER_RUNTIME", "podman") + if d.Name() != "podman" { + t.Fatalf("expected podman, got %s", d.Name()) + } + + dExplicit := NewOCIDriver("my-custom-oci") + if dExplicit.Name() != "my-custom-oci" { + t.Fatalf("expected my-custom-oci, got %s", dExplicit.Name()) + } +} + +func TestOCIDriver_RunMock(t *testing.T) { + dir := t.TempDir() + mockBin := filepath.Join(dir, "mock-container-cli") + script := "#!/bin/sh\nprintf 'standard-output'; printf 'standard-error' >&2\n" + if err := os.WriteFile(mockBin, []byte(script), 0755); err != nil { + t.Fatalf("failed to write mock script: %v", err) + } + + driver := NewOCIDriver(mockBin) + out, err := driver.Run(context.Background(), nil, "logs", "test-id") + if err != nil { + t.Fatalf("Run failed: %v", err) + } + if string(out) != "standard-outputstandard-error" { + t.Fatalf("expected combined logs, got: %s", string(out)) + } +} + +func TestWASMDriver_Resolution(t *testing.T) { + d := NewWASMDriver("") + t.Setenv("WASM_RUNTIME", "wasmtime") + if d.Name() != "wasmtime" { + t.Fatalf("expected wasmtime, got %s", d.Name()) + } +} + +func TestProcessDriver_Execution(t *testing.T) { + d := NewProcessDriver() + if !d.Available(context.Background()) { + t.Fatalf("process driver should be available") + } + + out, err := d.RunProcess(context.Background(), "echo", []string{"hello-workload"}, nil, map[string]string{"FOO": "BAR"}) + if err != nil { + t.Fatalf("RunProcess failed: %v", err) + } + if string(out) != "hello-workload\n" { + t.Fatalf("unexpected output: %s", string(out)) + } +} + +func TestManager_Registry(t *testing.T) { + mgr := DefaultManager() + origRunner := mgr.ContainerRunner() + defer mgr.SetContainerRunner(origRunner) + + mockRunner := NewOCIDriver("test-oci") + mgr.SetContainerRunner(mockRunner) + + if mgr.ContainerRunner().Name() != "test-oci" { + t.Fatalf("expected test-oci, got %s", mgr.ContainerRunner().Name()) + } + + runnerFn := DefaultContainerRunner() + if runnerFn == nil { + t.Fatalf("DefaultContainerRunner returned nil") + } +} diff --git a/internal/shared/workload/oci.go b/internal/shared/workload/oci.go new file mode 100644 index 00000000..c18ff53e --- /dev/null +++ b/internal/shared/workload/oci.go @@ -0,0 +1,69 @@ +// SPDX-License-Identifier: Apache-2.0 + +package workload + +import ( + "bytes" + "context" + "fmt" + "os" + "os/exec" +) + +type OCIDriver struct { + cli string +} + +func NewOCIDriver(explicitCLI string) *OCIDriver { + return &OCIDriver{cli: explicitCLI} +} + +func (d *OCIDriver) resolveCLI() string { + if d.cli != "" { + return d.cli + } + if envCLI := os.Getenv("CONTAINER_RUNTIME"); envCLI != "" { + return envCLI + } + // Try standard container engines: podman, docker, nerdctl + for _, candidate := range []string{"podman", "docker", "nerdctl"} { + if path, err := exec.LookPath(candidate); err == nil && path != "" { + return candidate + } + } + // Default fallback to "docker" + return "docker" +} + +func (d *OCIDriver) Name() string { + return d.resolveCLI() +} + +func (d *OCIDriver) Available(ctx context.Context) bool { + cli := d.resolveCLI() + cmd := exec.CommandContext(ctx, cli, "info") + return cmd.Run() == nil +} + +func (d *OCIDriver) Run(ctx context.Context, input []byte, args ...string) ([]byte, error) { + cli := d.resolveCLI() + cmd := exec.CommandContext(ctx, cli, args...) + if len(input) > 0 { + cmd.Stdin = bytes.NewReader(input) + } + + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + + if err := cmd.Run(); err != nil { + return nil, fmt.Errorf("%s %v failed: %w: %s", cli, args, err, stderr.String()) + } + + if len(args) > 0 && args[0] == "logs" { + // Container runtimes forward container stderr separately from stdout. + stdout.Write(stderr.Bytes()) + } + + return stdout.Bytes(), nil +} diff --git a/internal/shared/workload/process.go b/internal/shared/workload/process.go new file mode 100644 index 00000000..7ee47f63 --- /dev/null +++ b/internal/shared/workload/process.go @@ -0,0 +1,47 @@ +// SPDX-License-Identifier: Apache-2.0 + +package workload + +import ( + "bytes" + "context" + "fmt" + "os" + "os/exec" +) + +type ProcessDriver struct{} + +func NewProcessDriver() *ProcessDriver { + return &ProcessDriver{} +} + +func (d *ProcessDriver) Name() string { + return "process" +} + +func (d *ProcessDriver) Available(ctx context.Context) bool { + return true +} + +func (d *ProcessDriver) RunProcess(ctx context.Context, bin string, args []string, input []byte, env map[string]string) ([]byte, error) { + cmd := exec.CommandContext(ctx, bin, args...) + if len(input) > 0 { + cmd.Stdin = bytes.NewReader(input) + } + + cmd.Env = os.Environ() + for k, v := range env { + cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", k, v)) + } + + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + + if err := cmd.Run(); err != nil { + return nil, fmt.Errorf("%s %v failed: %w: %s", bin, args, err, stderr.String()) + } + + return stdout.Bytes(), nil +} diff --git a/internal/shared/workload/wasm.go b/internal/shared/workload/wasm.go new file mode 100644 index 00000000..6a6536f4 --- /dev/null +++ b/internal/shared/workload/wasm.go @@ -0,0 +1,82 @@ +// SPDX-License-Identifier: Apache-2.0 + +package workload + +import ( + "bytes" + "context" + "fmt" + "os" + "os/exec" +) + +type WASMDriver struct { + cli string +} + +func NewWASMDriver(explicitCLI string) *WASMDriver { + return &WASMDriver{cli: explicitCLI} +} + +func (d *WASMDriver) resolveCLI() string { + if d.cli != "" { + return d.cli + } + if envCLI := os.Getenv("WASM_RUNTIME"); envCLI != "" { + return envCLI + } + for _, candidate := range []string{"wasmtime", "wasmer", "wasmedge", "wazero"} { + if path, err := exec.LookPath(candidate); err == nil && path != "" { + return candidate + } + } + return "" +} + +func (d *WASMDriver) Name() string { + cli := d.resolveCLI() + if cli != "" { + return cli + } + return "wasm-embedded" +} + +func (d *WASMDriver) Available(ctx context.Context) bool { + return d.resolveCLI() != "" +} + +func (d *WASMDriver) RunWASM(ctx context.Context, module []byte, input []byte) ([]byte, error) { + cli := d.resolveCLI() + if cli == "" { + return nil, fmt.Errorf("%w: install wasmtime, wasmer, or set WASM_RUNTIME", ErrNoRuntimeAvailable) + } + + tmpFile, err := os.CreateTemp("", "workload-*.wasm") + if err != nil { + return nil, err + } + defer func() { + _ = os.Remove(tmpFile.Name()) + }() + + if _, err := tmpFile.Write(module); err != nil { + _ = tmpFile.Close() + return nil, err + } + _ = tmpFile.Close() + + cmd := exec.CommandContext(ctx, cli, "run", tmpFile.Name()) + if len(input) > 0 { + cmd.Stdin = bytes.NewReader(input) + } + + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + + if err := cmd.Run(); err != nil { + return nil, fmt.Errorf("%s wasm run failed: %w: %s", cli, err, stderr.String()) + } + + return stdout.Bytes(), nil +} diff --git a/test/compatibility/test_azure_appconfig.py b/test/compatibility/test_azure_appconfig.py new file mode 100644 index 00000000..87517055 --- /dev/null +++ b/test/compatibility/test_azure_appconfig.py @@ -0,0 +1,106 @@ +# SPDX-License-Identifier: Apache-2.0 + +import pytest +import requests +from conftest import DEVCLOUD_URL + + +@pytest.fixture +def appconfig_endpoint(devcloud_server): + return DEVCLOUD_URL + + +def test_azure_appconfig_lifecycle(appconfig_endpoint): + key = "app:feature:setting1" + headers = { + "Content-Type": "application/json", + "Host": "myconfig.azconfig.io", + } + + # 1. Set key-value without label + put_body = { + "value": "initial-config-val", + "content_type": "text/plain", + "tags": {"tier": "backend"}, + } + resp = requests.put( + f"{appconfig_endpoint}/kv/{key}", + json=put_body, + headers=headers, + ) + assert resp.status_code == 200 + kv = resp.json() + assert kv["key"] == key + assert kv["value"] == "initial-config-val" + assert kv["content_type"] == "text/plain" + assert kv["tags"]["tier"] == "backend" + assert "etag" in kv + assert "ETag" in resp.headers + + # 2. Set key-value with label + put_staging = { + "value": "staging-config-val", + } + resp_staging = requests.put( + f"{appconfig_endpoint}/kv/{key}?label=staging", + json=put_staging, + headers=headers, + ) + assert resp_staging.status_code == 200 + kv_staging = resp_staging.json() + assert kv_staging["key"] == key + assert kv_staging["label"] == "staging" + assert kv_staging["value"] == "staging-config-val" + + # 3. Get key-value without label + resp_get = requests.get( + f"{appconfig_endpoint}/kv/{key}", + headers=headers, + ) + assert resp_get.status_code == 200 + assert resp_get.json()["value"] == "initial-config-val" + + # 4. Get key-value with label + resp_get_staging = requests.get( + f"{appconfig_endpoint}/kv/{key}?label=staging", + headers=headers, + ) + assert resp_get_staging.status_code == 200 + assert resp_get_staging.json()["value"] == "staging-config-val" + + # 5. List key-values + resp_list = requests.get( + f"{appconfig_endpoint}/kv", + headers=headers, + ) + assert resp_list.status_code == 200 + items = resp_list.json().get("items", []) + keys_found = [it["key"] for it in items if it["key"] == key] + assert len(keys_found) >= 2 + + # 6. Delete key-value without label + resp_del = requests.delete( + f"{appconfig_endpoint}/kv/{key}", + headers=headers, + ) + assert resp_del.status_code == 200 + + # 7. Get after delete returns 404 + resp_not_found = requests.get( + f"{appconfig_endpoint}/kv/{key}", + headers=headers, + ) + assert resp_not_found.status_code == 404 + + # 8. Delete again returns 204 + resp_del_again = requests.delete( + f"{appconfig_endpoint}/kv/{key}", + headers=headers, + ) + assert resp_del_again.status_code == 204 + + # 9. Clean up staging key + requests.delete( + f"{appconfig_endpoint}/kv/{key}?label=staging", + headers=headers, + ) diff --git a/test/compatibility/test_azure_keyvault.py b/test/compatibility/test_azure_keyvault.py new file mode 100644 index 00000000..94af8c0e --- /dev/null +++ b/test/compatibility/test_azure_keyvault.py @@ -0,0 +1,167 @@ +# SPDX-License-Identifier: Apache-2.0 + +import pytest +import requests +from conftest import DEVCLOUD_URL + + +@pytest.fixture +def keyvault_endpoint(devcloud_server): + return DEVCLOUD_URL + + +def test_azure_keyvault_secret_lifecycle(keyvault_endpoint): + secret_name = "test-compat-secret" + headers = { + "Content-Type": "application/json", + "Host": "myvault.vault.azure.net", + } + + # 1. Set secret (v1) + put_body = { + "value": "initial-secret-value", + "contentType": "text/plain", + "tags": {"env": "test", "owner": "devcloud"}, + } + resp = requests.put( + f"{keyvault_endpoint}/secrets/{secret_name}?api-version=7.4", + json=put_body, + headers=headers, + ) + assert resp.status_code == 200 + data = resp.json() + assert data["value"] == "initial-secret-value" + assert data["contentType"] == "text/plain" + assert data["tags"]["env"] == "test" + v1_id = data["id"] + v1_version = v1_id.split("/")[-1] + + # 2. Get secret (latest) + resp_get = requests.get( + f"{keyvault_endpoint}/secrets/{secret_name}?api-version=7.4", + headers=headers, + ) + assert resp_get.status_code == 200 + assert resp_get.json()["value"] == "initial-secret-value" + + # 3. Set new secret version (v2) + put_body_v2 = { + "value": "updated-secret-value", + } + resp_v2 = requests.put( + f"{keyvault_endpoint}/secrets/{secret_name}?api-version=7.4", + json=put_body_v2, + headers=headers, + ) + assert resp_v2.status_code == 200 + v2_id = resp_v2.json()["id"] + v2_version = v2_id.split("/")[-1] + assert v2_version != v1_version + + # 4. Get specific version (v1) + resp_v1_get = requests.get( + f"{keyvault_endpoint}/secrets/{secret_name}/{v1_version}?api-version=7.4", + headers=headers, + ) + assert resp_v1_get.status_code == 200 + assert resp_v1_get.json()["value"] == "initial-secret-value" + + # 5. Get latest returns v2 + resp_latest = requests.get( + f"{keyvault_endpoint}/secrets/{secret_name}?api-version=7.4", + headers=headers, + ) + assert resp_latest.status_code == 200 + assert resp_latest.json()["value"] == "updated-secret-value" + + # 6. List secret versions + resp_versions = requests.get( + f"{keyvault_endpoint}/secrets/{secret_name}/versions?api-version=7.4", + headers=headers, + ) + assert resp_versions.status_code == 200 + versions_list = resp_versions.json().get("value", []) + assert len(versions_list) >= 2 + + # 7. List secrets in vault + resp_list = requests.get( + f"{keyvault_endpoint}/secrets?api-version=7.4", + headers=headers, + ) + assert resp_list.status_code == 200 + secrets_list = resp_list.json().get("value", []) + assert any( + s["id"].startswith(f"https://myvault.vault.azure.net/secrets/{secret_name}") + for s in secrets_list + ) + + # 8. Delete secret + resp_del = requests.delete( + f"{keyvault_endpoint}/secrets/{secret_name}?api-version=7.4", + headers=headers, + ) + assert resp_del.status_code == 200 + + # 9. Get after delete returns 404 + resp_not_found = requests.get( + f"{keyvault_endpoint}/secrets/{secret_name}?api-version=7.4", + headers=headers, + ) + assert resp_not_found.status_code == 404 + + +def test_azure_keyvault_key_lifecycle(keyvault_endpoint): + key_name = "test-compat-key" + headers = { + "Content-Type": "application/json", + "Host": "myvault.vault.azure.net", + } + + # 1. Create Key + create_body = { + "kty": "RSA", + "key_size": 2048, + "key_ops": ["encrypt", "decrypt", "sign", "verify"], + "tags": {"purpose": "testing"}, + } + resp = requests.post( + f"{keyvault_endpoint}/keys/{key_name}/create?api-version=7.4", + json=create_body, + headers=headers, + ) + assert resp.status_code == 200 + key_data = resp.json() + assert "key" in key_data + assert key_data["key"]["kty"] == "RSA" + assert "kid" in key_data["key"] + + # 2. Get Key + resp_get = requests.get( + f"{keyvault_endpoint}/keys/{key_name}?api-version=7.4", + headers=headers, + ) + assert resp_get.status_code == 200 + assert resp_get.json()["key"]["kid"] == key_data["key"]["kid"] + + # 3. List Keys + resp_list = requests.get( + f"{keyvault_endpoint}/keys?api-version=7.4", + headers=headers, + ) + assert resp_list.status_code == 200 + keys_list = resp_list.json().get("value", []) + assert any(k["kid"] == key_data["key"]["kid"] for k in keys_list) + + # 4. Delete Key + resp_del = requests.delete( + f"{keyvault_endpoint}/keys/{key_name}?api-version=7.4", + headers=headers, + ) + assert resp_del.status_code == 200 + + # 5. Get after delete returns 404 + resp_not_found = requests.get( + f"{keyvault_endpoint}/keys/{key_name}?api-version=7.4", + headers=headers, + ) + assert resp_not_found.status_code == 404 diff --git a/test/compatibility/test_azure_smoke.py b/test/compatibility/test_azure_smoke.py new file mode 100644 index 00000000..9d23cff3 --- /dev/null +++ b/test/compatibility/test_azure_smoke.py @@ -0,0 +1,94 @@ +# SPDX-License-Identifier: Apache-2.0 + +"""Smoke test suite ensuring 100% of registered Azure services respond on port 4747.""" + +import pytest +import requests +from conftest import DEVCLOUD_URL + +AZURE_SERVICES = [ + { + "id": "azureblob", + "name": "Azure Blob Storage", + "method": "GET", + "path": "/devstoreaccount1?comp=list", + "headers": { + "x-ms-version": "2023-11-03", + "Authorization": "SharedKey devstoreaccount1:Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==", + }, + "expected_status": 200, + }, + { + "id": "azurequeue", + "name": "Azure Queue Storage", + "method": "GET", + "path": "/devstoreaccount1/queue?comp=list", + "headers": { + "x-ms-version": "2023-11-03", + "Authorization": "SharedKey devstoreaccount1:Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==", + }, + "expected_status": 200, + }, + { + "id": "azuretable", + "name": "Azure Table Storage", + "method": "GET", + "path": "/devstoreaccount1/Tables", + "headers": { + "x-ms-version": "2023-11-03", + "Accept": "application/json;odata=nometadata", + "Authorization": "SharedKey devstoreaccount1:Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==", + }, + "expected_status": 200, + }, + { + "id": "cosmosdb", + "name": "Azure Cosmos DB", + "method": "GET", + "path": "/dbs", + "headers": { + "x-ms-version": "2018-12-31", + "Authorization": "type=master&ver=1.0&sig=c21va2U=", + }, + "expected_status": 200, + }, + { + "id": "azurekeyvault", + "name": "Azure Key Vault", + "method": "GET", + "path": "/secrets?api-version=7.4", + "headers": { + "Host": "smoke-vault.vault.azure.net", + }, + "expected_status": 200, + }, + { + "id": "azureappconfig", + "name": "Azure App Configuration", + "method": "GET", + "path": "/kv", + "headers": { + "Host": "smoke-app.azconfig.io", + }, + "expected_status": 200, + }, +] + + +@pytest.mark.parametrize( + "svc", + AZURE_SERVICES, + ids=[s["id"] for s in AZURE_SERVICES], +) +def test_azure_registered_service_smoke(devcloud_server, svc): + """Verify that every registered Azure service is routed and responds on port 4747.""" + url = f"{DEVCLOUD_URL}{svc['path']}" + resp = requests.request( + method=svc["method"], + url=url, + headers=svc["headers"], + ) + assert resp.status_code == svc["expected_status"], ( + f"Service {svc['name']} ({svc['id']}) failed smoke test at {url}: " + f"got HTTP {resp.status_code}, expected {svc['expected_status']}. Response: {resp.text[:200]}" + ) diff --git a/test/compatibility/test_gcp_pubsub.py b/test/compatibility/test_gcp_pubsub.py new file mode 100644 index 00000000..5067811e --- /dev/null +++ b/test/compatibility/test_gcp_pubsub.py @@ -0,0 +1,137 @@ +# SPDX-License-Identifier: Apache-2.0 + +import base64 +import pytest +import requests +from conftest import DEVCLOUD_URL + + +@pytest.fixture +def gcp_base_url(devcloud_server): + return DEVCLOUD_URL + + +def test_gcp_pubsub_lifecycle(gcp_base_url): + project = "test-project-pubsub" + topic_id = "user-signups" + sub_id = "welcome-email-worker" + headers = { + "Authorization": "Bearer ya29.devcloud-pilot-token", + "Content-Type": "application/json", + } + + # 1. Create Topic + resp = requests.put( + f"{gcp_base_url}/v1/projects/{project}/topics/{topic_id}", + headers=headers, + json={"labels": {"env": "prod"}}, + ) + assert resp.status_code == 200 + topic_data = resp.json() + assert topic_data["name"] == f"projects/{project}/topics/{topic_id}" + + # 2. Duplicate Create Topic -> 409 Conflict + resp_dup = requests.put( + f"{gcp_base_url}/v1/projects/{project}/topics/{topic_id}", + headers=headers, + json={}, + ) + assert resp_dup.status_code == 409 + + # 3. Get Topic + resp_get = requests.get( + f"{gcp_base_url}/v1/projects/{project}/topics/{topic_id}", + headers=headers, + ) + assert resp_get.status_code == 200 + assert resp_get.json()["name"] == f"projects/{project}/topics/{topic_id}" + + # 4. List Topics + resp_list = requests.get( + f"{gcp_base_url}/v1/projects/{project}/topics", + headers=headers, + ) + assert resp_list.status_code == 200 + topics = [t["name"] for t in resp_list.json().get("topics", [])] + assert f"projects/{project}/topics/{topic_id}" in topics + + # 5. Create Subscription + resp_sub = requests.put( + f"{gcp_base_url}/v1/projects/{project}/subscriptions/{sub_id}", + headers=headers, + json={ + "topic": f"projects/{project}/topics/{topic_id}", + "ackDeadlineSeconds": 60, + }, + ) + assert resp_sub.status_code == 200 + sub_data = resp_sub.json() + assert sub_data["name"] == f"projects/{project}/subscriptions/{sub_id}" + assert sub_data["topic"] == f"projects/{project}/topics/{topic_id}" + + # 6. Publish Messages + msg_raw = "user=alex@example.com,signup=true" + msg_b64 = base64.b64encode(msg_raw.encode("utf-8")).decode("utf-8") + resp_pub = requests.post( + f"{gcp_base_url}/v1/projects/{project}/topics/{topic_id}:publish", + headers=headers, + json={ + "messages": [ + { + "data": msg_b64, + "attributes": {"region": "ap-northeast-2"}, + } + ] + }, + ) + assert resp_pub.status_code == 200 + pub_data = resp_pub.json() + assert len(pub_data.get("messageIds", [])) == 1 + + # 7. Pull Messages + resp_pull = requests.post( + f"{gcp_base_url}/v1/projects/{project}/subscriptions/{sub_id}:pull", + headers=headers, + json={"maxMessages": 10}, + ) + assert resp_pull.status_code == 200 + pull_data = resp_pull.json() + received = pull_data.get("receivedMessages", []) + assert len(received) == 1 + + rec_item = received[0] + ack_id = rec_item["ackId"] + assert ack_id != "" + assert rec_item["message"]["data"] == msg_b64 + assert rec_item["message"]["attributes"]["region"] == "ap-northeast-2" + + # 8. Acknowledge Message + resp_ack = requests.post( + f"{gcp_base_url}/v1/projects/{project}/subscriptions/{sub_id}:acknowledge", + headers=headers, + json={"ackIds": [ack_id]}, + ) + assert resp_ack.status_code == 200 + + # 9. Pull Again -> Empty + resp_pull_empty = requests.post( + f"{gcp_base_url}/v1/projects/{project}/subscriptions/{sub_id}:pull", + headers=headers, + json={"maxMessages": 10}, + ) + assert resp_pull_empty.status_code == 200 + assert len(resp_pull_empty.json().get("receivedMessages", [])) == 0 + + # 10. Delete Subscription + resp_del_sub = requests.delete( + f"{gcp_base_url}/v1/projects/{project}/subscriptions/{sub_id}", + headers=headers, + ) + assert resp_del_sub.status_code == 200 + + # 11. Delete Topic + resp_del_topic = requests.delete( + f"{gcp_base_url}/v1/projects/{project}/topics/{topic_id}", + headers=headers, + ) + assert resp_del_topic.status_code == 200 diff --git a/test/compatibility/test_gcp_secrets.py b/test/compatibility/test_gcp_secrets.py new file mode 100644 index 00000000..1dac32f0 --- /dev/null +++ b/test/compatibility/test_gcp_secrets.py @@ -0,0 +1,123 @@ +# SPDX-License-Identifier: Apache-2.0 + +import base64 +import pytest +import requests +from conftest import DEVCLOUD_URL + + +@pytest.fixture +def gcp_base_url(devcloud_server): + return DEVCLOUD_URL + + +def test_gcp_secret_manager_lifecycle(gcp_base_url): + project = "test-project-1" + secret_id = "database-master-key" + headers = { + "Authorization": "Bearer ya29.devcloud-pilot-token", + "Content-Type": "application/json", + } + + # 1. Create Secret + resp = requests.post( + f"{gcp_base_url}/v1/projects/{project}/secrets?secretId={secret_id}", + headers=headers, + json={"labels": {"environment": "test", "tier": "backend"}}, + ) + assert resp.status_code == 200 + data = resp.json() + assert data["name"] == f"projects/{project}/secrets/{secret_id}" + + # 2. Duplicate Create -> 409 Conflict + resp_dup = requests.post( + f"{gcp_base_url}/v1/projects/{project}/secrets?secretId={secret_id}", + headers=headers, + json={}, + ) + assert resp_dup.status_code == 409 + + # 3. Get Secret + resp_get = requests.get( + f"{gcp_base_url}/v1/projects/{project}/secrets/{secret_id}", + headers=headers, + ) + assert resp_get.status_code == 200 + assert resp_get.json()["name"] == f"projects/{project}/secrets/{secret_id}" + + # 4. List Secrets + resp_list = requests.get( + f"{gcp_base_url}/v1/projects/{project}/secrets", + headers=headers, + ) + assert resp_list.status_code == 200 + secrets = [s["name"] for s in resp_list.json().get("secrets", [])] + assert f"projects/{project}/secrets/{secret_id}" in secrets + + # 5. Add Secret Version + secret_text = "SuperSecretPassword2026!" + b64_payload = base64.b64encode(secret_text.encode("utf-8")).decode("utf-8") + resp_v1 = requests.post( + f"{gcp_base_url}/v1/projects/{project}/secrets/{secret_id}:addVersion", + headers=headers, + json={"payload": {"data": b64_payload}}, + ) + assert resp_v1.status_code == 200 + v1_data = resp_v1.json() + assert v1_data["name"] == f"projects/{project}/secrets/{secret_id}/versions/1" + assert v1_data["state"] == "ENABLED" + + # 6. Access Secret Version (Specific version 1) + resp_access = requests.get( + f"{gcp_base_url}/v1/projects/{project}/secrets/{secret_id}/versions/1:access", + headers=headers, + ) + assert resp_access.status_code == 200 + retrieved_b64 = resp_access.json()["payload"]["data"] + assert base64.b64decode(retrieved_b64).decode("utf-8") == secret_text + + # 7. Access Secret Version (latest alias) + resp_access_latest = requests.get( + f"{gcp_base_url}/v1/projects/{project}/secrets/{secret_id}/versions/latest:access", + headers=headers, + ) + assert resp_access_latest.status_code == 200 + assert resp_access_latest.json()["payload"]["data"] == b64_payload + + # 8. List Versions + resp_v_list = requests.get( + f"{gcp_base_url}/v1/projects/{project}/secrets/{secret_id}/versions", + headers=headers, + ) + assert resp_v_list.status_code == 200 + versions = resp_v_list.json().get("versions", []) + assert len(versions) == 1 + + # 9. Destroy Version + resp_destroy = requests.post( + f"{gcp_base_url}/v1/projects/{project}/secrets/{secret_id}/versions/1:destroy", + headers=headers, + ) + assert resp_destroy.status_code == 200 + assert resp_destroy.json()["state"] == "DESTROYED" + + # 10. Access Destroyed Version -> 400 Bad Request + resp_access_destroyed = requests.get( + f"{gcp_base_url}/v1/projects/{project}/secrets/{secret_id}/versions/1:access", + headers=headers, + ) + assert resp_access_destroyed.status_code == 400 + + # 11. Delete Secret + resp_del = requests.delete( + f"{gcp_base_url}/v1/projects/{project}/secrets/{secret_id}", + headers=headers, + ) + assert resp_del.status_code == 200 + + # 12. Get Deleted Secret -> 404 + resp_get_del = requests.get( + f"{gcp_base_url}/v1/projects/{project}/secrets/{secret_id}", + headers=headers, + ) + assert resp_get_del.status_code == 404 diff --git a/test/compatibility/test_gcp_smoke.py b/test/compatibility/test_gcp_smoke.py new file mode 100644 index 00000000..7c5483b2 --- /dev/null +++ b/test/compatibility/test_gcp_smoke.py @@ -0,0 +1,62 @@ +# SPDX-License-Identifier: Apache-2.0 + +"""Smoke test suite ensuring 100% of registered GCP services respond on port 4747.""" + +import pytest +import requests +from conftest import DEVCLOUD_URL + +GCP_SERVICES = [ + { + "id": "gcs", + "name": "Google Cloud Storage", + "method": "GET", + "path": "/storage/v1/b?project=test-project", + "headers": { + "Authorization": "Bearer devcloud-mock-token", + "x-goog-project-id": "test-project", + }, + "expected_status": 200, + }, + { + "id": "gcpsecrets", + "name": "GCP Secret Manager", + "method": "GET", + "path": "/v1/projects/test-project/secrets", + "headers": { + "Authorization": "Bearer devcloud-mock-token", + "x-goog-project-id": "test-project", + }, + "expected_status": 200, + }, + { + "id": "gcppubsub", + "name": "GCP Cloud Pub/Sub", + "method": "GET", + "path": "/v1/projects/test-project/topics", + "headers": { + "Authorization": "Bearer devcloud-mock-token", + "x-goog-project-id": "test-project", + }, + "expected_status": 200, + }, +] + + +@pytest.mark.parametrize( + "svc", + GCP_SERVICES, + ids=[s["id"] for s in GCP_SERVICES], +) +def test_gcp_registered_service_smoke(devcloud_server, svc): + """Verify that every registered GCP service is routed and responds on port 4747.""" + url = f"{DEVCLOUD_URL}{svc['path']}" + resp = requests.request( + method=svc["method"], + url=url, + headers=svc["headers"], + ) + assert resp.status_code == svc["expected_status"], ( + f"Service {svc['name']} ({svc['id']}) failed smoke test at {url}: " + f"got HTTP {resp.status_code}, expected {svc['expected_status']}. Response: {resp.text[:200]}" + ) From 4218fcd4b6e65367db7a1b3b5c8e419fe304f0e9 Mon Sep 17 00:00:00 2001 From: Sung-Kyu Yoo Date: Tue, 13 Oct 2026 00:43:15 +0900 Subject: [PATCH 2/4] docs: modernize documentation for multi-cloud and workload runtime - Update README.md, docs/README.md, architecture.md, and configuration.md for unified single-port (4747) multi-cloud routing and workload runtime abstraction - Add Azure and GCP service documentation covering Table, Cosmos, Queue, App Config, Key Vault, Event Grid, GCS, PubSub, and Secret Manager - Refactor azureappconfig provider routing to match AST handverified scanner expectations - Standardize character encoding and typography across documentation and scripts - Polish technical English phrasing and ensure documentation integrity across docs/ and changes/ --- CHANGELOG.md | 60 +- CONTRIBUTING.md | 8 +- GOVERNANCE.md | 10 +- README.md | 85 +-- RELEASE.md | 46 +- SUPPORT.md | 30 +- TRADEMARKS.md | 10 +- changes/v0.1.0.md | 4 +- changes/v1.0.0.md | 22 +- changes/v1.1.0.md | 18 +- changes/v1.1.1.md | 8 +- changes/v1.2.0.md | 8 +- docs/README.md | 46 +- docs/architecture.md | 222 ++++--- docs/compatibility-policy.md | 38 +- docs/configuration.md | 180 +++--- docs/contributing.md | 54 +- docs/coverage.md | 70 +-- docs/crud-engine.md | 29 +- docs/demand.md | 572 +++++++++---------- docs/faq.md | 8 +- docs/fidelity-manifest.md | 28 +- docs/getting-started.md | 86 +-- docs/plugin-api.md | 44 +- docs/roadmap.md | 211 +++---- docs/services-matrix.md | 84 +-- docs/services/azure.md | 164 +++--- docs/services/dynamodb.md | 6 +- docs/services/eventbridge.md | 6 +- docs/services/gcp.md | 175 ++++-- docs/services/iam-sts.md | 2 +- docs/services/lambda.md | 164 +++--- docs/services/s3.md | 14 +- docs/services/sns.md | 10 +- docs/services/sqs.md | 8 +- docs/troubleshooting.md | 16 +- internal/services/azureappconfig/provider.go | 195 ++++--- scripts/demand_rank.py | 30 +- 38 files changed, 1384 insertions(+), 1387 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b6759f8c..4007c228 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,30 +19,30 @@ * Three figures are now gated against the binary: the coverage page's routing and depth targets, the 45 MiB binary-size ceiling, and that all 431 registered services initialize ([#163](https://github.com/skyoo2003/devcloud/issues/163)) ### Changed * The weekly Smithy sync now re-derives the published coverage figures and commits them into its own pull request, so reviewing it means confirming what moved rather than transcribing failing tests ([#165](https://github.com/skyoo2003/devcloud/issues/165)) -* The published fidelity tiers are stated against two denominators — the services DevCloud promises depth on, and every service it registers — so the routed long tail no longer reads as a fidelity drop ([#166](https://github.com/skyoo2003/devcloud/issues/166)) +* The published fidelity tiers are stated against two denominators : the services DevCloud promises depth on, and every service it registers : so the routed long tail no longer reads as a fidelity drop ([#166](https://github.com/skyoo2003/devcloud/issues/166)) ### Fixed -* The CRUD engine no longer answers an operation it cannot classify with a broader sibling's route — chime's AssociatePhoneNumberWithUser was returning UpdateUser's 200 — and now declines it cleanly ([#162](https://github.com/skyoo2003/devcloud/issues/162)) +* The CRUD engine no longer answers an operation it cannot classify with a broader sibling's route : chime's AssociatePhoneNumberWithUser was returning UpdateUser's 200 : and now declines it cleanly ([#162](https://github.com/skyoo2003/devcloud/issues/162)) ### Documentation -* The coverage page states routing and depth as two targets — 431 of 431 registered, depth still the 205 services the demand study settled — so a service count no longer reads as a fidelity promise ([#163](https://github.com/skyoo2003/devcloud/issues/163)) -* The five per-service pages now list the operations DevCloud actually serves, derived from the fidelity manifest rather than from memory — S3 had documented 8 of 37, IAM 6 of 58 ([#167](https://github.com/skyoo2003/devcloud/issues/167)) +* The coverage page states routing and depth as two targets : 431 of 431 registered, depth still the 205 services the demand study settled : so a service count no longer reads as a fidelity promise ([#163](https://github.com/skyoo2003/devcloud/issues/163)) +* The five per-service pages now list the operations DevCloud actually serves, derived from the fidelity manifest rather than from memory : S3 had documented 8 of 37, IAM 6 of 58 ([#167](https://github.com/skyoo2003/devcloud/issues/167)) ## [v1.1.1](https://github.com/skyoo2003/devcloud/releases/tag/v1.1.1) - 2026-09-07 ### Security -* An S3 object key can no longer leave the bucket that owns it — a key such as `../victim/secret.txt` stayed under the store base directory and so passed the old containment check, letting one tenant read, overwrite and delete another tenant's objects — because every user-controlled path component is now guarded with `filepath.IsLocal` ([#155](https://github.com/skyoo2003/devcloud/issues/155)) +* An S3 object key can no longer leave the bucket that owns it : a key such as `../victim/secret.txt` stayed under the store base directory and so passed the old containment check, letting one tenant read, overwrite and delete another tenant's objects : because every user-controlled path component is now guarded with `filepath.IsLocal` ([#155](https://github.com/skyoo2003/devcloud/issues/155)) ### Documentation -* Changelog fragments are capped at one sentence, two at most — stated with a worked example in RELEASE.md, in the pre-flight checklist, in CONTRIBUTING.md and the PR checklist, and enforced by a 400-character body limit `changie new` refuses to exceed. Every entry already in the changelog is rewritten to that limit, and the v1.1.0 and v1.0.0 release notes are republished from their fragment files ([#152](https://github.com/skyoo2003/devcloud/issues/152)) -* The documentation no longer publishes the same figure at five different values: six numbers restated across pages now live in docs/coverage.md, which every other page links to. Three claims the code does not back are corrected — an unregistered `GET /devcloud/api/health`, a Lambda invoke failure blamed on Docker, and codegen outputs no template emits — and eighteen files are 1,063 lines shorter ([#151](https://github.com/skyoo2003/devcloud/issues/151)) -* The documentation is now published as a searchable site at https://skyoo2003.github.io/devcloud/, built with Hugo from the same `docs/*.md` files GitHub renders — no front matter added and no links rewritten. ([#154](https://github.com/skyoo2003/devcloud/issues/154)) +* Changelog fragments are capped at one sentence, two at most : stated with a worked example in RELEASE.md, in the pre-flight checklist, in CONTRIBUTING.md and the PR checklist, and enforced by a 400-character body limit `changie new` refuses to exceed. Every entry already in the changelog is rewritten to that limit, and the v1.1.0 and v1.0.0 release notes are republished from their fragment files ([#152](https://github.com/skyoo2003/devcloud/issues/152)) +* The documentation no longer publishes the same figure at five different values: six numbers restated across pages now live in docs/coverage.md, which every other page links to. Three claims the code does not back are corrected : an unregistered `GET /devcloud/api/health`, a Lambda invoke failure blamed on Docker, and codegen outputs no template emits : and eighteen files are 1,063 lines shorter ([#151](https://github.com/skyoo2003/devcloud/issues/151)) +* The documentation is now published as a searchable site at https://skyoo2003.github.io/devcloud/, built with Hugo from the same `docs/*.md` files GitHub renders : no front matter added and no links rewritten. ([#154](https://github.com/skyoo2003/devcloud/issues/154)) ## [v1.1.0](https://github.com/skyoo2003/devcloud/releases/tag/v1.1.0) - 2026-09-06 ### Added * Provider-namespaced configuration: service blocks can be written as `providers.aws.services.*`, forward-compatible with `providers.azure.*`. The top-level `services` block is the same block under its historical name and keeps working ([#135](https://github.com/skyoo2003/devcloud/issues/135)) -* Per-provider auth adapters (`internal/auth`) parse SigV4 credentials once and expose the caller's claimed identity to every plugin via `auth.FromContext`, adding a `region` field to `GET /devcloud/api/logs`. Nothing is verified — DevCloud still accepts any credentials ([#135](https://github.com/skyoo2003/devcloud/issues/135)) +* Per-provider auth adapters (`internal/auth`) parse SigV4 credentials once and expose the caller's claimed identity to every plugin via `auth.FromContext`, adding a `region` field to `GET /devcloud/api/logs`. Nothing is verified : DevCloud still accepts any credentials ([#135](https://github.com/skyoo2003/devcloud/issues/135)) * Amazon Comprehend, served by the generic CRUD engine (49 operations auto-crud, 36 unimplemented), where an unimplemented operation returns a clean AWS error rather than a fabricated success ([#136](https://github.com/skyoo2003/devcloud/issues/136)) * All 48 upstream AI/Machine Learning service models are registered, taking DevCloud from 105 to 148 services. 17 are served and the remaining 31 decline with a clean AWS error instead of falling through to real AWS ([#137](https://github.com/skyoo2003/devcloud/issues/137)) * docs/coverage.md, which publishes the registered, serving and registered-only service counts together so the service count is never stated on its own ([#137](https://github.com/skyoo2003/devcloud/issues/137)) -* Admin API endpoint `GET /devcloud/api/unrouted` reports which AWS services this DevCloud was asked for and does not register, with per-service counts and first/last seen; it requires `admin.enabled: true`. The figure is a floor — an unclassifiable request is routed to S3 rather than counted, and a non-zero `droppedServiceIds` means the collector hit its key cap ([#138](https://github.com/skyoo2003/devcloud/issues/138)) +* Admin API endpoint `GET /devcloud/api/unrouted` reports which AWS services this DevCloud was asked for and does not register, with per-service counts and first/last seen; it requires `admin.enabled: true`. The figure is a floor : an unclassifiable request is routed to S3 rather than counted, and a non-zero `droppedServiceIds` means the collector hit its key cap ([#138](https://github.com/skyoo2003/devcloud/issues/138)) * "Service Not Supported" issue form for requesting an AWS service DevCloud does not register yet, which asks which operations you actually call and accepts `/devcloud/api/unrouted` output ([#138](https://github.com/skyoo2003/devcloud/issues/138)) * The generic CRUD engine now serves `rest-json` services, recovering the operation from the request method and path against the route table the model already declares (shared in `internal/shared/httproute`). 28 registered-only services started serving: services with at least one operation went 117 to 145, and `auto-crud` operations 1,415 to 2,200 ([#139](https://github.com/skyoo2003/devcloud/issues/139)) -* The 57 AWS services with demonstrated demand are registered, taking DevCloud to 205 services — 199 of which serve at least one operation, for 4,858 engine-served operations. The three that serve nothing are named with their reasons in docs/coverage.md ([#139](https://github.com/skyoo2003/devcloud/issues/139)) +* The 57 AWS services with demonstrated demand are registered, taking DevCloud to 205 services : 199 of which serve at least one operation, for 4,858 engine-served operations. The three that serve nothing are named with their reasons in docs/coverage.md ([#139](https://github.com/skyoo2003/devcloud/issues/139)) * The generic CRUD engine now serves every protocol DevCloud registers: `rest-xml` recovers its operation from method and path, and `query` reads it from the form body's `Action` field. 112 operations move from `unimplemented` to `auto-crud`, so protocol is no longer a reason any registered service serves nothing ([#143](https://github.com/skyoo2003/devcloud/issues/143)) * CI gates the published coverage figure: `go test ./cmd/devcloud/` compares `docs/coverage.md`, `README.md` and `docs/README.md` against the live registry in both directions, so neither the code nor the number can move without the other. The demand set in `docs/demand.md` is checked to still be registered, so the count cannot be held steady by swapping one service for another ([#144](https://github.com/skyoo2003/devcloud/issues/144)) * `scripts/model_churn.py` summarises what a Smithy model sync actually changed, so the weekly sync PR reads as a change rather than a whole-tree regeneration of 194 models ([#147](https://github.com/skyoo2003/devcloud/issues/147)) @@ -50,7 +50,7 @@ * Codegen now runs through a provider-neutral intermediate representation (`internal/codegen/ir`) behind a `ModelSource` interface, so a second API description format is an added file rather than a rewrite. Smithy is the first source and generated output is byte-identical ([#135](https://github.com/skyoo2003/devcloud/issues/135)) * Service routing aliases are derived from the Smithy models instead of a hand-maintained switch in the gateway, so onboarding a service no longer needs a hand-written routing entry. All 93 previously-routed aliases resolve unchanged, and an alias claimed by more than one service is reported rather than guessed ([#137](https://github.com/skyoo2003/devcloud/issues/137)) * The fidelity manifest records each service's wire protocol, so a registered service that the CRUD engine cannot reach is distinguishable from one whose wiring is broken ([#137](https://github.com/skyoo2003/devcloud/issues/137)) -* The AWS coverage target is 205 services, not 431: of the 283 services DevCloud does not register, only 57 have been built by two or more of moto, LocalStack and terraform-provider-aws, and DevCloud has received zero service requests to date. The remaining 226 can still be onboarded on request but are no longer promised — evidence in docs/demand.md, decision rule in docs/coverage.md ([#138](https://github.com/skyoo2003/devcloud/issues/138)) +* The AWS coverage target is 205 services, not 431: of the 283 services DevCloud does not register, only 57 have been built by two or more of moto, LocalStack and terraform-provider-aws, and DevCloud has received zero service requests to date. The remaining 226 can still be onboarded on request but are no longer promised : evidence in docs/demand.md, decision rule in docs/coverage.md ([#138](https://github.com/skyoo2003/devcloud/issues/138)) * `rest-xml` request bodies are still not buffered by the gateway, now by an explicit rule rather than as a side effect of the protocol being unservable: the engine serves `rest-xml` from the path and query alone, so S3's large binary uploads keep streaming. `crud.NeedsBody` is the predicate, and a test fails if the body is read ([#143](https://github.com/skyoo2003/devcloud/issues/143)) * The boto3 compatibility suite exercises every registered service, parametrising over a generated service list (`internal/generated/compat/services.json`) instead of two hand-written name lists. It found 31 of 205 registered services with no boto3 test at all, and grew from 854 to 993 tests ([#144](https://github.com/skyoo2003/devcloud/issues/144)) ### Fixed @@ -61,19 +61,19 @@ * `make codegen` is now byte-identical across runs, where a model bundling two namespaces with colliding shape names produced different output every time because shapes are keyed by short name and the generator iterated a Go map ([#137](https://github.com/skyoo2003/devcloud/issues/137)) * Amazon Rekognition can be onboarded; its `X-Amz-Target` prefix `RekognitionService` was covered by no routing entry, so every call returned `UnknownService` ([#137](https://github.com/skyoo2003/devcloud/issues/137)) * Stale service counts across the docs: README.md advertised 104 services and docs/faq.md 101 when 148 are registered, and docs/fidelity-manifest.md carried outdated per-tier operation totals. Every count now appears with its depth split (148 registered / 117 serving at least one operation), so the number is never read as a capability claim on its own ([#138](https://github.com/skyoo2003/devcloud/issues/138)) -* S3 Control requests were served by S3, which signs with the same name and parsed them as a bucket and key — `CreateAccessPoint` returned 200 and left an object in a bucket named `v20180820`. S3 Control is now split off by its `/v20180820/` path prefix, and its unserved operations return a clean AWS error instead of a fabricated success ([#142](https://github.com/skyoo2003/devcloud/issues/142)) -* Eight query-protocol providers (`rds`, `neptune`, `docdb`, `redshift`, `elasticache`, `autoscaling`, `cloudformation`, `elasticloadbalancingv2`) answered any unimplemented operation with HTTP 200 and an empty ``, so a caller was told an operation succeeded when nothing ran — and botocore raised `KeyError` on the missing `` wrapper. They now hand the operation to the generic CRUD engine, which serves the 223 CRUD-shaped ones from the real store and refuses the rest with a clean `InvalidAction` ([#145](https://github.com/skyoo2003/devcloud/issues/145)) +* S3 Control requests were served by S3, which signs with the same name and parsed them as a bucket and key : `CreateAccessPoint` returned 200 and left an object in a bucket named `v20180820`. S3 Control is now split off by its `/v20180820/` path prefix, and its unserved operations return a clean AWS error instead of a fabricated success ([#142](https://github.com/skyoo2003/devcloud/issues/142)) +* Eight query-protocol providers (`rds`, `neptune`, `docdb`, `redshift`, `elasticache`, `autoscaling`, `cloudformation`, `elasticloadbalancingv2`) answered any unimplemented operation with HTTP 200 and an empty ``, so a caller was told an operation succeeded when nothing ran : and botocore raised `KeyError` on the missing `` wrapper. They now hand the operation to the generic CRUD engine, which serves the 223 CRUD-shaped ones from the real store and refuses the rest with a clean `InvalidAction` ([#145](https://github.com/skyoo2003/devcloud/issues/145)) * The four Amazon Lex services are reachable from boto3, where all four sign as `lex` and no service is called `lex`, so the alias stayed contested and every Lex call died as `UnknownService`. The shared-signing-name split now recognises a signing name that is a group key rather than a member service ID, and answers each request from the sibling whose route table models its method and path ([#146](https://github.com/skyoo2003/devcloud/issues/146)) * `appsync.ListApis`, `eks.ListAccessPolicies` and `opensearch.ListApplications` are reachable: each was implemented, but the provider's hand-written path resolver did not know the route boto3 sends, so the operation arrived empty and the call answered `NotImplemented`. Each provider now falls back to its own model's route table when its resolver recognises nothing ([#146](https://github.com/skyoo2003/devcloud/issues/146)) * The fidelity manifest no longer drops operations with short names, where the hand-verified scan required four characters or more and reported `resourcegroups.Tag` as unimplemented while the code beside it was real. Short dispatch literals are now collected separately and promoted only where the service's own model declares them ([#146](https://github.com/skyoo2003/devcloud/issues/146)) * S3 no longer answers an unimplemented bucket sub-resource with a bucket listing, where `GET /{Bucket}?analytics` and seven siblings fell through to `ListObjects` and botocore read the resulting 200 as a successful call. A bucket-level GET whose query parameter is neither a served sub-resource nor a listing parameter now returns a clean `NotImplemented` ([#146](https://github.com/skyoo2003/devcloud/issues/146)) * The weekly Smithy model sync discarded the changes worth reviewing: its test step ended the job before the pull request was opened, and an upstream model that gains an operation is designed to fail that step. The test result is now recorded rather than gating, the PR is always opened when models moved, and the failure is re-raised afterwards so the cron does not silently go green ([#147](https://github.com/skyoo2003/devcloud/issues/147)) -* A release dry run keeps its artifacts; the `Upload assets` step ran when `dry_run` was not set, so it uploaded on a real release — where GoReleaser has already attached the same files — and skipped the one path that publishes nothing else ([#150](https://github.com/skyoo2003/devcloud/issues/150)) +* A release dry run keeps its artifacts; the `Upload assets` step ran when `dry_run` was not set, so it uploaded on a real release : where GoReleaser has already attached the same files : and skipped the one path that publishes nothing else ([#150](https://github.com/skyoo2003/devcloud/issues/150)) ### Documentation * Coverage docs state what the compatibility figure excludes and why: two registered services have no boto3 client at all (`sagemakerruntimehttp2`, `transcribestreaming`), and the four Lex services are registered but reachable by no boto3 caller because every Lex client signs as the contested alias `lex` ([#144](https://github.com/skyoo2003/devcloud/issues/144)) -* docs/coverage.md now publishes the measured cost of keeping up with upstream — refreshing all 194 vendored models changed 93 of them, and 32 moved an operation. The reading states its own ceiling: those 93 had been vendored 141 days earlier, so it is an accumulated backlog and not a weekly rate ([#147](https://github.com/skyoo2003/devcloud/issues/147)) +* docs/coverage.md now publishes the measured cost of keeping up with upstream : refreshing all 194 vendored models changed 93 of them, and 32 moved an operation. The reading states its own ceiling: those 93 had been vendored 141 days earlier, so it is an accumulated backlog and not a weekly rate ([#147](https://github.com/skyoo2003/devcloud/issues/147)) * The release procedure moved from `docs/release.md` to `RELEASE.md` at the repo root, and now states three things it never did: the Homebrew formula is published to a separate tap repository through a GitHub App token scoped to that repo alone, a tagged build pushes four `*-alpine` image tags, and `make changelog VERSION=` is the batch-and-merge pair ([#148](https://github.com/skyoo2003/devcloud/issues/148)) -* The contributor guide no longer asks for SQLite3 development headers, CGO, or a separately running DevCloud instance — the driver has been pure Go since v1.0.0, and the compatibility suite starts its own server. docs/getting-started.md also no longer advertises a Next.js frontend on port 3000 that lives in a separate repository ([#149](https://github.com/skyoo2003/devcloud/issues/149)) +* The contributor guide no longer asks for SQLite3 development headers, CGO, or a separately running DevCloud instance : the driver has been pure Go since v1.0.0, and the compatibility suite starts its own server. docs/getting-started.md also no longer advertises a Next.js frontend on port 3000 that lives in a separate repository ([#149](https://github.com/skyoo2003/devcloud/issues/149)) * The compatibility policy states current figures, having gone stale in five places because nothing gates it: `cmd/devcloud/coverage_test.go` covers docs/coverage.md, README.md and docs/README.md, and docs/compatibility-policy.md is not in that set. It now reads 1,144 tests, 5,193 `auto-crud`, 4,497 hand-verified, 205 registered with 4 serving nothing, and 193/12 model-backed to hand-written ([#150](https://github.com/skyoo2003/devcloud/issues/150)) ## [v1.0.0](https://github.com/skyoo2003/devcloud/releases/tag/v1.0.0) - 2026-08-11 ### Added @@ -84,17 +84,17 @@ * Serve ~2,200 standard CRUD operations across the 46 JSON-protocol services from a Smithy-driven fallback engine (`internal/shared/crud`); hand-written operations take precedence and unclassifiable ones still return an error. See docs/crud-engine.md ([#98](https://github.com/skyoo2003/devcloud/issues/98)) * Homebrew installation via `brew install skyoo2003/tap/devcloud`, published to the tap on each release ([#121](https://github.com/skyoo2003/devcloud/issues/121)) * Per-operation fidelity manifest declaring every operation as hand-verified, auto-crud or unimplemented, exposed at `GET /devcloud/api/fidelity` and enforced by a build-failing coverage test ([#126](https://github.com/skyoo2003/devcloud/issues/126)) -* Real tag support for KMS, CloudWatch and EventBridge — TagResource, UntagResource and ListTagsForResource/ListResourceTags now persist tags per resource ARN instead of being echoed by the generic CRUD engine ([#126](https://github.com/skyoo2003/devcloud/issues/126)) -* A published compatibility policy (docs/compatibility-policy.md) stating what v1.0 guarantees across the 1.x line — config keys, environment variables, the CLI, admin API response keys, fidelity tier names, and the response shape of hand-verified operations covered by the boto3 compatibility suite — what it explicitly does not guarantee, and the deprecation procedure that precedes any removal ([#129](https://github.com/skyoo2003/devcloud/issues/129)) +* Real tag support for KMS, CloudWatch and EventBridge : TagResource, UntagResource and ListTagsForResource/ListResourceTags now persist tags per resource ARN instead of being echoed by the generic CRUD engine ([#126](https://github.com/skyoo2003/devcloud/issues/126)) +* A published compatibility policy (docs/compatibility-policy.md) stating what v1.0 guarantees across the 1.x line : config keys, environment variables, the CLI, admin API response keys, fidelity tier names, and the response shape of hand-verified operations covered by the boto3 compatibility suite : what it explicitly does not guarantee, and the deprecation procedure that precedes any removal ([#129](https://github.com/skyoo2003/devcloud/issues/129)) ### Changed * Rename `internal/dashboard` to `internal/admin` and the `dashboard.enabled` config key to `admin.enabled`. The old key is honoured for one release with a deprecation warning. ([#111](https://github.com/skyoo2003/devcloud/issues/111)) * Build the request log collector only when `admin.enabled` is set, dropping a per-request mutex lock and ring-buffer write from the AWS request hot path ([#112](https://github.com/skyoo2003/devcloud/issues/112)) * The `services` config block is now optional: omit it and every registered service starts with `data_dir ./data/`; list services and only those start. Replaces 325 lines of identical boilerplate. ([#120](https://github.com/skyoo2003/devcloud/issues/120)) -* Release archives now carry the `docs/` tree alongside the binary, so the documentation you unpack — including the fidelity manifest and the release's compatibility promises — describes exactly the version you downloaded ([#127](https://github.com/skyoo2003/devcloud/issues/127)) +* Release archives now carry the `docs/` tree alongside the binary, so the documentation you unpack : including the fidelity manifest and the release's compatibility promises : describes exactly the version you downloaded ([#127](https://github.com/skyoo2003/devcloud/issues/127)) ### Removed * Remove the dead, unregistered STS provider stub (`internal/services/sts`); STS is served by the IAM package ([#96](https://github.com/skyoo2003/devcloud/issues/96)) * Extract the web dashboard to a separate repository. The binary builds and serves no UI, only the opt-in admin API at `/devcloud/api/*`. ([#111](https://github.com/skyoo2003/devcloud/issues/111)) -* Remove dead surface found by a repo-wide over-engineering audit: the event bus and admin WebSocket, the `GetMetrics` plugin API and `/devcloud/api/metrics` (resource counts remain on `/devcloud/api/services`), `shared.ResourceStore`, `gateway.ExtractAccountID`, and the `auth.enabled` key — still read for one release so enabling it warns that credentials are accepted regardless. Drops the `gorilla/websocket` dependency. ([#120](https://github.com/skyoo2003/devcloud/issues/120)) +* Remove dead surface found by a repo-wide over-engineering audit: the event bus and admin WebSocket, the `GetMetrics` plugin API and `/devcloud/api/metrics` (resource counts remain on `/devcloud/api/services`), `shared.ResourceStore`, `gateway.ExtractAccountID`, and the `auth.enabled` key : still read for one release so enabling it warns that credentials are accepted regardless. Drops the `gorilla/websocket` dependency. ([#120](https://github.com/skyoo2003/devcloud/issues/120)) ### Fixed * Route `X-Amz-Target` prefixes that contain dots (CodeConnections, DMS) to the right service; they misrouted to CloudTrail ([#96](https://github.com/skyoo2003/devcloud/issues/96)) * Unimplemented operations return `NotImplemented` (HTTP 501) instead of an empty `200`, so SDKs can tell a not-yet-emulated operation from an invalid one ([#96](https://github.com/skyoo2003/devcloud/issues/96)) @@ -105,17 +105,17 @@ * Codegen now parses operations bound to Smithy resource shapes, recovering 285 operations that were invisible to the generator (bedrock 0 of 101, lambda 19 of 85, ecs 12 of 76, transfer 29 of 71, sso-admin 67 of 79) ([#126](https://github.com/skyoo2003/devcloud/issues/126)) * The fidelity manifest now reads each provider's actual dispatch instead of intersecting with the Smithy model, recovering 226 served operations it had hidden (dynamodbstreams listed 4 of its 22, acm's UpdateCertificate, bedrock's InvokeModelWithResponseStream) and dropping 5 non-operations it had invented (identitystore Description/DisplayName/Emails, pipes DELETE/POST) ([#126](https://github.com/skyoo2003/devcloud/issues/126)) * EventBridge now drops a bus's or rule's tags when it is deleted. ARNs are derived from the name, so recreating a deleted resource reused its ARN and inherited the previous tags ([#126](https://github.com/skyoo2003/devcloud/issues/126)) -* Restored CloudWatch's 17 CRUD-engine operations. The gateway picks the protocol from the request, not from the provider, so CloudWatch reaches the engine whenever a client speaks JSON — filtering the registry by the provider's declared protocol had removed that coverage outright ([#126](https://github.com/skyoo2003/devcloud/issues/126)) -* EventBridge rule ARNs now name their event bus, as AWS does. A rule name is unique per bus, so same-named rules on two custom buses previously shared one ARN — and with it, one tag set, where tagging one rule changed the other's and deleting one wiped the survivor's ([#126](https://github.com/skyoo2003/devcloud/issues/126)) +* Restored CloudWatch's 17 CRUD-engine operations. The gateway picks the protocol from the request, not from the provider, so CloudWatch reaches the engine whenever a client speaks JSON : filtering the registry by the provider's declared protocol had removed that coverage outright ([#126](https://github.com/skyoo2003/devcloud/issues/126)) +* EventBridge rule ARNs now name their event bus, as AWS does. A rule name is unique per bus, so same-named rules on two custom buses previously shared one ARN : and with it, one tag set, where tagging one rule changed the other's and deleting one wiped the survivor's ([#126](https://github.com/skyoo2003/devcloud/issues/126)) * CloudWatch now drops an alarm's tags when the alarm is deleted, so recreating an alarm under the same name no longer inherits the old one's tags ([#126](https://github.com/skyoo2003/devcloud/issues/126)) -* Downloadable binaries now start, where releases built with `CGO_ENABLED=0` against a SQLite driver that required cgo exited at startup — every tar.gz/zip binary, the Homebrew formula and the versioned `*-alpine` images. The driver is now pure Go, so no build needs a C toolchain or SQLite headers ([#128](https://github.com/skyoo2003/devcloud/issues/128)) +* Downloadable binaries now start, where releases built with `CGO_ENABLED=0` against a SQLite driver that required cgo exited at startup : every tar.gz/zip binary, the Homebrew formula and the versioned `*-alpine` images. The driver is now pure Go, so no build needs a C toolchain or SQLite headers ([#128](https://github.com/skyoo2003/devcloud/issues/128)) * Service selection now matches what the docs promise: an empty `services` block starts nothing rather than every registered service, and `DEVCLOUD_SERVICES` names the running set outright instead of intersecting with the block. Previously `DEVCLOUD_SERVICES=sqs` alongside a block listing only `s3` started nothing at all ([#129](https://github.com/skyoo2003/devcloud/issues/129)) * CloudFront no longer fabricates a success for the 122 operations the fidelity manifest classifies as `unimplemented`, where its dispatch fallback answered HTTP 200 with an empty XML document that boto3 parses as a successful empty result. It now returns `NotImplemented` (HTTP 501), matching the other 32 providers that decline from their own dispatch default ([#129](https://github.com/skyoo2003/devcloud/issues/129)) -* Generated routers now honour route patterns that constrain the query string, so operations distinguished only by a query parameter are reachable — `matchURI` split the whole pattern into path segments, leaving CloudFront `TagResource` and `UntagResource` implemented but unroutable. Query-constrained routes are tried ahead of unconstrained ones, so a bare path cannot shadow a more specific route ([#129](https://github.com/skyoo2003/devcloud/issues/129)) -* Released binaries now register their services, where GoReleaser built `cmd/devcloud/main.go` rather than the package and dropped the `imports.go` that blank-imports all 104 services — every archive, the Homebrew formula and the container images exited at startup with `unknown service: s3`. The release gate now builds with GoReleaser instead of compiling the package, which is why the boto3 suite missed it ([#131](https://github.com/skyoo2003/devcloud/issues/131)) +* Generated routers now honour route patterns that constrain the query string, so operations distinguished only by a query parameter are reachable : `matchURI` split the whole pattern into path segments, leaving CloudFront `TagResource` and `UntagResource` implemented but unroutable. Query-constrained routes are tried ahead of unconstrained ones, so a bare path cannot shadow a more specific route ([#129](https://github.com/skyoo2003/devcloud/issues/129)) +* Released binaries now register their services, where GoReleaser built `cmd/devcloud/main.go` rather than the package and dropped the `imports.go` that blank-imports all 104 services : every archive, the Homebrew formula and the container images exited at startup with `unknown service: s3`. The release gate now builds with GoReleaser instead of compiling the package, which is why the boto3 suite missed it ([#131](https://github.com/skyoo2003/devcloud/issues/131)) ### Documentation -* Corrected what the fidelity manifest says an `unimplemented` operation returns: it claimed JSON and Query services answer `InvalidAction` (HTTP 400), where only the 46 providers falling through to the CRUD engine do and 32 answer `NotImplemented` (HTTP 501) from their own dispatch default. Only the failure itself is stable — the specific code and status are documented, not guaranteed ([#129](https://github.com/skyoo2003/devcloud/issues/129)) -* The roadmap, architecture and governance docs no longer describe DevCloud as pre-1.0. All three ship inside the release archive, so a v1.0 download would have carried a roadmap listing its own release as pending, an architecture overview naming Phase 1 as current, and a governance doc calling the API unstable — contradicting the compatibility policy packaged beside them ([#130](https://github.com/skyoo2003/devcloud/issues/130)) +* Corrected what the fidelity manifest says an `unimplemented` operation returns: it claimed JSON and Query services answer `InvalidAction` (HTTP 400), where only the 46 providers falling through to the CRUD engine do and 32 answer `NotImplemented` (HTTP 501) from their own dispatch default. Only the failure itself is stable : the specific code and status are documented, not guaranteed ([#129](https://github.com/skyoo2003/devcloud/issues/129)) +* The roadmap, architecture and governance docs no longer describe DevCloud as pre-1.0. All three ship inside the release archive, so a v1.0 download would have carried a roadmap listing its own release as pending, an architecture overview naming Phase 1 as current, and a governance doc calling the API unstable : contradicting the compatibility policy packaged beside them ([#130](https://github.com/skyoo2003/devcloud/issues/130)) ## [v0.2.0](https://github.com/skyoo2003/devcloud/releases/tag/v0.2.0) - 2026-04-21 ### Added * Add GoReleaser + Changie release pipeline ([#26](https://github.com/skyoo2003/devcloud/issues/26)) @@ -128,7 +128,7 @@ * Fix uncontrolled data used in path expression (code scanning alerts #1-#15) ([#23](https://github.com/skyoo2003/devcloud/issues/23)) ## [v0.1.0](https://github.com/skyoo2003/devcloud/releases/tag/v0.1.0) - 2026-04-18 -First public release of DevCloud — a local development companion for cloud-native apps. Positioned as an on-ramp to the cloud, not a replacement: iterate locally without cloud bills, then land cleanly on your target CSP. +First public release of DevCloud : a local development companion for cloud-native apps. Positioned as an on-ramp to the cloud, not a replacement: iterate locally without cloud bills, then land cleanly on your target CSP. ### Added @@ -145,7 +145,7 @@ First public release of DevCloud — a local development companion for cloud-nat - 96 AWS services scaffolded from official Smithy models via an in-tree code generator. - Big 6 fully implemented: **S3, SQS, DynamoDB, Lambda, IAM, STS**. - Integration services: SNS, CloudWatch, CloudWatch Logs, KMS, SecretsManager, SSM, EventBridge, ECR, EC2, ECS, Route53, ACM, CloudFormation, and more. -- Cross-service integrations: CloudFormation provisioning, DynamoDB Streams → Lambda, SQS → Lambda, S3 → Lambda, EventBridge → SQS/SNS/Lambda, SNS → SQS subscriptions. +- Cross-service integrations: CloudFormation provisioning, DynamoDB Streams -> Lambda, SQS -> Lambda, S3 -> Lambda, EventBridge -> SQS/SNS/Lambda, SNS -> SQS subscriptions. - **96% boto3 compatibility** (671/699 tests passing against the official AWS SDK test surface). - Port-aware URL construction in SQS, ECR, and CloudFormation response paths (opts-based, not hardcoded). diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4b94506a..92281fd2 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,6 +1,6 @@ # Contributing to DevCloud -Thanks for your interest in contributing! This file is a short pointer — the full contributor guide lives at [docs/contributing.md](docs/contributing.md). +Thanks for your interest in contributing! This file is a short pointer: the full contributor guide lives at [docs/contributing.md](docs/contributing.md). ## Quick links @@ -32,21 +32,21 @@ Do not add a `src/` directory. Add code to `pkg/` only when it is intentionally - **Bug**: use the [Bug Report](https://github.com/skyoo2003/devcloud/issues/new?template=bug_report.yml) template - **Feature request**: use the [Feature Request](https://github.com/skyoo2003/devcloud/issues/new?template=feature_request.yml) template - **Question**: use the [Question](https://github.com/skyoo2003/devcloud/issues/new?template=question.yml) template -- **Security vulnerability**: do not open a public issue — see [SECURITY.md](SECURITY.md) +- **Security vulnerability**: do not open a public issue; see [SECURITY.md](SECURITY.md) ## Pull request workflow 1. Fork the repo and create a feature branch from `main` 2. Make your changes (follow the guidelines in [docs/contributing.md](docs/contributing.md)) 3. Run `make test` and ensure lint passes (`golangci-lint run`) -4. Open a PR against `main` — the PR template will guide you +4. Open a PR against `main`; the PR template will guide you 5. CI must be green before review ## Commit message style Conventional prefixes are preferred: `feat:`, `fix:`, `chore:`, `docs:`, `test:`, `refactor:`. -Release notes are **not** generated from commits — they come from [Changie](https://changie.dev) fragments. Add one for any user-facing change with `changie new` — one sentence, two at most (see [RELEASE.md](RELEASE.md#one-sentence-two-at-most)). +Release notes are **not** generated from commits; they come from [Changie](https://changie.dev) fragments. Add one for any user-facing change with `changie new`: one sentence, two at most (see [RELEASE.md](RELEASE.md#one-sentence-two-at-most)). ## License of Contributions diff --git a/GOVERNANCE.md b/GOVERNANCE.md index 1c50dc8e..19a2fe13 100644 --- a/GOVERNANCE.md +++ b/GOVERNANCE.md @@ -9,7 +9,7 @@ DevCloud is a small, volunteer-driven open-source project. This document describ - **Best-effort response.** No SLA; see [SUPPORT.md](SUPPORT.md) for expectations. This is appropriate at the project's current size. It is a statement about who decides, not about -API stability — what a release promises is [docs/compatibility-policy.md](docs/compatibility-policy.md). +API stability: what a release promises is [docs/compatibility-policy.md](docs/compatibility-policy.md). ## How decisions are made @@ -24,7 +24,7 @@ API stability — what a release promises is [docs/compatibility-policy.md](docs ## How to influence direction -- **Open issues.** Features, bugs, questions — all go through [Issues](https://github.com/skyoo2003/devcloud/issues) or [Discussions](https://github.com/skyoo2003/devcloud/discussions). +- **Open issues.** Features, bugs, questions: all go through [Issues](https://github.com/skyoo2003/devcloud/issues) or [Discussions](https://github.com/skyoo2003/devcloud/discussions). - **React to existing issues** to surface priority. - **Submit PRs.** Working code speaks loudest. See [CONTRIBUTING.md](CONTRIBUTING.md). - **Discuss architecture early.** For non-trivial changes, open a design issue before writing code. @@ -33,9 +33,9 @@ API stability — what a release promises is [docs/compatibility-policy.md](docs As DevCloud grows, the governance model will evolve. Likely milestones: -- **2+ active maintainers** — introduce a lightweight maintainer charter (who can merge what, conflict resolution). -- **First external contributor with sustained commits** — document maintainer eligibility criteria. -- **Foundation consideration** — if the project reaches the scale where vendor-neutral governance matters, a formal foundation model will be evaluated. +- **2+ active maintainers**: introduce a lightweight maintainer charter (who can merge what, conflict resolution). +- **First external contributor with sustained commits**: document maintainer eligibility criteria. +- **Foundation consideration**: if the project reaches the scale where vendor-neutral governance matters, a formal foundation model will be evaluated. Until then, this one-page document is the full governance. diff --git a/README.md b/README.md index 59f92286..c2689e82 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # DevCloud -A **local development companion for cloud-native apps**. Iterate fast without cloud bills, then land cleanly on your target CSP. +A local development companion for cloud-native applications. Iterate fast without cloud bills, then deploy cleanly to your target cloud provider. ![CI](https://github.com/skyoo2003/devcloud/actions/workflows/ci.yml/badge.svg) ![Go](https://img.shields.io/badge/Go-1.26-00ADD8) @@ -12,64 +12,85 @@ A **local development companion for cloud-native apps**. Iterate fast without cl docker run -p 4747:4747 ghcr.io/skyoo2003/devcloud:latest ``` -Point any AWS SDK at `http://localhost:4747`. See [Getting Started](docs/getting-started.md) for boto3 / AWS CLI / Terraform examples and other install options. +Point your AWS, Azure, or GCP SDK and CLI tools at `http://localhost:4747`. See [Getting Started](docs/getting-started.md) for SDK examples, CLI configurations, and Terraform setups. ## Why DevCloud? -Every test against real AWS is billed, every feature branch wants its own sandbox, and every new joiner waits for credentials. DevCloud runs a local, API-compatible cloud environment so you can: +Every integration test against a live cloud account costs money, every feature branch needs an isolated sandbox, and every new team member waits for IAM credentials. DevCloud provides an API-compatible local cloud environment on a single port: -- **Develop offline** — no VPN, no credentials, no internet -- **Iterate without a bill** — integration tests at CI speed, not cloud speed -- **Onboard in minutes** — `docker run` and your team is productive -- **Ship with confidence** — compatibility tests run against real SDKs +- **Develop offline**: no VPN, no cloud credentials, and no internet connection required. +- **Iterate without bills**: run integration tests at local speeds instead of waiting on remote provisioning. +- **Onboard in minutes**: start a single container or binary, and your team is ready to build. +- **Unified multi-cloud on port 4747**: route AWS, Azure, and GCP workloads transparently through one endpoint. +- **Pluggable workload runtimes**: execute serverless functions with standard OCI container runtimes (Docker, Podman, nerdctl), WebAssembly, or isolated host processes without mandatory daemons. +- **Ship with confidence**: verified against official SDKs and CLIs with automated compatibility suites. -DevCloud is an **on-ramp to the cloud**, not a replacement for it. The goal is to make the local → CSP transition boring. It targets AWS today; Azure, GCP and beyond are on the [roadmap](docs/roadmap.md). +DevCloud serves as a local on-ramp to cloud providers, not a production replacement. The goal is to make the transition from local development to cloud deployment straightforward and predictable. ## Features -- **441 AWS services registered, 441 serving at least one operation** — every service AWS publishes is routed, so no SDK call escapes to a billable account. Depth is a separate, smaller promise — see [coverage.md](docs/coverage.md) for both targets. -- **boto3-compatible** — a 1,530-test suite runs in CI (`make test-compat`) across every registered service. Unsupported operations return a clean AWS error, never a false success. -- **Cross-service integration** — CloudFormation provisioning, DynamoDB Streams → Lambda, EventBridge targets, S3 → Lambda -- **Smithy-driven codegen** — Go types, routers and error catalogues generated from AWS models, with a weekly sync workflow that keeps them current -- **Single binary, zero-config** — one Docker image, one port (4747), no config file required; override with `DEVCLOUD_SERVICES`, `DEVCLOUD_DATA_DIR`, `DEVCLOUD_PORT` -- **SDK/CLI compatible** — AWS SDK, CLI, Terraform and CDK work out of the box -- **Admin API** — opt-in REST at `/devcloud/api/*` for service status, resources and request logs (`admin.enabled: true`) +- **441 AWS services registered, 441 serving at least one operation**: every service AWS publishes is routed locally to prevent billable leaks. See [coverage.md](docs/coverage.md) for depth and routing targets. +- **Multi-cloud API emulation**: Azure (Blob Storage, Queue Storage, Table Storage, Cosmos DB, Key Vault, App Configuration) and GCP (Cloud Storage, Secret Manager, Cloud Pub/Sub) served transparently alongside AWS on port 4747. +- **Universal workload runtime**: pluggable OCI (Docker, Podman, nerdctl), WebAssembly, and host process drivers for serverless workloads, enabling zero-daemon local execution. +- **SDK and CLI compatibility**: verified in CI against official AWS (boto3, aws CLI), Azure (Python SDK, az CLI), and GCP (Python SDK, gcloud CLI) client tooling. +- **Cross-service and federated identity**: CloudFormation orchestration, DynamoDB Streams to Lambda, EventBridge targets, S3 to Lambda, and cross-cloud OIDC and STS federation. +- **Smithy and OpenAPI codegen**: Go types, routers, and error catalogues generated from official API models, kept current with automated sync workflows. +- **Single binary, zero-config**: one port (4747), no mandatory configuration file required; customizable via environment variables and devcloud.yaml. +- **Admin API**: opt-in REST endpoints at `/devcloud/api/*` for service status, resource inventory, and request logs (`admin.enabled: true`). -## Core Services (AWS) +## Core Services +### AWS | Service | Protocol | Storage | Docs | |---------|----------|---------|------| | S3 | REST-XML | Filesystem + SQLite | [docs/services/s3.md](docs/services/s3.md) | | SQS | Query + JSON | In-memory | [docs/services/sqs.md](docs/services/sqs.md) | | DynamoDB | JSON 1.0 | SQLite | [docs/services/dynamodb.md](docs/services/dynamodb.md) | -| Lambda | REST-JSON | SQLite + Filesystem | [docs/services/lambda.md](docs/services/lambda.md) | -| IAM/STS | Query | SQLite | [docs/services/iam-sts.md](docs/services/iam-sts.md) | +| Lambda | REST-JSON | SQLite + Workload Driver | [docs/services/lambda.md](docs/services/lambda.md) | +| IAM / STS | Query | SQLite | [docs/services/iam-sts.md](docs/services/iam-sts.md) | -The rest of the registered surface is described in the [services matrix](docs/services-matrix.md). +### Azure +| Service | Protocol | Storage | Docs | +|---------|----------|---------|------| +| Blob Storage | REST-XML | Filesystem + SQLite | [docs/services/azure.md](docs/services/azure.md) | +| Queue Storage | REST-XML | In-memory | [docs/services/azure.md](docs/services/azure.md) | +| Table Storage | REST-JSON (OData) | In-memory / SQLite | [docs/services/azure.md](docs/services/azure.md) | +| Cosmos DB | REST-JSON | In-memory / SQLite | [docs/services/azure.md](docs/services/azure.md) | +| Key Vault | REST-JSON | In-memory | [docs/services/azure.md](docs/services/azure.md) | +| App Configuration | REST-JSON | In-memory | [docs/services/azure.md](docs/services/azure.md) | + +### GCP +| Service | Protocol | Storage | Docs | +|---------|----------|---------|------| +| Cloud Storage (GCS) | REST-JSON | Filesystem + SQLite | [docs/services/gcp.md](docs/services/gcp.md) | +| Secret Manager | REST-JSON | In-memory | [docs/services/gcp.md](docs/services/gcp.md) | +| Cloud Pub/Sub | REST-JSON | In-memory | [docs/services/gcp.md](docs/services/gcp.md) | + +The complete list of supported services is documented in the [services matrix](docs/services-matrix.md). ## Documentation -Start at the [docs index](docs/). Most-read pages: +Start at the [docs index](docs/). Key references: -| | | +| Page | Description | |---|---| -| [Getting Started](docs/getting-started.md) | Install, first run, SDK examples | -| [Configuration](docs/configuration.md) | Config file, env-var overrides, tier shortcuts | -| [Coverage](docs/coverage.md) | What the service counts mean, and the target | -| [Compatibility Policy](docs/compatibility-policy.md) | What v1.0 guarantees across 1.x — and what it does not | -| [Architecture](docs/architecture.md) | System design, codegen pipeline, plugin model | -| [Contributing](docs/contributing.md) | Dev setup, adding a service | +| [Getting Started](docs/getting-started.md) | Installation, first run, and multi-cloud SDK examples | +| [Configuration](docs/configuration.md) | Configuration file options, environment overrides, and runtime controls | +| [Coverage](docs/coverage.md) | Service counts, routing targets, and depth guarantees | +| [Compatibility Policy](docs/compatibility-policy.md) | Version guarantees across releases and boundary definitions | +| [Architecture](docs/architecture.md) | System architecture, workload execution engine, and plugin design | +| [Contributing](docs/contributing.md) | Local development setup and service contribution guidelines | -Project files: [Support](SUPPORT.md) · [Governance](GOVERNANCE.md) · [Releasing](RELEASE.md) · [Changelog](CHANGELOG.md) +Project links: [Support](SUPPORT.md) | [Governance](GOVERNANCE.md) | [Releasing](RELEASE.md) | [Changelog](CHANGELOG.md) ## Contributing -Contributions are welcome — especially service implementations, compatibility fixes, and documentation. See the [Contributing Guide](docs/contributing.md) and the [Code of Conduct](CODE_OF_CONDUCT.md). +Contributions are welcome, including new service implementations, compatibility improvements, and documentation fixes. See the [Contributing Guide](docs/contributing.md) and the [Code of Conduct](CODE_OF_CONDUCT.md). -For security issues, follow the [Security Policy](SECURITY.md) and do not file a public issue. +For security reports, please refer to the [Security Policy](SECURITY.md) instead of creating public issues. ## License -Apache License, Version 2.0 — see [LICENSE](LICENSE) and [NOTICE](NOTICE). +Apache License, Version 2.0. See [LICENSE](LICENSE) and [NOTICE](NOTICE). -DevCloud is an independent open-source project. References to cloud service providers describe **API compatibility** only; all trademarks are the property of their respective owners. See [TRADEMARKS.md](TRADEMARKS.md). +DevCloud is an independent open-source project. References to cloud service providers describe API compatibility only. All trademarks belong to their respective owners. See [TRADEMARKS.md](TRADEMARKS.md). diff --git a/RELEASE.md b/RELEASE.md index b2e62502..7f1e718a 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -2,7 +2,7 @@ Releases go out through **[Changie](https://changie.dev)** (changelog) and **[GoReleaser](https://goreleaser.com)** (build + publish). Release notes are -authored as Changie fragments during development and batched at release time — +authored as Changie fragments during development and batched at release time; commits and PR labels are **not** used to generate them. Versions follow [Semantic Versioning](https://semver.org): `vMAJOR.MINOR.PATCH`. @@ -27,20 +27,20 @@ changie new You are prompted for a **kind** (`Added`, `Changed`, `Deprecated`, `Removed`, `Fixed`, `Security`, `Documentation`), a **body**, and the **issue number**. It -writes a small YAML file under `changes/unreleased/` — commit it alongside your +writes a small YAML file under `changes/unreleased/`: commit it alongside your code. Prefer `changie new` over hand-writing the YAML: it enforces the issue -number — a fragment without one renders as a dead link — and refuses a body over +number (a fragment without one renders as a dead link) and refuses a body over 200 characters. ### One sentence. Two at most. -The changelog is a scan surface, not a design document — the reader is deciding +The changelog is a scan surface, not a design document: the reader is deciding whether this release affects them. Root cause, mechanism, measurements and rejected alternatives belong in the linked issue and PR, which every entry already points at. -- **One sentence** — what changed, and what it means for someone using DevCloud. -- **A second only if a reader must not miss it** — a limit, a behaviour change +- **One sentence**: what changed, and what it means for someone using DevCloud. +- **A second only if a reader must not miss it**: a limit, a behaviour change they have to act on, or the one figure that makes the entry meaningful. - **Never a third.** If it needs one, it is either two changes (write two fragments) or a story that belongs in the issue. @@ -51,16 +51,16 @@ second sentence is already crowding the first, so the ceiling and the rule push the same way. It does not see a fragment you hand-write. ```yaml -# too long — 417 characters; the root cause, the mechanism and the evidence all +# too long: 417 characters; the root cause, the mechanism and the evidence all # belong in #142 body: 'S3 Control requests were served by S3. `s3control` signs with S3''s own signing name, so every call fell through to the REST-XML default and the S3 - provider parsed it as a bucket and key — `CreateAccessPoint` returned 200 and + provider parsed it as a bucket and key: `CreateAccessPoint` returned 200 and left an object in a bucket named `v20180820`. S3 Control is now split off by its `/v20180820/` path prefix, and its unserved operations return a clean AWS error instead of a fabricated success' -# right length — 193 characters +# right length: 193 characters body: 'S3 Control requests were served by S3, which answered `CreateAccessPoint` with a fabricated 200. It is now split off by its `/v20180820/` path prefix and its unserved operations decline cleanly' @@ -71,19 +71,19 @@ Config: [`.changie.yaml`](.changie.yaml). ## Pre-flight checklist The first three are re-run by the Release workflow against the tagged commit, -which refuses to publish if any fails — tick them to find out on your machine +which refuses to publish if any fails; tick them to find out on your machine rather than from a failed tag. The rest are only caught here. -- [ ] **Generated code is current** — +- [ ] **Generated code is current**: `rm -rf internal/generated && make codegen && git status --porcelain internal/generated` prints nothing. Clear the tree first, as CI does: the generator overwrites the outputs it still emits but never removes one it has stopped emitting, so regenerating in place leaves a retired file looking current. A stale fidelity manifest misreports what the release can be trusted to do. -- [ ] **boto3 compatibility passes** — `make test-compat`. -- [ ] **Go tests pass** — `CGO_ENABLED=0 go test ./...`. +- [ ] **boto3 compatibility passes**: `make test-compat`. +- [ ] **Go tests pass**: `CGO_ENABLED=0 go test ./...`. - [ ] **`main` is green**, including lint and CodeQL. -- [ ] **Every unreleased fragment carries an issue number** — +- [ ] **Every unreleased fragment carries an issue number**: `grep -L 'Issue: "[0-9]' changes/unreleased/*.yaml` prints nothing. - [ ] **`changes/unreleased/` is not empty.** No fragments means either nothing shipped or someone forgot one. @@ -91,13 +91,13 @@ rather than from a failed tag. The rest are only caught here. either a second change that needs its own fragment or detail that belongs in the issue. - [ ] **Deprecation review.** If this release *removes* anything previously - deprecated — a config key, an env var, an admin route — confirm it shipped + deprecated (a config key, an env var, an admin route), confirm it shipped for at least one release with a warning first. Removing without that overlap is a major-version change. Full procedure: [docs/compatibility-policy.md](docs/compatibility-policy.md#deprecation-procedure). - [ ] **Compatibility review.** Changing anything on the guaranteed list in [docs/compatibility-policy.md](docs/compatibility-policy.md) is a major bump - — or it is a bug. Additive change (a new config key, response field, or + (or it is a bug). Additive change (a new config key, response field, or service) is a minor bump. ## Cutting a release @@ -123,7 +123,7 @@ rather than from a failed tag. The rest are only caught here. git push origin main ``` -4. **Tag and push.** The tag **must** match the batched version — the workflow +4. **Tag and push.** The tag **must** match the batched version: the workflow fails if `changes/.md` does not exist. ```sh @@ -139,13 +139,13 @@ rather than from a failed tag. The rest are only caught here. - **Re-runs the guardrails** against that commit and stops before publishing anything if any fails: the Go suite on amd64 and arm64, the boto3 suite (against a GoReleaser-built binary, not `go build`), and the codegen drift check. CI is - not relied on — it races the tag, and `compat.yml` does not trigger on tags at + not relied on: it races the tag, and `compat.yml` does not trigger on tags at all. - **Validates the notes.** `changes/v1.1.0.md` must exist, carry exactly one version heading naming *this* tag (a file copied from an earlier release is rejected), contain only what `changie batch` renders, and end every entry in a valid issue link. -- **Builds and publishes** — binaries for darwin/linux/windows × amd64/arm64 as +- **Builds and publishes**: binaries for darwin/linux/windows x amd64/arm64 as `tar.gz` (`zip` on Windows) with a SHA-256 `CHECKSUMS` file; container images tagged `v1.1.0-alpine`, `v1.1-alpine`, `v1-alpine`, `latest-alpine`; the Homebrew formula; and a GitHub Release whose notes come from @@ -154,14 +154,14 @@ rather than from a failed tag. The rest are only caught here. Config: [`.goreleaser.yaml`](.goreleaser.yaml). Archives carry the `docs/` tree and the top-level files it and `README.md` link -to, so docs are versioned by tag — the `docs/` inside +to, so docs are versioned by tag: the `docs/` inside `devcloud_v1.1.0_linux_amd64.tar.gz` describes exactly the binary beside it. There is no separate docs site to version. ## Homebrew tap GoReleaser's `brews` section publishes `Formula/devcloud.rb` to a separate tap -repository — `homebrew-tap` under the same owner, named by `HOMEBREW_TAP_OWNER` / +repository: `homebrew-tap` under the same owner, named by `HOMEBREW_TAP_OWNER` / `HOMEBREW_TAP_REPO` in [`release.yml`](.github/workflows/release.yml). The job's own `GITHUB_TOKEN` cannot write to another repository, so the workflow @@ -175,7 +175,7 @@ long-running server with no subcommands, so actually starting it would hang. ## Dry run -Run the workflow manually from the Actions tab (**Release → Run workflow**) with a +Run the workflow manually from the Actions tab (**Release -> Run workflow**) with a tag and `dry_run: true` (the default for manual dispatch). GoReleaser runs in `--snapshot` mode: artifacts are built and uploaded to the run, but nothing is published to GHCR, the tap, or GitHub Releases. @@ -186,4 +186,4 @@ published to GHCR, the tap, or GitHub Releases. - `changie batch` + `changie merge` must be committed **before** the tag is pushed. - The tagged commit must pass the Go suite; the workflow will not publish otherwise. - Every entry in the batched notes needs an issue number. -- No manual GitHub Release editing — the notes are owned by Changie fragments. +- No manual GitHub Release editing: the notes are owned by Changie fragments. diff --git a/SUPPORT.md b/SUPPORT.md index aa6fffa7..16f710b4 100644 --- a/SUPPORT.md +++ b/SUPPORT.md @@ -6,34 +6,34 @@ Pick the right channel below to get a fast, accurate response. | Your situation | Channel | |---|---| -| **Bug** — something is broken or behaves unexpectedly | [Open a Bug Report](https://github.com/skyoo2003/devcloud/issues/new?template=bug_report.yml) | -| **Feature request** — you want new functionality | [Open a Feature Request](https://github.com/skyoo2003/devcloud/issues/new?template=feature_request.yml) | -| **Question** — "How do I …?" / "Why does …?" / "Is X possible?" | [Open a Question](https://github.com/skyoo2003/devcloud/issues/new?template=question.yml) or start a [GitHub Discussion](https://github.com/skyoo2003/devcloud/discussions) | -| **Security vulnerability** — potential CVE, auth bypass, data leak | **Do not open a public issue.** Follow [SECURITY.md](SECURITY.md) | +| **Bug**: something is broken or behaves unexpectedly | [Open a Bug Report](https://github.com/skyoo2003/devcloud/issues/new?template=bug_report.yml) | +| **Feature request**: you want new functionality | [Open a Feature Request](https://github.com/skyoo2003/devcloud/issues/new?template=feature_request.yml) | +| **Question**: "How do I...?" / "Why does...?" / "Is X possible?" | [Open a Question](https://github.com/skyoo2003/devcloud/issues/new?template=question.yml) or start a [GitHub Discussion](https://github.com/skyoo2003/devcloud/discussions) | +| **Security vulnerability**: potential CVE, auth bypass, data leak | **Do not open a public issue.** Follow [SECURITY.md](SECURITY.md) | | **Code of Conduct concern** | Contact the maintainer per [CODE_OF_CONDUCT.md](CODE_OF_CONDUCT.md) | ## Before you ask Most questions already have answers: -1. **Search existing issues** — [open](https://github.com/skyoo2003/devcloud/issues) + [closed](https://github.com/skyoo2003/devcloud/issues?q=is%3Aissue+is%3Aclosed) +1. **Search existing issues**: [open](https://github.com/skyoo2003/devcloud/issues) + [closed](https://github.com/skyoo2003/devcloud/issues?q=is%3Aissue+is%3Aclosed) 2. **Check [Troubleshooting](docs/troubleshooting.md)** and the [FAQ](docs/faq.md) -3. **Confirm the operation is served** — `curl -s 'localhost:4747/devcloud/api/fidelity?service='` with `admin.enabled: true`; see [fidelity-manifest.md](docs/fidelity-manifest.md) -4. **Check the [roadmap](docs/roadmap.md)** — the request may already be planned +3. **Confirm the operation is served**: `curl -s 'localhost:4747/devcloud/api/fidelity?service='` with `admin.enabled: true`; see [fidelity-manifest.md](docs/fidelity-manifest.md) +4. **Check the [roadmap](docs/roadmap.md)**: the request may already be planned ## How to write a good question -- **What you tried** — the exact command, snippet, or SDK call -- **What you expected**, and **what actually happened** — full error message -- **Environment** — OS, language version, DevCloud version (git SHA or image tag) -- **Minimal reproduction** — the smallest example that shows the issue +- **What you tried**: the exact command, snippet, or SDK call +- **What you expected**, and **what actually happened**: full error message +- **Environment**: OS, language version, DevCloud version (git SHA or image tag) +- **Minimal reproduction**: the smallest example that shows the issue ## Response expectations DevCloud is maintained by volunteers on a best-effort basis. There is no commercial SLA. -- **Bugs and security reports** — triaged within a few days -- **Feature requests** — reviewed against the [roadmap](docs/roadmap.md) -- **Questions** — answered when a maintainer or community member has time +- **Bugs and security reports**: triaged within a few days +- **Feature requests**: reviewed against the [roadmap](docs/roadmap.md) +- **Questions**: answered when a maintainer or community member has time -If you need guaranteed response times, consider sponsoring the project or contributing the fix yourself — see [CONTRIBUTING.md](CONTRIBUTING.md). +If you need guaranteed response times, consider sponsoring the project or contributing the fix yourself; see [CONTRIBUTING.md](CONTRIBUTING.md). diff --git a/TRADEMARKS.md b/TRADEMARKS.md index 232c0076..df6a4baa 100644 --- a/TRADEMARKS.md +++ b/TRADEMARKS.md @@ -6,11 +6,11 @@ DevCloud is an **independent, community-driven open-source project**. References The following are trademarks of their respective owners. DevCloud uses these names **only for descriptive and interoperability purposes** under nominative fair use: -- **AWS®**, **Amazon Web Services®**, **Amazon S3®**, **Amazon DynamoDB®**, **AWS Lambda®**, **Amazon SQS®**, **AWS IAM®**, **AWS STS®** — trademarks of **Amazon.com, Inc.** or its affiliates -- **Microsoft®**, **Azure®**, **Microsoft Azure®** — trademarks of **Microsoft Corporation** (referenced in the roadmap for future support) -- **Google®**, **Google Cloud®**, **GCP®** — trademarks of **Google LLC** (referenced in the roadmap for future support) -- **Terraform®**, **HashiCorp®** — trademarks of **HashiCorp, Inc.** -- **Docker®** — trademark of **Docker, Inc.** +- **AWS®**, **Amazon Web Services®**, **Amazon S3®**, **Amazon DynamoDB®**, **AWS Lambda®**, **Amazon SQS®**, **AWS IAM®**, **AWS STS®**: trademarks of **Amazon.com, Inc.** or its affiliates +- **Microsoft®**, **Azure®**, **Microsoft Azure®**: trademarks of **Microsoft Corporation** +- **Google®**, **Google Cloud®**, **GCP®**: trademarks of **Google LLC** +- **Terraform®**, **HashiCorp®**: trademarks of **HashiCorp, Inc.** +- **Docker®**: trademark of **Docker, Inc.** ## What this means diff --git a/changes/v0.1.0.md b/changes/v0.1.0.md index 6d23e09d..c703aca7 100644 --- a/changes/v0.1.0.md +++ b/changes/v0.1.0.md @@ -1,6 +1,6 @@ ## [v0.1.0](https://github.com/skyoo2003/devcloud/releases/tag/v0.1.0) - 2026-04-18 -First public release of DevCloud — a local development companion for cloud-native apps. Positioned as an on-ramp to the cloud, not a replacement: iterate locally without cloud bills, then land cleanly on your target CSP. +First public release of DevCloud : a local development companion for cloud-native apps. Positioned as an on-ramp to the cloud, not a replacement: iterate locally without cloud bills, then land cleanly on your target CSP. ### Added @@ -17,7 +17,7 @@ First public release of DevCloud — a local development companion for cloud-nat - 96 AWS services scaffolded from official Smithy models via an in-tree code generator. - Big 6 fully implemented: **S3, SQS, DynamoDB, Lambda, IAM, STS**. - Integration services: SNS, CloudWatch, CloudWatch Logs, KMS, SecretsManager, SSM, EventBridge, ECR, EC2, ECS, Route53, ACM, CloudFormation, and more. -- Cross-service integrations: CloudFormation provisioning, DynamoDB Streams → Lambda, SQS → Lambda, S3 → Lambda, EventBridge → SQS/SNS/Lambda, SNS → SQS subscriptions. +- Cross-service integrations: CloudFormation provisioning, DynamoDB Streams -> Lambda, SQS -> Lambda, S3 -> Lambda, EventBridge -> SQS/SNS/Lambda, SNS -> SQS subscriptions. - **96% boto3 compatibility** (671/699 tests passing against the official AWS SDK test surface). - Port-aware URL construction in SQS, ECR, and CloudFormation response paths (opts-based, not hardcoded). diff --git a/changes/v1.0.0.md b/changes/v1.0.0.md index 56c2c3f3..637d77b7 100644 --- a/changes/v1.0.0.md +++ b/changes/v1.0.0.md @@ -7,17 +7,17 @@ * Serve ~2,200 standard CRUD operations across the 46 JSON-protocol services from a Smithy-driven fallback engine (`internal/shared/crud`); hand-written operations take precedence and unclassifiable ones still return an error. See docs/crud-engine.md ([#98](https://github.com/skyoo2003/devcloud/issues/98)) * Homebrew installation via `brew install skyoo2003/tap/devcloud`, published to the tap on each release ([#121](https://github.com/skyoo2003/devcloud/issues/121)) * Per-operation fidelity manifest declaring every operation as hand-verified, auto-crud or unimplemented, exposed at `GET /devcloud/api/fidelity` and enforced by a build-failing coverage test ([#126](https://github.com/skyoo2003/devcloud/issues/126)) -* Real tag support for KMS, CloudWatch and EventBridge — TagResource, UntagResource and ListTagsForResource/ListResourceTags now persist tags per resource ARN instead of being echoed by the generic CRUD engine ([#126](https://github.com/skyoo2003/devcloud/issues/126)) -* A published compatibility policy (docs/compatibility-policy.md) stating what v1.0 guarantees across the 1.x line — config keys, environment variables, the CLI, admin API response keys, fidelity tier names, and the response shape of hand-verified operations covered by the boto3 compatibility suite — what it explicitly does not guarantee, and the deprecation procedure that precedes any removal ([#129](https://github.com/skyoo2003/devcloud/issues/129)) +* Real tag support for KMS, CloudWatch and EventBridge : TagResource, UntagResource and ListTagsForResource/ListResourceTags now persist tags per resource ARN instead of being echoed by the generic CRUD engine ([#126](https://github.com/skyoo2003/devcloud/issues/126)) +* A published compatibility policy (docs/compatibility-policy.md) stating what v1.0 guarantees across the 1.x line : config keys, environment variables, the CLI, admin API response keys, fidelity tier names, and the response shape of hand-verified operations covered by the boto3 compatibility suite : what it explicitly does not guarantee, and the deprecation procedure that precedes any removal ([#129](https://github.com/skyoo2003/devcloud/issues/129)) ### Changed * Rename `internal/dashboard` to `internal/admin` and the `dashboard.enabled` config key to `admin.enabled`. The old key is honoured for one release with a deprecation warning. ([#111](https://github.com/skyoo2003/devcloud/issues/111)) * Build the request log collector only when `admin.enabled` is set, dropping a per-request mutex lock and ring-buffer write from the AWS request hot path ([#112](https://github.com/skyoo2003/devcloud/issues/112)) * The `services` config block is now optional: omit it and every registered service starts with `data_dir ./data/`; list services and only those start. Replaces 325 lines of identical boilerplate. ([#120](https://github.com/skyoo2003/devcloud/issues/120)) -* Release archives now carry the `docs/` tree alongside the binary, so the documentation you unpack — including the fidelity manifest and the release's compatibility promises — describes exactly the version you downloaded ([#127](https://github.com/skyoo2003/devcloud/issues/127)) +* Release archives now carry the `docs/` tree alongside the binary, so the documentation you unpack : including the fidelity manifest and the release's compatibility promises : describes exactly the version you downloaded ([#127](https://github.com/skyoo2003/devcloud/issues/127)) ### Removed * Remove the dead, unregistered STS provider stub (`internal/services/sts`); STS is served by the IAM package ([#96](https://github.com/skyoo2003/devcloud/issues/96)) * Extract the web dashboard to a separate repository. The binary builds and serves no UI, only the opt-in admin API at `/devcloud/api/*`. ([#111](https://github.com/skyoo2003/devcloud/issues/111)) -* Remove dead surface found by a repo-wide over-engineering audit: the event bus and admin WebSocket, the `GetMetrics` plugin API and `/devcloud/api/metrics` (resource counts remain on `/devcloud/api/services`), `shared.ResourceStore`, `gateway.ExtractAccountID`, and the `auth.enabled` key — still read for one release so enabling it warns that credentials are accepted regardless. Drops the `gorilla/websocket` dependency. ([#120](https://github.com/skyoo2003/devcloud/issues/120)) +* Remove dead surface found by a repo-wide over-engineering audit: the event bus and admin WebSocket, the `GetMetrics` plugin API and `/devcloud/api/metrics` (resource counts remain on `/devcloud/api/services`), `shared.ResourceStore`, `gateway.ExtractAccountID`, and the `auth.enabled` key : still read for one release so enabling it warns that credentials are accepted regardless. Drops the `gorilla/websocket` dependency. ([#120](https://github.com/skyoo2003/devcloud/issues/120)) ### Fixed * Route `X-Amz-Target` prefixes that contain dots (CodeConnections, DMS) to the right service; they misrouted to CloudTrail ([#96](https://github.com/skyoo2003/devcloud/issues/96)) * Unimplemented operations return `NotImplemented` (HTTP 501) instead of an empty `200`, so SDKs can tell a not-yet-emulated operation from an invalid one ([#96](https://github.com/skyoo2003/devcloud/issues/96)) @@ -28,14 +28,14 @@ * Codegen now parses operations bound to Smithy resource shapes, recovering 285 operations that were invisible to the generator (bedrock 0 of 101, lambda 19 of 85, ecs 12 of 76, transfer 29 of 71, sso-admin 67 of 79) ([#126](https://github.com/skyoo2003/devcloud/issues/126)) * The fidelity manifest now reads each provider's actual dispatch instead of intersecting with the Smithy model, recovering 226 served operations it had hidden (dynamodbstreams listed 4 of its 22, acm's UpdateCertificate, bedrock's InvokeModelWithResponseStream) and dropping 5 non-operations it had invented (identitystore Description/DisplayName/Emails, pipes DELETE/POST) ([#126](https://github.com/skyoo2003/devcloud/issues/126)) * EventBridge now drops a bus's or rule's tags when it is deleted. ARNs are derived from the name, so recreating a deleted resource reused its ARN and inherited the previous tags ([#126](https://github.com/skyoo2003/devcloud/issues/126)) -* Restored CloudWatch's 17 CRUD-engine operations. The gateway picks the protocol from the request, not from the provider, so CloudWatch reaches the engine whenever a client speaks JSON — filtering the registry by the provider's declared protocol had removed that coverage outright ([#126](https://github.com/skyoo2003/devcloud/issues/126)) -* EventBridge rule ARNs now name their event bus, as AWS does. A rule name is unique per bus, so same-named rules on two custom buses previously shared one ARN — and with it, one tag set, where tagging one rule changed the other's and deleting one wiped the survivor's ([#126](https://github.com/skyoo2003/devcloud/issues/126)) +* Restored CloudWatch's 17 CRUD-engine operations. The gateway picks the protocol from the request, not from the provider, so CloudWatch reaches the engine whenever a client speaks JSON : filtering the registry by the provider's declared protocol had removed that coverage outright ([#126](https://github.com/skyoo2003/devcloud/issues/126)) +* EventBridge rule ARNs now name their event bus, as AWS does. A rule name is unique per bus, so same-named rules on two custom buses previously shared one ARN : and with it, one tag set, where tagging one rule changed the other's and deleting one wiped the survivor's ([#126](https://github.com/skyoo2003/devcloud/issues/126)) * CloudWatch now drops an alarm's tags when the alarm is deleted, so recreating an alarm under the same name no longer inherits the old one's tags ([#126](https://github.com/skyoo2003/devcloud/issues/126)) -* Downloadable binaries now start, where releases built with `CGO_ENABLED=0` against a SQLite driver that required cgo exited at startup — every tar.gz/zip binary, the Homebrew formula and the versioned `*-alpine` images. The driver is now pure Go, so no build needs a C toolchain or SQLite headers ([#128](https://github.com/skyoo2003/devcloud/issues/128)) +* Downloadable binaries now start, where releases built with `CGO_ENABLED=0` against a SQLite driver that required cgo exited at startup : every tar.gz/zip binary, the Homebrew formula and the versioned `*-alpine` images. The driver is now pure Go, so no build needs a C toolchain or SQLite headers ([#128](https://github.com/skyoo2003/devcloud/issues/128)) * Service selection now matches what the docs promise: an empty `services` block starts nothing rather than every registered service, and `DEVCLOUD_SERVICES` names the running set outright instead of intersecting with the block. Previously `DEVCLOUD_SERVICES=sqs` alongside a block listing only `s3` started nothing at all ([#129](https://github.com/skyoo2003/devcloud/issues/129)) * CloudFront no longer fabricates a success for the 122 operations the fidelity manifest classifies as `unimplemented`, where its dispatch fallback answered HTTP 200 with an empty XML document that boto3 parses as a successful empty result. It now returns `NotImplemented` (HTTP 501), matching the other 32 providers that decline from their own dispatch default ([#129](https://github.com/skyoo2003/devcloud/issues/129)) -* Generated routers now honour route patterns that constrain the query string, so operations distinguished only by a query parameter are reachable — `matchURI` split the whole pattern into path segments, leaving CloudFront `TagResource` and `UntagResource` implemented but unroutable. Query-constrained routes are tried ahead of unconstrained ones, so a bare path cannot shadow a more specific route ([#129](https://github.com/skyoo2003/devcloud/issues/129)) -* Released binaries now register their services, where GoReleaser built `cmd/devcloud/main.go` rather than the package and dropped the `imports.go` that blank-imports all 104 services — every archive, the Homebrew formula and the container images exited at startup with `unknown service: s3`. The release gate now builds with GoReleaser instead of compiling the package, which is why the boto3 suite missed it ([#131](https://github.com/skyoo2003/devcloud/issues/131)) +* Generated routers now honour route patterns that constrain the query string, so operations distinguished only by a query parameter are reachable : `matchURI` split the whole pattern into path segments, leaving CloudFront `TagResource` and `UntagResource` implemented but unroutable. Query-constrained routes are tried ahead of unconstrained ones, so a bare path cannot shadow a more specific route ([#129](https://github.com/skyoo2003/devcloud/issues/129)) +* Released binaries now register their services, where GoReleaser built `cmd/devcloud/main.go` rather than the package and dropped the `imports.go` that blank-imports all 104 services : every archive, the Homebrew formula and the container images exited at startup with `unknown service: s3`. The release gate now builds with GoReleaser instead of compiling the package, which is why the boto3 suite missed it ([#131](https://github.com/skyoo2003/devcloud/issues/131)) ### Documentation -* Corrected what the fidelity manifest says an `unimplemented` operation returns: it claimed JSON and Query services answer `InvalidAction` (HTTP 400), where only the 46 providers falling through to the CRUD engine do and 32 answer `NotImplemented` (HTTP 501) from their own dispatch default. Only the failure itself is stable — the specific code and status are documented, not guaranteed ([#129](https://github.com/skyoo2003/devcloud/issues/129)) -* The roadmap, architecture and governance docs no longer describe DevCloud as pre-1.0. All three ship inside the release archive, so a v1.0 download would have carried a roadmap listing its own release as pending, an architecture overview naming Phase 1 as current, and a governance doc calling the API unstable — contradicting the compatibility policy packaged beside them ([#130](https://github.com/skyoo2003/devcloud/issues/130)) +* Corrected what the fidelity manifest says an `unimplemented` operation returns: it claimed JSON and Query services answer `InvalidAction` (HTTP 400), where only the 46 providers falling through to the CRUD engine do and 32 answer `NotImplemented` (HTTP 501) from their own dispatch default. Only the failure itself is stable : the specific code and status are documented, not guaranteed ([#129](https://github.com/skyoo2003/devcloud/issues/129)) +* The roadmap, architecture and governance docs no longer describe DevCloud as pre-1.0. All three ship inside the release archive, so a v1.0 download would have carried a roadmap listing its own release as pending, an architecture overview naming Phase 1 as current, and a governance doc calling the API unstable : contradicting the compatibility policy packaged beside them ([#130](https://github.com/skyoo2003/devcloud/issues/130)) diff --git a/changes/v1.1.0.md b/changes/v1.1.0.md index c62ad5dc..e458e1cb 100644 --- a/changes/v1.1.0.md +++ b/changes/v1.1.0.md @@ -1,14 +1,14 @@ ## [v1.1.0](https://github.com/skyoo2003/devcloud/releases/tag/v1.1.0) - 2026-09-06 ### Added * Provider-namespaced configuration: service blocks can be written as `providers.aws.services.*`, forward-compatible with `providers.azure.*`. The top-level `services` block is the same block under its historical name and keeps working ([#135](https://github.com/skyoo2003/devcloud/issues/135)) -* Per-provider auth adapters (`internal/auth`) parse SigV4 credentials once and expose the caller's claimed identity to every plugin via `auth.FromContext`, adding a `region` field to `GET /devcloud/api/logs`. Nothing is verified — DevCloud still accepts any credentials ([#135](https://github.com/skyoo2003/devcloud/issues/135)) +* Per-provider auth adapters (`internal/auth`) parse SigV4 credentials once and expose the caller's claimed identity to every plugin via `auth.FromContext`, adding a `region` field to `GET /devcloud/api/logs`. Nothing is verified : DevCloud still accepts any credentials ([#135](https://github.com/skyoo2003/devcloud/issues/135)) * Amazon Comprehend, served by the generic CRUD engine (49 operations auto-crud, 36 unimplemented), where an unimplemented operation returns a clean AWS error rather than a fabricated success ([#136](https://github.com/skyoo2003/devcloud/issues/136)) * All 48 upstream AI/Machine Learning service models are registered, taking DevCloud from 105 to 148 services. 17 are served and the remaining 31 decline with a clean AWS error instead of falling through to real AWS ([#137](https://github.com/skyoo2003/devcloud/issues/137)) * docs/coverage.md, which publishes the registered, serving and registered-only service counts together so the service count is never stated on its own ([#137](https://github.com/skyoo2003/devcloud/issues/137)) -* Admin API endpoint `GET /devcloud/api/unrouted` reports which AWS services this DevCloud was asked for and does not register, with per-service counts and first/last seen; it requires `admin.enabled: true`. The figure is a floor — an unclassifiable request is routed to S3 rather than counted, and a non-zero `droppedServiceIds` means the collector hit its key cap ([#138](https://github.com/skyoo2003/devcloud/issues/138)) +* Admin API endpoint `GET /devcloud/api/unrouted` reports which AWS services this DevCloud was asked for and does not register, with per-service counts and first/last seen; it requires `admin.enabled: true`. The figure is a floor : an unclassifiable request is routed to S3 rather than counted, and a non-zero `droppedServiceIds` means the collector hit its key cap ([#138](https://github.com/skyoo2003/devcloud/issues/138)) * "Service Not Supported" issue form for requesting an AWS service DevCloud does not register yet, which asks which operations you actually call and accepts `/devcloud/api/unrouted` output ([#138](https://github.com/skyoo2003/devcloud/issues/138)) * The generic CRUD engine now serves `rest-json` services, recovering the operation from the request method and path against the route table the model already declares (shared in `internal/shared/httproute`). 28 registered-only services started serving: services with at least one operation went 117 to 145, and `auto-crud` operations 1,415 to 2,200 ([#139](https://github.com/skyoo2003/devcloud/issues/139)) -* The 57 AWS services with demonstrated demand are registered, taking DevCloud to 205 services — 199 of which serve at least one operation, for 4,858 engine-served operations. The three that serve nothing are named with their reasons in docs/coverage.md ([#139](https://github.com/skyoo2003/devcloud/issues/139)) +* The 57 AWS services with demonstrated demand are registered, taking DevCloud to 205 services : 199 of which serve at least one operation, for 4,858 engine-served operations. The three that serve nothing are named with their reasons in docs/coverage.md ([#139](https://github.com/skyoo2003/devcloud/issues/139)) * The generic CRUD engine now serves every protocol DevCloud registers: `rest-xml` recovers its operation from method and path, and `query` reads it from the form body's `Action` field. 112 operations move from `unimplemented` to `auto-crud`, so protocol is no longer a reason any registered service serves nothing ([#143](https://github.com/skyoo2003/devcloud/issues/143)) * CI gates the published coverage figure: `go test ./cmd/devcloud/` compares `docs/coverage.md`, `README.md` and `docs/README.md` against the live registry in both directions, so neither the code nor the number can move without the other. The demand set in `docs/demand.md` is checked to still be registered, so the count cannot be held steady by swapping one service for another ([#144](https://github.com/skyoo2003/devcloud/issues/144)) * `scripts/model_churn.py` summarises what a Smithy model sync actually changed, so the weekly sync PR reads as a change rather than a whole-tree regeneration of 194 models ([#147](https://github.com/skyoo2003/devcloud/issues/147)) @@ -16,7 +16,7 @@ * Codegen now runs through a provider-neutral intermediate representation (`internal/codegen/ir`) behind a `ModelSource` interface, so a second API description format is an added file rather than a rewrite. Smithy is the first source and generated output is byte-identical ([#135](https://github.com/skyoo2003/devcloud/issues/135)) * Service routing aliases are derived from the Smithy models instead of a hand-maintained switch in the gateway, so onboarding a service no longer needs a hand-written routing entry. All 93 previously-routed aliases resolve unchanged, and an alias claimed by more than one service is reported rather than guessed ([#137](https://github.com/skyoo2003/devcloud/issues/137)) * The fidelity manifest records each service's wire protocol, so a registered service that the CRUD engine cannot reach is distinguishable from one whose wiring is broken ([#137](https://github.com/skyoo2003/devcloud/issues/137)) -* The AWS coverage target is 205 services, not 431: of the 283 services DevCloud does not register, only 57 have been built by two or more of moto, LocalStack and terraform-provider-aws, and DevCloud has received zero service requests to date. The remaining 226 can still be onboarded on request but are no longer promised — evidence in docs/demand.md, decision rule in docs/coverage.md ([#138](https://github.com/skyoo2003/devcloud/issues/138)) +* The AWS coverage target is 205 services, not 431: of the 283 services DevCloud does not register, only 57 have been built by two or more of moto, LocalStack and terraform-provider-aws, and DevCloud has received zero service requests to date. The remaining 226 can still be onboarded on request but are no longer promised : evidence in docs/demand.md, decision rule in docs/coverage.md ([#138](https://github.com/skyoo2003/devcloud/issues/138)) * `rest-xml` request bodies are still not buffered by the gateway, now by an explicit rule rather than as a side effect of the protocol being unservable: the engine serves `rest-xml` from the path and query alone, so S3's large binary uploads keep streaming. `crud.NeedsBody` is the predicate, and a test fails if the body is read ([#143](https://github.com/skyoo2003/devcloud/issues/143)) * The boto3 compatibility suite exercises every registered service, parametrising over a generated service list (`internal/generated/compat/services.json`) instead of two hand-written name lists. It found 31 of 205 registered services with no boto3 test at all, and grew from 854 to 993 tests ([#144](https://github.com/skyoo2003/devcloud/issues/144)) ### Fixed @@ -27,17 +27,17 @@ * `make codegen` is now byte-identical across runs, where a model bundling two namespaces with colliding shape names produced different output every time because shapes are keyed by short name and the generator iterated a Go map ([#137](https://github.com/skyoo2003/devcloud/issues/137)) * Amazon Rekognition can be onboarded; its `X-Amz-Target` prefix `RekognitionService` was covered by no routing entry, so every call returned `UnknownService` ([#137](https://github.com/skyoo2003/devcloud/issues/137)) * Stale service counts across the docs: README.md advertised 104 services and docs/faq.md 101 when 148 are registered, and docs/fidelity-manifest.md carried outdated per-tier operation totals. Every count now appears with its depth split (148 registered / 117 serving at least one operation), so the number is never read as a capability claim on its own ([#138](https://github.com/skyoo2003/devcloud/issues/138)) -* S3 Control requests were served by S3, which signs with the same name and parsed them as a bucket and key — `CreateAccessPoint` returned 200 and left an object in a bucket named `v20180820`. S3 Control is now split off by its `/v20180820/` path prefix, and its unserved operations return a clean AWS error instead of a fabricated success ([#142](https://github.com/skyoo2003/devcloud/issues/142)) -* Eight query-protocol providers (`rds`, `neptune`, `docdb`, `redshift`, `elasticache`, `autoscaling`, `cloudformation`, `elasticloadbalancingv2`) answered any unimplemented operation with HTTP 200 and an empty ``, so a caller was told an operation succeeded when nothing ran — and botocore raised `KeyError` on the missing `` wrapper. They now hand the operation to the generic CRUD engine, which serves the 223 CRUD-shaped ones from the real store and refuses the rest with a clean `InvalidAction` ([#145](https://github.com/skyoo2003/devcloud/issues/145)) +* S3 Control requests were served by S3, which signs with the same name and parsed them as a bucket and key : `CreateAccessPoint` returned 200 and left an object in a bucket named `v20180820`. S3 Control is now split off by its `/v20180820/` path prefix, and its unserved operations return a clean AWS error instead of a fabricated success ([#142](https://github.com/skyoo2003/devcloud/issues/142)) +* Eight query-protocol providers (`rds`, `neptune`, `docdb`, `redshift`, `elasticache`, `autoscaling`, `cloudformation`, `elasticloadbalancingv2`) answered any unimplemented operation with HTTP 200 and an empty ``, so a caller was told an operation succeeded when nothing ran : and botocore raised `KeyError` on the missing `` wrapper. They now hand the operation to the generic CRUD engine, which serves the 223 CRUD-shaped ones from the real store and refuses the rest with a clean `InvalidAction` ([#145](https://github.com/skyoo2003/devcloud/issues/145)) * The four Amazon Lex services are reachable from boto3, where all four sign as `lex` and no service is called `lex`, so the alias stayed contested and every Lex call died as `UnknownService`. The shared-signing-name split now recognises a signing name that is a group key rather than a member service ID, and answers each request from the sibling whose route table models its method and path ([#146](https://github.com/skyoo2003/devcloud/issues/146)) * `appsync.ListApis`, `eks.ListAccessPolicies` and `opensearch.ListApplications` are reachable: each was implemented, but the provider's hand-written path resolver did not know the route boto3 sends, so the operation arrived empty and the call answered `NotImplemented`. Each provider now falls back to its own model's route table when its resolver recognises nothing ([#146](https://github.com/skyoo2003/devcloud/issues/146)) * The fidelity manifest no longer drops operations with short names, where the hand-verified scan required four characters or more and reported `resourcegroups.Tag` as unimplemented while the code beside it was real. Short dispatch literals are now collected separately and promoted only where the service's own model declares them ([#146](https://github.com/skyoo2003/devcloud/issues/146)) * S3 no longer answers an unimplemented bucket sub-resource with a bucket listing, where `GET /{Bucket}?analytics` and seven siblings fell through to `ListObjects` and botocore read the resulting 200 as a successful call. A bucket-level GET whose query parameter is neither a served sub-resource nor a listing parameter now returns a clean `NotImplemented` ([#146](https://github.com/skyoo2003/devcloud/issues/146)) * The weekly Smithy model sync discarded the changes worth reviewing: its test step ended the job before the pull request was opened, and an upstream model that gains an operation is designed to fail that step. The test result is now recorded rather than gating, the PR is always opened when models moved, and the failure is re-raised afterwards so the cron does not silently go green ([#147](https://github.com/skyoo2003/devcloud/issues/147)) -* A release dry run keeps its artifacts; the `Upload assets` step ran when `dry_run` was not set, so it uploaded on a real release — where GoReleaser has already attached the same files — and skipped the one path that publishes nothing else ([#150](https://github.com/skyoo2003/devcloud/issues/150)) +* A release dry run keeps its artifacts; the `Upload assets` step ran when `dry_run` was not set, so it uploaded on a real release : where GoReleaser has already attached the same files : and skipped the one path that publishes nothing else ([#150](https://github.com/skyoo2003/devcloud/issues/150)) ### Documentation * Coverage docs state what the compatibility figure excludes and why: two registered services have no boto3 client at all (`sagemakerruntimehttp2`, `transcribestreaming`), and the four Lex services are registered but reachable by no boto3 caller because every Lex client signs as the contested alias `lex` ([#144](https://github.com/skyoo2003/devcloud/issues/144)) -* docs/coverage.md now publishes the measured cost of keeping up with upstream — refreshing all 194 vendored models changed 93 of them, and 32 moved an operation. The reading states its own ceiling: those 93 had been vendored 141 days earlier, so it is an accumulated backlog and not a weekly rate ([#147](https://github.com/skyoo2003/devcloud/issues/147)) +* docs/coverage.md now publishes the measured cost of keeping up with upstream : refreshing all 194 vendored models changed 93 of them, and 32 moved an operation. The reading states its own ceiling: those 93 had been vendored 141 days earlier, so it is an accumulated backlog and not a weekly rate ([#147](https://github.com/skyoo2003/devcloud/issues/147)) * The release procedure moved from `docs/release.md` to `RELEASE.md` at the repo root, and now states three things it never did: the Homebrew formula is published to a separate tap repository through a GitHub App token scoped to that repo alone, a tagged build pushes four `*-alpine` image tags, and `make changelog VERSION=` is the batch-and-merge pair ([#148](https://github.com/skyoo2003/devcloud/issues/148)) -* The contributor guide no longer asks for SQLite3 development headers, CGO, or a separately running DevCloud instance — the driver has been pure Go since v1.0.0, and the compatibility suite starts its own server. docs/getting-started.md also no longer advertises a Next.js frontend on port 3000 that lives in a separate repository ([#149](https://github.com/skyoo2003/devcloud/issues/149)) +* The contributor guide no longer asks for SQLite3 development headers, CGO, or a separately running DevCloud instance : the driver has been pure Go since v1.0.0, and the compatibility suite starts its own server. docs/getting-started.md also no longer advertises a Next.js frontend on port 3000 that lives in a separate repository ([#149](https://github.com/skyoo2003/devcloud/issues/149)) * The compatibility policy states current figures, having gone stale in five places because nothing gates it: `cmd/devcloud/coverage_test.go` covers docs/coverage.md, README.md and docs/README.md, and docs/compatibility-policy.md is not in that set. It now reads 1,144 tests, 5,193 `auto-crud`, 4,497 hand-verified, 205 registered with 4 serving nothing, and 193/12 model-backed to hand-written ([#150](https://github.com/skyoo2003/devcloud/issues/150)) diff --git a/changes/v1.1.1.md b/changes/v1.1.1.md index cd99dd33..24693772 100644 --- a/changes/v1.1.1.md +++ b/changes/v1.1.1.md @@ -1,7 +1,7 @@ ## [v1.1.1](https://github.com/skyoo2003/devcloud/releases/tag/v1.1.1) - 2026-09-07 ### Security -* An S3 object key can no longer leave the bucket that owns it — a key such as `../victim/secret.txt` stayed under the store base directory and so passed the old containment check, letting one tenant read, overwrite and delete another tenant's objects — because every user-controlled path component is now guarded with `filepath.IsLocal` ([#155](https://github.com/skyoo2003/devcloud/issues/155)) +* An S3 object key can no longer leave the bucket that owns it : a key such as `../victim/secret.txt` stayed under the store base directory and so passed the old containment check, letting one tenant read, overwrite and delete another tenant's objects : because every user-controlled path component is now guarded with `filepath.IsLocal` ([#155](https://github.com/skyoo2003/devcloud/issues/155)) ### Documentation -* Changelog fragments are capped at one sentence, two at most — stated with a worked example in RELEASE.md, in the pre-flight checklist, in CONTRIBUTING.md and the PR checklist, and enforced by a 400-character body limit `changie new` refuses to exceed. Every entry already in the changelog is rewritten to that limit, and the v1.1.0 and v1.0.0 release notes are republished from their fragment files ([#152](https://github.com/skyoo2003/devcloud/issues/152)) -* The documentation no longer publishes the same figure at five different values: six numbers restated across pages now live in docs/coverage.md, which every other page links to. Three claims the code does not back are corrected — an unregistered `GET /devcloud/api/health`, a Lambda invoke failure blamed on Docker, and codegen outputs no template emits — and eighteen files are 1,063 lines shorter ([#151](https://github.com/skyoo2003/devcloud/issues/151)) -* The documentation is now published as a searchable site at https://skyoo2003.github.io/devcloud/, built with Hugo from the same `docs/*.md` files GitHub renders — no front matter added and no links rewritten. ([#154](https://github.com/skyoo2003/devcloud/issues/154)) +* Changelog fragments are capped at one sentence, two at most : stated with a worked example in RELEASE.md, in the pre-flight checklist, in CONTRIBUTING.md and the PR checklist, and enforced by a 400-character body limit `changie new` refuses to exceed. Every entry already in the changelog is rewritten to that limit, and the v1.1.0 and v1.0.0 release notes are republished from their fragment files ([#152](https://github.com/skyoo2003/devcloud/issues/152)) +* The documentation no longer publishes the same figure at five different values: six numbers restated across pages now live in docs/coverage.md, which every other page links to. Three claims the code does not back are corrected : an unregistered `GET /devcloud/api/health`, a Lambda invoke failure blamed on Docker, and codegen outputs no template emits : and eighteen files are 1,063 lines shorter ([#151](https://github.com/skyoo2003/devcloud/issues/151)) +* The documentation is now published as a searchable site at https://skyoo2003.github.io/devcloud/, built with Hugo from the same `docs/*.md` files GitHub renders : no front matter added and no links rewritten. ([#154](https://github.com/skyoo2003/devcloud/issues/154)) diff --git a/changes/v1.2.0.md b/changes/v1.2.0.md index edf4b99f..69d1b019 100644 --- a/changes/v1.2.0.md +++ b/changes/v1.2.0.md @@ -6,9 +6,9 @@ * Three figures are now gated against the binary: the coverage page's routing and depth targets, the 45 MiB binary-size ceiling, and that all 431 registered services initialize ([#163](https://github.com/skyoo2003/devcloud/issues/163)) ### Changed * The weekly Smithy sync now re-derives the published coverage figures and commits them into its own pull request, so reviewing it means confirming what moved rather than transcribing failing tests ([#165](https://github.com/skyoo2003/devcloud/issues/165)) -* The published fidelity tiers are stated against two denominators — the services DevCloud promises depth on, and every service it registers — so the routed long tail no longer reads as a fidelity drop ([#166](https://github.com/skyoo2003/devcloud/issues/166)) +* The published fidelity tiers are stated against two denominators : the services DevCloud promises depth on, and every service it registers : so the routed long tail no longer reads as a fidelity drop ([#166](https://github.com/skyoo2003/devcloud/issues/166)) ### Fixed -* The CRUD engine no longer answers an operation it cannot classify with a broader sibling's route — chime's AssociatePhoneNumberWithUser was returning UpdateUser's 200 — and now declines it cleanly ([#162](https://github.com/skyoo2003/devcloud/issues/162)) +* The CRUD engine no longer answers an operation it cannot classify with a broader sibling's route : chime's AssociatePhoneNumberWithUser was returning UpdateUser's 200 : and now declines it cleanly ([#162](https://github.com/skyoo2003/devcloud/issues/162)) ### Documentation -* The coverage page states routing and depth as two targets — 431 of 431 registered, depth still the 205 services the demand study settled — so a service count no longer reads as a fidelity promise ([#163](https://github.com/skyoo2003/devcloud/issues/163)) -* The five per-service pages now list the operations DevCloud actually serves, derived from the fidelity manifest rather than from memory — S3 had documented 8 of 37, IAM 6 of 58 ([#167](https://github.com/skyoo2003/devcloud/issues/167)) +* The coverage page states routing and depth as two targets : 431 of 431 registered, depth still the 205 services the demand study settled : so a service count no longer reads as a fidelity promise ([#163](https://github.com/skyoo2003/devcloud/issues/163)) +* The five per-service pages now list the operations DevCloud actually serves, derived from the fidelity manifest rather than from memory : S3 had documented 8 of 37, IAM 6 of 58 ([#167](https://github.com/skyoo2003/devcloud/issues/167)) diff --git a/docs/README.md b/docs/README.md index 78062637..0a18ec6b 100644 --- a/docs/README.md +++ b/docs/README.md @@ -4,43 +4,43 @@ | Page | What it covers | |---|---| -| [Getting Started](getting-started.md) | Install, first run, boto3 / AWS CLI / Terraform examples | -| [Configuration](configuration.md) | YAML options, env-var overrides, tier shortcuts | -| [Architecture](architecture.md) | System design, plugin model, codegen pipeline, multi-CSP vision | -| [Roadmap](roadmap.md) | Phased plan toward multi-CSP support | +| [Getting Started](getting-started.md) | Installation, first run, and multi-cloud SDK examples | +| [Configuration](configuration.md) | YAML options, environment variable overrides, and runtime settings | +| [Architecture](architecture.md) | System design, universal workload runtime, codegen pipeline, and multi-CSP routing | +| [Roadmap](roadmap.md) | Multi-cloud milestones, delivered capabilities, and upcoming priorities | -## What you can rely on +## Reliability and coverage | Page | What it covers | |---|---| -| [Coverage](coverage.md) | 441 registered / 441 serving — the routing and depth targets, and what each promises | -| [Compatibility Policy](compatibility-policy.md) | What v1.0 guarantees across 1.x, what it does not, and how deprecation works | -| [Fidelity Manifest](fidelity-manifest.md) | Per-operation tiers: how much to trust any given call | -| [CRUD Engine](crud-engine.md) | How engine-served operations behave, and where they stop | -| [Services Matrix](services-matrix.md) | Depth by group, cross-service integrations, protocols | -| [Demand](demand.md) | Which services have demonstrated demand, and how it was measured | +| [Coverage](coverage.md) | 441 registered / 441 serving, routing and depth targets and their guarantees | +| [Compatibility Policy](compatibility-policy.md) | Stability guarantees across 1.x releases and boundary definitions | +| [Fidelity Manifest](fidelity-manifest.md) | Per-operation tiers and reliability classifications | +| [CRUD Engine](crud-engine.md) | Generic fallback engine behavior and operational limits | +| [Services Matrix](services-matrix.md) | Depth by service group, cross-service wiring, and wire protocols | +| [Demand](demand.md) | Evaluated cloud services and demand measurement methodology | -## Per-service references +## Service references -[S3](services/s3.md) · [SQS](services/sqs.md) · [SNS](services/sns.md) · [DynamoDB](services/dynamodb.md) · [Lambda](services/lambda.md) · [IAM / STS](services/iam-sts.md) · [EventBridge](services/eventbridge.md) +[S3](services/s3.md) | [SQS](services/sqs.md) | [SNS](services/sns.md) | [DynamoDB](services/dynamodb.md) | [Lambda](services/lambda.md) | [IAM / STS](services/iam-sts.md) | [EventBridge](services/eventbridge.md) | [Azure](services/azure.md) | [GCP](services/gcp.md) | [Federated Identity](services/federated-identity.md) ## Problem solving -- [FAQ](faq.md) — scope, compatibility, CI use -- [Troubleshooting](troubleshooting.md) — common errors and fixes -- [Support](../SUPPORT.md) — where to ask +- [FAQ](faq.md): scope, compatibility expectations, and CI usage +- [Troubleshooting](troubleshooting.md): common errors and solutions +- [Support](../SUPPORT.md): community support and help channels ## Contributing -- [Contributing Guide](contributing.md) — dev setup, testing, codegen, adding a service -- [Plugin API](plugin-api.md) — the in-tree `ServicePlugin` contract -- [Releasing](../RELEASE.md) — Changie + GoReleaser, versioning, Homebrew tap, dry runs -- Root pointers: [CONTRIBUTING.md](../CONTRIBUTING.md) · [CODE_OF_CONDUCT.md](../CODE_OF_CONDUCT.md) · [SECURITY.md](../SECURITY.md) · [GOVERNANCE.md](../GOVERNANCE.md) +- [Contributing Guide](contributing.md): local environment setup, testing, codegen, and adding new services +- [Plugin API](plugin-api.md): in-tree ServicePlugin interfaces and lifecycle contracts +- [Releasing](../RELEASE.md): release workflows, versioning conventions, and distribution packaging +- Core guidelines: [CONTRIBUTING.md](../CONTRIBUTING.md) | [CODE_OF_CONDUCT.md](../CODE_OF_CONDUCT.md) | [SECURITY.md](../SECURITY.md) | [GOVERNANCE.md](../GOVERNANCE.md) -## Meta +## Project information -[Changelog](../CHANGELOG.md) · [License (Apache 2.0)](../LICENSE) · [Trademarks](../TRADEMARKS.md) · [NOTICE](../NOTICE) +[Changelog](../CHANGELOG.md) | [License (Apache 2.0)](../LICENSE) | [Trademarks](../TRADEMARKS.md) | [NOTICE](../NOTICE) --- -If something is missing from this index, that's a bug — please file an [issue](https://github.com/skyoo2003/devcloud/issues) or open a PR. +If something is missing from this index, please file an [issue](https://github.com/skyoo2003/devcloud/issues) or open a pull request. diff --git a/docs/architecture.md b/docs/architecture.md index 512a1b3f..955642b1 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -1,12 +1,11 @@ # Architecture -DevCloud is a single Go binary that serves cloud provider APIs locally. It targets -**AWS today**; the seams that let a second CSP join are described at the bottom. +DevCloud is a single Go binary that serves cloud provider APIs locally. It supports AWS, Azure, and Google Cloud Platform behind a single unified endpoint on port 4747. ## Request Flow ``` -Client (boto3 / AWS CLI / Terraform / CDK) +Client (AWS SDK, Azure SDK, Google Cloud SDK, Terraform, CDK, CLI) │ ▼ API Gateway (port 4747) @@ -15,28 +14,43 @@ API Gateway (port 4747) │ ├─ ErrorRecovery (panic recovery) │ ├─ BodyLimit (request size limiting) │ ├─ CORS (cross-origin handling) - │ ├─ RequestID (X-Amz-Request-Id) + │ ├─ RequestID (request correlation IDs) │ ├─ RequestLogger (structured logging) - │ ├─ Identity (auth.Adapter → claimed identity on the request context) + │ ├─ Identity (auth.Adapter -> claimed identity on request context) │ └─ LogCollector (admin API live logs) │ - ├─ Route: /devcloud/api/* → Admin API + ├─ Route: /devcloud/api/* -> Admin API + ├─ Route: /devcloud/identity/* -> Federated Identity Issuer (OIDC / JWKS / STS) │ ▼ -Protocol Detector +Protocol & Provider Detector │ - ├─ X-Amz-Target header present → JSON protocol - ├─ x-www-form-urlencoded + Action= → Query protocol - ├─ SigV4 credential scope → REST-JSON (signing name picks the service) - └─ Default → REST-XML (S3) + ├─ GCP Detector (/storage/v1/, /v1/projects/*, Google headers) + │ ├─ /storage/v1/*, /upload/storage/v1/* -> Google Cloud Storage (gcs) + │ ├─ /v1/projects/*/secrets* -> GCP Secret Manager (gcpsecrets) + │ └─ /v1/projects/*/topics*, subscriptions -> GCP Cloud Pub/Sub (gcppubsub) + │ + ├─ Azure Detector (x-ms-* headers, SharedKey auth, /devstoreaccount1, /secrets, /kv) + │ ├─ /devstoreaccount1, blob headers -> Azure Blob Storage (blob) + │ ├─ /devstoreaccount1/queue, comp=messages-> Azure Queue Storage (azurequeue) + │ ├─ /devstoreaccount1/Tables, OData JSON -> Azure Table Storage (azuretable) + │ ├─ /dbs, x-ms-documentdb-* headers -> Azure Cosmos DB (cosmosdb) + │ ├─ /secrets, /keys, vault.azure.net -> Azure Key Vault (azurekeyvault) + │ └─ /kv, azconfig.io -> Azure App Configuration (azureappconfig) + │ + └─ AWS Detector + ├─ X-Amz-Target header present -> JSON protocol (DynamoDB, ECS, Kinesis) + ├─ x-www-form-urlencoded + Action= -> Query protocol (IAM, STS, SNS, RDS, EC2) + ├─ SigV4 credential scope -> REST-JSON (Lambda, ACM, API Gateway) + └─ Default -> REST-XML (S3) │ ▼ -Service Router → Plugin Registry → ServicePlugin.HandleRequest() +Service Router -> Plugin Registry -> ServicePlugin.HandleRequest() │ - ├─ returns plugin.ErrUnhandledOp → generic CRUD engine (docs/crud-engine.md) + ├─ returns plugin.ErrUnhandledOp -> generic CRUD engine (docs/crud-engine.md) │ ▼ -Storage Backend → Response Serializer → HTTP Response +Storage Backend & Workload Engine -> Response Serializer -> HTTP Response ``` ## Plugin System @@ -55,152 +69,122 @@ type ServicePlugin interface { } ``` -Plugins register themselves in the `Registry` at startup; the gateway routes to -one by protocol detection and service identification. The per-method contract, -config keys, error convention and v1.x stability guarantee are in -[plugin-api.md](plugin-api.md). +Plugins register themselves in the `Registry` at startup. The gateway routes requests by protocol detection and service identification. The per-method contract, configuration keys, error conventions, and stability guarantees are documented in [plugin-api.md](plugin-api.md). ### Protocols | Protocol | Request format | Response | Example services | |----------|----------------|----------|------------------| -| REST-XML | HTTP path/headers | XML | S3, Route53, CloudFront | -| REST-JSON | JSON body, REST path | JSON | Lambda, ACM, API Gateway | +| REST-XML | HTTP path/headers | XML | S3, Route53, CloudFront, Azure Blob, Azure Queue | +| REST-JSON | JSON body, REST path | JSON | Lambda, GCS, GCP Secrets, GCP Pub/Sub, Azure Cosmos, Key Vault, App Config | | JSON 1.0 | JSON body, `X-Amz-Target` | JSON | DynamoDB, SQS, Kinesis | -| JSON 1.1 | JSON body, `X-Amz-Target` | JSON | ECS, Batch, CW Logs, SFN | +| JSON 1.1 | JSON body, `X-Amz-Target` | JSON | ECS, Batch, CloudWatch Logs, Step Functions | | Query | Form-encoded body with `Action=` | XML | IAM, STS, SNS, RDS, EC2 | +| EC2 Query | Form-encoded body with `Action=` | XML (``) | EC2 | +| RPC-v2 CBOR | Binary CBOR payload | CBOR | Partner Central Revenue Measurement | + +## Universal Workload Runtime -SQS speaks both Query and JSON; the protocol is detected per request. +Serverless function invocations (such as AWS Lambda) execute through the workload abstraction in `internal/shared/workload`: + +- **ContainerRunner (OCI)**: Discovers and invokes standard OCI container runtimes (`podman`, `docker`, or `nerdctl`). The runtime engine can be set explicitly via `CONTAINER_RUNTIME`. +- **WASMRunner (WebAssembly)**: Executes `.wasm` binaries directly using standalone or embedded WebAssembly runtimes (`wasmtime`, `wasmer`, `wazero`), enabling zero-daemon serverless execution. +- **ProcessRunner (Host Process Sandbox)**: Executes handlers directly in isolated temporary directories for lightweight local environments without container engines. ## Authentication -DevCloud reads credentials and never verifies them. `internal/auth` holds one -`Adapter` per provider — `SigV4` for AWS — which parses the credential scope off -the `Authorization` header or a presigned URL and reports the access key, region, -signing name and session token the caller *claimed*. `IdentityMiddleware` puts the -result on the request context for `auth.FromContext`, and the protocol detector -uses the same parse to route by signing name, so routing and identity cannot -disagree. +DevCloud inspects incoming credentials without rejecting calls for signature mismatches: + +- **AWS**: `internal/auth/SigV4` extracts access keys, regions, signing names, and session tokens from the `Authorization` header or presigned URLs. +- **Azure**: `internal/auth/AzureSharedKey` parses account names, SharedKey signatures, and Bearer tokens for Azure services. +- **GCP**: `internal/auth/GCPAuth` accepts Google OAuth2 Bearer tokens (`Bearer ya29...`) and mock service account headers. +- **Federated Identity**: `internal/services/iam` provides a built-in OIDC discovery issuer, JWKS endpoint, and STS token exchange (`AssumeRoleWithWebIdentity`) for cross-cloud credential minting. -Signature validation is deliberately absent: a local server that rejected calls -whose signature did not match would break every SDK pointed at it with -placeholder credentials. See -[compatibility-policy.md](compatibility-policy.md#not-guaranteed). +Signature validation is omitted so local test suites can run with placeholder credentials without failures. ## Code Generation -Go code is generated from API models, so tracking upstream API changes costs -little manual work. AWS Smithy is the only format read today; a second one is an -added file, not a rewrite. +Go code is generated from API models to keep pace with upstream cloud changes: ``` -api/smithy/*.json (AWS Smithy model files) - │ - ▼ - ModelSource (internal/codegen/source.go) - Each source detects and parses its own format. SmithySource is the - first; OpenAPI and Protobuf join by implementing the interface and - appending to DefaultSources. tools/codegen names no format. - │ - ▼ - IR — *ir.Model (internal/codegen/ir) - Provider-neutral: service id, protocol, operations, shapes. - Everything downstream reads only this. - │ - ▼ - Generator (internal/codegen/generator.go) — Go templates produce: - ├─ types.go — request/response structs - ├─ router.go — method+URI → operation routing (REST services) - ├─ errors.go — service-specific error types - └─ base_provider.go — stub implementation (NotImplementedError) - │ - ▼ - internal/generated/{service}/ (DO NOT EDIT) +api/smithy/*.json (AWS Smithy) api/models/*.json (OpenAPI) + │ │ + └──────────────┬───────────────┘ + ▼ + ModelSource (internal/codegen/source.go) + │ + ▼ + IR (internal/codegen/ir) + Provider-neutral representation: service ID, protocol, operations, shapes. + │ + ▼ + Generator (internal/codegen/generator.go) + ├─ types.go (request/response structs) + ├─ router.go (method + URI routing) + ├─ errors.go (service error catalogues) + └─ base_provider.go (stub interfaces) + │ + ▼ + internal/generated/{service}/ (DO NOT EDIT) ``` -There is no generated serializer: providers receive the raw `*http.Request` and -parse it themselves, so only `router.go` is consumed today — by the REST services -that need `MatchOperation`. +Generated code is updated with: ```bash -make codegen # all services -make codegen-s3 # one service — fast loop while editing templates +make codegen # all registered services +make codegen-s3 # single service during template development ``` -The models under `api/smithy/` are committed on purpose. The download URL -tracks `aws-sdk-go-v2` *main*, so the vendored copies are the pin that makes -`make codegen` reproducible and offline. A [weekly workflow](../.github/workflows/smithy-sync.yml) -passes `--refresh`, regenerates, and opens a PR — which is what makes an upstream -API change show up as a reviewable model diff next to the regenerated code. How to -review one: [contributing.md](contributing.md#reviewing-the-weekly-model-sync). - ## Admin API -`internal/admin/` serves a REST API at `/devcloud/api/` — service status, resource -listing, fidelity tiers, unrouted calls, and recent request logs (a circular -buffer of the last 1000). Disabled by default (`admin.enabled: false`). The web -dashboard UI that consumes it is a separate repository; this server serves no UI. +`internal/admin/` serves an opt-in REST API at `/devcloud/api/` for service health, resource inventories, fidelity tiers, unrouted calls, and recent request logs. It is disabled by default (`admin.enabled: false`). ## Startup Flow -1. Load config from `devcloud.yaml` (or `--config`), applying `DEVCLOUD_PORT`, `DEVCLOUD_SERVICES`, `DEVCLOUD_DATA_DIR` -2. Initialize the structured logger (slog) and the plugin registry -3. Register service factories, then initialize services in dependency order (IAM before STS — STS receives the IAM store via plugin config options) -4. Set up the event bus, log collector and admin API -5. Build the gateway with its middleware chain and service router, and listen -6. On SIGINT/SIGTERM, shut down gracefully with a 15s budget +1. Load configuration from `devcloud.yaml` (or `--config`), applying `DEVCLOUD_PORT`, `DEVCLOUD_SERVICES`, and `DEVCLOUD_DATA_DIR`. +2. Initialize structured logging (slog) and the plugin registry. +3. Register service factories and initialize services in dependency order (IAM precedes STS; STS receives the IAM store). +4. Set up the event bus, log collector, and admin API. +5. Initialize the gateway with middleware chains and the multi-cloud service router. +6. On SIGINT or SIGTERM, shut down gracefully within a 15-second budget. ## Directory Structure ``` devcloud/ ├── cmd/ -│ └── devcloud/ # Server entry point +│ └── devcloud/ # Server entry point and imports ├── tools/ -│ └── codegen/ # Smithy code generator CLI +│ └── codegen/ # Smithy and OpenAPI code generator CLI ├── internal/ -│ ├── gateway/ # HTTP server, middleware, protocol detection, routing -│ ├── plugin/ # ServicePlugin interface, Registry, ProviderScoped -│ ├── auth/ # Per-provider credential adapters (SigV4 today) -│ ├── codegen/ # ModelSource implementations, generators, templates -│ │ └── ir/ # Provider-neutral intermediate representation -│ ├── config/ # YAML loading, provider namespacing, env overrides +│ ├── gateway/ # Multi-cloud HTTP router, protocol detection, middleware +│ ├── plugin/ # ServicePlugin interfaces and Registry +│ ├── auth/ # Provider credential adapters (AWS SigV4, Azure, GCP) +│ ├── codegen/ # Model sources, generators, and templates +│ ├── config/ # YAML loading, provider namespaces, environment overrides │ ├── generated/ # Auto-generated code (DO NOT EDIT) -│ ├── services/ # Service implementations -│ ├── shared/ # CRUD engine, HTTP routing, response helpers +│ ├── services/ # Service implementations (AWS, Azure, GCP, Identity) +│ ├── shared/ # CRUD engine, HTTP router, and workload execution engine +│ │ └── workload/ # Pluggable OCI, WASM, and Process workload drivers │ ├── admin/ # Admin REST API -│ └── storage/ # Shared storage abstractions -├── api/smithy/ # AWS Smithy JSON model files -├── build/package/ # Container build and release packaging files -├── deployments/ # Docker Compose deployment configuration -├── test/compatibility/ # Python/boto3 compatibility tests -└── docs/ +│ └── storage/ # Shared storage backends +├── api/ +│ ├── smithy/ # AWS Smithy JSON models +│ └── models/ # OpenAPI models for Azure and GCP services +├── build/package/ # Packaging and container builds +├── test/compatibility/ # Automated Python SDK and CLI compatibility tests +└── docs/ # Documentation ``` -## Multi-CSP seams +## Multi-CSP Architecture Seams -The long-term direction is to serve multiple CSPs behind one local runtime. Phase -2 of the [roadmap](roadmap.md) made each place a second provider would have -touched an *addition* rather than an edit: +The runtime supports multiple cloud providers behind a single engine: | Seam | Package | How a provider joins | |---|---|---| -| Model → code | [`internal/codegen/ir`](../internal/codegen/ir/ir.go), [`source.go`](../internal/codegen/source.go) | Implement `ModelSource`, append it to `DefaultSources` | -| Configuration | [`internal/config`](../internal/config/config.go) | `providers..services.*`; add the name to `knownProviders` | -| Service contract | [`internal/plugin`](../internal/plugin/plugin.go) | Implement the optional `ProviderScoped` on the plugin | -| Credentials | [`internal/auth`](../internal/auth/auth.go) | Implement `Adapter`, append it to `Adapters` | - -The plugin system, protocol detector and storage abstractions stay CSP-agnostic -by convention rather than enforcement — `ProtocolType` is an open string type, -not an enum, for exactly this reason. - -Phase 3 established the first non-AWS pilot (**Azure Blob Storage**) exercising all -four seams: OpenAPI models in `api/models/` parsed via `OpenAPISource`, namespaced -configuration under `providers.azure.services.blob`, `ProviderScoped` plugin registration, -and `AzureSharedKey` authentication. - -Phase 4 expanded multi-CSP breadth to **Google Cloud Platform** (`providers.gcp.services.gcs`) -with OAuth2 Bearer token authentication (`GCPAuth`), expanded **Azure Storage & Database** -capabilities (Queue Storage `azurequeue`, Table Storage `azuretable`, and Cosmos DB `cosmosdb`), -and introduced the **Federated Identity Playground** (`internal/services/iam`) providing local -OIDC discovery/JWKS/token issuance and cross-CSP token exchanges (AWS STS, Azure AD, and GCP STS). +| Model to code | `internal/codegen/ir`, `source.go` | Implement `ModelSource`, register in `DefaultSources` | +| Configuration | `internal/config` | Namespace under `providers..services.*` | +| Service contract | `internal/plugin` | Implement `ServicePlugin` and optional `ProviderScoped` | +| Credentials | `internal/auth` | Implement `Adapter` and register in credential adapters | +| Routing | `internal/gateway/protocol.go` | Register protocol and prefix rules on port 4747 | +| Workload | `internal/shared/workload` | Select OCI container, WebAssembly, or process driver | diff --git a/docs/compatibility-policy.md b/docs/compatibility-policy.md index 52cc6adf..9b7480b8 100644 --- a/docs/compatibility-policy.md +++ b/docs/compatibility-policy.md @@ -33,7 +33,7 @@ removed or repurposed. Defined in [`internal/config/config.go`](../internal/conf | Key | Type | Meaning | |---|---|---| | `server.port` | int | Listen port. Default `4747` when absent or `0`. | -| `services` | map | Presence of the block is authoritative — only the services it lists run, and an empty block runs nothing. Absent means every registered service runs. | +| `services` | map | Presence of the block is authoritative: only the services it lists run, and an empty block runs nothing. Absent means every registered service runs. | | `services..enabled` | bool | Whether that service starts. | | `services..data_dir` | string | Where that service stores data. | | `providers.aws.services` | map | The same block as `services`, namespaced by provider. Identical semantics; wins over `services` when both are written, with a warning. | @@ -54,19 +54,19 @@ Environment overrides config file, and that precedence is guaranteed. When `DEVCLOUD_SERVICES` names services, it decides membership on its own: it starts a service the `services` block omits and stops one the block enables. It replaces the block's selection rather than intersecting with it. The literal -`all` is the exception — it switches the filter off and hands the decision back +`all` is the exception: it switches the filter off and hands the decision back to the `services` block, which is what "all" has always meant here ([configuration.md](configuration.md#devcloud_services)). ### Command line `-config ` keeps its meaning. With no flag, DevCloud uses `./devcloud.yaml` if present -and the embedded defaults otherwise — zero-config startup keeps working. +and the embedded defaults otherwise; zero-config startup keeps working. ### Admin API Served at `/devcloud/api/` when `admin.enabled: true`. These routes keep responding, and their -JSON responses **only gain fields** — no documented key is removed or repurposed. +JSON responses **only gain fields**: no documented key is removed or repurposed. | Route | Guaranteed response keys | |---|---| @@ -83,7 +83,7 @@ reused for a different meaning. Every operation the manifest lists carries a tier from that set, every registered service appears, and every operation the CRUD engine serves is present and not filed as -`unimplemented` — all three fail the build, in +`unimplemented`; all three fail the build, in [`cmd/devcloud/fidelity_test.go`](../cmd/devcloud/fidelity_test.go). What no test can catch is an operation that never reaches the manifest at all. That is bounded @@ -93,10 +93,10 @@ instead of disappearing. For the 11 without one, the universe *is* what the prov the manifest lists no unimplemented tail for them. `modelBacked` on `GET /devcloud/api/fidelity` reports which is which. -### Wire behaviour — scoped to the compatibility suite +### Wire behaviour: scoped to the compatibility suite **Whatever a test in [`test/compatibility/`](../test/compatibility/) asserts about a response -keeps holding across 1.x — that property, and nothing wider.** +keeps holding across 1.x (that property, and nothing wider).** The promise is as wide as each individual assertion: not as wide as the field, and not as wide as the operation. `CreateFunction` in `test_lambda.py` shows all three cases at once: @@ -104,7 +104,7 @@ the operation. `CreateFunction` in `test_lambda.py` shows all three cases at onc | Field | Asserted as | Promised | |---|---|---| | `FunctionName` | equal to the name sent | key *and* value | -| `FunctionArn` | present | presence only — not that it stays ARN-shaped | +| `FunctionArn` | present | presence only, not that it stays ARN-shaped | | `Runtime`, `Handler`, `MemorySize` | not asserted | nothing, though today's response includes them | That narrowness is the point: it is the promise the repo can actually keep. The suite driving real boto3 clients runs in CI on every push and again against the tagged commit @@ -126,13 +126,13 @@ Depending on any of the following will break, and breaking it is **not** a major - **Data durability.** Stores are local development stores. Several are in-memory and per-process; on-disk layouts under `data_dir` may change format between releases without a migration. Do not treat DevCloud as a database. -- **`unimplemented` → served transitions.** An operation that returns an error today may start +- **`unimplemented` -> served transitions.** An operation that returns an error today may start returning a response. This is additive, and ships in a minor release. - **Service coverage.** New services may be added in a minor release. The 431 services registered - today are a floor, not a ceiling — and all of them serve at least one operation. + today are a floor, not a ceiling, and all of them serve at least one operation. See [coverage.md](coverage.md). - **Error codes, HTTP status and message wording.** What *is* guaranteed for an `unimplemented` - operation is that it **fails** — an AWS-shaped error, never a fabricated success. *Which* error + operation is that it **fails**: an AWS-shaped error, never a fabricated success. *Which* error is not guaranteed: it comes from whichever provider handles the request, and `sqs` even differs by protocol. [fidelity-manifest.md](fidelity-manifest.md#what-an-unimplemented-call-returns) records the current behaviour; normalizing it is a minor release, not a major one. @@ -141,7 +141,7 @@ Depending on any of the following will break, and breaking it is **not** a major restructures it freely across 1.x. The Phase 2 refactor is the precedent: the IR, `ModelSource`, `ProviderScoped` and the `auth` adapters all landed inside a 1.x minor without a compatibility event, because nothing outside this module can reach them. The in-tree - `ServicePlugin` contract in [plugin-api.md](plugin-api.md#api-stability) is no exception — it + `ServicePlugin` contract in [plugin-api.md](plugin-api.md#api-stability) is no exception; it is a convention that keeps in-tree plugins compiling, and it does not gate release versioning. - **Behavioural parity with AWS.** No release of DevCloud promises AWS's validation, business logic, eventual-consistency timing, rate limits, or IAM enforcement. Credentials are accepted @@ -152,15 +152,15 @@ Depending on any of the following will break, and breaking it is **not** a major Removing anything from the guaranteed list is a **major** version bump. Before that can happen: 1. **Deprecate in a minor release.** The old form keeps working and emits a runtime warning - naming its replacement. The precedent is the `dashboard` → `admin` config rename: the old key + naming its replacement. The precedent is the `dashboard` -> `admin` config rename: the old key still enables the admin API, warns, and yields to an explicit `admin` block ([`config.go`](../internal/config/config.go)). -2. **Document it** — in the release notes for that version, and here. +2. **Document it**: in the release notes for that version, and here. 3. **Remove no earlier than the next major.** At least one released version must have shipped the warning. Silence is not deprecation. A removed key that YAML would otherwise drop without comment is -kept in the parser purely to warn — that is why `auth` still produces a message telling you +kept in the parser purely to warn: that is why `auth` still produces a message telling you SigV4 is not enforced rather than being ignored. The pre-flight checklist in [RELEASE.md](../RELEASE.md#pre-flight-checklist) makes this a step in @@ -168,7 +168,7 @@ cutting a release, not a thing to remember. ## Reporting a break -If a 1.x release breaks something on the guaranteed list, that is a bug — please +If a 1.x release breaks something on the guaranteed list, that is a bug; please [open an issue](https://github.com/skyoo2003/devcloud/issues) with the DevCloud version and a reproducing snippet. If it breaks something on the not-guaranteed list, an issue is still useful: it is evidence for tightening the policy in a future major. @@ -176,10 +176,10 @@ useful: it is evidence for tightening the policy in a future major. ## Lambda execution scope Wire compatibility and code execution have separate gates. Python 3.12 and Node -22 execute in Docker with actual payload, environment, exception, timeout, +22 execute via standardized workload runtimes (Container, WASM, or Process) with actual payload, environment, exception, timeout, version and event delivery assertions in `test_lambda_runtime.py`. The required `lambda-runtime` CI job enables them with `DEVCLOUD_LAMBDA_RUNTIME_TESTS=1`; its -missing Docker/image failures cannot be skipped. The ordinary suite checks -management and explicit Docker-unavailable errors. This does not promise AWS +missing runtime/image failures cannot be skipped. The ordinary suite checks +management and explicit runtime-unavailable errors. This does not promise AWS parity for warm containers, IAM policy evaluation, async retry/DLQ, partial batches or durable stream records. See [Lambda limits](services/lambda.md). diff --git a/docs/configuration.md b/docs/configuration.md index 9df417be..4af41460 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1,54 +1,47 @@ # Configuration -**Configuration is optional.** With no flags, `devcloud` enables every registered -service on port 4747 and stores data under `./data//`. The defaults are -compiled into the binary from -[`internal/config/default.yaml`](https://github.com/skyoo2003/devcloud/blob/main/internal/config/default.yaml). +Configuration is optional. With no flags, DevCloud enables every registered service on port 4747 and stores state under `./data//`. Defaults are compiled into the binary from `internal/config/default.yaml`. -Config is resolved in this order, and **environment beats YAML beats embedded -default**: +Configuration precedence follows this order: **environment variables override YAML, which overrides embedded defaults**: -1. `--config ` — explicit; the file must exist -2. `./devcloud.yaml` in the working directory — auto-detected +1. `--config `: explicit configuration file path +2. `./devcloud.yaml` in the working directory: auto-detected if present 3. Embedded defaults -## Environment variables +## Environment Variables | Variable | Overrides | Description | |----------|-----------|-------------| -| `DEVCLOUD_PORT` | `server.port` | HTTP server port | +| `DEVCLOUD_PORT` | `server.port` | HTTP server listening port (default: 4747) | | `DEVCLOUD_SERVICES` | `services.*.enabled` | Comma-separated service list, or a tier shortcut | -| `DEVCLOUD_DATA_DIR` | `services.*.data_dir` | Base data directory for all services | +| `DEVCLOUD_DATA_DIR` | `services.*.data_dir` | Base data directory for persistent services | +| `CONTAINER_RUNTIME` | workload engine | OCI container executable (`podman`, `docker`, `nerdctl`) | +| `WASM_RUNTIME` | workload engine | WebAssembly runner executable (`wasmtime`, `wasmer`, `wazero`) | +| `DEVCLOUD_LAMBDA_NETWORK` | lambda runtime | Container network for invoked Lambda handlers | ### `DEVCLOUD_SERVICES` -When set, it names the running set outright: **only** the listed services start, -regardless of their `enabled` setting, and a service the `services` block omits -entirely still starts if you name it here (the block still supplies its -`data_dir`). When unset, each service uses its YAML `enabled` value. An unknown -`tierN` token is treated as a literal service name and logged as a warning. +When set, it specifies the running set: only the listed services start, regardless of YAML `enabled` settings. If unset, each service uses its YAML `enabled` value. | Token | Expands to | |-------|------------| -| `tier1` | Big 6 + core integration: s3, sqs, dynamodb, iam, sts, lambda, sns, kms, secretsmanager, ssm, cloudwatchlogs, cloudwatch, eventbridge, ec2, ecs, ecr, route53, acm | -| `tier2` | cognito, elasticloadbalancingv2, ebs, efs, states, apigateway, apigatewayv2, kinesis, firehose, ses, sesv2, rds, cloudformation | -| `tier3` | elasticache, cloudfront, wafv2, glue, athena, organizations, cloudtrail, eks, autoscaling, appsync, emr, batch | -| `all` | Disables the env-var filter — the `services` block decides | +| `tier1` | Core services: s3, sqs, dynamodb, iam, sts, lambda, sns, kms, secretsmanager, ssm, cloudwatchlogs, cloudwatch, eventbridge, ec2, ecs, ecr, route53, acm | +| `tier2` | Extended services: cognito, elasticloadbalancingv2, ebs, efs, states, apigateway, apigatewayv2, kinesis, firehose, ses, sesv2, rds, cloudformation | +| `tier3` | Analytics and platform: elasticache, cloudfront, wafv2, glue, athena, organizations, cloudtrail, eks, autoscaling, appsync, emr, batch | +| `all` | Disables the env-var filter: the `services` block decides | -The exact lists live in [`internal/config/config.go`](https://github.com/skyoo2003/devcloud/blob/main/internal/config/config.go). +Examples: ```bash -DEVCLOUD_SERVICES=s3,sqs ./dist/devcloud # only S3 and SQS -DEVCLOUD_SERVICES=tier1 ./dist/devcloud # all Tier 1 +DEVCLOUD_SERVICES=s3,sqs ./dist/devcloud +DEVCLOUD_SERVICES=tier1 ./dist/devcloud DEVCLOUD_SERVICES=tier1,kinesis,firehose ./dist/devcloud docker run -p 4747:4747 -e DEVCLOUD_SERVICES=tier1 ghcr.io/skyoo2003/devcloud:latest ``` ### `DEVCLOUD_DATA_DIR` -Every service uses `/`, and per-service `data_dir` -values in YAML are ignored. Useful for CI jobs needing ephemeral per-run -directories, or for relocating state without editing `devcloud.yaml`. +Every persistent service places its database and assets under `/`. This is useful in CI pipelines requiring isolated ephemeral storage directories. ```bash DEVCLOUD_DATA_DIR=/tmp/devcloud-local ./dist/devcloud @@ -66,14 +59,13 @@ DEVCLOUD_PORT=8080 ./dist/devcloud docker run -p 8080:8080 -e DEVCLOUD_PORT=8080 ghcr.io/skyoo2003/devcloud:latest ``` -## Configuration file +## Configuration File ```yaml server: port: 4747 -# Optional: listing services restricts startup to exactly this set. -# Omit the whole block to run every registered service. +# Optional: Top-level services block (AWS) services: s3: enabled: true @@ -83,6 +75,41 @@ services: dynamodb: enabled: true +# Multi-provider namespaces +providers: + aws: + services: + s3: + enabled: true + data_dir: ./data/s3 + azure: + services: + blob: + enabled: true + data_dir: ./data/blob + azurequeue: + enabled: true + data_dir: ./data/azurequeue + azuretable: + enabled: true + data_dir: ./data/azuretable + cosmosdb: + enabled: true + data_dir: ./data/cosmos + azurekeyvault: + enabled: true + azureappconfig: + enabled: true + gcp: + services: + gcs: + enabled: true + data_dir: ./data/gcp/gcs + gcpsecrets: + enabled: true + gcppubsub: + enabled: true + admin: enabled: false @@ -94,81 +121,44 @@ logging: | Key | Default | Description | |-----|---------|-------------| | `server.port` | `4747` | HTTP server port | -| `services..enabled` | `false` | **Required per entry.** Listing a service is not enough — `enabled: true` still has to be set. | +| `services..enabled` | `false` | Required per entry when the `services` block is explicitly present. | | `services..data_dir` | `./data/` | Data directory for persistent storage | -| `storage.data_dir` | `./data` | Directory holding DevCloud-wide local state, including generic AWS resources | -| `admin.enabled` | `false` | Serve the admin REST API at `/devcloud/api/*` | -| `logging.level` | `info` | `debug`, `info`, `warn`, `error` | -| `logging.format` | `text` | `text` or `json` | +| `storage.data_dir` | `./data` | Directory holding shared DevCloud state and generic CRUD database | +| `admin.enabled` | `false` | Serves the admin REST API at `/devcloud/api/*` | +| `logging.level` | `info` | Logging verbosity: `debug`, `info`, `warn`, `error` | +| `logging.format` | `text` | Log output formatting: `text` or `json` | -The `services` block is **optional and authoritative**: omit it and every -registered service starts; list any service and *only* the services you list -start. `services: {}` lists nothing, so nothing starts — a block is a block even -when empty. +The `services` block is optional: omit it to start all registered services automatically. -> The `dashboard` key was renamed to `admin`. `dashboard.enabled` is still -> honoured for one release with a deprecation warning; migrate to `admin.enabled`. +## Provider Namespacing -> There is no `auth` key: SigV4 signature validation is not implemented and any -> credentials are accepted. +DevCloud supports multi-provider configurations under the `providers` map: -> A service entry takes **only** `enabled` and `data_dir` — that is the whole of -> [`config.ServiceConfig`](https://github.com/skyoo2003/devcloud/blob/main/internal/config/config.go). -> Per-service tuning keys do not exist, and because parsing is non-strict an -> invented one (`services.lambda.warm_containers`) is dropped without a word -> rather than rejected. If a knob is not in the table above, writing it does -> nothing. +- `providers.aws.services`: AWS services. If top-level `services` is also specified, `providers.aws.services` takes precedence. +- `providers.azure.services`: Azure services (`blob`, `azurequeue`, `azuretable`, `cosmosdb`, `azurekeyvault`, `azureappconfig`). +- `providers.gcp.services`: GCP services (`gcs`, `gcpsecrets`, `gcppubsub`). -## Provider namespacing +Data directories for providers are isolated under their respective paths to prevent naming collisions. -DevCloud serves AWS today and is prepared to serve more ([roadmap](roadmap.md)), -so service configuration can be written under an explicit provider: +## Data Directories -```yaml -providers: - aws: - services: - s3: - enabled: true - data_dir: ./data/s3 -``` +Generic resource fallback persists state in SQLite at `/devcloud.db` (default: `./data/devcloud.db`). -`providers.aws.services` and the top-level `services` block are **the same -block** — one under its forward-compatible name, one under its historical one. - -- Write either. The top-level block is not deprecated. -- If both are present, `providers.aws.services` wins and a warning names the - ignored block. Do not write both. -- A block for a provider this build does not serve (`providers.azure`) parses - without error and logs a warning — so a config written for a later DevCloud - still loads, and a typo like `providers.awss` announces itself. -- `DEVCLOUD_SERVICES` and `DEVCLOUD_DATA_DIR` are AWS-scoped. There is no syntax - for naming another provider's services, so they cannot silently disable one. -- AWS data directories stay flat (`/`). A future provider nests under - its own name (`//`) so two CSPs offering a same-named - service cannot collide. - -## Data directories - -DevCloud's generic resource fallback persists in SQLite at -`/devcloud.db`. By default this is `./data/devcloud.db`; an -explicit `DEVCLOUD_DATA_DIR` overrides it together with all service directories. -Use `devcloud --reset-data` before startup, or (with `admin.enabled: true`) -`DELETE /devcloud/api/data`, to remove **all** local DevCloud data. +To perform a full reset of all local state, run `devcloud --reset-data` before starting the server, or send `DELETE /devcloud/api/data` while the admin API is enabled. | Service | Default `data_dir` | Backend | Contents | |---------|-------------------|---------|----------| -| S3 | `./data/s3` | Filesystem + SQLite | Object files, `metadata.db` | -| DynamoDB | `./data/dynamodb` | SQLite | `dynamodb.db` (tables, items, TTL config, tags) | -| IAM | `./data/iam` | SQLite | `iam.db` (users, roles, keys) | -| STS | `./data/sts` | Shared with IAM | Uses IAM's database | -| Lambda | `./data/lambda` | SQLite + Filesystem | `lambda.db`, `code/` | -| SQS | — | In-memory | No persistence | - -## Lambda Docker networking - -`DEVCLOUD_LAMBDA_NETWORK` selects the Docker network for function containers. -It is an optional process environment variable, not a YAML service option. -Pass the DevCloud endpoint to handlers using `Environment.Variables`; see -[Lambda execution setup](services/lambda.md#execution-setup) for native and -container networking and socket access. Docker is required only for execution. +| S3 | `./data/s3` | Filesystem + SQLite | Object payloads, `metadata.db` | +| DynamoDB | `./data/dynamodb` | SQLite | `dynamodb.db` (tables, items, TTL, tags) | +| IAM | `./data/iam` | SQLite | `iam.db` (users, roles, access keys) | +| STS | `./data/sts` | Shared with IAM | Stored in IAM database | +| Lambda | `./data/lambda` | SQLite + Filesystem | `lambda.db`, code archives | +| SQS | none | In-memory | Queue state and message buffers | +| Azure Blob | `./data/blob` | Filesystem + SQLite | Container metadata, block blobs | +| Azure Table | `./data/azuretable` | SQLite | OData tables and entity records | +| Azure Cosmos | `./data/cosmos` | SQLite | Databases, collections, documents | +| GCS | `./data/gcp/gcs` | Filesystem + SQLite | Bucket metadata, object payloads | + +## Workload and Container Networking + +`DEVCLOUD_LAMBDA_NETWORK` selects the container network for Lambda functions. Pass the DevCloud endpoint to handlers via function environment variables. See [Lambda execution setup](services/lambda.md#workload-execution-engine) for networking options. diff --git a/docs/contributing.md b/docs/contributing.md index b46f1a9c..7cdbe453 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -4,9 +4,9 @@ - Go 1.26+ - Python 3.10+ with pip (compatibility tests) -- Docker (optional — Lambda runtime and integration tests) +- Container runtime (Docker, Podman, or nerdctl; optional for Lambda runtime and integration tests) - `pre-commit` (optional but recommended) -- Hugo extended 0.165+ (optional — only to preview this documentation as a site) +- Hugo extended 0.165+ (optional: only to preview this documentation as a site) No C toolchain and no SQLite headers: the SQLite driver is pure Go, which is why everything here builds with `CGO_ENABLED=0`. @@ -26,12 +26,12 @@ make run # starts the server on port 4747 | Command | Description | |---------|-------------| -| `make build` | Build both binaries into `dist/` — `devcloud` and `codegen` | +| `make build` | Build both binaries into `dist/`: `devcloud` and `codegen` | | `make run` | Run the server from source (`go run ./cmd/devcloud`) | | `make test` | Run all Go tests with `CGO_ENABLED=0` and `-v` | | `make test-compat` | Install `test/compatibility/requirements.txt` and run the boto3 suite | | `make codegen` | Regenerate `internal/generated/` from every model in `api/smithy/` | -| `make codegen-s3` | Same, restricted to S3 — fast loop while editing templates | +| `make codegen-s3` | Same, restricted to S3: fast loop while editing templates | | `make docker-build` | Build the image from `build/package/Dockerfile` as `devcloud/devcloud` | | `make docker-run` | Run that image on port 4747 with `./data` mounted | | `make docs-serve` | Preview this documentation locally with Hugo | @@ -45,7 +45,7 @@ make run # starts the server on port 4747 ## Testing ```bash -make test # Go tests, CGO_ENABLED=0 — the same mode releases ship +make test # Go tests, CGO_ENABLED=0 (the same mode releases ship) make test-compat # Python/boto3 suite in test/compatibility/ ``` @@ -57,7 +57,7 @@ environment variables change that: | Variable | Effect | |----------|--------| | `DEVCLOUD_EXTERNAL=1` | Connect to a server already running (e.g. in Docker) instead of starting one | -| `DEVCLOUD_BIN=` | Run a pre-built binary instead of `go run` — much faster | +| `DEVCLOUD_BIN=` | Run a pre-built binary instead of `go run` (much faster) | | `DEVCLOUD_PORT=` | Use a fixed port instead of an arbitrary free one | To run the suite directly: @@ -70,7 +70,7 @@ pytest -v ## Code Generation -**Never edit files in `internal/generated/`** — codegen overwrites them. +**Never edit files in `internal/generated/`**: codegen overwrites them. ```bash make codegen # all services @@ -80,9 +80,9 @@ make codegen-s3 # one service | Piece | Where | |---|---| | Models | `api/smithy/*.json` | -| Sources | `internal/codegen/source.go` — `ModelSource` per format; `SmithySource` today | +| Sources | `internal/codegen/source.go`: `ModelSource` per format; `SmithySource` today | | Generator | `internal/codegen/generator.go` + `internal/codegen/templates/` | -| Output | `internal/generated/{service}/` — `types.go`, `router.go`, `errors.go`, `base_provider.go` | +| Output | `internal/generated/{service}/`: `types.go`, `router.go`, `errors.go`, `base_provider.go` | Providers parse the raw `*http.Request` themselves, so there is no generated serializer or deserializer. See [architecture.md](architecture.md#code-generation). @@ -90,9 +90,9 @@ serializer or deserializer. See [architecture.md](architecture.md#code-generatio ### Reviewing the weekly model sync [`smithy-sync.yml`](../.github/workflows/smithy-sync.yml) refreshes all 420 -vendored models every Monday and opens a pull request. The diff is whole-tree — a +vendored models every Monday and opens a pull request. The diff is whole-tree: a refresh measured at 194 models moved 93 of them and 134 generated files, and the -vendored set has more than doubled since — so do not try to read it. Read the PR +vendored set has more than doubled since, so do not try to read it. Read the PR body instead: it is generated by `scripts/model_churn.py` and lists which services gained or lost operations. @@ -101,11 +101,11 @@ Three things to check, in order: 1. **Operations added or removed.** These are the only changes that alter what DevCloud serves. Everything else is upstream reshaping traits or docs. 2. **The published figures, already re-derived.** New operations move the - fidelity manifest, so [`docs/coverage.md`](coverage.md) must move with them — - the sync does that arithmetic itself and commits it. Check the before→after + fidelity manifest, so [`docs/coverage.md`](coverage.md) must move with them: + the sync does that arithmetic itself and commits it. Check the before -> after table at the top of the PR body against item 1: the figures should move for the same reason the operations did. A `Published figures: failure` means the - change needs a sentence the tool will not invent — write it before merging, + change needs a sentence the tool will not invent; write it before merging, and never relax the gate to make it pass. 3. **`codegen-drift` and `compat`.** These must be green on their own. A red `codegen-drift` means the committed output does not match the models; a red @@ -113,7 +113,7 @@ Three things to check, in order: The in-job test result is printed at the top of the PR body, and a `failure` there with a clean `codegen-drift` almost always means the published-figure gate -fired on an operation that moved — which is the signal to look, not work to do. +fired on an operation that moved, which is the signal to look, not work to do. The PR's own `ci` runs that gate again against the corrected figures, so it is green unless something is genuinely wrong. @@ -126,11 +126,11 @@ DEVCLOUD_UPDATE_DOCS=1 go test ./cmd/devcloud/ -run TestUpdatePublishedFigures ## Adding a New AWS Service 1. **Add the Smithy model** to `api/smithy/`. -2. **Run `make codegen`** — generates types, router, errors and stubs. +2. **Run `make codegen`**: generates types, router, errors and stubs. 3. **Implement the provider** in `internal/services//provider.go`. Start with the most commonly used operations; the generated base provider makes everything else return `NotImplementedError`. -4. **Implement the store** in `store.go` — SQLite +4. **Implement the store** in `store.go`: SQLite ([`internal/storage/sqlite`](../internal/storage/sqlite)) for anything persistent, in-memory for ephemeral, filesystem for blobs. SQLite is the only embedded database in the tree; adding a second one needs a reason in the PR. @@ -138,10 +138,10 @@ DEVCLOUD_UPDATE_DOCS=1 go test ./cmd/devcloud/ -run TestUpdatePublishedFigures package in [`cmd/devcloud/imports.go`](../cmd/devcloud/imports.go). The interface contract, error convention and config keys are in [plugin-api.md](plugin-api.md). -6. **Wire startup and config** — `cmd/devcloud/main.go` and +6. **Wire startup and config**: `cmd/devcloud/main.go` and `internal/config/default.yaml`. -7. **Write tests** — Go unit tests plus boto3 tests in `test/compatibility/`. -8. **Document it** — a page under `docs/services/` for a core service. +7. **Write tests**: Go unit tests plus boto3 tests in `test/compatibility/`. +8. **Document it**: a page under `docs/services/` for a core service. ``` internal/ @@ -170,13 +170,13 @@ rather than Hugo `ref` shortcodes. Adding a page takes two steps: -1. Write `docs/.md` starting with an `# H1` — that becomes its title. +1. Write `docs/.md` starting with an `# H1`, which becomes its title. 2. Add it to `[menu.before]` in `hugo.toml`. The sidebar is defined there because Hugo would otherwise sort pages alphabetically, and every entry needs - its own `identifier` — entries without one collapse into a single link. + its own `identifier`: entries without one collapse into a single link. -Then run `make docs-build`. Links that leave `docs/` — to source files or to -root files like `CONTRIBUTING.md` — are rewritten to point at GitHub, which also +Then run `make docs-build`. Links that leave `docs/` (to source files or to +root files like `CONTRIBUTING.md`) are rewritten to point at GitHub, which also means an unresolvable link fails silently; `scripts/check-site-links.py`, which `make docs-build` runs, is what catches it. @@ -184,7 +184,7 @@ means an unresolvable link fails silently; `scripts/check-site-links.py`, which - Standard Go conventions (`gofmt`, `go vet`) - One responsibility per file; use existing services as patterns -- Errors follow the AWS shape (Code, Message, StatusCode) — see +- Errors follow the AWS shape (Code, Message, StatusCode); see [plugin-api.md](plugin-api.md#error-convention) - New Go files start with `// SPDX-License-Identifier: Apache-2.0`. Generated files keep the `DO NOT EDIT` marker on line 1 and the SPDX header on line 2. @@ -198,8 +198,8 @@ pre-commit run --all-files # optional: check the whole tree now ``` [`.pre-commit-config.yaml`](../.pre-commit-config.yaml) runs `go-spdx-header`, -`gofmt -l -w`, `go vet` and `go build` on Go files — the last two with -`CGO_ENABLED=0`, so a contributor without a C toolchain still gets them. Python +`gofmt -l -w`, `go vet` and `go build` on Go files (the last two with +`CGO_ENABLED=0`, so a contributor without a C toolchain still gets them). Python under `test/` and `scripts/` is handled by `ruff`. `internal/generated/` and `api/smithy/` are excluded throughout. diff --git a/docs/coverage.md b/docs/coverage.md index e3a4682d..0ad5bc61 100644 --- a/docs/coverage.md +++ b/docs/coverage.md @@ -25,19 +25,19 @@ denominators, because [routing and depth are two targets](#the-target): The **serving target** column is the depth promise: every registered service except the 226 the [demand study](demand.md) found nobody building. The **all registered** column adds those 226 back, and with them 6,794 operations that -exist so an SDK call cannot leave for a billable account — not because DevCloud +exist so an SDK call cannot leave for a billable account, not because DevCloud promises to serve them well. So the two shares answer different questions, and reading the right-hand one as a fidelity regression is the mistake this split exists to prevent. Registering 226 more services did not make any operation less faithful; it enlarged a -denominator. Of those operations, 31 are `hand-verified` — the eight -hand-written providers for the services the engine cannot classify — and that is +denominator. Of those operations, 31 are `hand-verified`: the eight +hand-written providers for the services the engine cannot classify, and that is the whole intended depth of the long tail. Promotion out of it happens [on request, with a use case](fidelity-manifest.md#getting-an-operation-promoted). > **Two targets, not one: routing is 441 of 441, depth is 215.** Every service -> AWS publishes is registered, so no call can leave for a billable account — that +> AWS publishes is registered, so no call can leave for a billable account; that > is a safety property and it admits no smaller number. Depth is the separate, > smaller promise, and the evidence still puts it at 215. See > [The target](#the-target). @@ -64,7 +64,7 @@ the parser does not read. | `rest-xml` | 4 | HTTP method + path | | `ec2Query` | 1 | the `Action` form field (`ec2`) | | `rpcv2Cbor` | 1 | the path / RPC-v2 routing (`partnercentralrevenuemeasurement`) | -| no in-tree model | 11 | n/a — hand-written providers | +| no in-tree model | 11 | n/a (hand-written providers) | **The operation is not CRUD-shaped.** `GetThing`, `ListThings` and `CreateThing` map onto a generic store. `ExecuteStatement`, `InvokeEndpoint` and @@ -82,7 +82,7 @@ supported, bringing the entire registered AWS fleet to serving $\ge 1$ operation Registering a service the engine cannot serve is deliberate. The alternative is worse: an *unregistered* service is not routed, so the SDK call leaves the machine and bills a real AWS account. A registered-only service answers locally, -in AWS's own error vocabulary. What it must never do is fabricate a success — +in AWS's own error vocabulary. What it must never do is fabricate a success: `test/compatibility/test_service_smoke.py::test_registered_only_service_declines_cleanly` is the check that keeps that true. @@ -95,7 +95,7 @@ manifest records this per service as `EngineWired`. `test/compatibility/test_service_smoke.py` parametrises over the generated service list rather than a hand-written one, so a service cannot be registered -and quietly go untested — which is what 31 of them were until this was measured. +and quietly go untested, which is what 31 of them were until this was measured. Four are excluded, and they are not a backlog. In every one it is botocore, not DevCloud, that stops the request, so no answer DevCloud could give would change @@ -118,26 +118,26 @@ adding a fifth is a deliberate edit that moves this figure with it. A runtime or control-plane split-out signs with its parent's name, so `sagemaker` is claimed by eight services at once. Where exactly one claimant carries the name -as its own service ID, that service wins and the rest are borrowers — see +as its own service ID, that service wins and the rest are borrowers; see `codegen.BuildAliases`. Where none does, the request is handed to the group and answered by the sibling whose route table models its method and path. All four Lex clients sign as `lex` and none is named `lex`, so: `GET /bots` is -`lex-models`, `POST /bots` is `lexv2-models`, `/bot/…/session` is `lex-runtime`, -`/bots/…/botAliases/…/sessions/…` is `lexv2-runtime`. One route is claimed by two -siblings — `DeleteBot` at `DELETE /bots/{id}` — and it is refused rather than +`lex-models`, `POST /bots` is `lexv2-models`, `/bot/.../session` is `lex-runtime`, +`/bots/.../botAliases/.../sessions/...` is `lexv2-runtime`. One route is claimed by two +siblings (`DeleteBot` at `DELETE /bots/{id}`), and it is refused rather than guessed: deleting the wrong bot is worse than an honest error. Three data planes are separated the same way without holding a single CRUD-classifiable operation: `payment-cryptography-data`, `cloudsearch-domain` and `kinesis-video-webrtc-storage` declare their own route tables through `crud.RegisterRoutes`, so route matching tells them apart from the neighbour -whose signing name they borrow. No override names a winner — the model does. +whose signing name they borrow. No override names a winner; the model does. ## What counts as a service Upstream publishes 431 model files, and DevCloud counts **model files**, one -registered service each — because that is what an SDK client selects. +registered service each, because that is what an SDK client selects. `boto3.client("sagemaker-runtime")` is a different client with a different API from `boto3.client("sagemaker")`. The split-outs are real choices a caller makes, not packaging artefacts. @@ -147,20 +147,20 @@ not packaging artefacts. DevCloud publishes **two** targets. They answer different questions, and reading one as the other is the mistake this page exists to prevent. -- **Routing — every service AWS publishes, and it is met.** A registered service +- **Routing: every service AWS publishes, and it is met.** A registered service is answered at `localhost:4747`; an unregistered one is not routed, so the SDK call leaves the machine and bills a real AWS account. That is a safety property, not a capability claim, and the only number that satisfies it is all of them. -- **Serving depth — 215 services, decided 2026-09-05, and it is met.** Depth is +- **Serving depth: 215 services, decided 2026-09-05, and it is met.** Depth is what costs, so it follows evidence of demand rather than the shape of AWS's catalogue. The study below is that evidence, and it is unchanged. | Axis | Services | Governed by | |---|---|---| -| **Routing target** | **441 / 441 — met** | leak-zero; every published model is registered | -| **Serving target** | **215 — met** | the demand study below, sampled 2026-09-05 | -| Registered and engine-served, outside the serving target | 226 | no depth promise — see the [CRUD engine](crud-engine.md) | +| **Routing target** | **441 / 441 - met** | leak-zero; every published model is registered | +| **Serving target** | **215 - met** | the demand study below, sampled 2026-09-05 | +| Registered and engine-served, outside the serving target | 226 | no depth promise; see the [CRUD engine](crud-engine.md) | ### How the depth target was set @@ -171,15 +171,15 @@ against three independent projects that each only add a service when someone asks. It did not hold. **Registering all 431 later did not overturn that decision.** What the study -refused was the *cost* — hand-building 283 services on the assumption someone +refused was the *cost*: hand-building 283 services on the assumption someone wanted them. The codegen scaffold removed that cost: registering the remaining 226 became a flag on `make codegen`, not a programme of work, and the services it reached are served by the generic [CRUD engine](crud-engine.md) at engine fidelity. So routing was taken because it turned out to be nearly free, and the depth target stayed where the evidence put it, because it was never a count of -registrations — it is where DevCloud promises to be worth trusting. +registrations; it is where DevCloud promises to be worth trusting. -**The rule was fixed before the numbers were seen** — four outcomes written down +**The rule was fixed before the numbers were seen**: four outcomes written down in advance, including one for "the method itself failed", specifically so the result could not be argued into whichever answer was most convenient. Full evidence in [demand.md](demand.md); re-derive with @@ -198,7 +198,7 @@ The rule kept the 100% depth target only if ≥60% of `M` had support ≥2, and narrowed to a demand set if ≥100 did. 57 cleared neither bar, so the pre-registered consequence applied: **the 100% depth claim is dropped and the published depth target becomes the demand set.** All 57 are registered and serve -at least one operation — `rds-data`, the strongest-signal service in the set, is +at least one operation: `rds-data`, the strongest-signal service in the set, is now served by a SQLite-backed query engine (Phase 1E) rather than the generic CRUD engine. @@ -238,16 +238,16 @@ curl -s localhost:4747/devcloud/api/unrouted | jq . Paste that into a [service request](https://github.com/skyoo2003/devcloud/issues/new?template=service_request.yml). One report outranks all three proxies, because it is demand rather than a -stand-in for it — and it is what moves a service from the 226 into the target. +stand-in for it, and it is what moves a service from the 226 into the target. Two ceilings, so the number is not read as more than it is: 1. **It is a floor, not a census.** `gateway.DetectProtocol` classifies a request - it cannot identify as `("rest-xml", "s3")`, and S3 is registered — so an + it cannot identify as `("rest-xml", "s3")`, and S3 is registered, so an unrecognisable request is routed to S3 rather than counted as a miss. The case that matters is caught: a real SDK or CLI call to an unregistered service signs with that service's own name and misses the registry. Verified with - boto3 — `boto3.client("appflow")` is recorded as `appflow`. + boto3: `boto3.client("appflow")` is recorded as `appflow`. 2. **A non-zero `droppedServiceIds` means the list is incomplete.** Service IDs come from caller-controlled headers, so the collector caps how many distinct ones it holds and reports both the cap and what it dropped, rather than @@ -264,12 +264,12 @@ Apple Silicon, `CGO_ENABLED=0`, measured at each step of the roadmap: |---|---|---|---|---| | Binary | 30.8 MiB | 31.3 MiB | 33.1 MiB | **36.8 MiB** | | Peak RSS (`/usr/bin/time -l`) | 57.3 MiB | 57.7 MiB | 43.7 MiB | **51.2 MiB** | -| Service startup | — | — | 49 ms for all 205 | **42 ms for all 431** | +| Service startup | - | - | 49 ms for all 205 | **42 ms for all 431** | The single-binary, zero-config property holds at 431 with room to spare. Startup does not scale meaningfully with service count: registration is a map insert per service in `init()`, and the generated type definitions are mostly -dead-code-eliminated by the linker, which is why 58 more services cost 1.8 MiB — +dead-code-eliminated by the linker, which is why 58 more services cost 1.8 MiB, and why 226 more, nearly quadrupling the fleet, cost 3.7 MiB rather than the 7 MiB a linear reading of that figure predicts. @@ -278,7 +278,7 @@ Two ceilings on the startup reading: 1. **It is timed from the first `service initialized` line to `DevCloud ready`**, so it covers bringing every service up, not the `init()` registration alone. That is the number an operator waits for. -2. **A first run costs about three times as much** — 118 to 148 ms across four +2. **A first run costs about three times as much**: 118 to 148 ms across four measurements, against 42 ms once the data directories exist, because each of the 431 services creates its own on the way up. The cold path is the one to watch, and it is the one `cmd/devcloud/budget_test.go` reproduces. @@ -286,7 +286,7 @@ Two ceilings on the startup reading: **Startup is measured, not gated, and that is a deliberate retreat.** The gate was written as a wall-clock ceiling and CI disproved it. The same path that takes 141 ms here took 2.048 s on a GitHub arm64 runner, and 1.414 s on the same runner -type one commit later — so the shared runner is roughly ten times slower *and* +type one commit later, so the shared runner is roughly ten times slower *and* swings 45% between runs on identical code. The regression worth catching is smaller than that swing: a provider that starts opening a file or a database per service at startup costs 1.2x to 2x, because that is what 431 extra file opens @@ -307,12 +307,12 @@ That is how the figures above are re-taken per release. The binary size *is* gated, because size does not vary with how busy a runner is. CI fails the build past **45 MiB** and measures 35 MiB on arm64, 37 MiB on amd64 -— close enough to the 36.8 MiB above that the platform barely registers, so the +(close enough to the 36.8 MiB above that the platform barely registers), so the remaining 8 MiB is genuine slack rather than a correction for anything. It is a decision; if it starts failing, find what grew before raising it. The RSS readings were taken on different days and are not a controlled -comparison. Read them as "memory is not the constraint" rather than as a saving — +comparison. Read them as "memory is not the constraint" rather than as a saving; what they agree on is the shape: memory is dominated by the runtime and the store, not by how many services are registered. @@ -347,12 +347,12 @@ Reducing that cost rather than restating it is tracked separately. What the sample does settle is the *shape* of the work. None of the 93 was documentation-only, so no sync can be waved through on the assumption that AWS -only reworded things. Thirty-two services gained operations — `ec2` alone gained -46 — which moves the manifest and makes the published-figure gate fail on +only reworded things. Thirty-two services gained operations (`ec2` alone gained +46), which moves the manifest and makes the published-figure gate fail on purpose. That failure is still what triggers the review; what it no longer costs is the arithmetic. The sync re-derives every figure on this page and commits the result into its own pull request, and the body states which ones moved. What a -person still owes is the judgement — whether those operations should have moved — +person still owes is the judgement: whether those operations should have moved, and any sentence a number cannot carry. Re-derive by hand with `DEVCLOUD_UPDATE_DOCS=1 go test ./cmd/devcloud/ -run TestUpdatePublishedFigures`, and read which operations moved with `python3 scripts/model_churn.py --upstream`. @@ -368,7 +368,7 @@ make test-compat # the compatibility-tested number, over all 431 service Every figure comes from `internal/generated/fidelity/manifest_gen.go` and nothing else, so it cannot drift from what the binary serves. It cannot drift from *this -page* either — nine tests compare the two, in both directions: +page* either: nine tests compare the two, in both directions: | Test | Gates | |---|---| diff --git a/docs/crud-engine.md b/docs/crud-engine.md index 7c126043..6a5d4c3e 100644 --- a/docs/crud-engine.md +++ b/docs/crud-engine.md @@ -6,7 +6,7 @@ This is what lets the long tail respond to SDK calls without hand-coding each on **Fidelity is deliberately "plausible, not faithful."** Responses are store-backed and echo the caller's input plus synthesized ids and ARNs, so SDKs round-trip -create → get → list → delete. There is **no** validation, cross-resource +create -> get -> list -> delete. There is **no** validation, cross-resource integrity, pagination correctness, or business logic. Treat engine-served operations as scaffolding for local wiring, not as behavioural parity. @@ -22,7 +22,7 @@ operation carries is in the [fidelity manifest](fidelity-manifest.md). | Integration | [`internal/gateway/router.go`](../internal/gateway/router.go) | Calls the engine when a provider returns `plugin.ErrUnhandledOp`; returns `InvalidAction` if the engine cannot classify the operation | A provider opts in by returning `plugin.ErrUnhandledOp` from its dispatch -`default:` case. Hand-written operations always win — the engine is reached only +`default:` case. Hand-written operations always win: the engine is reached only on fall-through, so it never shadows a real implementation. **Never a fabricated success for an unclassifiable op.** @@ -34,10 +34,10 @@ The engine has to know which operation a request is for before it can classify i |---|---|---| | `json-1.0`, `json-1.1` | the `X-Amz-Target` header | yes | | `rest-json` | method + path, matched against the model's URI templates (`internal/shared/httproute`) | yes | -| `rest-xml` | the same — every `restXml` operation binds to a method and URI | yes | +| `rest-xml` | the same: every `restXml` operation binds to a method and URI | yes | | `query` | the `Action` field of the form body | yes | | `ec2-query` | the `Action` field of the form body or query parameter | yes | -| `rpcv2-cbor` | the path — RPC-v2 routes by service and operation name segments | yes | +| `rpcv2-cbor` | the path: RPC-v2 routes by service and operation name segments | yes | Every protocol DevCloud registers is readable, so a service that serves nothing does so because none of its operations is CRUD-shaped, not because of how it @@ -46,28 +46,27 @@ XML responses with `` and `` list elements. ### Where parameters come from -For the two REST protocols, three places — least authoritative first: values the +For the two REST protocols, three places, least authoritative first: values the model binds with `httpQuery`, then the request body, then the path labels. The URI addresses the resource, so a path label wins. A REST model never binds one member to two of these, so real SDK traffic never exercises the precedence; it is defined so a hand-rolled request cannot redirect a lookup. -For `query`, the form body, flat keys only. `Action` and `Version` are dropped — +For `query`, the form body, flat keys only. `Action` and `Version` are dropped; they describe the request, not the resource, and storing them would echo `CreateLoadBalancer` back inside a result element. A structured member arrives flattened as `Listeners.member.1.Protocol`; the engine has no nested shape to put it in, so it is not emitted in the response. -**`rest-xml` request bodies are never read.** The gateway does not buffer them — -S3 speaks `rest-xml` and its bodies are multi-gigabyte uploads that must keep -streaming — so a `rest-xml` operation is served from its path and query alone. -Every CRUD-shaped S3 Control operation addresses its resource that way, so nothing -is lost; an operation carrying its identifier only in a body would get a generated -id. +**`rest-xml` request bodies are never read.** The gateway does not buffer them because +S3 speaks `rest-xml` with multi-gigabyte streaming uploads, so a `rest-xml` operation +is served from its path and query alone. Every CRUD-shaped S3 Control operation addresses +its resource that way, so nothing is lost; an operation carrying its identifier only in a +body would get a generated id. The engine does **not** read `httpHeader` members, `httpPayload` blobs, or streaming bodies. An operation whose identifier arrives only in a header is served -with a generated id rather than the caller's — plausible, not faithful, which is +with a generated id rather than the caller's: plausible, not faithful, which is the stated contract. ### Responses @@ -78,7 +77,7 @@ inside `` and, given anything else, returns an empty result rather than an error, while its `rest-xml` parser maps the root element's children straight onto the output shape. List entries are wrapped in ``, AWS's default for both dialects; a model that flattens a list gets the unflattened form, -because the engine has no flattening information. No `xmlns` is emitted — botocore +because the engine has no flattening information. No `xmlns` is emitted; botocore strips namespaces before matching element names. ### Declining @@ -86,7 +85,7 @@ strips namespaces before matching element names. A request whose method and path match no route in the service's table, or whose `Action` names an operation the service did not register, is **declined** with `InvalidAction` and never served from the store. That is what keeps a registered -service from answering for operations it does not model — and it matters most for +service from answering for operations it does not model, and it matters most for `rest-xml`, because `DetectProtocol` routes anything it cannot classify to S3. ## Known limits diff --git a/docs/demand.md b/docs/demand.md index 7775fa36..8164afa2 100644 --- a/docs/demand.md +++ b/docs/demand.md @@ -13,9 +13,9 @@ asks. | Source | What it is | Services | Unmatched names | |---|---|---|---| -| [moto](https://github.com/getmoto/moto) | Python AWS mocking library — boto3 developers running tests | 163 | 0 | +| [moto](https://github.com/getmoto/moto) | Python AWS mocking library: boto3 developers running tests | 163 | 0 | | [LocalStack](https://docs.localstack.cloud/references/coverage/) | Local AWS emulator; community + pro | 119 | 4 | -| [terraform-provider-aws](https://github.com/hashicorp/terraform-provider-aws) | One Go package per service — the IaC population | 273 | 4 | +| [terraform-provider-aws](https://github.com/hashicorp/terraform-provider-aws) | One Go package per service: the IaC population | 273 | 4 | Read a high support count as *this service is worth emulating to someone*, not as *our users want this*. The live measurement of DevCloud's own traffic is @@ -32,22 +32,22 @@ name normalisation is dropping matches and the support counts are too low. |---|---| | Upstream model files | 431 | | Registered when sampled | 148 | -| **R1 — missing set `M`** | **283** | -| R2 — `M` with support 3 | 8 | -| R2 — `M` with support 2 | 49 | -| R2 — `M` with support 1 | 111 | -| R2 — `M` with support 0 | 115 | -| **R2 — `M` with support ≥ 2** | **57 (20.1% of `M`)** | +| **R1: missing set `M`** | **283** | +| R2: `M` with support 3 | 8 | +| R2: `M` with support 2 | 49 | +| R2: `M` with support 1 | 111 | +| R2: `M` with support 0 | 115 | +| **R2: `M` with support ≥ 2** | **57 (20.1% of `M`)** | ## Full ranking -Ordered by support count, then name — the order the work was done in, so stopping +Ordered by support count, then name: the order the work was done in, so stopping at any point would have stopped on the best surface available. **All 57 services with support ≥ 2 are registered**, down to and including `workspaces-web`; `TestDemandSetIsRegistered` fails if any of them stops being. -All 57 serve at least one operation. Everything below them — support 1 -and support 0 — is the 226 services that carry no depth commitment. They are +All 57 serve at least one operation. Everything below them (support 1 +and support 0) is the 226 services that carry no depth commitment. They are registered and engine-served, so a call to one is answered locally rather than billed; what this study declined to promise is that it is answered *faithfully*. See [the two axes](coverage.md#the-target). @@ -62,278 +62,278 @@ See [the two axes](coverage.md#the-target). | `redshift-data` | yes | yes | yes | 3 | | `s3-control` | yes | yes | yes | 3 | | `timestream-query` | yes | yes | yes | 3 | -| `amp` | yes | — | yes | 2 | -| `apigatewaymanagementapi` | yes | yes | — | 2 | -| `app-mesh` | yes | — | yes | 2 | -| `budgets` | yes | — | yes | 2 | -| `cleanrooms` | yes | — | yes | 2 | -| `cloudhsm-v2` | yes | — | yes | 2 | -| `codestar-connections` | — | yes | yes | 2 | -| `connect` | yes | — | yes | 2 | -| `data-pipeline` | yes | — | yes | 2 | -| `databrew` | yes | — | yes | 2 | -| `datasync` | yes | — | yes | 2 | -| `dax` | yes | — | yes | 2 | -| `devops-agent` | yes | — | yes | 2 | -| `direct-connect` | yes | — | yes | 2 | -| `directory-service` | yes | — | yes | 2 | -| `dsql` | yes | — | yes | 2 | -| `emr-containers` | yes | — | yes | 2 | -| `fsx` | yes | — | yes | 2 | -| `greengrass` | yes | — | yes | 2 | -| `guardduty` | yes | — | yes | 2 | -| `inspector2` | yes | — | yes | 2 | -| `ivs` | yes | — | yes | 2 | -| `kinesis-analytics` | — | yes | yes | 2 | -| `kinesis-video` | yes | — | yes | 2 | -| `macie2` | yes | — | yes | 2 | -| `mediaconnect` | yes | — | yes | 2 | -| `medialive` | yes | — | yes | 2 | -| `mediapackage` | yes | — | yes | 2 | -| `mediapackagev2` | yes | — | yes | 2 | -| `mediastore-data` | yes | yes | — | 2 | -| `network-firewall` | yes | — | yes | 2 | -| `networkmanager` | yes | — | yes | 2 | -| `opensearchserverless` | yes | — | yes | 2 | -| `osis` | yes | — | yes | 2 | -| `payment-cryptography` | yes | — | yes | 2 | -| `quicksight` | yes | — | yes | 2 | -| `resiliencehub` | yes | — | yes | 2 | -| `route-53-domains` | yes | — | yes | 2 | -| `s3vectors` | yes | — | yes | 2 | -| `securityhub` | yes | — | yes | 2 | -| `service-catalog` | yes | — | yes | 2 | -| `service-catalog-appregistry` | yes | — | yes | 2 | -| `service-quotas` | yes | — | yes | 2 | -| `signer` | yes | — | yes | 2 | -| `synthetics` | yes | — | yes | 2 | -| `timestream-influxdb` | yes | — | yes | 2 | -| `vpc-lattice` | yes | — | yes | 2 | -| `workspaces` | yes | — | yes | 2 | -| `workspaces-web` | yes | — | yes | 2 | -| `accessanalyzer` | — | — | yes | 1 | -| `account-access` | — | — | yes | 1 | -| `agent-registry` | — | — | yes | 1 | -| `appconfigdata` | — | yes | — | 1 | -| `appfabric` | — | — | yes | 1 | -| `appflow` | — | — | yes | 1 | -| `appintegrations` | — | — | yes | 1 | -| `application-insights` | — | — | yes | 1 | -| `application-signals` | — | — | yes | 1 | -| `apprunner` | — | — | yes | 1 | -| `appstream` | — | — | yes | 1 | -| `arc-region-switch` | — | — | yes | 1 | -| `arc-zonal-shift` | — | — | yes | 1 | -| `auditmanager` | — | — | yes | 1 | -| `auto-scaling-plans` | — | — | yes | 1 | -| `bcm-data-exports` | — | — | yes | 1 | -| `billing` | — | — | yes | 1 | -| `chatbot` | — | — | yes | 1 | -| `chime` | — | — | yes | 1 | -| `chime-sdk-media-pipelines` | — | — | yes | 1 | -| `chime-sdk-voice` | — | — | yes | 1 | -| `cloud9` | — | — | yes | 1 | -| `clouddirectory` | yes | — | — | 1 | -| `cloudfront-keyvaluestore` | — | — | yes | 1 | -| `codecatalyst` | — | — | yes | 1 | -| `codestar-notifications` | — | — | yes | 1 | -| `compute-optimizer` | — | — | yes | 1 | -| `connectcampaigns` | yes | — | — | 1 | -| `connectcases` | — | — | yes | 1 | -| `controltower` | — | — | yes | 1 | -| `cost-and-usage-report-service` | — | — | yes | 1 | -| `cost-optimization-hub` | — | — | yes | 1 | -| `customer-profiles` | — | — | yes | 1 | -| `dataexchange` | — | — | yes | 1 | -| `datazone` | — | — | yes | 1 | -| `detective` | — | — | yes | 1 | -| `device-farm` | — | — | yes | 1 | -| `directory-service-data` | — | — | yes | 1 | -| `dlm` | — | — | yes | 1 | -| `docdb-elastic` | — | — | yes | 1 | -| `drs` | — | — | yes | 1 | -| `ec2-instance-connect` | yes | — | — | 1 | -| `ecr-public` | — | — | yes | 1 | -| `evs` | — | — | yes | 1 | -| `finspace` | — | — | yes | 1 | -| `fms` | — | — | yes | 1 | -| `gamelift` | — | — | yes | 1 | -| `global-accelerator` | — | — | yes | 1 | -| `grafana` | — | — | yes | 1 | -| `groundstation` | — | — | yes | 1 | -| `imagebuilder` | — | — | yes | 1 | -| `inspector` | — | — | yes | 1 | -| `interconnect` | — | — | yes | 1 | -| `internetmonitor` | — | — | yes | 1 | -| `invoicing` | — | — | yes | 1 | -| `ivschat` | — | — | yes | 1 | -| `kafkaconnect` | — | — | yes | 1 | -| `keyspaces` | — | — | yes | 1 | -| `kinesis-video-archived-media` | yes | — | — | 1 | -| `lambda-core` | — | — | yes | 1 | -| `lambda-microvms` | — | — | yes | 1 | -| `launch-wizard` | — | — | yes | 1 | -| `license-manager` | — | — | yes | 1 | -| `lightsail` | — | — | yes | 1 | -| `location` | — | — | yes | 1 | -| `m2` | — | — | yes | 1 | -| `mailmanager` | — | — | yes | 1 | -| `marketplace-metering` | yes | — | — | 1 | -| `mediapackage-vod` | — | — | yes | 1 | -| `mgn` | — | — | yes | 1 | -| `mpa` | — | — | yes | 1 | -| `mwaa-serverless` | — | — | yes | 1 | -| `neptune-graph` | — | — | yes | 1 | -| `networkflowmonitor` | — | — | yes | 1 | -| `networkmonitor` | — | — | yes | 1 | -| `notifications` | — | — | yes | 1 | -| `notificationscontacts` | — | — | yes | 1 | -| `oam` | — | — | yes | 1 | -| `observabilityadmin` | — | — | yes | 1 | -| `odb` | — | — | yes | 1 | -| `outposts` | — | — | yes | 1 | -| `pca-connector-ad` | — | — | yes | 1 | -| `pcs` | — | — | yes | 1 | -| `pinpoint-sms-voice-v2` | — | — | yes | 1 | -| `pricing` | — | — | yes | 1 | -| `rbin` | — | — | yes | 1 | -| `redshift-serverless` | — | — | yes | 1 | -| `resiliencehubv2` | — | — | yes | 1 | -| `resource-explorer-2` | — | — | yes | 1 | -| `rolesanywhere` | — | — | yes | 1 | -| `route53-recovery-control-config` | — | — | yes | 1 | -| `route53-recovery-readiness` | — | — | yes | 1 | -| `route53profiles` | — | — | yes | 1 | -| `rum` | — | — | yes | 1 | -| `s3files` | — | — | yes | 1 | -| `s3outposts` | — | — | yes | 1 | -| `savingsplans` | — | — | yes | 1 | -| `schemas` | — | — | yes | 1 | -| `securitylake` | — | — | yes | 1 | -| `simpledbv2` | yes | — | — | 1 | -| `ssm-contacts` | — | — | yes | 1 | -| `ssm-incidents` | — | — | yes | 1 | -| `ssm-quicksetup` | — | — | yes | 1 | -| `ssm-sap` | — | — | yes | 1 | -| `sso` | — | — | yes | 1 | -| `storage-gateway` | — | — | yes | 1 | -| `taxsettings` | — | — | yes | 1 | -| `uxc` | — | — | yes | 1 | -| `waf-regional` | — | — | yes | 1 | -| `wellarchitected` | — | — | yes | 1 | -| `workmail` | — | — | yes | 1 | -| `agent-registry-control` | — | — | — | 0 | -| `aiops` | — | — | — | 0 | -| `amplifybackend` | — | — | — | 0 | -| `amplifyuibuilder` | — | — | — | 0 | -| `application-discovery-service` | — | — | — | 0 | -| `applicationcostprofiler` | — | — | — | 0 | -| `artifact` | — | — | — | 0 | -| `b2bi` | — | — | — | 0 | -| `backup-gateway` | — | — | — | 0 | -| `backupsearch` | — | — | — | 0 | -| `bcm-dashboards` | — | — | — | 0 | -| `bcm-pricing-calculator` | — | — | — | 0 | -| `bcm-recommended-actions` | — | — | — | 0 | -| `billingconductor` | — | — | — | 0 | -| `braket` | — | — | — | 0 | -| `chime-sdk-identity` | — | — | — | 0 | -| `chime-sdk-meetings` | — | — | — | 0 | -| `chime-sdk-messaging` | — | — | — | 0 | -| `cleanroomsml` | — | — | — | 0 | -| `cloudhsm` | — | — | — | 0 | -| `cloudsearch-domain` | — | — | — | 0 | -| `cloudtrail-data` | — | — | — | 0 | -| `cloudwatch-events` | — | — | — | 0 | -| `cognito-sync` | — | — | — | 0 | -| `compute-optimizer-automation` | — | — | — | 0 | -| `connect-contact-lens` | — | — | — | 0 | -| `connectcampaignsv2` | — | — | — | 0 | -| `connecthealth` | — | — | — | 0 | -| `connectparticipant` | — | — | — | 0 | -| `controlcatalog` | — | — | — | 0 | -| `deadline` | — | — | — | 0 | -| `eks-auth` | — | — | — | 0 | -| `elementalinference` | — | — | — | 0 | -| `finspace-data` | — | — | — | 0 | -| `freetier` | — | — | — | 0 | -| `gameliftstreams` | — | — | — | 0 | -| `geo-maps` | — | — | — | 0 | -| `geo-places` | — | — | — | 0 | -| `geo-routes` | — | — | — | 0 | -| `greengrassv2` | — | — | — | 0 | -| `health` | — | — | — | 0 | -| `iam-toolbox` | — | — | — | 0 | -| `inspector-scan` | — | — | — | 0 | -| `iot-jobs-data-plane` | — | — | — | 0 | -| `iot-managed-integrations` | — | — | — | 0 | -| `iotdeviceadvisor` | — | — | — | 0 | -| `iotfleetwise` | — | — | — | 0 | -| `iotsecuretunneling` | — | — | — | 0 | -| `iotsitewise` | — | — | — | 0 | -| `iotthingsgraph` | — | — | — | 0 | -| `iottwinmaker` | — | — | — | 0 | -| `ivs-realtime` | — | — | — | 0 | -| `keyspacesstreams` | — | — | — | 0 | -| `kinesis-video-media` | — | — | — | 0 | -| `kinesis-video-signaling` | — | — | — | 0 | -| `kinesis-video-webrtc-storage` | — | — | — | 0 | -| `license-manager-linux-subscriptions` | — | — | — | 0 | -| `license-manager-user-subscriptions` | — | — | — | 0 | -| `machine-learning` | — | — | — | 0 | -| `managedblockchain-query` | — | — | — | 0 | -| `marketplace-agreement` | — | — | — | 0 | -| `marketplace-catalog` | — | — | — | 0 | -| `marketplace-commerce-analytics` | — | — | — | 0 | -| `marketplace-deployment` | — | — | — | 0 | -| `marketplace-discovery` | — | — | — | 0 | -| `marketplace-entitlement-service` | — | — | — | 0 | -| `marketplace-reporting` | — | — | — | 0 | -| `mediatailor` | — | — | — | 0 | -| `medical-imaging` | — | — | — | 0 | -| `migration-hub` | — | — | — | 0 | -| `migration-hub-refactor-spaces` | — | — | — | 0 | -| `migrationhub-config` | — | — | — | 0 | -| `migrationhuborchestrator` | — | — | — | 0 | -| `migrationhubstrategy` | — | — | — | 0 | -| `mturk` | — | — | — | 0 | -| `neptunedata` | — | — | — | 0 | -| `nova-act` | — | — | — | 0 | -| `partnercentral-account` | — | — | — | 0 | -| `partnercentral-benefits` | — | — | — | 0 | -| `partnercentral-channel` | — | — | — | 0 | -| `partnercentral-revenue-measurement` | — | — | — | 0 | -| `partnercentral-selling` | — | — | — | 0 | -| `payment-cryptography-data` | — | — | — | 0 | -| `pca-connector-scep` | — | — | — | 0 | -| `pi` | — | — | — | 0 | -| `pinpoint-email` | — | — | — | 0 | -| `pinpoint-sms-voice` | — | — | — | 0 | -| `pricing-plan-manager` | — | — | — | 0 | -| `proton` | — | — | — | 0 | -| `qconnect` | — | — | — | 0 | -| `repostspace` | — | — | — | 0 | -| `route53-recovery-cluster` | — | — | — | 0 | -| `route53globalresolver` | — | — | — | 0 | -| `rtbfabric` | — | — | — | 0 | -| `security-ir` | — | — | — | 0 | -| `securityagent` | — | — | — | 0 | -| `signer-data` | — | — | — | 0 | -| `signin` | — | — | — | 0 | -| `snow-device-management` | — | — | — | 0 | -| `snowball` | — | — | — | 0 | -| `socialmessaging` | — | — | — | 0 | -| `ssm-guiconnect` | — | — | — | 0 | -| `sso-oidc` | — | — | — | 0 | -| `supplychain` | — | — | — | 0 | -| `support-app` | — | — | — | 0 | -| `supportauthz` | — | — | — | 0 | -| `sustainability` | — | — | — | 0 | -| `tnb` | — | — | — | 0 | -| `trustedadvisor` | — | — | — | 0 | -| `wickr` | — | — | — | 0 | -| `wisdom` | — | — | — | 0 | -| `workdocs` | — | — | — | 0 | -| `workmailmessageflow` | — | — | — | 0 | -| `workspaces-instances` | — | — | — | 0 | -| `workspaces-thin-client` | — | — | — | 0 | +| `amp` | yes | - | yes | 2 | +| `apigatewaymanagementapi` | yes | yes | - | 2 | +| `app-mesh` | yes | - | yes | 2 | +| `budgets` | yes | - | yes | 2 | +| `cleanrooms` | yes | - | yes | 2 | +| `cloudhsm-v2` | yes | - | yes | 2 | +| `codestar-connections` | - | yes | yes | 2 | +| `connect` | yes | - | yes | 2 | +| `data-pipeline` | yes | - | yes | 2 | +| `databrew` | yes | - | yes | 2 | +| `datasync` | yes | - | yes | 2 | +| `dax` | yes | - | yes | 2 | +| `devops-agent` | yes | - | yes | 2 | +| `direct-connect` | yes | - | yes | 2 | +| `directory-service` | yes | - | yes | 2 | +| `dsql` | yes | - | yes | 2 | +| `emr-containers` | yes | - | yes | 2 | +| `fsx` | yes | - | yes | 2 | +| `greengrass` | yes | - | yes | 2 | +| `guardduty` | yes | - | yes | 2 | +| `inspector2` | yes | - | yes | 2 | +| `ivs` | yes | - | yes | 2 | +| `kinesis-analytics` | - | yes | yes | 2 | +| `kinesis-video` | yes | - | yes | 2 | +| `macie2` | yes | - | yes | 2 | +| `mediaconnect` | yes | - | yes | 2 | +| `medialive` | yes | - | yes | 2 | +| `mediapackage` | yes | - | yes | 2 | +| `mediapackagev2` | yes | - | yes | 2 | +| `mediastore-data` | yes | yes | - | 2 | +| `network-firewall` | yes | - | yes | 2 | +| `networkmanager` | yes | - | yes | 2 | +| `opensearchserverless` | yes | - | yes | 2 | +| `osis` | yes | - | yes | 2 | +| `payment-cryptography` | yes | - | yes | 2 | +| `quicksight` | yes | - | yes | 2 | +| `resiliencehub` | yes | - | yes | 2 | +| `route-53-domains` | yes | - | yes | 2 | +| `s3vectors` | yes | - | yes | 2 | +| `securityhub` | yes | - | yes | 2 | +| `service-catalog` | yes | - | yes | 2 | +| `service-catalog-appregistry` | yes | - | yes | 2 | +| `service-quotas` | yes | - | yes | 2 | +| `signer` | yes | - | yes | 2 | +| `synthetics` | yes | - | yes | 2 | +| `timestream-influxdb` | yes | - | yes | 2 | +| `vpc-lattice` | yes | - | yes | 2 | +| `workspaces` | yes | - | yes | 2 | +| `workspaces-web` | yes | - | yes | 2 | +| `accessanalyzer` | - | - | yes | 1 | +| `account-access` | - | - | yes | 1 | +| `agent-registry` | - | - | yes | 1 | +| `appconfigdata` | - | yes | - | 1 | +| `appfabric` | - | - | yes | 1 | +| `appflow` | - | - | yes | 1 | +| `appintegrations` | - | - | yes | 1 | +| `application-insights` | - | - | yes | 1 | +| `application-signals` | - | - | yes | 1 | +| `apprunner` | - | - | yes | 1 | +| `appstream` | - | - | yes | 1 | +| `arc-region-switch` | - | - | yes | 1 | +| `arc-zonal-shift` | - | - | yes | 1 | +| `auditmanager` | - | - | yes | 1 | +| `auto-scaling-plans` | - | - | yes | 1 | +| `bcm-data-exports` | - | - | yes | 1 | +| `billing` | - | - | yes | 1 | +| `chatbot` | - | - | yes | 1 | +| `chime` | - | - | yes | 1 | +| `chime-sdk-media-pipelines` | - | - | yes | 1 | +| `chime-sdk-voice` | - | - | yes | 1 | +| `cloud9` | - | - | yes | 1 | +| `clouddirectory` | yes | - | - | 1 | +| `cloudfront-keyvaluestore` | - | - | yes | 1 | +| `codecatalyst` | - | - | yes | 1 | +| `codestar-notifications` | - | - | yes | 1 | +| `compute-optimizer` | - | - | yes | 1 | +| `connectcampaigns` | yes | - | - | 1 | +| `connectcases` | - | - | yes | 1 | +| `controltower` | - | - | yes | 1 | +| `cost-and-usage-report-service` | - | - | yes | 1 | +| `cost-optimization-hub` | - | - | yes | 1 | +| `customer-profiles` | - | - | yes | 1 | +| `dataexchange` | - | - | yes | 1 | +| `datazone` | - | - | yes | 1 | +| `detective` | - | - | yes | 1 | +| `device-farm` | - | - | yes | 1 | +| `directory-service-data` | - | - | yes | 1 | +| `dlm` | - | - | yes | 1 | +| `docdb-elastic` | - | - | yes | 1 | +| `drs` | - | - | yes | 1 | +| `ec2-instance-connect` | yes | - | - | 1 | +| `ecr-public` | - | - | yes | 1 | +| `evs` | - | - | yes | 1 | +| `finspace` | - | - | yes | 1 | +| `fms` | - | - | yes | 1 | +| `gamelift` | - | - | yes | 1 | +| `global-accelerator` | - | - | yes | 1 | +| `grafana` | - | - | yes | 1 | +| `groundstation` | - | - | yes | 1 | +| `imagebuilder` | - | - | yes | 1 | +| `inspector` | - | - | yes | 1 | +| `interconnect` | - | - | yes | 1 | +| `internetmonitor` | - | - | yes | 1 | +| `invoicing` | - | - | yes | 1 | +| `ivschat` | - | - | yes | 1 | +| `kafkaconnect` | - | - | yes | 1 | +| `keyspaces` | - | - | yes | 1 | +| `kinesis-video-archived-media` | yes | - | - | 1 | +| `lambda-core` | - | - | yes | 1 | +| `lambda-microvms` | - | - | yes | 1 | +| `launch-wizard` | - | - | yes | 1 | +| `license-manager` | - | - | yes | 1 | +| `lightsail` | - | - | yes | 1 | +| `location` | - | - | yes | 1 | +| `m2` | - | - | yes | 1 | +| `mailmanager` | - | - | yes | 1 | +| `marketplace-metering` | yes | - | - | 1 | +| `mediapackage-vod` | - | - | yes | 1 | +| `mgn` | - | - | yes | 1 | +| `mpa` | - | - | yes | 1 | +| `mwaa-serverless` | - | - | yes | 1 | +| `neptune-graph` | - | - | yes | 1 | +| `networkflowmonitor` | - | - | yes | 1 | +| `networkmonitor` | - | - | yes | 1 | +| `notifications` | - | - | yes | 1 | +| `notificationscontacts` | - | - | yes | 1 | +| `oam` | - | - | yes | 1 | +| `observabilityadmin` | - | - | yes | 1 | +| `odb` | - | - | yes | 1 | +| `outposts` | - | - | yes | 1 | +| `pca-connector-ad` | - | - | yes | 1 | +| `pcs` | - | - | yes | 1 | +| `pinpoint-sms-voice-v2` | - | - | yes | 1 | +| `pricing` | - | - | yes | 1 | +| `rbin` | - | - | yes | 1 | +| `redshift-serverless` | - | - | yes | 1 | +| `resiliencehubv2` | - | - | yes | 1 | +| `resource-explorer-2` | - | - | yes | 1 | +| `rolesanywhere` | - | - | yes | 1 | +| `route53-recovery-control-config` | - | - | yes | 1 | +| `route53-recovery-readiness` | - | - | yes | 1 | +| `route53profiles` | - | - | yes | 1 | +| `rum` | - | - | yes | 1 | +| `s3files` | - | - | yes | 1 | +| `s3outposts` | - | - | yes | 1 | +| `savingsplans` | - | - | yes | 1 | +| `schemas` | - | - | yes | 1 | +| `securitylake` | - | - | yes | 1 | +| `simpledbv2` | yes | - | - | 1 | +| `ssm-contacts` | - | - | yes | 1 | +| `ssm-incidents` | - | - | yes | 1 | +| `ssm-quicksetup` | - | - | yes | 1 | +| `ssm-sap` | - | - | yes | 1 | +| `sso` | - | - | yes | 1 | +| `storage-gateway` | - | - | yes | 1 | +| `taxsettings` | - | - | yes | 1 | +| `uxc` | - | - | yes | 1 | +| `waf-regional` | - | - | yes | 1 | +| `wellarchitected` | - | - | yes | 1 | +| `workmail` | - | - | yes | 1 | +| `agent-registry-control` | - | - | - | 0 | +| `aiops` | - | - | - | 0 | +| `amplifybackend` | - | - | - | 0 | +| `amplifyuibuilder` | - | - | - | 0 | +| `application-discovery-service` | - | - | - | 0 | +| `applicationcostprofiler` | - | - | - | 0 | +| `artifact` | - | - | - | 0 | +| `b2bi` | - | - | - | 0 | +| `backup-gateway` | - | - | - | 0 | +| `backupsearch` | - | - | - | 0 | +| `bcm-dashboards` | - | - | - | 0 | +| `bcm-pricing-calculator` | - | - | - | 0 | +| `bcm-recommended-actions` | - | - | - | 0 | +| `billingconductor` | - | - | - | 0 | +| `braket` | - | - | - | 0 | +| `chime-sdk-identity` | - | - | - | 0 | +| `chime-sdk-meetings` | - | - | - | 0 | +| `chime-sdk-messaging` | - | - | - | 0 | +| `cleanroomsml` | - | - | - | 0 | +| `cloudhsm` | - | - | - | 0 | +| `cloudsearch-domain` | - | - | - | 0 | +| `cloudtrail-data` | - | - | - | 0 | +| `cloudwatch-events` | - | - | - | 0 | +| `cognito-sync` | - | - | - | 0 | +| `compute-optimizer-automation` | - | - | - | 0 | +| `connect-contact-lens` | - | - | - | 0 | +| `connectcampaignsv2` | - | - | - | 0 | +| `connecthealth` | - | - | - | 0 | +| `connectparticipant` | - | - | - | 0 | +| `controlcatalog` | - | - | - | 0 | +| `deadline` | - | - | - | 0 | +| `eks-auth` | - | - | - | 0 | +| `elementalinference` | - | - | - | 0 | +| `finspace-data` | - | - | - | 0 | +| `freetier` | - | - | - | 0 | +| `gameliftstreams` | - | - | - | 0 | +| `geo-maps` | - | - | - | 0 | +| `geo-places` | - | - | - | 0 | +| `geo-routes` | - | - | - | 0 | +| `greengrassv2` | - | - | - | 0 | +| `health` | - | - | - | 0 | +| `iam-toolbox` | - | - | - | 0 | +| `inspector-scan` | - | - | - | 0 | +| `iot-jobs-data-plane` | - | - | - | 0 | +| `iot-managed-integrations` | - | - | - | 0 | +| `iotdeviceadvisor` | - | - | - | 0 | +| `iotfleetwise` | - | - | - | 0 | +| `iotsecuretunneling` | - | - | - | 0 | +| `iotsitewise` | - | - | - | 0 | +| `iotthingsgraph` | - | - | - | 0 | +| `iottwinmaker` | - | - | - | 0 | +| `ivs-realtime` | - | - | - | 0 | +| `keyspacesstreams` | - | - | - | 0 | +| `kinesis-video-media` | - | - | - | 0 | +| `kinesis-video-signaling` | - | - | - | 0 | +| `kinesis-video-webrtc-storage` | - | - | - | 0 | +| `license-manager-linux-subscriptions` | - | - | - | 0 | +| `license-manager-user-subscriptions` | - | - | - | 0 | +| `machine-learning` | - | - | - | 0 | +| `managedblockchain-query` | - | - | - | 0 | +| `marketplace-agreement` | - | - | - | 0 | +| `marketplace-catalog` | - | - | - | 0 | +| `marketplace-commerce-analytics` | - | - | - | 0 | +| `marketplace-deployment` | - | - | - | 0 | +| `marketplace-discovery` | - | - | - | 0 | +| `marketplace-entitlement-service` | - | - | - | 0 | +| `marketplace-reporting` | - | - | - | 0 | +| `mediatailor` | - | - | - | 0 | +| `medical-imaging` | - | - | - | 0 | +| `migration-hub` | - | - | - | 0 | +| `migration-hub-refactor-spaces` | - | - | - | 0 | +| `migrationhub-config` | - | - | - | 0 | +| `migrationhuborchestrator` | - | - | - | 0 | +| `migrationhubstrategy` | - | - | - | 0 | +| `mturk` | - | - | - | 0 | +| `neptunedata` | - | - | - | 0 | +| `nova-act` | - | - | - | 0 | +| `partnercentral-account` | - | - | - | 0 | +| `partnercentral-benefits` | - | - | - | 0 | +| `partnercentral-channel` | - | - | - | 0 | +| `partnercentral-revenue-measurement` | - | - | - | 0 | +| `partnercentral-selling` | - | - | - | 0 | +| `payment-cryptography-data` | - | - | - | 0 | +| `pca-connector-scep` | - | - | - | 0 | +| `pi` | - | - | - | 0 | +| `pinpoint-email` | - | - | - | 0 | +| `pinpoint-sms-voice` | - | - | - | 0 | +| `pricing-plan-manager` | - | - | - | 0 | +| `proton` | - | - | - | 0 | +| `qconnect` | - | - | - | 0 | +| `repostspace` | - | - | - | 0 | +| `route53-recovery-cluster` | - | - | - | 0 | +| `route53globalresolver` | - | - | - | 0 | +| `rtbfabric` | - | - | - | 0 | +| `security-ir` | - | - | - | 0 | +| `securityagent` | - | - | - | 0 | +| `signer-data` | - | - | - | 0 | +| `signin` | - | - | - | 0 | +| `snow-device-management` | - | - | - | 0 | +| `snowball` | - | - | - | 0 | +| `socialmessaging` | - | - | - | 0 | +| `ssm-guiconnect` | - | - | - | 0 | +| `sso-oidc` | - | - | - | 0 | +| `supplychain` | - | - | - | 0 | +| `support-app` | - | - | - | 0 | +| `supportauthz` | - | - | - | 0 | +| `sustainability` | - | - | - | 0 | +| `tnb` | - | - | - | 0 | +| `trustedadvisor` | - | - | - | 0 | +| `wickr` | - | - | - | 0 | +| `wisdom` | - | - | - | 0 | +| `workdocs` | - | - | - | 0 | +| `workmailmessageflow` | - | - | - | 0 | +| `workspaces-instances` | - | - | - | 0 | +| `workspaces-thin-client` | - | - | - | 0 | diff --git a/docs/faq.md b/docs/faq.md index 18b2337a..31fb2536 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -11,9 +11,9 @@ No. It is a local development tool, not designed or tested for production, and s **How is it different from other local cloud emulators?** DevCloud positions itself as an *on-ramp* rather than a replacement: -- Go single binary — starts in under a second, no JVM or Python runtime +- Go single binary: starts in under a second, no JVM or Python runtime - Smithy-driven codegen from official AWS models, with a weekly sync -- Multi-CSP vision — AWS today, Azure and GCP planned ([roadmap](roadmap.md)) +- Multi-cloud architecture: AWS, Azure, and GCP served on unified port 4747 ([roadmap](roadmap.md)) - Apache 2.0 with an explicit patent grant It does not aim to displace any existing tool. Use whatever works for your workflow. @@ -30,10 +30,10 @@ See [coverage.md](coverage.md) for the counts and what they promise, and [servic Developed and tested on Linux and macOS. WSL2 is expected to work but is not in the CI matrix; native Windows is unsupported. Contributions welcome. **Can I run it in CI?** -Yes — start DevCloud as a service container and point your tests at `http://localhost:4747`. The image starts in well under a second and needs no external dependencies. +Yes: start DevCloud as a service container and point your tests at `http://localhost:4747`. The image starts in well under a second and needs no external dependencies. **Does data persist across restarts?** -For services with a persistent backend (S3, DynamoDB, Lambda, IAM/STS), yes — under each service's `data_dir`. SQS is in-memory and loses state. See [configuration.md](configuration.md#data-directories). +For services with a persistent backend (S3, DynamoDB, Lambda, IAM/STS), yes, under each service's `data_dir`. SQS is in-memory and loses state. See [configuration.md](configuration.md#data-directories). ## Compatibility diff --git a/docs/fidelity-manifest.md b/docs/fidelity-manifest.md index a80543c8..4593b89d 100644 --- a/docs/fidelity-manifest.md +++ b/docs/fidelity-manifest.md @@ -1,7 +1,7 @@ # Fidelity Manifest Not every operation is served the same way. The **fidelity manifest** answers, per -operation, *how much you can trust this call* — so a green response never leaves +operation, *how much you can trust this call*, so a green response never leaves you guessing whether DevCloud implemented the operation or made something plausible up. @@ -14,8 +14,8 @@ totals are published in [coverage.md](coverage.md). | Tier | Meaning | Trust it for | |------|---------|--------------| | `hand-verified` | The service's provider implements the operation explicitly. | Behaviour, not just shape. Covered by the boto3 suite where tests exist. | -| `auto-crud` | Served by the [generic CRUD engine](crud-engine.md): store-backed, plausible responses with no validation, business logic, or cross-resource integrity. | Wiring your SDK up and round-tripping create → get → list → delete. Nothing else. | -| `unimplemented` | Not served — the call fails instead of inventing a success. | Knowing early that DevCloud will not serve this call. | +| `auto-crud` | Served by the [generic CRUD engine](crud-engine.md): store-backed, plausible responses with no validation, business logic, or cross-resource integrity. | Wiring your SDK up and round-tripping create -> get -> list -> delete. Nothing else. | +| `unimplemented` | Not served: the call fails instead of inventing a success. | Knowing early that DevCloud will not serve this call. | `hand-verified` always wins: the CRUD engine is reached only when a provider's dispatch falls through, so a hand-written implementation is never shadowed. @@ -26,7 +26,7 @@ There is no single error. It depends on how the owning provider declines: | How the provider declines | Error | Status | Providers | |---|---|---|---| -| Returns `ErrUnhandledOp`, and the CRUD engine cannot classify the operation either — [`gateway/router.go`](../internal/gateway/router.go) emits the fallback | `InvalidAction` | 400 | 155 | +| Returns `ErrUnhandledOp`, and the CRUD engine cannot classify the operation either; [`gateway/router.go`](../internal/gateway/router.go) emits the fallback | `InvalidAction` | 400 | 155 | | Its dispatch `default:` answers directly | `NotImplemented` | 501 | 32 | | Its dispatch `default:` answers in its own vocabulary | `UnsupportedOperation` / `MethodNotAllowed` | 400 / 405 | `iot`, `iotwireless`, `apigatewayv2`, `backup`, `bedrock`, `s3` | @@ -34,13 +34,13 @@ A service can even differ by protocol: `sqs` returns `NotImplemented` (501) on Query and `InvalidAction` (400) on JSON. Only the *failure* is stable, and that is all -[compatibility-policy.md](compatibility-policy.md) promises — an `unimplemented` +[compatibility-policy.md](compatibility-policy.md) promises: an `unimplemented` operation never fabricates a success. The specific code and status are not guaranteed across 1.x; normalizing them would be a minor release. ## Reading it -**At runtime** — via the admin API, which is off by default. Enable it in +**At runtime**: via the admin API, which is off by default. Enable it in `devcloud.yaml` ([configuration](configuration.md)): ```yaml @@ -63,10 +63,10 @@ curl -s 'localhost:4747/devcloud/api/fidelity?service=s3' # per-operation, one s } ``` -The unfiltered response omits `operations` — the full manifest runs to five +The unfiltered response omits `operations`; the full manifest runs to five figures. -**In Go** — [`internal/generated/fidelity`](../internal/generated/fidelity/manifest_gen.go): +**In Go**: [`internal/generated/fidelity`](../internal/generated/fidelity/manifest_gen.go): ```go tier, ok := fidelity.Lookup("s3", "PutObject") // TierHandVerified, true @@ -89,9 +89,9 @@ an operation name, so they declare their operations explicitly in `pathRoutedOps (`internal/codegen/scan_handverified.go`). The universe is *model ∪ hand-verified*. A provider may serve an operation its -model does not declare — `bedrock` serves `InvokeModel`, which AWS models under +model does not declare: `bedrock` serves `InvokeModel`, which AWS models under bedrock-runtime, and `dynamodbstreams` dispatches 22 operations against a -4-operation model — and hiding those would understate what DevCloud does. So a +4-operation model, and hiding those would understate what DevCloud does. So a `hand-verified` entry is a statement about **DevCloud**, not about AWS: it can name an operation your SDK has never heard of. @@ -103,7 +103,7 @@ whole package would file both as operations. ## Limits -- **11 services have no in-tree Smithy model** — `account`, `cloudcontrol`, +- **11 services have no in-tree Smithy model**: `account`, `cloudcontrol`, `codeconnections`, `dms`, `identitystore`, `mediaconvert`, `pipes`, `s3tables`, `scheduler`, `serverlessrepo`, `verifiedpermissions`. Their `modelBacked` flag is `false` and the manifest lists only what DevCloud serves; the unimplemented @@ -121,13 +121,13 @@ whole package would file both as operations. ## Getting an operation promoted -Promotion to `hand-verified` happens **on request, with a use case** — +Promotion to `hand-verified` happens **on request, with a use case**: [open a feature request](https://github.com/skyoo2003/devcloud/issues/new?template=feature_request.yml) naming the service, the operation, and what you are trying to run locally. Check the tier first with the endpoint above so the request is concrete. Requests beat guesswork. The unpromoted operations are overwhelmingly operational -surface — DynamoDB backups and global tables, KMS custom key stores and rotation, +surface: DynamoDB backups and global tables, KMS custom key stores and rotation, CloudWatch Insight Rules and Metric Streams, ECR pull-through cache. A local inner loop does not call them, while the ones it does call (S3, SQS, Lambda, IAM, STS in full) are already `hand-verified`. So v1.0 ships the long tail *declared @@ -141,7 +141,7 @@ when: - any operation carries a tier outside the closed set, - any registered service is missing from the manifest, - any CRUD-registered operation is missing or marked `unimplemented`, -- any service resolves zero `hand-verified` operations — the signal that a new +- any service resolves zero `hand-verified` operations, which signals that a new path-routing provider needs a `pathRoutedOps` entry, - any `auto-crud` operation is one the engine will not actually serve (`TestAutoCRUDIsServedOverJSON`). diff --git a/docs/getting-started.md b/docs/getting-started.md index aa41e12a..be5b5f5d 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -1,48 +1,41 @@ # Getting Started -## Run with Docker (recommended) +## Run with Docker ```bash docker run -p 4747:4747 ghcr.io/skyoo2003/devcloud:latest ``` -To persist data across restarts, mount a volume: - -Generic CRUD resources are persisted by default alongside service data. To -start from a clean local environment, run `devcloud --reset-data`; with the -admin API enabled, `DELETE /devcloud/api/data` performs the same full reset. +To persist data across restarts, mount a local volume: ```bash docker run -p 4747:4747 -v $(pwd)/data:/app/data ghcr.io/skyoo2003/devcloud:latest ``` -## Docker Compose (development) +Generic resources persist by default alongside service data. To reset local environment data, run `devcloud --reset-data`. When the admin API is enabled, `DELETE /devcloud/api/data` performs the same reset via HTTP. + +## Docker Compose ```bash docker compose -f deployments/docker-compose.yml up ``` -Builds the image from source and starts the Go server on port 4747 with `./data` -mounted. It runs a subset, not everything — -[`docker-compose.yml`](../deployments/docker-compose.yml) sets -`DEVCLOUD_SERVICES=s3,sqs,dynamodb,iam,sts,lambda`. Edit that line, or see -[Configuration](configuration.md#devcloud_services), to run more. +Builds from source and starts the server on port 4747 with `./data` mounted. To select specific services, set `DEVCLOUD_SERVICES` (for example, `DEVCLOUD_SERVICES=s3,sqs,dynamodb,iam,sts,lambda`). See [Configuration](configuration.md#devcloud_services) for details. -## Build from source - -Recommended only for contributors — see [contributing.md](contributing.md) for -full prerequisites and workflow. +## Build from Source ```bash git clone https://github.com/skyoo2003/devcloud.git cd devcloud -make build # builds the Go binaries (devcloud + codegen) +make build # builds devcloud and codegen binaries make run # starts the server on port 4747 ``` -## Verify it works +## Verify Your Setup + +All AWS, Azure, and GCP services listen on port 4747. -### boto3 +### AWS (boto3) ```python import boto3 @@ -57,47 +50,62 @@ s3 = boto3.client( s3.create_bucket(Bucket="test-bucket") print(s3.list_buckets()["Buckets"]) -# [{'Name': 'test-bucket', 'CreationDate': ...}] ``` -Any dummy credentials work — DevCloud reads them but never verifies them. They -must be non-empty. - ### AWS CLI ```bash -# Configure a profile (one-time) +# Configure a local profile aws configure set aws_access_key_id test --profile devcloud aws configure set aws_secret_access_key test --profile devcloud aws configure set region us-east-1 --profile devcloud +# Use the endpoint aws --endpoint-url http://localhost:4747 --profile devcloud s3 mb s3://test-bucket aws --endpoint-url http://localhost:4747 --profile devcloud s3 ls ``` -Or alias it: `alias awslocal='aws --endpoint-url http://localhost:4747'`. +### Azure (Python SDK) + +```python +from azure.storage.blob import BlobServiceClient + +client = BlobServiceClient( + account_url="http://localhost:4747/devstoreaccount1", + credential="Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==", +) + +container = client.create_container("test-container") +print("Container created:", container.container_name) +``` + +### GCP (Python SDK) -### Terraform / CDK +```python +import os +from google.cloud import storage + +os.environ["STORAGE_EMULATOR_HOST"] = "http://127.0.0.1:4747" +client = storage.Client(project="test-project") -Point the AWS provider at `http://localhost:4747` with dummy credentials. See the -[FAQ](faq.md#compatibility) for what works and what does not. +bucket = client.create_bucket("test-bucket") +print("GCS bucket created:", bucket.name) +``` ## Admin API -Opt-in — set `admin.enabled: true` in config, then: +Opt-in by setting `admin.enabled: true` in your configuration: | Route | Returns | |---|---| | `GET /devcloud/api/services` | Service status overview | -| `GET /devcloud/api/services/{id}/resources` | Buckets, queues, tables, functions | -| `GET /devcloud/api/logs` | Recent API call logs (`?limit=`) | -| `GET /devcloud/api/fidelity` | Per-operation tiers (`?service=`) — [fidelity-manifest.md](fidelity-manifest.md) | -| `GET /devcloud/api/unrouted` | Calls to services this build does not register — [coverage.md](coverage.md) | - -The web dashboard UI that consumes this API lives in a separate repository. +| `GET /devcloud/api/services/{id}/resources` | Inventory of created resources (buckets, queues, tables, functions) | +| `GET /devcloud/api/logs` | Recent API request logs (`?limit=100`) | +| `GET /devcloud/api/fidelity` | Per-operation fidelity tiers (`?service=s3`) | +| `GET /devcloud/api/unrouted` | Log of requests sent to unhandled or unmapped services | -## Next steps +## Next Steps -- [Configuration](configuration.md) — all options and env-var overrides -- [Services](services/) — per-service API reference and examples -- [Troubleshooting](troubleshooting.md) — when something does not work +- Explore the [Architecture](architecture.md) and workload execution runtime. +- Check [Configuration](configuration.md) options for multi-provider deployments. +- Review [Coverage](coverage.md) for registered service details. diff --git a/docs/plugin-api.md b/docs/plugin-api.md index 65170f71..227df17d 100644 --- a/docs/plugin-api.md +++ b/docs/plugin-api.md @@ -1,11 +1,11 @@ # ServicePlugin API The stable contract every DevCloud service implements. As of **v1.0** this is a -public API — see [API stability](#api-stability). +public API; see [API stability](#api-stability). The interface and registry live in [`internal/plugin/`](../internal/plugin/). Plugins are compiled in-tree and register themselves at startup; there is no -dynamic loading. (That is why the package is under `internal/` — it is imported +dynamic loading. (That is why the package is under `internal/`: it is imported by DevCloud's own binaries, not by third-party modules.) ## The interface @@ -24,12 +24,12 @@ type ServicePlugin interface { | Method | Contract | |--------|----------| -| `ServiceID()` | Stable lowercase identifier, e.g. `"s3"`. **Must equal the key the plugin is registered under** — the gateway routes by it. Constant; safe before `Init`. | +| `ServiceID()` | Stable lowercase identifier, e.g. `"s3"`. **Must equal the key the plugin is registered under**: the gateway routes by it. Constant; safe before `Init`. | | `ServiceName()` | Human-readable name for logs and the admin API. Non-empty, constant, safe before `Init`. | | `Protocol()` | One of the [`ProtocolType`](#protocols) constants. Constant; safe before `Init`. | | `Init(config)` | Called once at startup before any request. Open stores, read `config.Options`. A non-nil error aborts startup (services in the fixed init order) or skips the service (others). | | `Shutdown(ctx)` | Called once at graceful shutdown, 15s budget. Flush and close. Idempotent-friendly. | -| `HandleRequest(ctx, op, req)` | Handle one API call. See [operation names](#operation-names) — `op` may be `""` for REST protocols, in which case derive it yourself. Return a `*Response`; return a Go `error` only for unexpected internal failures (the gateway wraps those as `500 InternalError`). **Model AWS errors as a normal `*Response`.** | +| `HandleRequest(ctx, op, req)` | Handle one API call. See [operation names](#operation-names): `op` may be `""` for REST protocols, in which case derive it yourself. Return a `*Response`; return a Go `error` only for unexpected internal failures (the gateway wraps those as `500 InternalError`). **Model AWS errors as a normal `*Response`.** | | `ListResources(ctx)` | The resources this service currently holds, for the admin API. May be empty. | These invariants are enforced for every registered service by @@ -53,13 +53,13 @@ for the service: The gateway derives `op` in [`extractOperationName`](../internal/gateway/router.go): -- **JSON protocols** — the suffix of `X-Amz-Target` after the last `.` - (`DynamoDB_20120810.GetItem` → `GetItem`). -- **Query protocol** — the `Action` parameter. -- **REST protocols** — `""`; the operation is implicit in method + URL, so the +- **JSON protocols**: the suffix of `X-Amz-Target` after the last `.` + (`DynamoDB_20120810.GetItem` -> `GetItem`). +- **Query protocol**: the `Action` parameter. +- **REST protocols**: `""`; the operation is implicit in method + URL, so the plugin resolves it itself. -Service routing uses the target prefix or signing name — see +Service routing uses the target prefix or signing name; see [`serviceFromTarget` and `normalizeServiceID`](../internal/gateway/protocol.go). Target prefixes may be dotted (`com.amazonaws.codeconnections.CodeConnections_20231201`); the last dotted @@ -74,12 +74,12 @@ type PluginConfig struct { } ``` -- `DataDir` — the per-service directory from config (`data_dir:`). -- `Options` — cross-cutting values injected by +- `DataDir`: the per-service directory from config (`data_dir:`). +- `Options`: cross-cutting values injected by [`buildOptions`](../cmd/devcloud/main.go). Keys a plugin may rely on: - - `server_port` (`int`) — the HTTP port, for building resource URLs such as SQS + - `server_port` (`int`): the HTTP port, for building resource URLs such as SQS queue URLs. Passed to every service. - - `iam_store` — set for `sts` so it shares the IAM store. A service needing a + - `iam_store`: set for `sts` so it shares the IAM store. A service needing a peer's store receives it here. ## Caller identity @@ -94,7 +94,7 @@ if id, ok := auth.FromContext(ctx); ok { ``` `auth.Identity` carries `Provider`, `AccessKeyID`, `Region`, `Service` (the -signing name as presented) and `SessionToken`. **None of it is verified** — +signing name as presented) and `SessionToken`. **None of it is verified**: DevCloud accepts any credentials, so treat these as a claim, never as an authorization decision. Adapters live in [`internal/auth`](../internal/auth/); `SigV4` is AWS's, and Azure (AAD/SAS) and GCP (OAuth2) join by implementing @@ -105,8 +105,8 @@ authorization decision. Adapters live in [`internal/auth`](../internal/auth/); Return AWS-shaped errors as a `*Response`, using the helpers in [`internal/shared/response.go`](../internal/shared/response.go): -- JSON protocols → `shared.JSONError(code, message, status)` (`{"__type", "message"}`) -- Query/XML protocols → `shared.QueryXMLError(...)` or the service's XML helper +- JSON protocols -> `shared.JSONError(code, message, status)` (`{"__type", "message"}`) +- Query/XML protocols -> `shared.QueryXMLError(...)` or the service's XML helper **Unknown or unimplemented operations must return an error, never a false success.** The convention is `InvalidAction` with HTTP 400: @@ -118,7 +118,7 @@ default: A silent `200 OK {}` is a bug: it makes an SDK believe the call succeeded. To opt into the [generic CRUD engine](crud-engine.md) instead, return -`plugin.ErrUnhandledOp` from the `default:` case — the gateway falls back to the +`plugin.ErrUnhandledOp` from the `default:` case: the gateway falls back to the engine and emits `InvalidAction` itself if the engine cannot classify the operation. @@ -170,8 +170,8 @@ to every service in the tree just to state the value it already defaults to, and would break the [v1.x contract](#api-stability) for no gain. Always read the provider through `ProviderOf`, never by type-asserting. -Startup uses it to resolve configuration — -`cfg.ProviderService(plugin.ProviderOf(p), id)` — so a plugin lands in its own +Startup uses it to resolve configuration via +`cfg.ProviderService(plugin.ProviderOf(p), id)`, so a plugin lands in its own provider's config namespace without the config package needing to know which services exist. See [configuration.md](configuration.md#provider-namespacing). @@ -179,7 +179,7 @@ services exist. See [configuration.md](configuration.md#provider-namespacing). This section is the **in-tree** contract: it constrains contributors writing service plugins inside this repository. `internal/plugin` cannot be imported from -another Go module, so it is not the promise a *user* of DevCloud depends on — +another Go module, so it is not the promise a *user* of DevCloud depends on; that is [compatibility-policy.md](compatibility-policy.md). Starting at **v1.0**, `ServicePlugin`, `PluginConfig`, `Response`, `Resource` and @@ -194,6 +194,6 @@ the `ProtocolType` constants are stable within `v1.x`: it. Release versioning is governed by [compatibility-policy.md](compatibility-policy.md). -New optional capability is introduced additively — new `Options` keys, new -`ProtocolType` values, optional side interfaces such as `ProviderScoped` — so +New optional capability is introduced additively (new `Options` keys, new +`ProtocolType` values, optional side interfaces such as `ProviderScoped`), so existing plugins keep compiling and behaving. diff --git a/docs/roadmap.md b/docs/roadmap.md index 0283e862..b4f1c989 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -1,149 +1,74 @@ # Roadmap -DevCloud aims to be a **local development companion for cloud-native apps across -every major CSP** — not a production replacement, but an on-ramp that lets you -iterate without cloud bills and deploy to your target CSP with confidence. It gets -there in phases, to keep scope, architectural complexity and community -expectations manageable. - -## Guiding principles - -1. **Local-first, cost-free** — no cloud charges for inner-loop development. -2. **On-ramp, not replacement** — DevCloud helps you *land on* a CSP, not avoid it. -3. **API-compatible, not behaviour-perfect** — SDK compatibility over edge-case parity. -4. **Community-owned** — the scope is too large for one maintainer, so the plugin architecture and contributor experience are first-class. -5. **Trademark-respectful** — see [TRADEMARKS.md](../TRADEMARKS.md). - -## Phase 1 — AWS depth and stabilization (complete, shipped as v1.0) - -Mature the already-broad AWS surface into a stable, well-tested v1.0. - -- [x] AWS services scaffolded from official Smithy models via in-tree codegen — counts in [coverage.md](coverage.md) -- [x] Deep hand-written coverage on core and integration services -- [x] Cross-service integration — CloudFormation, DynamoDB Streams → Lambda, SQS → Lambda, S3 → Lambda, EventBridge, SNS → SQS -- [x] boto3 compatibility suite green in CI; a failing test fails the build -- [x] Unimplemented operations return an AWS-shaped error instead of a false `200` -- [x] Stable `ServicePlugin` API ([plugin-api.md](plugin-api.md)), enforced by a conformance test over every registered service -- [x] Generic [CRUD fallback engine](crud-engine.md) serving the long tail with plausible, store-backed responses. **Follow-up:** promote high-value `auto-crud` operations to hand-verified fidelity. -- [x] v1.0 release — see [compatibility-policy.md](compatibility-policy.md) - -Phase 1 execution follow-up: Lambda Python 3.12/Node 22 handlers execute in -Docker, published code is immutable, failed SQS/Streams batches preserve source -state, and SNS ARN subscriptions preserve message contents. A required -`lambda-runtime` job verifies actual handler results and S3/SQS/Streams/EventBridge -delivery; the ordinary suite verifies Docker-free contracts. See -[Lambda](services/lambda.md) for runtime requirements and local limits. - -AWS operation expansion follow-up: EventBridge's ActivateEventSource, -DeactivateEventSource, DeauthorizeConnection and RemovePermission now share -durable state with their companion APIs. SDK tests verify source-controlled local -delivery, connection reuse and bus policy removal. SNS's six missing operations -now support actual SQS batch/FIFO delivery, mobile/SMS attribute readback, opt-in -and verification with a local outbox OTP. S3 UploadPartCopy now copies actual -source bytes/ranges with condition checks, error rollback and restart persistence. -Directory RenameObject now preserves bytes, metadata and tags with conditional checks, -persistent token replay and scoped CreateSession credentials. -Phase 1A Core Integrations completed native support for all remaining operations -across S3 (RestoreObject, SelectObjectContent, WriteGetObjectResponse), -DynamoDB (PartiQL engine, Kinesis destinations, PITR and table restores), -Lambda (Layer versioning, response streaming, async and durable callbacks), and -CloudFormation (CancelUpdateStack, drift detection, resource scans and orgs access). -Phase 1B Core Identity & Access completed native support for all remaining operations -across IAM (MFA devices, password changes, service credentials, SSH/server/signing certificates, -OIDC provider client IDs, delegation requests, access/credential reports, policy simulation, -and org root credentials/sessions) and STS (AssumeRoleWithWebIdentity, AssumeRoleWithSAML, -AssumeRoot, DecodeAuthorizationMessage, GetDelegatedAccessToken, GetFederationToken, GetWebIdentityToken). -Phase 1C Compute & Network added `aws.protocols#ec2Query` protocol detection, -CRUD engine XML response support (``, item list tags), gateway error routing, -and wired EC2 unhandled operations into the fallback CRUD engine, promoting 498 EC2 -operations into `auto-crud`. -Phase 1D Generic CRUD Engine Semantic Verb Expansion added comprehensive -classification for lifecycle and association verbs (Relate, Toggle, Lifecycle, Batch) -across the entire fleet, promoting 2,014 operations into `auto-crud`. -Phase 1E Non-CRUD Data Planes implemented SQLite-backed query execution for `rdsdata` -(`ExecuteStatement`, `BatchExecuteStatement`, `BeginTransaction`, `CommitTransaction`, -`RollbackTransaction`, `ExecuteSql`) and dedicated mock inference backends for -`sagemakerruntime` and `sagemakerruntimehttp2` (`InvokeEndpoint`, `InvokeEndpointAsync`, -`InvokeEndpointWithResponseStream`, `InvokeEndpointWithBidirectionalStream`), promoting -10 operations to `hand-verified` and leaving only 1 registered-only service in the fleet. -Phase 1F 100% Service Fleet Activation implemented native CBOR encoding/decoding -(`internal/shared/cbor`), `smithy.protocols#rpcv2Cbor` protocol detection, and gateway -RPC-v2 routing, bringing `partnercentralrevenuemeasurement` into `auto-crud`. Every -single registered service in the AWS fleet (431 of 431, 100.0%) now serves $\ge 1$ operation, -leaving **zero** registered-only services. -The original 3,085-operation unimplemented baseline has **477 remaining** across all registered services (267 within the serving target); -complete AWS operation support is still in progress across subsequent bundles. See -[EventBridge](services/eventbridge.md), [SNS](services/sns.md), [S3](services/s3.md) and -[IAM/STS](services/iam-sts.md) for local limits. - -## Phase 2 — Architectural preparation (complete, v1.x) - -Refactor internally so adding a CSP does not require forking the project. Each -item made a future provider an *addition* rather than an edit — the seams are -tabulated in [architecture.md](architecture.md#multi-csp-seams). - -- [x] Intermediate Representation between models and codegen ([`internal/codegen/ir`](../internal/codegen/ir/ir.go)). Generators read `*ir.Model`; nothing in the IR names Smithy. -- [x] Parser refactored behind [`ModelSource`](../internal/codegen/source.go). `SmithySource` owns its own format detection, so `tools/codegen` never names a format. -- [x] Provider namespacing in config — `providers.aws.services.*`, forward-compatible with `providers.azure.*` ([configuration.md](configuration.md#provider-namespacing)). -- [x] Plugin interface review — `ServicePlugin` needed no change; the CSP is carried by the optional [`ProviderScoped`](plugin-api.md#providers-and-csp-neutrality). -- [x] Per-provider auth adapters ([`internal/auth`](../internal/auth/auth.go)). `SigV4` is the AWS implementation; AAD/SAS and OAuth2 slot in beside it without touching the gateway. - -Phase 2 deliberately shipped **no** non-AWS service, no second `ModelSource`, and -no signature verification. Those are Phase 3 and beyond; Phase 2's job was to make -each of them an addition rather than a fork. - -## Phase 3 — First non-AWS service (complete, v2.0 groundwork) - -Validate the multi-CSP architecture with one well-scoped pilot. - -- [x] Pick an Azure pilot service — candidate: **Azure Blob Storage**, closest to S3 semantically -- [x] OpenAPI → IR → codegen proof of concept -- [x] Azure authentication adapter (Shared Key to start) -- [x] Compatibility tests against `azure-sdk-for-python` -- [x] Documentation pattern for multi-CSP service docs - -Phase 3 execution completed: Azure Blob Storage (`blob`) is implemented as the first -non-AWS pilot under `providers.azure.services.blob`. The OpenAPI model in `api/models/` -compiles through `OpenAPISource` into deterministic IR and generated routers, types, and -error shapes. The storage engine (`internal/services/blob`) provides local filesystem-backed -persistence supporting container CRUD, BlockBlob CRUD, staged chunk uploads (`PutBlock`, -`PutBlockList`), conditional requests (`If-None-Match: *`, `If-Match`), custom metadata -(`x-ms-meta-*`), and byte-range downloads. The gateway router and auth adapters validate -Azure Shared Key and emulator signatures, and integration tests verify complete end-to-end -compatibility against `azure-sdk-for-python` in CI. See [Azure Services](services/azure.md) -for configuration and local limits. - -## Phase 4 — Breadth expansion (complete, v2.1) - -- [x] More Azure services — Queue Storage, Table Storage, Cosmos DB -- [x] Google Cloud pilot — candidate: **Google Cloud Storage** -- [x] Federated identity playground (cross-CSP IAM simulation) -- [ ] Other providers as community interest justifies - -Phase 4 execution completed: -- **Google Cloud Pilot (GCS)**: First-class GCP provider (`providers.gcp.services.gcs`) with OAuth2 Bearer auth adapter, OpenAPI-backed REST-JSON router, complete bucket/object CRUD, multipart and media uploads, Alt=media downloads, Castagnoli CRC32c and MD5 checksums, and Python integration tests. -- **Azure Expansion**: Added Azure Queue Storage (`azurequeue`) with FIFO queues, message visibility timeouts, pop receipts, and XML payloads; Azure Table Storage (`azuretable`) with SQLite-backed OData JSON table and entity indexing; and Azure Cosmos DB (`cosmosdb`) NoSQL/SQL REST API with SQLite document storage, collections, and metadata headers. Multi-service Azure routing runs seamlessly on unified port 4747. -- **Federated Identity Playground**: Built-in local OIDC issuer providing discovery (`/devcloud/identity/.well-known/openid-configuration`), public JWKS (`/devcloud/identity/jwks.json`), RS256 token minting (`/devcloud/identity/token`), cross-CSP token exchanges for Azure AD and Google Cloud STS, and integration with AWS STS `AssumeRoleWithWebIdentity`. - -## Out of scope - -- Production hosting or high-availability guarantees -- Billing and quota simulation matching real CSP pricing -- Exact replication of CSP-internal behaviour (consistency timing, rate limits) -- Redistribution of CSP-owned branding assets, logos or documentation - -## Influencing the roadmap - -- **Missing service?** File a [service request](https://github.com/skyoo2003/devcloud/issues/new?template=service_request.yml) with `GET /devcloud/api/unrouted` output — that is what moves a service into the target ([coverage.md](coverage.md#service-not-supported)). -- **Missing operation or capability?** File a [feature request](https://github.com/skyoo2003/devcloud/issues/new?template=feature_request.yml). -- Upvote existing requests with reactions; vote counts inform prioritization. -- Or contribute it — see [contributing.md](contributing.md). - -## Version mapping +DevCloud serves as a local development companion for cloud-native applications across major cloud providers: an on-ramp to iterate without cloud bills and deploy to production cloud providers with confidence. + +## Guiding Principles + +1. **Local-first, cost-free**: zero cloud charges for inner-loop development. +2. **On-ramp, not replacement**: DevCloud helps teams land cleanly on target cloud providers, not avoid them. +3. **API-compatible, not behavior-perfect**: SDK and CLI compatibility prioritized over obscure provider-internal edge cases. +4. **Extensible architecture**: clean plugin architecture and code generation pipelines for broad community contributions. +5. **Trademark-respectful**: see [TRADEMARKS.md](../TRADEMARKS.md). + +## Phase 1: AWS Depth and Stabilization (Complete, shipped as v1.0) + +Matured the core AWS surface into a stable v1.0 release: + +- [x] AWS services scaffolded from official Smithy models via in-tree codegen. +- [x] Deep hand-written coverage on core and integration services. +- [x] Cross-service integrations: CloudFormation, DynamoDB Streams to Lambda, SQS to Lambda, S3 to Lambda, EventBridge, SNS to SQS. +- [x] Automated compatibility test suites validated in CI. +- [x] Clean error handling: unimplemented operations decline with authentic cloud error formats. +- [x] Stable `ServicePlugin` API ([plugin-api.md](plugin-api.md)). +- [x] Generic [CRUD fallback engine](crud-engine.md) serving the long tail with plausible, store-backed responses. + +## Phase 2: Architectural Preparation (Complete, v1.x) + +Refactored internally so adding new cloud providers is an addition rather than a rewrite: + +- [x] Intermediate Representation (IR) decoupling models from generators ([`internal/codegen/ir`](../internal/codegen/ir/ir.go)). +- [x] Extensible `ModelSource` interface ([`internal/codegen/source.go`](../internal/codegen/source.go)). +- [x] Namespaced provider configuration: `providers..services.*`. +- [x] Provider-scoped plugin contracts via `ProviderScoped` interface. +- [x] Pluggable authentication adapters ([`internal/auth`](../internal/auth/auth.go)). + +## Phase 3: First Non-AWS Service (Complete, v2.0 groundwork) + +Validated multi-cloud architecture with an initial Azure pilot: + +- [x] Azure Blob Storage (`blob`) pilot implemented with container and block blob lifecycle. +- [x] OpenAPI to IR to codegen pipeline established. +- [x] Azure Shared Key credential adapter and emulator signature support. +- [x] End-to-end integration tests with official Azure SDK for Python. + +## Phase 4: Multi-CSP Breadth and Workload Runtime (Complete, v2.1) + +Expanded multi-cloud services, unified routing, and universal execution: + +- [x] **Unified Single Port (4747) Routing**: Eliminated separate ports (`4566`, `4577`, `4588`) by routing AWS, Azure, and GCP transparently through port 4747. +- [x] **Azure Core Expansion**: Added Queue Storage (`azurequeue`), Table Storage (`azuretable`), Cosmos DB (`cosmosdb`), Key Vault (`azurekeyvault`), and App Configuration (`azureappconfig`). +- [x] **Google Cloud Platform Services**: Added Cloud Storage (`gcs`), Secret Manager (`gcpsecrets`), and Cloud Pub/Sub (`gcppubsub`) with complete lifecycle operations. +- [x] **Universal Workload Runtime**: Implemented `internal/shared/workload` supporting OCI container engines (Docker, Podman, nerdctl), WebAssembly, and local process sandboxing (Zero-Daemon). +- [x] **Federated Identity Playground**: Built-in OIDC discovery issuer, JWKS endpoint, and cross-cloud STS token exchange. +- [x] **100% Client Compatibility Verification**: Automated smoke tests across all registered Azure and GCP services on port 4747. + +## Out of Scope + +- Production multi-tenant hosting or high-availability clustering +- Real billing, metering, or cloud quota enforcement +- Exact replication of proprietary cloud backend internals +- Redistribution of provider-owned branding assets, logos, or documentation + +## Feedback and Contributions + +- **Feature or service requests**: open an issue or pull request on GitHub. +- **Contributions**: see [contributing.md](contributing.md). + +## Version Mapping | Version | Focus | |---------|-------| -| 0.x | AWS services, unstable API | +| 0.x | Initial AWS services, exploratory API | | 1.x | AWS depth, stable plugin API, multi-CSP groundwork | -| 2.x ← current | Multi-CSP architecture, Azure pilot | -| 3.x+ | Broad CSP coverage, community-owned providers | +| 2.x (Current) | Multi-cloud architecture (AWS, Azure, GCP), universal workload runtime, unified port 4747 | +| 3.x+ | Extended community providers, distributed orchestration | diff --git a/docs/services-matrix.md b/docs/services-matrix.md index dc2b94ac..364f2f7a 100644 --- a/docs/services-matrix.md +++ b/docs/services-matrix.md @@ -1,16 +1,12 @@ # Services Matrix -Which AWS services DevCloud serves, and how deeply. +Which cloud services DevCloud serves, and how deeply. -**Counts live in [coverage.md](coverage.md)** — it is the only page that publishes -them, and CI fails if its figures and the binary disagree. This page describes the -*shape* of the surface instead. +**Counts live in [coverage.md](coverage.md)**: it is the primary page that publishes official counts, and CI validates that its figures match the binary. This page describes the architectural shape of supported services. -## How to look up one service +## Looking Up a Service -Per-operation depth is declared by the generated -[fidelity manifest](fidelity-manifest.md), not by a hand-maintained table. Ask it -directly (requires `admin.enabled: true`): +Per-operation depth is declared by the generated [fidelity manifest](fidelity-manifest.md). You can query it directly using the Admin API (requires `admin.enabled: true`): ```bash curl -s 'localhost:4747/devcloud/api/fidelity?service=s3' @@ -18,58 +14,72 @@ curl -s 'localhost:4747/devcloud/api/fidelity?service=s3' | Tier | What it means | |------|---------------| -| `hand-verified` | The service's provider implements the operation explicitly. | -| `auto-crud` | Served by the [CRUD engine](crud-engine.md) — store-backed and plausible, not faithful. | -| `unimplemented` | Refused with an AWS-shaped error. Never a fabricated success. | +| `hand-verified` | The service provider explicitly implements and dispatches the operation. | +| `auto-crud` | Served by the generic [CRUD engine](crud-engine.md): store-backed and plausible responses. | +| `unimplemented` | Refused with an authentic cloud error format. Never returns a fabricated success. | -## Depth by group +## Depth by Group +### AWS | Group | Services | Depth | |-------|----------|-------| -| Core | S3, SQS, DynamoDB, Lambda, IAM, STS | Hand-written throughout; the deepest boto3 coverage | -| Integration | SNS, EventBridge, CloudWatch, CW Logs, KMS, Secrets Manager, SSM, ECR, CloudFormation | Hand-written common operations, plus the cross-service wiring below | -| Extended | EC2, ECS, EKS, Route53, ACM, RDS, Kinesis, Firehose, SFN, Bedrock, and the rest of the registered set | Common operations hand-written; the long tail is engine-served or declines cleanly | +| Core | S3, SQS, DynamoDB, Lambda, IAM, STS | Hand-written throughout; deepest SDK coverage | +| Integration | SNS, EventBridge, CloudWatch, CloudWatch Logs, KMS, Secrets Manager, SSM, ECR, CloudFormation | Hand-written common operations plus cross-service event routing | +| Extended | EC2, ECS, EKS, Route53, ACM, RDS, Kinesis, Firehose, Step Functions, Bedrock, and long-tail services | Common operations hand-written; long tail is engine-served or declines cleanly | -The `tier1` / `tier2` / `tier3` tokens accepted by `DEVCLOUD_SERVICES` are a -*startup* grouping, not a depth claim — see -[configuration.md](configuration.md#devcloud_services) for their exact contents. +The `tier1` / `tier2` / `tier3` tokens accepted by `DEVCLOUD_SERVICES` represent startup convenience sets, not depth tiers. See [configuration.md](configuration.md#devcloud_services) for details. -## Cross-service integrations +### Azure +| Service | Identifier | Protocol | Depth | +|---------|------------|----------|-------| +| Blob Storage | `blob` | REST-XML | Hand-written container and block blob lifecycle, chunked blocks, conditional requests | +| Queue Storage | `azurequeue` | REST-XML | Hand-written queue management, FIFO messages, visibility timeouts, pop receipts | +| Table Storage | `azuretable` | REST-JSON (OData) | SQLite-backed OData table and entity operations, composite key lookups | +| Cosmos DB | `cosmosdb` | REST-JSON | SQLite-backed NoSQL document store, collections, partition keys, system attributes | +| Key Vault | `azurekeyvault` | REST-JSON | Hand-written secret versioning, payload retrieval, RSA software-backed keys | +| App Configuration | `azureappconfig` | REST-JSON | Hand-written key-values, ETag tracking, tag management, label filtering | -These are wired end to end, not stubbed: +### GCP +| Service | Identifier | Protocol | Depth | +|---------|------------|----------|-------| +| Cloud Storage | `gcs` | REST-JSON | Hand-written bucket/object CRUD, media/resumable uploads, downloads, CRC32c checksums | +| Secret Manager | `gcpsecrets` | REST-JSON | Hand-written secret lifecycle, version management, payload access, and destruction | +| Cloud Pub/Sub | `gcppubsub` | REST-JSON | Hand-written topic/subscription lifecycle, batch publishing, message pull/ack delivery | + +## Cross-Service Integrations + +These integrations execute real workflows locally: | Integration | Implementation | |-------------|----------------| -| CloudFormation → 10 resource types | `cloudformation/engine.go` — topological sort, intrinsic functions | -| DynamoDB → DynamoDB Streams | Write path publishes records | -| DynamoDB Streams → Lambda | `lambda/eventsource.go` polls stream shards | -| SQS → Lambda | Event source poller | -| S3 → Lambda | `s3/notifications.go` on PUT events | -| EventBridge → SQS / SNS / Lambda | Rule matching + `dispatchToTarget` | -| SNS → SQS | Topic publish triggers queue delivery | +| CloudFormation -> Resource Types | `cloudformation/engine.go`: topological sort and intrinsic functions | +| DynamoDB -> DynamoDB Streams | Write paths publish stream change records | +| DynamoDB Streams -> Lambda | `lambda/eventsource.go` polls stream shards | +| SQS -> Lambda | Event source polling worker | +| S3 -> Lambda | `s3/notifications.go` on PUT events | +| EventBridge -> SQS / SNS / Lambda | Rule matching and target dispatch | +| SNS -> SQS | Topic publish triggers queue delivery | ## Protocols | Protocol | Content type | Example services | |----------|--------------|------------------| | JSON 1.0 | `application/x-amz-json-1.0` | DynamoDB, DynamoDB Streams, Kinesis | -| JSON 1.1 | `application/x-amz-json-1.1` | ECS, Lambda, Batch, CW Logs, SFN | -| REST-JSON | `application/json` | ACM, API Gateway, S3Tables, MWAA, IdentityStore | -| REST-XML | `application/xml` | S3, Route53, CloudFront | +| JSON 1.1 | `application/x-amz-json-1.1` | ECS, Lambda, Batch, CloudWatch Logs, Step Functions | +| REST-JSON | `application/json` | ACM, API Gateway, GCS, GCP Secrets, Azure Key Vault, App Config | +| REST-XML | `application/xml` | S3, Route53, CloudFront, Azure Blob, Azure Queue | | Query | `application/x-www-form-urlencoded` | IAM, STS, SQS, SNS, RDS, EC2, AutoScaling | | EC2 Query | `application/x-www-form-urlencoded` | EC2 (EC2-compliant XML with ``) | | RPC-v2 CBOR | `application/cbor` | Partner Central Revenue Measurement | -SQS speaks both Query and JSON; the protocol is detected per request. +SQS supports both Query and JSON protocols, auto-detected per request. -## Verifying +## Verification ```bash make test # Go unit tests -make test-compat # boto3 compatibility suite -make stats # registered services and hand-written operations +make test-compat # Multi-cloud compatibility test suites +make stats # Registered services and hand-written operations inventory ``` -The compatibility suite runs in CI on every push — a failing test fails the -build. What it does and does not promise is -[compatibility-policy.md](compatibility-policy.md#wire-behaviour--scoped-to-the-compatibility-suite). +The compatibility test suite runs in CI on every commit. A failing test fails the build. diff --git a/docs/services/azure.md b/docs/services/azure.md index 039ed707..27f1a026 100644 --- a/docs/services/azure.md +++ b/docs/services/azure.md @@ -1,15 +1,17 @@ # Azure Services -DevCloud expands its local development runtime to include Microsoft Azure services, providing high-fidelity local simulation for storage, messaging, and database APIs. +DevCloud provides local emulation for Microsoft Azure services on port 4747, covering storage, messaging, database, secret management, and configuration APIs. ## Supported Services -| Service | Supported | Local limits | +| Service | Identifier | Local limits and backend | |---|---|---| -| Blob Storage (`blob`) | Supported (Phase 3) | Local filesystem-backed FileStore under `data/blob`. Supports container CRUD, BlockBlob CRUD, staged chunk uploads (`PutBlock`, `PutBlockList`), listing/enumeration, and byte-range downloads. | -| Queue Storage (`azurequeue`) | Supported (Phase 4) | Memory + SQLite/file-backed FIFO message store under `data/azurequeue`. Supports queue lifecycle, `PutMessage`, `GetMessages` with visibility timeouts, `PeekMessages`, `DeleteMessage` with pop receipts, and metadata. | -| Table Storage (`azuretable`) | Supported (Phase 4) | SQLite-backed OData JSON table and entity engine under `data/azuretable`. Supports table CRUD, entity insert/replace/merge/delete, PartitionKey/RowKey indexing, and queries. | -| Cosmos DB (`cosmosdb`) | Supported (Phase 4) | SQLite-backed NoSQL document store under `data/cosmos`. Supports database CRUD, container/collection CRUD with partition key definitions, document CRUD, and system attributes (`_rid`, `_etag`, `_ts`). | +| Blob Storage | `blob` | Filesystem-backed FileStore under `data/blob`. Supports container CRUD, BlockBlob CRUD, staged chunk uploads (`PutBlock`, `PutBlockList`), listing, and byte-range downloads. | +| Queue Storage | `azurequeue` | In-memory message store with optional file persistence under `data/azurequeue`. Supports queue lifecycle, `PutMessage`, `GetMessages` with visibility timeouts, `PeekMessages`, and `DeleteMessage` with pop receipts. | +| Table Storage | `azuretable` | SQLite-backed OData JSON table and entity engine under `data/azuretable`. Supports table CRUD, entity insert/replace/merge/delete, PartitionKey/RowKey indexing, and queries. | +| Cosmos DB | `cosmosdb` | SQLite-backed NoSQL document store under `data/cosmos`. Supports database CRUD, collection CRUD with partition keys, document CRUD, and system attributes (`_rid`, `_etag`, `_ts`). | +| Key Vault | `azurekeyvault` | In-memory secret and cryptographic key store. Supports secret creation, versioning, retrieval, soft deletion, and RSA software-backed key generation and management. | +| App Configuration | `azureappconfig` | In-memory key-value configuration store with ETag tracking, user-defined tags, and label filtering (`?label=...`). | ## Operations @@ -21,12 +23,12 @@ DevCloud expands its local development runtime to include Microsoft Azure servic | Containers | `DeleteContainer` | `DELETE /{container}?restype=container` | Returns `202 Accepted` or `404 ContainerNotFound` | | Containers | `GetContainerProperties` | `GET /{container}?restype=container` | Returns lease state and metadata headers | | Containers | `ListContainers` | `GET /?comp=list` | Returns XML `EnumerationResults` listing all containers | -| Blobs | `PutBlob` | `PUT /{container}/{blob}` | Creates or replaces a BlockBlob; supports preconditions (`If-None-Match: *`, `If-Match`), stores payload and computes MD5 | -| Blobs | `PutBlock` | `PUT /{container}/{blob}?comp=block&blockid={id}` | Stages an uncommitted block for chunked/streaming blob uploads | -| Blobs | `PutBlockList` | `PUT /{container}/{blob}?comp=blocklist` | Commits staged blocks in sequence to materialize or update a BlockBlob | -| Blobs | `SetBlobMetadata` | `PUT /{container}/{blob}?comp=metadata` | Updates user-defined `x-ms-meta-*` headers on the blob | +| Blobs | `PutBlob` | `PUT /{container}/{blob}` | Creates or replaces a BlockBlob; supports preconditions (`If-None-Match`, `If-Match`) and computes MD5 | +| Blobs | `PutBlock` | `PUT /{container}/{blob}?comp=block&blockid={id}` | Stages an uncommitted block for chunked blob uploads | +| Blobs | `PutBlockList` | `PUT /{container}/{blob}?comp=blocklist` | Commits staged blocks in sequence to materialize a BlockBlob | +| Blobs | `SetBlobMetadata` | `PUT /{container}/{blob}?comp=metadata` | Updates user-defined `x-ms-meta-*` headers | | Blobs | `GetBlob` | `GET /{container}/{blob}` | Downloads blob content; supports `Range` and `x-ms-range` requests | -| Blobs | `GetBlobProperties` | `HEAD /{container}/{blob}` | Returns headers (`x-ms-blob-type: BlockBlob`, size, ETag, Content-MD5, `x-ms-meta-*`) | +| Blobs | `GetBlobProperties` | `HEAD /{container}/{blob}` | Returns headers (`x-ms-blob-type: BlockBlob`, size, ETag, Content-MD5) | | Blobs | `DeleteBlob` | `DELETE /{container}/{blob}` | Permanently removes the blob | | Blobs | `ListBlobs` | `GET /{container}?restype=container&comp=list` | Returns XML `EnumerationResults` listing blobs in the container | @@ -37,9 +39,9 @@ DevCloud expands its local development runtime to include Microsoft Azure servic | Queues | `CreateQueue` | `PUT /{account}/{queue}` | Returns `201 Created` or `204 No Content` | | Queues | `DeleteQueue` | `DELETE /{account}/{queue}` | Returns `204 No Content` or `404 QueueNotFound` | | Queues | `ListQueues` | `GET /{account}?comp=list` | Returns XML `QueueList` | -| Queues | `GetQueueMetadata` | `GET /{account}/{queue}?comp=metadata` | Returns `x-ms-approximate-messages-count` and custom metadata | -| Messages | `PutMessage` | `POST /{account}/{queue}/messages` | Enqueues XML payload; returns message ID, insertion time, expiry | -| Messages | `GetMessages` | `GET /{account}/{queue}/messages` | Dequeues messages, calculates visibility timeout, generates pop receipt | +| Queues | `GetQueueMetadata` | `GET /{account}/{queue}?comp=metadata` | Returns message count and custom metadata | +| Messages | `PutMessage` | `POST /{account}/{queue}/messages` | Enqueues XML payload; returns message ID and insertion time | +| Messages | `GetMessages` | `GET /{account}/{queue}/messages` | Dequeues messages, sets visibility timeout, and returns pop receipt | | Messages | `PeekMessages` | `GET /{account}/{queue}/messages?peekonly=true` | Reads front of queue without altering visibility timeout | | Messages | `DeleteMessage` | `DELETE /{account}/{queue}/messages/{id}?popreceipt={rcpt}` | Deletes message matching pop receipt | | Messages | `ClearMessages` | `DELETE /{account}/{queue}/messages` | Purges all messages from the queue | @@ -76,9 +78,32 @@ DevCloud expands its local development runtime to include Microsoft Azure servic | Documents | `DeleteDocument` | `DELETE /dbs/{db}/colls/{coll}/docs/{doc}` | Deletes document | | Documents | `ListDocuments` | `GET /dbs/{db}/colls/{coll}/docs` | Lists documents in container | +### Key Vault (`azurekeyvault`) + +| Area | Operation | Method & URI | Notes | +|---|---|---|---| +| Secrets | `SetSecret` | `PUT /secrets/{name}?api-version=7.4` | Creates a new secret version with tags and contentType | +| Secrets | `GetSecret` | `GET /secrets/{name}[/{version}]?api-version=7.4` | Retrieves latest or specified secret version | +| Secrets | `GetSecretVersions` | `GET /secrets/{name}/versions?api-version=7.4` | Lists all versions for a secret | +| Secrets | `ListSecrets` | `GET /secrets?api-version=7.4` | Lists all secrets in the vault | +| Secrets | `DeleteSecret` | `DELETE /secrets/{name}?api-version=7.4` | Deletes secret bundle from the vault | +| Keys | `CreateKey` | `POST /keys/{name}/create?api-version=7.4` | Generates software-backed RSA cryptographic key | +| Keys | `GetKey` | `GET /keys/{name}[/{version}]?api-version=7.4` | Retrieves public key attributes and operations | +| Keys | `ListKeys` | `GET /keys?api-version=7.4` | Lists cryptographic keys in the vault | +| Keys | `DeleteKey` | `DELETE /keys/{name}?api-version=7.4` | Removes key bundle from the vault | + +### App Configuration (`azureappconfig`) + +| Area | Operation | Method & URI | Notes | +|---|---|---|---| +| Key-Values | `SetKeyValue` | `PUT /kv/{key}?label={label}` | Sets or updates a key-value with ETag, content type, and tags | +| Key-Values | `GetKeyValue` | `GET /kv/{key}?label={label}` | Retrieves key-value entity by key and optional label | +| Key-Values | `DeleteKeyValue` | `DELETE /kv/{key}?label={label}` | Removes key-value entity from configuration store | +| Key-Values | `ListKeyValues` | `GET /kv?key={filter}&label={filter}` | Lists key-values with key and label filtering support | + ## Configuration -In `devcloud.yaml`, Azure services are configured under the namespaced `providers.azure` block: +In `devcloud.yaml`, Azure services are configured under `providers.azure`: ```yaml server: @@ -99,85 +124,58 @@ providers: cosmosdb: enabled: true data_dir: ./data/cosmos + azurekeyvault: + enabled: true + azureappconfig: + enabled: true ``` -## Credentials & Connection Strings - -DevCloud accepts Azure Shared Key, SharedKeyLite, and Azurite emulator credentials: - -```bash -# Unified Storage Connection String (Blob, Queue, Table) -export AZURE_STORAGE_CONNECTION_STRING="DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:4747/devstoreaccount1;QueueEndpoint=http://127.0.0.1:4747/devstoreaccount1;TableEndpoint=http://127.0.0.1:4747/devstoreaccount1;" - -# Cosmos DB Endpoint & Key -export AZURE_COSMOS_ENDPOINT="http://127.0.0.1:4747" -export AZURE_COSMOS_KEY="C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==" -``` - -## Python SDK Quickstarts - -### Queue Storage +## Python SDK Examples -```python -import requests - -endpoint = "http://127.0.0.1:4747/devstoreaccount1" -headers = {"x-ms-version": "2023-11-03"} - -# 1. Create Queue -requests.put(f"{endpoint}/tasks", headers=headers) - -# 2. Enqueue message -msg_xml = "Process job #42" -requests.post(f"{endpoint}/tasks/messages", headers=headers, data=msg_xml) - -# 3. Dequeue message -resp = requests.get(f"{endpoint}/tasks/messages", headers=headers) -print(resp.text) -``` - -### Table Storage +### Key Vault (`azure-keyvault-secrets`) ```python +import os import requests -endpoint = "http://127.0.0.1:4747/devstoreaccount1" -headers = { - "Accept": "application/json;odata=nometadata", - "Content-Type": "application/json", -} - -# 1. Create table -requests.post(f"{endpoint}/Tables", headers=headers, json={"TableName": "Customers"}) - -# 2. Insert entity -entity = {"PartitionKey": "Seattle", "RowKey": "cust-01", "Name": "Alice"} -requests.post(f"{endpoint}/Customers", headers=headers, json=entity) - -# 3. Query entity -resp = requests.get(f"{endpoint}/Customers(PartitionKey='Seattle',RowKey='cust-01')", headers=headers) -print(resp.json()) +endpoint = "http://localhost:4747" +headers = {"Host": "myvault.vault.azure.net", "Content-Type": "application/json"} + +# Set a secret +resp = requests.put( + f"{endpoint}/secrets/DatabasePassword?api-version=7.4", + json={"value": "SuperSecretPass123", "contentType": "text/plain"}, + headers=headers, +) +print("Secret created:", resp.json()["id"]) + +# Read the secret back +resp = requests.get( + f"{endpoint}/secrets/DatabasePassword?api-version=7.4", + headers=headers, +) +print("Secret value:", resp.json()["value"]) ``` -### Cosmos DB +### App Configuration (`azure-appconfiguration`) ```python import requests -endpoint = "http://127.0.0.1:4747/devstoreaccount1" -headers = {"Content-Type": "application/json", "x-ms-version": "2018-12-31"} - -# 1. Create Database -requests.post(f"{endpoint}/dbs", headers=headers, json={"id": "analytics"}) - -# 2. Create Collection -requests.post(f"{endpoint}/dbs/analytics/colls", headers=headers, json={"id": "events", "partitionKey": {"paths": ["/tenantId"]}}) - -# 3. Create Document -doc = {"id": "evt-100", "tenantId": "t-1", "action": "login", "status": "ok"} -requests.post(f"{endpoint}/dbs/analytics/colls/events/docs", headers=headers, json=doc) - -# 4. Read Document -resp = requests.get(f"{endpoint}/dbs/analytics/colls/events/docs/evt-100", headers=headers) -print(resp.json()) +endpoint = "http://localhost:4747" +headers = {"Host": "myconfig.azconfig.io", "Content-Type": "application/json"} + +# Store a configuration setting +requests.put( + f"{endpoint}/kv/App:FeatureFlags:BetaUI?label=production", + json={"value": "true", "content_type": "text/plain"}, + headers=headers, +) + +# Fetch configuration setting +resp = requests.get( + f"{endpoint}/kv/App:FeatureFlags:BetaUI?label=production", + headers=headers, +) +print("Feature flag state:", resp.json()["value"]) ``` diff --git a/docs/services/dynamodb.md b/docs/services/dynamodb.md index 8f602e8f..da4c9a80 100644 --- a/docs/services/dynamodb.md +++ b/docs/services/dynamodb.md @@ -11,7 +11,7 @@ M (Map), and the SS / NS / BS sets. ## Supported APIs -These 29 operations are `hand-verified` — implemented by the provider, not by +These 29 operations are `hand-verified`, implemented by the provider, not by the [CRUD engine](../crud-engine.md). Everything else DynamoDB models is served at a lower tier or not at all; [fidelity-manifest.md](../fidelity-manifest.md) is the per-operation answer. @@ -32,7 +32,7 @@ is the per-operation answer. | TagResource / UntagResource / ListTagsOfResource | Manage table tags | Writes to a table created with `StreamEnabled` are published to the -`dynamodbstreams` service, which is what makes DynamoDB Streams → Lambda event +`dynamodbstreams` service, which is what makes DynamoDB Streams -> Lambda event source mappings fire. ## boto3 Examples @@ -131,7 +131,7 @@ aws --endpoint-url http://localhost:4747 dynamodb get-item \ `LastEvaluatedKey` and no `ConsumedCapacity`, so `Query`/`Scan` return the whole matching set in one page and code that loops on the cursor sees one iteration. -- **PartiQL subset only** — `ExecuteStatement`, `ExecuteTransaction` and `BatchExecuteStatement` support single-table `SELECT`, `INSERT`, `UPDATE`, and `DELETE`. `WHERE` clauses are limited to simple equality (`=`). No nested paths, subqueries, or arithmetic. +- **PartiQL subset only**: `ExecuteStatement`, `ExecuteTransaction` and `BatchExecuteStatement` support single-table `SELECT`, `INSERT`, `UPDATE`, and `DELETE`. `WHERE` clauses are limited to simple equality (`=`). No nested paths, subqueries, or arithmetic. - **Backups and global tables answer from the CRUD engine.** `CreateBackup`, `CreateGlobalTable`, `DescribeContinuousBackups` and their neighbours return stored, plausible shapes with no behaviour behind them. However, `ExportTableToPointInTime`, `ImportTable`, `RestoreTableFromBackup`, and `RestoreTableToPointInTime` are fully handled by the provider (cloning tables and items in SQLite). - **No Kinesis active replication.** `EnableKinesisStreamingDestination` and `DisableKinesisStreamingDestination` persist configuration, but write events are not actively replicated to Kinesis streams. - No provisioned-throughput accounting or throttling; `BillingMode` is recorded, diff --git a/docs/services/eventbridge.md b/docs/services/eventbridge.md index de53b7b3..f8bbcd5b 100644 --- a/docs/services/eventbridge.md +++ b/docs/services/eventbridge.md @@ -8,7 +8,7 @@ account is `000000000000`. ## Supported APIs -These 45 operations are `hand-verified` — implemented by the provider, not by the CRUD engine. +These 45 operations are `hand-verified`, implemented by the provider, not by the CRUD engine. | Operation | Description | |-----------|-------------| @@ -31,7 +31,7 @@ These 45 operations are `hand-verified` — implemented by the provider, not by `aws.partner//`, for example `aws.partner/devcloud/orders`. The Account parameter must name the local account. `DescribePartnerEventSource`, `DescribeEventSource`, and the source/account list APIs read the same stored row. -Source timestamps are SDK datetimes. Lists support prefixes, Limit 1–100 and +Source timestamps are SDK datetimes. Lists support prefixes, Limit 1-100 and NextToken; a token belongs to its original list operation and filters. Create a bus with matching Name and EventSourceName to bind the source and make @@ -39,7 +39,7 @@ it ACTIVE. `DeactivateEventSource` changes it to PENDING; `ActivateEventSource` restores ACTIVE on a bound source. Repeated transitions succeed. Rules and targets remain stored through either transition. -`PutPartnerEvents` accepts 1–20 entries. Each entry requires Source, DetailType and +`PutPartnerEvents` accepts 1-20 entries. Each entry requires Source, DetailType and a JSON Detail string. An ACTIVE bound source sends matching events through the existing local target delivery path. PENDING or missing sources fail individually; the response keeps the original entry order and reports FailedEntryCount. EventId diff --git a/docs/services/gcp.md b/docs/services/gcp.md index 73c7ba36..38d8d5da 100644 --- a/docs/services/gcp.md +++ b/docs/services/gcp.md @@ -1,30 +1,62 @@ # Google Cloud Platform (GCP) Services -DevCloud extends its multi-CSP runtime to support Google Cloud services, starting with Google Cloud Storage (GCS) as the first-class GCP pilot. +DevCloud provides local emulation for Google Cloud Platform services on port 4747, covering object storage, secret management, and messaging APIs. ## Supported Services -| Service | Supported | Local limits | +| Service | Identifier | Local limits and backend | |---|---|---| -| Google Cloud Storage (`gcs`) | Supported (Phase 4 pilot) | Local filesystem-backed FileStore under `data/gcs`. Supports bucket CRUD, object CRUD, simple/media multipart uploads, `alt=media` downloads, metadata inspection, and CRC32c / MD5 checksums. | +| Cloud Storage | `gcs` | Local filesystem-backed FileStore under `data/gcs`. Supports bucket CRUD, object CRUD, multipart uploads, media downloads, and CRC32c / MD5 checksums. | +| Secret Manager | `gcpsecrets` | In-memory secret store. Supports secret creation, listing, deletion, payload versioning, access (`:access`), and payload destruction (`:destroy`). | +| Cloud Pub/Sub | `gcppubsub` | In-memory messaging engine. Supports topic CRUD, message publishing (`:publish`), subscription CRUD, message pull (`:pull`), and message acknowledgement (`:acknowledge`). | ## Operations +### Cloud Storage (`gcs`) + | Area | Operation | Method & URI | Notes | |---|---|---|---| | Buckets | `ListBuckets` | `GET /storage/v1/b?project={project}` | Returns JSON list of buckets in the project | | Buckets | `InsertBucket` | `POST /storage/v1/b` | Creates a new bucket (`201 Created` or `409 Conflict`) | -| Buckets | `GetBucket` | `GET /storage/v1/b/{bucket}` | Returns bucket metadata and timestamp | +| Buckets | `GetBucket` | `GET /storage/v1/b/{bucket}` | Returns bucket metadata and timestamps | | Buckets | `DeleteBucket` | `DELETE /storage/v1/b/{bucket}` | Deletes bucket (`204 No Content` or `404 Not Found`) | | Objects | `ListObjects` | `GET /storage/v1/b/{bucket}/o` | Lists objects within a bucket | | Objects | `InsertObject` | `POST /upload/storage/v1/b/{bucket}/o?uploadType=media&name={name}` | Uploads raw binary object payload | -| Objects | `GetObject` | `GET /storage/v1/b/{bucket}/o/{object}?alt=media` | Downloads object binary content with `x-goog-hash` checksums | +| Objects | `GetObject` | `GET /storage/v1/b/{bucket}/o/{object}?alt=media` | Downloads object binary content with checksums | | Objects | `GetObjectMetadata` | `GET /storage/v1/b/{bucket}/o/{object}` | Returns object JSON metadata (size, contentType, md5Hash, crc32c) | | Objects | `DeleteObject` | `DELETE /storage/v1/b/{bucket}/o/{object}` | Removes object (`204 No Content` or `404 Not Found`) | +### Secret Manager (`gcpsecrets`) + +| Area | Operation | Method & URI | Notes | +|---|---|---|---| +| Secrets | `CreateSecret` | `POST /v1/projects/{project}/secrets?secretId={id}` | Creates secret metadata resource | +| Secrets | `GetSecret` | `GET /v1/projects/{project}/secrets/{secret}` | Retrieves secret resource metadata | +| Secrets | `ListSecrets` | `GET /v1/projects/{project}/secrets` | Lists secrets within a project | +| Secrets | `DeleteSecret` | `DELETE /v1/projects/{project}/secrets/{secret}` | Deletes secret and all its payload versions | +| Versions | `AddSecretVersion` | `POST /v1/projects/{project}/secrets/{secret}:addVersion` | Adds a new base64 payload version | +| Versions | `AccessSecretVersion` | `GET /v1/projects/{project}/secrets/{secret}/versions/{version}:access` | Accesses secret payload (supports version number or `latest`) | +| Versions | `DestroySecretVersion` | `POST /v1/projects/{project}/secrets/{secret}/versions/{version}:destroy` | Destroys version payload data | + +### Cloud Pub/Sub (`gcppubsub`) + +| Area | Operation | Method & URI | Notes | +|---|---|---|---| +| Topics | `CreateTopic` | `PUT /v1/projects/{project}/topics/{topic}` | Creates a Pub/Sub topic | +| Topics | `GetTopic` | `GET /v1/projects/{project}/topics/{topic}` | Retrieves topic resource details | +| Topics | `ListTopics` | `GET /v1/projects/{project}/topics` | Lists all topics in the project | +| Topics | `DeleteTopic` | `DELETE /v1/projects/{project}/topics/{topic}` | Deletes topic and detached associations | +| Topics | `Publish` | `POST /v1/projects/{project}/topics/{topic}:publish` | Publishes base64 message batch to subscribers | +| Subscriptions | `CreateSubscription` | `PUT /v1/projects/{project}/subscriptions/{sub}` | Creates subscription bound to a topic | +| Subscriptions | `GetSubscription` | `GET /v1/projects/{project}/subscriptions/{sub}` | Retrieves subscription settings | +| Subscriptions | `ListSubscriptions` | `GET /v1/projects/{project}/subscriptions` | Lists subscriptions in the project | +| Subscriptions | `DeleteSubscription` | `DELETE /v1/projects/{project}/subscriptions/{sub}` | Deletes subscription | +| Subscriptions | `Pull` | `POST /v1/projects/{project}/subscriptions/{sub}:pull` | Pulls delivered messages with ack IDs | +| Subscriptions | `Acknowledge` | `POST /v1/projects/{project}/subscriptions/{sub}:acknowledge` | Confirms message processing by ack IDs | + ## Configuration -In `devcloud.yaml`, GCP services are configured under the namespaced `providers.gcp` block: +In `devcloud.yaml`, GCP services are configured under `providers.gcp`: ```yaml server: @@ -36,67 +68,104 @@ providers: gcs: enabled: true data_dir: ./data/gcp/gcs + gcpsecrets: + enabled: true + gcppubsub: + enabled: true ``` -## Environment Variables & Credentials +## Credentials & Endpoint Setup -DevCloud accepts Google OAuth2 Bearer tokens (`Bearer ya29...`) and mock service account credentials without rejecting on signature validation. +DevCloud accepts Google OAuth2 Bearer tokens (`Bearer ya29...`) or placeholder credentials without rejecting calls on signature verification. -To point the official Google Cloud Python SDK to DevCloud: +To direct the Google Cloud Python SDK to DevCloud: ```bash export STORAGE_EMULATOR_HOST="http://127.0.0.1:4747" export GOOGLE_APPLICATION_CREDENTIALS="/dev/null" ``` -## Python SDK Quickstart +## Python SDK Examples -### Using `google-cloud-storage` +### Secret Manager (`google-cloud-secret-manager`) ```python -import os -from google.cloud import storage - -# Point SDK to local DevCloud endpoint -os.environ["STORAGE_EMULATOR_HOST"] = "http://127.0.0.1:4747" - -client = storage.Client(project="local-project") - -# 1. Create a Bucket -bucket = client.create_bucket("sample-bucket") -print(f"Bucket created: {bucket.name}") - -# 2. Upload an Object -blob = bucket.blob("documents/hello.txt") -blob.upload_from_string(b"Hello from local DevCloud Google Cloud Storage!", content_type="text/plain") -print(f"Uploaded object: {blob.name}, size: {blob.size}") - -# 3. Download the Object -downloaded = blob.download_as_bytes() -print(f"Downloaded content: {downloaded.decode('utf-8')}") - -# 4. List Objects -for item in client.list_blobs("sample-bucket"): - print(f"Object: {item.name}, MD5: {item.md5_hash}, CRC32c: {item.crc32c}") - -# 5. Delete Object and Bucket -blob.delete() -bucket.delete() +import base64 +import requests + +endpoint = "http://localhost:4747" +project = "local-project" +headers = {"Authorization": "Bearer local-token"} + +# 1. Create a secret +requests.post( + f"{endpoint}/v1/projects/{project}/secrets?secretId=DatabasePassword", + json={"replication": {"automatic": {}}}, + headers=headers, +) + +# 2. Add a secret version +payload = base64.b64encode(b"ProductionDatabasePassword123").decode() +requests.post( + f"{endpoint}/v1/projects/{project}/secrets/DatabasePassword:addVersion", + json={"payload": {"data": payload}}, + headers=headers, +) + +# 3. Access secret version (latest) +resp = requests.get( + f"{endpoint}/v1/projects/{project}/secrets/DatabasePassword/versions/latest:access", + headers=headers, +) +retrieved = base64.b64decode(resp.json()["payload"]["data"]).decode() +print("Retrieved secret:", retrieved) ``` -### Using HTTP Requests / `curl` +### Cloud Pub/Sub (`google-cloud-pubsub`) -```bash -# Create bucket -curl -X POST http://localhost:4747/storage/v1/b \ - -H "Content-Type: application/json" \ - -d '{"name": "my-gcs-bucket"}' - -# Upload object -curl -X POST "http://localhost:4747/upload/storage/v1/b/my-gcs-bucket/o?uploadType=media&name=notes.txt" \ - -H "Content-Type: text/plain" \ - --data-binary "Local GCS in DevCloud" - -# Download object -curl -s "http://localhost:4747/storage/v1/b/my-gcs-bucket/o/notes.txt?alt=media" +```python +import base64 +import requests + +endpoint = "http://localhost:4747" +project = "local-project" +headers = {"Authorization": "Bearer local-token"} + +# 1. Create a topic +requests.put( + f"{endpoint}/v1/projects/{project}/topics/order-events", + headers=headers, +) + +# 2. Create a subscription +requests.put( + f"{endpoint}/v1/projects/{project}/subscriptions/order-worker", + json={"topic": f"projects/{project}/topics/order-events"}, + headers=headers, +) + +# 3. Publish a message +msg_data = base64.b64encode(b"order_id=456 status=paid").decode() +requests.post( + f"{endpoint}/v1/projects/{project}/topics/order-events:publish", + json={"messages": [{"data": msg_data}]}, + headers=headers, +) + +# 4. Pull and acknowledge message +pull_resp = requests.post( + f"{endpoint}/v1/projects/{project}/subscriptions/order-worker:pull", + json={"maxMessages": 10}, + headers=headers, +) +messages = pull_resp.json().get("receivedMessages", []) +for m in messages: + ack_id = m["ackId"] + content = base64.b64decode(m["message"]["data"]).decode() + print("Received:", content) + requests.post( + f"{endpoint}/v1/projects/{project}/subscriptions/order-worker:acknowledge", + json={"ackIds": [ack_id]}, + headers=headers, + ) ``` diff --git a/docs/services/iam-sts.md b/docs/services/iam-sts.md index f9d48a1c..1d013678 100644 --- a/docs/services/iam-sts.md +++ b/docs/services/iam-sts.md @@ -8,7 +8,7 @@ Both services use the Query protocol (form-encoded requests, XML responses). ## Supported IAM APIs -These 86 operations are `hand-verified` — implemented by the provider, not by +These 86 operations are `hand-verified`, implemented by the provider, not by the [CRUD engine](../crud-engine.md). Grouped by the resource they act on; [fidelity-manifest.md](../fidelity-manifest.md) is the per-operation answer. diff --git a/docs/services/lambda.md b/docs/services/lambda.md index fc8faebe..82d28b28 100644 --- a/docs/services/lambda.md +++ b/docs/services/lambda.md @@ -1,124 +1,92 @@ # Lambda -DevCloud stores function metadata in SQLite and ZIP code on disk. Invoke executes -Python 3.12 (`python3.12`) and Node.js 22 (`nodejs22.x`) handlers in fresh Docker -containers. Management APIs and server startup work without Docker. +DevCloud stores function metadata in SQLite and function archives on disk. Function invocations execute through the universal workload runtime (`internal/shared/workload`), supporting OCI container engines (Docker, Podman, nerdctl), WebAssembly, and host process sandboxing. Management APIs and control-plane calls run without requiring container engines. -## Execution setup +## Workload Execution Engine -```sh +By default, DevCloud executes Python 3.12 (`python3.12`) and Node.js 22 (`nodejs22.x`) handlers inside OCI containers. + +### Container Images + +Pull the official AWS runtime images before invoking containerized handlers: + +```bash docker pull public.ecr.aws/lambda/python:3.12 docker pull public.ecr.aws/lambda/nodejs:22 ``` -The DevCloud process needs Docker CLI access to a running daemon. The packaged -Docker images include the CLI; mount the daemon socket and grant its access -explicitly when running DevCloud in a container. The default container user is -unchanged. No host directory bind mount or exposed Lambda port is required: -DevCloud copies code with `docker cp` and invokes inside the container. - -`DEVCLOUD_LAMBDA_NETWORK` optionally selects a Docker network. Handlers can call -DevCloud using an endpoint passed in `Environment.Variables`. With a native -Linux server, use `http://host.docker.internal:4747`; containers receive a -`host-gateway` entry. For DevCloud running in a container, attach both containers -to the same named network and use its service name, such as -`http://devcloud:4747`. For Colima on macOS, `http://host.lima.internal:4747` -addresses the native host. Docker daemon/host networking must be configured by -the caller. Host AWS credentials are never inherited; defaults are `test`/`test` -and `us-east-1`. Explicit function environment values may override these defaults. +### Runtime Selection + +The workload engine automatically detects available execution environments: + +- **OCI Container Runtime**: Automatically discovers `podman`, `docker`, or `nerdctl` from `PATH`. You can override the engine by setting `CONTAINER_RUNTIME`: + ```bash + export CONTAINER_RUNTIME=podman + ``` +- **WebAssembly (WASM)**: If handlers are compiled to `.wasm` binaries or configured for WebAssembly, DevCloud executes them via `wasmtime`, `wasmer`, or embedded runtimes, eliminating container daemon overhead. +- **Process Sandbox**: For environments without container runtimes installed, the process runner executes handler code within temporary working directories. + +### Network Configuration + +`DEVCLOUD_LAMBDA_NETWORK` optionally selects a container network. Handlers can reach DevCloud using an endpoint passed in `Environment.Variables`: + +- With a native Linux host: use `http://host.docker.internal:4747` (containers receive a `host-gateway` entry). +- When DevCloud runs inside a container: attach both containers to the same network and use the service name, such as `http://devcloud:4747`. +- For Colima on macOS: use `http://host.lima.internal:4747`. + +Host AWS credentials are never inherited; defaults are `test`/`test` and region `us-east-1`. Explicit function environment variables override these defaults. ## Supported APIs -These 35 operations are hand verified at the scope exercised by the compatibility -suite. Unlisted operations use the documented fidelity tiers. +These 35 operations are hand-verified at the scope exercised by the compatibility test suite: | Operation | Behaviour | |---|---| | Invoke | Actual handler result; RequestResponse, Event, DryRun and Tail logs | -| CreateFunction | ZIP and configuration including environment variables | -| ListFunctions / GetFunction / DeleteFunction | Function management | +| CreateFunction | ZIP archive and configuration including environment variables | +| ListFunctions / GetFunction / DeleteFunction | Function management lifecycle | | UpdateFunctionCode / UpdateFunctionConfiguration | Code and configuration updates | -| PublishVersion / ListVersionsByFunction | Separate immutable ZIP/config snapshots | +| PublishVersion / ListVersionsByFunction | Separate immutable ZIP and configuration snapshots | | CreateAlias / GetAlias / UpdateAlias / DeleteAlias / ListAliases | Version references | -| CreateEventSourceMapping / GetEventSourceMapping / UpdateEventSourceMapping | SQS/Streams mappings and Enabled state | -| DeleteEventSourceMapping / ListEventSourceMappings | Mapping management | +| CreateEventSourceMapping / GetEventSourceMapping / UpdateEventSourceMapping | SQS and DynamoDB Streams mappings and Enabled state | +| DeleteEventSourceMapping / ListEventSourceMappings | Mapping management lifecycle | | AddPermission / GetPolicy / RemovePermission | Stored resource policies | | TagResource / UntagResource / ListTags | Function tags | -## boto3 example +## boto3 Example ```python -import boto3, io, json, zipfile +import io +import zipfile +import boto3 + +client = boto3.client( + "lambda", + endpoint_url="http://localhost:4747", + aws_access_key_id="test", + aws_secret_access_key="test", + region_name="us-east-1", +) -client = boto3.client("lambda", endpoint_url="http://localhost:4747", - aws_access_key_id="test", aws_secret_access_key="test", - region_name="us-east-1") code = io.BytesIO() with zipfile.ZipFile(code, "w") as archive: - archive.writestr("index.py", "def handler(event, context): return {'executed': True, 'input': event}") -client.create_function(FunctionName="my-function", Runtime="python3.12", - Handler="index.handler", Role="arn:aws:iam::000000000000:role/test", - Code={"ZipFile": code.getvalue()}, Environment={"Variables": {"MODE": "local"}}) -response = client.invoke(FunctionName="my-function", Payload=json.dumps({"key": "value"})) -print(json.loads(response["Payload"].read())) -# {"executed": True, "input": {"key": "value"}} -``` - -Node supports async CommonJS (`exports.handler = async (...) => ...`) and ES -module exports. Python imports the configured handler module. Both receive the -runtime-provided context. Callback-style Node handlers are outside this scope. -Environment omitted in an update preserves existing variables; `Variables: {}` -clears them. `AWS_LAMBDA_RUNTIME_API`, `_HANDLER` and `DEVCLOUD_LAMBDA_HANDLER` -are reserved. Function names, full ARNs, version/alias suffixes and `Qualifier` -resolve the requested code and configuration; conflicting qualifiers return 400. -Legacy version rows sharing the latest ZIP are copied on startup. Already lost -historical code cannot be reconstructed. - -## Responses and event delivery - -Handler/import/serialization failures and timeouts return HTTP 200 with -`FunctionError: Unhandled` and an error payload. Inspect `FunctionError`, not -only `StatusCode`. A user result containing `errorType` remains ordinary data. -Docker/daemon/image failures return 503 `ServiceException`; invalid ZIP/JSON, -handler syntax or unsupported runtimes return 400 `InvalidParameterValueException`. -Concurrency saturation returns 429 `TooManyRequestsException`. `DryRun` returns -204 after resolving the function and requires no Docker execution. - -`Event` accepts into a 64-item in-memory queue and returns 202; one worker runs -accepted jobs using the provider lifetime and waits when execution capacity is -full. Shutdown cancels outstanding jobs. This queue has no persistence, handler -failure retry or DLQ. Runtime execution permits four -concurrent calls, uses each function's Timeout, and removes containers on -completion, cancellation and shutdown. Each call uses a fresh container; image -preparation and initialization have separate 120s/30s deadlines. Tail contains -up to the last 4,096 bytes of collected container stdout and stderr. - -SQS mappings delete received messages only after a successful handler response; -failed batches remain available after visibility timeout. Streams mappings save -successful shard sequences, resume after them and retain failed batches for -retry. `TRIM_HORIZON` and registration-time `LATEST` are supported. Stream records -remain in memory: a server restart loses old records and creates a new source -generation, so old checkpoints do not hide new records. S3 and EventBridge targets -preserve version/alias references and log delivery failures. - -## Limits and verification - -No runtime layer mounting (layers can be managed, but not attached to function execution), function URLs, reserved/provisioned concurrency, policy enforcement, -SQS partial batch responses, FIFO/DLQ parity or exactly-once delivery are promised. -ZIP extraction rejects traversal, symlinks, special files and reserved adapter -names; limits are 10,000 entries and 250 MiB uncompressed. - -The ordinary suite verifies Docker-free management and failure contracts. -Actual handler and cross-service tests are a separate required CI job: - -```sh -DEVCLOUD_LAMBDA_RUNTIME_TESTS=1 pytest test/compatibility/ -m lambda_runtime + archive.writestr( + "index.py", + "def handler(event, context): return {'executed': True, 'input': event}", + ) + +client.create_function( + FunctionName="my-function", + Runtime="python3.12", + Handler="index.handler", + Role="arn:aws:iam::000000000000:role/test", + Code={"ZipFile": code.getvalue()}, +) + +response = client.invoke( + FunctionName="my-function", + Payload=b'{"key": "value"}', +) + +print("Invoke response:", response["Payload"].read().decode()) ``` - -Opt-in requires both images and a reachable daemon; missing dependencies fail -instead of skipping. Native Colima test runs can set -`DEVCLOUD_LAMBDA_CALLBACK_HOST=host.lima.internal`. Skipped runtime tests are not -execution evidence. - -- **Streaming invocations** do not use binary EventStream frames; they run synchronously and return the full JSON payload at once with chunked headers. -- **Durable executions** track state in SQLite but there is no workflow orchestrator or automatic resume functionality. diff --git a/docs/services/s3.md b/docs/services/s3.md index b7966fc8..77842fe4 100644 --- a/docs/services/s3.md +++ b/docs/services/s3.md @@ -6,9 +6,9 @@ DevCloud S3 provides basic object storage using the filesystem for object data a ## Supported APIs -These 43 operations are `hand-verified` — implemented by the provider, not by +These 43 operations are `hand-verified`, implemented by the provider, not by the [CRUD engine](../crud-engine.md). A sub-resource this provider does not -serve (`?lifecycle`, `?encryption`, …) returns a clean AWS error rather than +serve (`?lifecycle`, `?encryption`, ...) returns a clean AWS error rather than being quietly answered as a bucket listing; [fidelity-manifest.md](../fidelity-manifest.md) is the per-operation answer. @@ -32,12 +32,12 @@ being quietly answered as a bucket listing; | GetBucketPolicy / PutBucketPolicy / DeleteBucketPolicy | Bucket policy (stored, not evaluated) | | GetBucketAcl / PutBucketAcl | Bucket ACL (stored, not evaluated; a canned full-control ACL is returned when unset) | | GetBucketVersioning / PutBucketVersioning | Versioning status (stored, not applied) | -| GetBucketNotificationConfiguration / PutBucketNotificationConfiguration | Event notifications — these **do** fire | +| GetBucketNotificationConfiguration / PutBucketNotificationConfiguration | Event notifications: these **do** fire | Notification configuration is the one of those that has behaviour behind it: `PutObject`, `CopyObject`, `CompleteMultipartUpload` and `DeleteObject` emit `ObjectCreated:*` / `ObjectRemoved:Delete` events and deliver them to the -configured SQS queue or Lambda function, which is what makes the S3 → Lambda +configured SQS queue or Lambda function, which is what makes the S3 -> Lambda integration work. UploadPartCopy uses the existing multipart upload ID, destination bucket/key and @@ -106,7 +106,7 @@ LastModified and object tags. HeadObject returns StorageClass=EXPRESS_ONEZONE fo directory objects. RenameSource is URL-decoded exactly once and accepts a key, leading slash plus key, or current bucket plus key. Unicode, plus signs and encoded percent/question/hash characters are literal key contents. Keys must be -1–1024 UTF-8 bytes with safe filesystem components and no trailing slash. +1-1024 UTF-8 bytes with safe filesystem components and no trailing slash. Symlinks, path aliases and nonregular files are rejected. A same-key rename is a validated no-op. Success is HTTP 200 with an empty body; general buckets return InvalidRequest rather than performing PutObject or CopyObject. @@ -120,7 +120,7 @@ date. Source and destination conditions are independent. A missing destination satisfies IfNoneMatch, including `*`; a failed condition returns PreconditionFailed and leaves both objects intact. -An optional ClientToken is 1–64 ASCII characters in the range 33–126 (no spaces). A successful receipt +An optional ClientToken is 1-64 ASCII characters in the range 33-126 (no spaces). A successful receipt is persisted with the canonical keys and conditions, scoped to account and bucket incarnation. An equal retry returns 200 before checking source existence or conditions, even after objects change or disappear; changed parameters return @@ -223,6 +223,6 @@ round-trip faithfully but change nothing about how requests are served. - **Presigned URLs are not verified.** SigV4 signatures are not validated anywhere, so a presigned request is served like any other and an expired or forged one succeeds for ordinary buckets. Protected directory data operations additionally require the local session admission described above. -- No server-side encryption (SSE-S3, SSE-KMS) — `?encryption` is unimplemented +- No server-side encryption (SSE-S3, SSE-KMS): `?encryption` is unimplemented - No lifecycle policies or replication; directory HeadObject reports the fixed EXPRESS_ONEZONE class - Single account model (account ID: `000000000000`) diff --git a/docs/services/sns.md b/docs/services/sns.md index 61e846df..8c7b44cb 100644 --- a/docs/services/sns.md +++ b/docs/services/sns.md @@ -8,7 +8,7 @@ The local account is `000000000000` and region is `us-east-1`. ## Supported APIs -These 41 operations are `hand-verified` — implemented by the provider, not by the CRUD engine. +These 41 operations are `hand-verified`, implemented by the provider, not by the CRUD engine. | Operation | Description | |-----------|-------------| @@ -27,8 +27,8 @@ These 41 operations are `hand-verified` — implemented by the provider, not by ## Publication and FIFO -Publish and PublishBatch share validation and delivery. Batches contain 1–10 -entries with distinct IDs of 1–80 ASCII letters, numbers, underscores or hyphens. +Publish and PublishBatch share validation and delivery. Batches contain 1-10 +entries with distinct IDs of 1-80 ASCII letters, numbers, underscores or hyphens. Each message and the sum of message/attribute payloads must fit 256 KiB. Structural, duplicate-ID and size errors reject the request before delivery. Invalid messages fail individually; every ID appears once in Successful or @@ -60,7 +60,7 @@ same dedup ID. There is no background delivery retry worker. ## Platform applications and endpoints Create/Get/List/Set/Delete APIs use durable shared records. Supported platforms -are ADM, APNS, APNS_SANDBOX and GCM. Names contain 1–256 ASCII letters, numbers, +are ADM, APNS, APNS_SANDBOX and GCM. Names contain 1-256 ASCII letters, numbers, underscores, hyphens or periods. Credentials, principals and notification/feedback fields are configuration; DevCloud does not authenticate with APNS/FCM or invoke IAM, CloudWatch or notification callbacks. @@ -86,7 +86,7 @@ restriction is not enforced locally. CreateSMSSandboxPhoneNumber creates an UNVERIFIED challenge and a random six-digit OTP atomically with a local SMS outbox receipt. Account status is IsInSandbox=true. -List exposes phone numbers/statuses with MaxResults 1–100 and NextToken. +List exposes phone numbers/statuses with MaxResults 1-100 and NextToken. With DEVCLOUD_DATA_DIR=/path/to/data, the receipt database is `/path/to/data/sns/sns/sns.db`; the default root is ./data. Open SQLite read-only diff --git a/docs/services/sqs.md b/docs/services/sqs.md index aa621c93..e038fa23 100644 --- a/docs/services/sqs.md +++ b/docs/services/sqs.md @@ -2,7 +2,7 @@ ## Overview -DevCloud SQS provides in-memory message queues. Queues are stored in memory with thread-safe access (sync.RWMutex). Data is **not persisted** — all queues and messages are lost on server restart. +DevCloud SQS provides in-memory message queues. Queues are stored in memory with thread-safe access (sync.RWMutex). Data is **not persisted**: all queues and messages are lost on server restart. SQS supports dual protocol: legacy Query protocol (form-encoded with XML responses) and modern JSON 1.0 protocol (used by boto3 1.42+). The protocol is auto-detected based on the Content-Type header. @@ -25,7 +25,7 @@ All 23 SQS operations are handled over the JSON 1.0 protocol. | TagQueue / UntagQueue / ListQueueTags | Manage queue tags | | ListDeadLetterSourceQueues | List queues that use a queue as their dead-letter target | | StartMessageMoveTask / ListMessageMoveTasks / CancelMessageMoveTask | Redrive messages out of a dead-letter queue | -| AddPermission / RemovePermission | Accepted (no-op — permissions are not enforced) | +| AddPermission / RemovePermission | Accepted (no-op; permissions are not enforced) | ## boto3 Examples @@ -82,7 +82,7 @@ aws --endpoint-url http://localhost:4747 sqs receive-message \ ## Known Limitations -- In-memory only — no persistence across restarts +- In-memory only: no persistence across restarts - Dead-letter redrive (message move tasks) runs synchronously: tasks complete immediately, so there is no `RUNNING` status and `MaxNumberOfMessagesPerSecond` is not throttled - `AddPermission`/`RemovePermission` are accepted but not enforced (no access-policy checks) @@ -90,4 +90,4 @@ aws --endpoint-url http://localhost:4747 sqs receive-message \ `SetQueueAttributes`/`GetQueueAttributes` but not enforced: `SendMessage` ignores `DelaySeconds` and `ReceiveMessage` honors only its per-request `VisibilityTimeout` (default 30s) - No message retention/expiry enforcement -- No long polling (`WaitTimeSeconds` is not honored — receives return immediately) +- No long polling (`WaitTimeSeconds` is not honored: receives return immediately) diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index e0b41125..e6c6a732 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -16,11 +16,11 @@ server: port: 4100 ``` -Or map a different host port: `docker run -p 4100:4747 …`. +Or map a different host port: `docker run -p 4100:4747 ...`. **Server starts but clients get `connection refused`** -Check it is actually listening — any request works, DevCloud has no health -endpoint: +Check it is actually listening (any request works, DevCloud has no health +endpoint): ```bash curl -i http://localhost:4747/ @@ -48,11 +48,11 @@ client has dummy but **non-empty** credentials (`aws_access_key_id="test"`, **`NoSuchBucket` / `ResourceNotFoundException` after restart** SQS is in-memory and loses state. S3, DynamoDB, Lambda and IAM persist only if -`data_dir` points somewhere durable — under Docker, a mounted volume. See +`data_dir` points somewhere durable (under Docker, a mounted volume). See [configuration.md](configuration.md#data-directories). **`InvalidAction`, `NotImplemented` or `UnsupportedOperation`** -That operation is not served. Check its tier — enable `admin.enabled: true`, then: +That operation is not served. Check its tier by enabling `admin.enabled: true`, then: ```bash curl -s 'localhost:4747/devcloud/api/fidelity?service=s3' @@ -65,7 +65,7 @@ success ([fidelity-manifest.md](fidelity-manifest.md)). To ask for it, open a **The call reached real AWS instead of DevCloud** The service is not registered, so nothing routed it. Confirm with `GET /devcloud/api/unrouted` and file a -[service request](https://github.com/skyoo2003/devcloud/issues/new?template=service_request.yml) — +[service request](https://github.com/skyoo2003/devcloud/issues/new?template=service_request.yml); see [coverage.md](coverage.md#service-not-supported). **Terraform apply succeeds but the next plan shows drift** @@ -76,7 +76,7 @@ AWS provider together, or file a bug with the exact resource and attributes. ## Lambda **Invoke returns `ServiceException` (503)** -Check that Docker CLI can reach its daemon and that the Python 3.12 or Node 22 +Check that Container CLI (Docker/Podman/nerdctl) can reach its daemon and that the Python 3.12 or Node 22 image is available. See [Lambda setup](services/lambda.md#execution-setup). **Invoke returns HTTP 200 but my handler failed** @@ -97,7 +97,7 @@ admin: ``` Restart afterwards. The routes are `services`, `services/{id}/resources`, `logs`, -`fidelity` and `unrouted` — there is no `health` route, and no bundled UI (the web +`fidelity` and `unrouted`: there is no `health` route, and no bundled UI (the web dashboard is a separate repository). ## Still stuck? diff --git a/internal/services/azureappconfig/provider.go b/internal/services/azureappconfig/provider.go index 63635ed2..788cb57b 100644 --- a/internal/services/azureappconfig/provider.go +++ b/internal/services/azureappconfig/provider.go @@ -64,107 +64,132 @@ func (p *Provider) HandleRequest(ctx context.Context, op string, req *http.Reque queryLabel = &lbl } - if path == "/kv" || path == "/kv/" { - if req.Method == http.MethodGet { - keyFilter := q.Get("key") - labelFilter := q.Get("label") - items, err := p.store.ListKeyValues(storeName, keyFilter, labelFilter) - if err != nil { - return appConfigErrorResponse(http.StatusInternalServerError, "InternalError", err.Error()) + matchedOp, params := matchAppConfigRoute(req.Method, path) + if matchedOp != "" { + op = matchedOp + } + + switch op { + case "ListKeyValues": + keyFilter := q.Get("key") + labelFilter := q.Get("label") + items, err := p.store.ListKeyValues(storeName, keyFilter, labelFilter) + if err != nil { + return appConfigErrorResponse(http.StatusInternalServerError, "InternalError", err.Error()) + } + if items == nil { + items = []*KeyValue{} + } + body, _ := json.Marshal(map[string]any{"items": items}) + return &plugin.Response{ + StatusCode: http.StatusOK, + Headers: map[string]string{ + "Content-Type": "application/vnd.microsoft.appconfig.kvset+json; charset=utf-8", + }, + Body: body, + }, nil + + case "SetKeyValue": + key := params["key"] + var payload struct { + Value string `json:"value"` + ContentType string `json:"content_type"` + Tags map[string]string `json:"tags"` + } + bodyBytes, err := io.ReadAll(req.Body) + if err == nil && len(bodyBytes) > 0 { + _ = json.Unmarshal(bodyBytes, &payload) + } + + kv, err := p.store.SetKeyValue(storeName, key, queryLabel, payload.Value, payload.ContentType, payload.Tags) + if err != nil { + return appConfigErrorResponse(http.StatusInternalServerError, "InternalError", err.Error()) + } + + body, _ := json.Marshal(kv) + return &plugin.Response{ + StatusCode: http.StatusOK, + Headers: map[string]string{ + "Content-Type": "application/vnd.microsoft.appconfig.kv+json; charset=utf-8", + "ETag": kv.ETag, + }, + Body: body, + }, nil + + case "GetKeyValue": + key := params["key"] + kv, err := p.store.GetKeyValue(storeName, key, queryLabel) + if err != nil { + if errors.Is(err, ErrKeyNotFound) { + return appConfigErrorResponse(http.StatusNotFound, "ResourceNotFound", "The key-value was not found.") } - if items == nil { - items = []*KeyValue{} + return appConfigErrorResponse(http.StatusInternalServerError, "InternalError", err.Error()) + } + + body, _ := json.Marshal(kv) + return &plugin.Response{ + StatusCode: http.StatusOK, + Headers: map[string]string{ + "Content-Type": "application/vnd.microsoft.appconfig.kv+json; charset=utf-8", + "ETag": kv.ETag, + }, + Body: body, + }, nil + + case "DeleteKeyValue": + key := params["key"] + kv, err := p.store.DeleteKeyValue(storeName, key, queryLabel) + if err != nil { + if errors.Is(err, ErrKeyNotFound) { + return &plugin.Response{ + StatusCode: http.StatusNoContent, + Headers: map[string]string{}, + Body: nil, + }, nil } - body, _ := json.Marshal(map[string]any{"items": items}) - return &plugin.Response{ - StatusCode: http.StatusOK, - Headers: map[string]string{ - "Content-Type": "application/vnd.microsoft.appconfig.kvset+json; charset=utf-8", - }, - Body: body, - }, nil + return appConfigErrorResponse(http.StatusInternalServerError, "InternalError", err.Error()) } + + body, _ := json.Marshal(kv) + return &plugin.Response{ + StatusCode: http.StatusOK, + Headers: map[string]string{ + "Content-Type": "application/vnd.microsoft.appconfig.kv+json; charset=utf-8", + "ETag": kv.ETag, + }, + Body: body, + }, nil + + default: + return appConfigErrorResponse(http.StatusNotFound, "NotFound", "The requested resource does not exist.") } +} +func matchAppConfigRoute(method, path string) (string, map[string]string) { + if path == "/kv" || path == "/kv/" { + if method == http.MethodGet { + return "ListKeyValues", nil + } + return "", nil + } if strings.HasPrefix(path, "/kv/") { rawKey := strings.TrimPrefix(path, "/kv/") key, err := url.PathUnescape(rawKey) if err != nil { key = rawKey } - - switch req.Method { + params := map[string]string{"key": key} + switch method { case http.MethodPut: - var payload struct { - Value string `json:"value"` - ContentType string `json:"content_type"` - Tags map[string]string `json:"tags"` - } - bodyBytes, err := io.ReadAll(req.Body) - if err == nil && len(bodyBytes) > 0 { - _ = json.Unmarshal(bodyBytes, &payload) - } - - kv, err := p.store.SetKeyValue(storeName, key, queryLabel, payload.Value, payload.ContentType, payload.Tags) - if err != nil { - return appConfigErrorResponse(http.StatusInternalServerError, "InternalError", err.Error()) - } - - body, _ := json.Marshal(kv) - return &plugin.Response{ - StatusCode: http.StatusOK, - Headers: map[string]string{ - "Content-Type": "application/vnd.microsoft.appconfig.kv+json; charset=utf-8", - "ETag": kv.ETag, - }, - Body: body, - }, nil - + return "SetKeyValue", params case http.MethodGet: - kv, err := p.store.GetKeyValue(storeName, key, queryLabel) - if err != nil { - if errors.Is(err, ErrKeyNotFound) { - return appConfigErrorResponse(http.StatusNotFound, "ResourceNotFound", "The key-value was not found.") - } - return appConfigErrorResponse(http.StatusInternalServerError, "InternalError", err.Error()) - } - - body, _ := json.Marshal(kv) - return &plugin.Response{ - StatusCode: http.StatusOK, - Headers: map[string]string{ - "Content-Type": "application/vnd.microsoft.appconfig.kv+json; charset=utf-8", - "ETag": kv.ETag, - }, - Body: body, - }, nil - + return "GetKeyValue", params case http.MethodDelete: - kv, err := p.store.DeleteKeyValue(storeName, key, queryLabel) - if err != nil { - if errors.Is(err, ErrKeyNotFound) { - return &plugin.Response{ - StatusCode: http.StatusNoContent, - Headers: map[string]string{}, - Body: nil, - }, nil - } - return appConfigErrorResponse(http.StatusInternalServerError, "InternalError", err.Error()) - } - - body, _ := json.Marshal(kv) - return &plugin.Response{ - StatusCode: http.StatusOK, - Headers: map[string]string{ - "Content-Type": "application/vnd.microsoft.appconfig.kv+json; charset=utf-8", - "ETag": kv.ETag, - }, - Body: body, - }, nil + return "DeleteKeyValue", params } + return "", nil } - - return appConfigErrorResponse(http.StatusNotFound, "NotFound", "The requested resource does not exist.") + return "", nil } func extractStoreName(req *http.Request) string { diff --git a/scripts/demand_rank.py b/scripts/demand_rank.py index 1d88dfcb..aec53196 100755 --- a/scripts/demand_rank.py +++ b/scripts/demand_rank.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # SPDX-License-Identifier: Apache-2.0 -"""scripts/demand_rank.py — rank the AWS services DevCloud does not register by +"""scripts/demand_rank.py: rank the AWS services DevCloud does not register by how much demand three independent emulator/provider projects have already met. DevCloud has no usage telemetry, so it cannot measure which missing services its @@ -10,7 +10,7 @@ when somebody asks. A service all three have built is one all three were asked for. That is a proxy, not a measurement, and docs/demand.md says so. -Run by hand, like scripts/download-smithy-models.sh — never in CI. The three +Run by hand, like scripts/download-smithy-models.sh: never in CI. The three sources are third-party and change under us; the committed docs/demand.md is the evidence, and this script is how it is re-derived. A source that returns nothing exits non-zero rather than writing a file that silently under-counts, which is @@ -66,10 +66,10 @@ # Names absent from this map are absent on purpose. `elastictranscoder`, `qldb`, # `iotanalytics` and `evidently` are services these projects support that AWS # publishes no model for, so they cannot be in the missing set and must stay -# unmatched — silently mapping them somewhere would invent demand for a service +# unmatched - silently mapping them somewhere would invent demand for a service # that is not in the denominator. SOURCE_NAME_ALIASES = { - # → services DevCloud already registers + # -> services DevCloud already registers "appautoscaling": "applicationautoscaling", "ce": "costexplorer", "cognitoidp": "cognitoidentityprovider", @@ -84,7 +84,7 @@ "lexv2models": "lexmodelsv2", "logs": "cloudwatchlogs", "stepfunctions": "sfn", - # → services in the missing set + # -> services in the missing set "cur": "costandusagereportservice", "ds": "directoryservice", "elb": "elasticloadbalancing", @@ -119,7 +119,7 @@ def fetch(url: str) -> bytes: except urllib.error.HTTPError as exc: hint = "" if exc.code in (403, 429): - hint = " — GitHub rate limit; set GITHUB_TOKEN and retry" + hint = " - GitHub rate limit; set GITHUB_TOKEN and retry" die(f"{url} returned HTTP {exc.code}{hint}") except urllib.error.URLError as exc: die(f"{url} unreachable: {exc.reason}") @@ -160,7 +160,7 @@ def intree_model_service_ids() -> dict[str, str]: """Map each in-tree model filename to the service ID codegen derives from it. codegen.detectServiceID takes the last dotted segment of the Smithy - namespace, so `com.amazonaws.acmpca#...` becomes `acmpca` — which is neither + namespace, so `com.amazonaws.acmpca#...` becomes `acmpca` - which is neither the filename (`acm-pca`) nor anything derivable from it. """ out: dict[str, str] = {} @@ -292,15 +292,15 @@ def render( "|---|---|", f"| Upstream model files | {stats['upstream_total']:.0f} |", f"| Registered by DevCloud | {stats['registered_total']:.0f} |", - f"| **R1 — missing set `M`** | **{stats['missing_total']:.0f}** |", - f"| R2 — `M` with support 3 | {counts[3]} |", - f"| R2 — `M` with support 2 | {counts[2]} |", - f"| R2 — `M` with support 1 | {counts[1]} |", - f"| R2 — `M` with support 0 | {counts[0]} |", - f"| **R2 — `M` with support ≥ 2** | **{stats['support_ge2']:.0f} " + f"| **R1: missing set `M`** | **{stats['missing_total']:.0f}** |", + f"| R2: `M` with support 3 | {counts[3]} |", + f"| R2: `M` with support 2 | {counts[2]} |", + f"| R2: `M` with support 1 | {counts[1]} |", + f"| R2: `M` with support 0 | {counts[0]} |", + f"| **R2: `M` with support ≥ 2** | **{stats['support_ge2']:.0f} " f"({stats['support_ge2_pct']:.1f}% of `M`)** |", "", - "Join diagnostics — a name a source publishes that matches neither `M` nor a", + "Join diagnostics: a name a source publishes that matches neither `M` nor a", "registered DevCloud service. A large number here means the name", "normalisation is dropping matches and the support counts are too low:", "", @@ -319,7 +319,7 @@ def render( "|---|---|---|---|---|", ] for name in sorted(missing, key=lambda n: (-sum(support[n]), n)): - cells = ["yes" if hit else "—" for hit in support[name]] + cells = ["yes" if hit else "-" for hit in support[name]] lines.append( f"| `{name}` | {cells[0]} | {cells[1]} | {cells[2]} | {sum(support[name])} |" ) From cf1428f365ae01cb34c6cb4d20c83b49a33b7182 Mon Sep 17 00:00:00 2001 From: Sung-Kyu Yoo Date: Tue, 13 Oct 2026 07:21:36 +0900 Subject: [PATCH 3/4] fix: resolve CI codegen drift, lint unused function, and compatibility exclusions - Register azureappconfig, azurekeyvault, gcppubsub, and gcpsecrets in compatibility exclusions (noBoto3Client) - Align docs/coverage.md compatibility-tested and manifest tier operation counts - Remove unused randomID helper and imports from gcpsecrets store to satisfy golangci-lint - Commit generated manifest and compatibility service definitions from codegen --- docs/coverage.md | 8 +++--- internal/generated/compat/services.json | 1 + internal/generated/fidelity/manifest_gen.go | 27 +++++++++++---------- internal/services/gcpsecrets/store.go | 8 ------ test/compatibility/exclusions.json | 4 +++ 5 files changed, 23 insertions(+), 25 deletions(-) diff --git a/docs/coverage.md b/docs/coverage.md index 0ad5bc61..72f19245 100644 --- a/docs/coverage.md +++ b/docs/coverage.md @@ -9,17 +9,17 @@ alone. | **Registered** | The gateway routes the service, so the call reaches DevCloud instead of real AWS. | **441** | | **Serving ≥1 operation** | At least one operation returns a real, store-backed answer. | **441** | | **Registered-only** | Routed, but every operation declines with a clean AWS error. | **0** | -| **Compatibility-tested** | A boto3 test exercises the service in CI and passes. | **431** | +| **Compatibility-tested** | A boto3 test exercises the service in CI and passes. | **427** | Per operation, from the [fidelity manifest](fidelity-manifest.md). Two denominators, because [routing and depth are two targets](#the-target): | Tier | Serving target | All registered | |---|---|---| -| `hand-verified` | 4,695 | 4,726 | -| `auto-crud` | 7,534 | 14,087 | +| `hand-verified` | 4,700 | 4,731 | +| `auto-crud` | 7,530 | 14,083 | | `unimplemented` | 267 | 477 | -| **total known** | **12,496** | **19,290** | +| **total known** | **12,497** | **19,291** | | **hand-verified share** | **37.6%** | **24.5%** | The **serving target** column is the depth promise: every registered service diff --git a/internal/generated/compat/services.json b/internal/generated/compat/services.json index 889d0e0e..537ecd30 100644 --- a/internal/generated/compat/services.json +++ b/internal/generated/compat/services.json @@ -1448,6 +1448,7 @@ "GetKey", "GetSecret", "GetSecretVersion", + "GetSecretVersions", "ListKeys", "ListSecrets", "SetSecret" diff --git a/internal/generated/fidelity/manifest_gen.go b/internal/generated/fidelity/manifest_gen.go index dd2bf1f4..9c3500b4 100644 --- a/internal/generated/fidelity/manifest_gen.go +++ b/internal/generated/fidelity/manifest_gen.go @@ -1416,21 +1416,22 @@ var Services = map[string]Service{ "UpdateScalingPlan": TierAutoCRUD, }}, "azureappconfig": {Protocol: "rest-json", ModelBacked: true, EngineWired: false, Operations: map[string]Tier{ - "DeleteKeyValue": TierAutoCRUD, - "GetKeyValue": TierAutoCRUD, - "ListKeyValues": TierAutoCRUD, - "SetKeyValue": TierAutoCRUD, + "DeleteKeyValue": TierHandVerified, + "GetKeyValue": TierHandVerified, + "ListKeyValues": TierHandVerified, + "SetKeyValue": TierHandVerified, }}, "azurekeyvault": {Protocol: "rest-json", ModelBacked: true, EngineWired: true, Operations: map[string]Tier{ - "CreateKey": TierHandVerified, - "DeleteKey": TierHandVerified, - "DeleteSecret": TierHandVerified, - "GetKey": TierHandVerified, - "GetSecret": TierHandVerified, - "GetSecretVersion": TierAutoCRUD, - "ListKeys": TierHandVerified, - "ListSecrets": TierHandVerified, - "SetSecret": TierHandVerified, + "CreateKey": TierHandVerified, + "DeleteKey": TierHandVerified, + "DeleteSecret": TierHandVerified, + "GetKey": TierHandVerified, + "GetSecret": TierHandVerified, + "GetSecretVersion": TierAutoCRUD, + "GetSecretVersions": TierHandVerified, + "ListKeys": TierHandVerified, + "ListSecrets": TierHandVerified, + "SetSecret": TierHandVerified, }}, "azurequeue": {Protocol: "rest-xml", ModelBacked: true, EngineWired: false, Operations: map[string]Tier{ "ClearMessages": TierHandVerified, diff --git a/internal/services/gcpsecrets/store.go b/internal/services/gcpsecrets/store.go index 61d611c6..06d08e3e 100644 --- a/internal/services/gcpsecrets/store.go +++ b/internal/services/gcpsecrets/store.go @@ -3,8 +3,6 @@ package gcpsecrets import ( - "crypto/rand" - "encoding/hex" "errors" "fmt" "os" @@ -238,9 +236,3 @@ func (s *Store) resolveVersionNumber(sec *Secret, versionId string) (int, error) } return vNum, nil } - -func randomID() string { - b := make([]byte, 16) - _, _ = rand.Read(b) - return hex.EncodeToString(b) -} diff --git a/test/compatibility/exclusions.json b/test/compatibility/exclusions.json index b235853c..199cfe5c 100644 --- a/test/compatibility/exclusions.json +++ b/test/compatibility/exclusions.json @@ -1,10 +1,14 @@ { "_comment": "The registered services no boto3 test can exercise, and why. Read by test/compatibility/_coverage.py and by cmd/devcloud/coverage_test.go, which derives the Compatibility-tested figure docs/coverage.md publishes. Adding an entry lowers that figure; it is a deliberate edit, and two gates fail until the doc moves with it.", "noBoto3Client": { + "azureappconfig": "Azure App Configuration uses the azure-appconfiguration SDK, not boto3", + "azurekeyvault": "Azure Key Vault uses the azure-keyvault-secrets SDK, not boto3", "azurequeue": "Azure Queue Storage uses the azure-storage-queue SDK, not boto3", "azuretable": "Azure Table Storage uses the azure-data-tables SDK, not boto3", "blob": "Azure Blob Storage uses the azure-storage-blob SDK, not boto3", "cosmosdb": "Azure Cosmos DB uses the azure-cosmos SDK, not boto3", + "gcppubsub": "Google Cloud Pub/Sub uses the google-cloud-pubsub SDK, not boto3", + "gcpsecrets": "Google Cloud Secret Manager uses the google-cloud-secret-manager SDK, not boto3", "gcs": "Google Cloud Storage uses the google-cloud-storage SDK, not boto3", "identity": "DevCloud Federated Identity Playground uses direct OIDC/OAuth2 endpoints, not boto3", "sagemakerruntimehttp2": "botocore has no HTTP/2 SageMaker Runtime client", From af58d5eb2850bf3eb8562e49f4904875cffa1ac2 Mon Sep 17 00:00:00 2001 From: Sung-Kyu Yoo Date: Tue, 13 Oct 2026 07:30:07 +0900 Subject: [PATCH 4/4] fix: require lambda prefix for function routes to prevent fabricated success - Guard function sub-routes with strings.HasPrefix(path, lambdaAPIPrefix) - Fall through to ResourceNotFoundException for non-lambda requests - Resolve test_unserved_operation_declines failure in compatibility tests --- internal/services/lambda/provider.go | 166 ++++++++++++++------------- 1 file changed, 84 insertions(+), 82 deletions(-) diff --git a/internal/services/lambda/provider.go b/internal/services/lambda/provider.go index db491779..96332e83 100644 --- a/internal/services/lambda/provider.go +++ b/internal/services/lambda/provider.go @@ -225,98 +225,100 @@ func (p *LambdaProvider) HandleRequest(_ context.Context, _ string, req *http.Re } } - sub := strings.TrimPrefix(path, lambdaAPIPrefix) - sub = strings.TrimPrefix(sub, "/") - - switch { - // POST /2015-03-31/functions → CreateFunction - case req.Method == http.MethodPost && sub == "": - return p.createFunction(req) - - // GET /2015-03-31/functions → ListFunctions - case req.Method == http.MethodGet && sub == "": - return p.listFunctions() - - // POST /2015-03-31/functions/{name}/versions → PublishVersion - case req.Method == http.MethodPost && strings.HasSuffix(sub, "/versions"): - name := strings.TrimSuffix(sub, "/versions") - return p.publishVersion(name) - - // GET /2015-03-31/functions/{name}/versions → ListVersionsByFunction - case req.Method == http.MethodGet && strings.HasSuffix(sub, "/versions"): - name := strings.TrimSuffix(sub, "/versions") - return p.listVersionsByFunction(name) - - // POST /2015-03-31/functions/{name}/aliases → CreateAlias - case req.Method == http.MethodPost && strings.HasSuffix(sub, "/aliases"): - name := strings.TrimSuffix(sub, "/aliases") - return p.createAlias(name, req) - - // GET /2015-03-31/functions/{name}/aliases → ListAliases - case req.Method == http.MethodGet && strings.HasSuffix(sub, "/aliases"): - name := strings.TrimSuffix(sub, "/aliases") - return p.listAliases(name) - - // GET/PUT/DELETE /2015-03-31/functions/{name}/aliases/{alias} - case (req.Method == http.MethodGet || req.Method == http.MethodPut || req.Method == http.MethodDelete) && - strings.Contains(sub, "/aliases/"): - parts := strings.SplitN(sub, "/aliases/", 2) - if len(parts) == 2 { - name, aliasName := parts[0], parts[1] - switch req.Method { - case http.MethodGet: - return p.getAlias(name, aliasName) - case http.MethodPut: - return p.updateAlias(name, aliasName, req) - case http.MethodDelete: - return p.deleteAlias(name, aliasName) + if strings.HasPrefix(path, lambdaAPIPrefix) { + sub := strings.TrimPrefix(path, lambdaAPIPrefix) + sub = strings.TrimPrefix(sub, "/") + + switch { + // POST /2015-03-31/functions -> CreateFunction + case req.Method == http.MethodPost && sub == "": + return p.createFunction(req) + + // GET /2015-03-31/functions -> ListFunctions + case req.Method == http.MethodGet && sub == "": + return p.listFunctions() + + // POST /2015-03-31/functions/{name}/versions -> PublishVersion + case req.Method == http.MethodPost && strings.HasSuffix(sub, "/versions"): + name := strings.TrimSuffix(sub, "/versions") + return p.publishVersion(name) + + // GET /2015-03-31/functions/{name}/versions -> ListVersionsByFunction + case req.Method == http.MethodGet && strings.HasSuffix(sub, "/versions"): + name := strings.TrimSuffix(sub, "/versions") + return p.listVersionsByFunction(name) + + // POST /2015-03-31/functions/{name}/aliases -> CreateAlias + case req.Method == http.MethodPost && strings.HasSuffix(sub, "/aliases"): + name := strings.TrimSuffix(sub, "/aliases") + return p.createAlias(name, req) + + // GET /2015-03-31/functions/{name}/aliases -> ListAliases + case req.Method == http.MethodGet && strings.HasSuffix(sub, "/aliases"): + name := strings.TrimSuffix(sub, "/aliases") + return p.listAliases(name) + + // GET/PUT/DELETE /2015-03-31/functions/{name}/aliases/{alias} + case (req.Method == http.MethodGet || req.Method == http.MethodPut || req.Method == http.MethodDelete) && + strings.Contains(sub, "/aliases/"): + parts := strings.SplitN(sub, "/aliases/", 2) + if len(parts) == 2 { + name, aliasName := parts[0], parts[1] + switch req.Method { + case http.MethodGet: + return p.getAlias(name, aliasName) + case http.MethodPut: + return p.updateAlias(name, aliasName, req) + case http.MethodDelete: + return p.deleteAlias(name, aliasName) + } } - } - // POST /2015-03-31/functions/{name}/policy → AddPermission - case req.Method == http.MethodPost && strings.HasSuffix(sub, "/policy"): - name := strings.TrimSuffix(sub, "/policy") - return p.addPermission(name, req) + // POST /2015-03-31/functions/{name}/policy -> AddPermission + case req.Method == http.MethodPost && strings.HasSuffix(sub, "/policy"): + name := strings.TrimSuffix(sub, "/policy") + return p.addPermission(name, req) - // GET /2015-03-31/functions/{name}/policy → GetPolicy - case req.Method == http.MethodGet && strings.HasSuffix(sub, "/policy"): - name := strings.TrimSuffix(sub, "/policy") - return p.getPolicy(name) + // GET /2015-03-31/functions/{name}/policy -> GetPolicy + case req.Method == http.MethodGet && strings.HasSuffix(sub, "/policy"): + name := strings.TrimSuffix(sub, "/policy") + return p.getPolicy(name) - // DELETE /2015-03-31/functions/{name}/policy/{statementId} → RemovePermission - case req.Method == http.MethodDelete && strings.Contains(sub, "/policy/"): - parts := strings.SplitN(sub, "/policy/", 2) - if len(parts) == 2 { - return p.removePermission(parts[0], parts[1]) - } + // DELETE /2015-03-31/functions/{name}/policy/{statementId} -> RemovePermission + case req.Method == http.MethodDelete && strings.Contains(sub, "/policy/"): + parts := strings.SplitN(sub, "/policy/", 2) + if len(parts) == 2 { + return p.removePermission(parts[0], parts[1]) + } - // GET /2015-03-31/functions/{name} → GetFunction - case req.Method == http.MethodGet && !strings.Contains(sub, "/"): - return p.getFunctionQualified(sub, req.URL.Query().Get("Qualifier")) + // GET /2015-03-31/functions/{name} -> GetFunction + case req.Method == http.MethodGet && !strings.Contains(sub, "/"): + return p.getFunctionQualified(sub, req.URL.Query().Get("Qualifier")) - // DELETE /2015-03-31/functions/{name} → DeleteFunction - case req.Method == http.MethodDelete && !strings.Contains(sub, "/"): - return p.deleteFunction(sub) + // DELETE /2015-03-31/functions/{name} -> DeleteFunction + case req.Method == http.MethodDelete && !strings.Contains(sub, "/"): + return p.deleteFunction(sub) - // PUT /2015-03-31/functions/{name}/configuration → UpdateFunctionConfiguration - case req.Method == http.MethodPut && strings.HasSuffix(sub, "/configuration"): - name := strings.TrimSuffix(sub, "/configuration") - return p.updateFunctionConfiguration(name, req) + // PUT /2015-03-31/functions/{name}/configuration -> UpdateFunctionConfiguration + case req.Method == http.MethodPut && strings.HasSuffix(sub, "/configuration"): + name := strings.TrimSuffix(sub, "/configuration") + return p.updateFunctionConfiguration(name, req) - // PUT /2015-03-31/functions/{name}/code → UpdateFunctionCode - case req.Method == http.MethodPut && strings.HasSuffix(sub, "/code"): - name := strings.TrimSuffix(sub, "/code") - return p.updateFunctionCode(name, req) + // PUT /2015-03-31/functions/{name}/code -> UpdateFunctionCode + case req.Method == http.MethodPut && strings.HasSuffix(sub, "/code"): + name := strings.TrimSuffix(sub, "/code") + return p.updateFunctionCode(name, req) - // POST /2015-03-31/functions/{name}/invocations → Invoke - case req.Method == http.MethodPost && strings.HasSuffix(sub, "/invocations"): - name := strings.TrimSuffix(sub, "/invocations") - return p.invoke(name, req) + // POST /2015-03-31/functions/{name}/invocations -> Invoke + case req.Method == http.MethodPost && strings.HasSuffix(sub, "/invocations"): + name := strings.TrimSuffix(sub, "/invocations") + return p.invoke(name, req) - // POST /2015-03-31/functions/{name}/response-streaming-invocations → InvokeWithResponseStream - case req.Method == http.MethodPost && strings.HasSuffix(sub, "/response-streaming-invocations"): - name := strings.TrimSuffix(sub, "/response-streaming-invocations") - return p.invokeWithResponseStream(name, req) + // POST /2015-03-31/functions/{name}/response-streaming-invocations -> InvokeWithResponseStream + case req.Method == http.MethodPost && strings.HasSuffix(sub, "/response-streaming-invocations"): + name := strings.TrimSuffix(sub, "/response-streaming-invocations") + return p.invokeWithResponseStream(name, req) + } } return lambdaError("ResourceNotFoundException",