From 5d6e085098c617387e511acffabe5ea565c96465 Mon Sep 17 00:00:00 2001 From: Fred E <7602667+wallscaler@users.noreply.github.com> Date: Wed, 16 Sep 2026 15:23:14 -0400 Subject: [PATCH 1/9] checkpoint(runtime): recover durable Cathedral create operations --- docs/ARCHITECTURE.md | 23 +- packages/api/internal/api/api.gen.go | 1193 ++++++++++++----- .../handlers/cathedral_sandbox_operations.go | 250 ++++ .../cathedral_sandbox_operations_test.go | 111 ++ .../api/internal/handlers/sandbox_create.go | 107 +- .../handlers/sandbox_create_fcgate_test.go | 2 +- .../internal/handlers/sandbox_create_test.go | 8 +- packages/api/internal/middleware/cors.go | 3 + ...80217_add_cathedral_sandbox_operations.sql | 39 + .../cathedral_sandbox_operations_test.go | 143 ++ packages/db/pkg/testutils/queries/models.go | 14 + .../cathedral_sandbox_operations.sql.go | 204 +++ packages/db/queries/models.go | 14 + .../cathedral_sandbox_operations.sql | 65 + spec/openapi.yml | 123 ++ 15 files changed, 1947 insertions(+), 352 deletions(-) create mode 100644 packages/api/internal/handlers/cathedral_sandbox_operations.go create mode 100644 packages/api/internal/handlers/cathedral_sandbox_operations_test.go create mode 100644 packages/db/migrations/20260916080217_add_cathedral_sandbox_operations.sql create mode 100644 packages/db/pkg/tests/cathedral_sandbox_operations_test.go create mode 100644 packages/db/queries/cathedral_sandbox_operations.sql.go create mode 100644 packages/db/queries/sandboxes/cathedral_sandbox_operations.sql diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 21e7dddfa8..52ac5ca9ca 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -147,7 +147,12 @@ The control-plane entry point (Gin, OpenAPI-generated from `spec/openapi.yml`, p sandbox→node **routing catalog** (`sandbox:catalog:{id}`) in Redis that client-proxy reads. This API-written record is the default routing source; the orchestrator-written `sandbox:routing:{id}` is a flag-gated alternative (see "Sandbox routing records"). Persistent - entities (templates, builds, snapshots, teams) live in Postgres. + entities (templates, builds, snapshots, teams) live in Postgres. Creates carrying an + `Idempotency-Key` also bind the authenticated team, canonical request digest, and one generated + sandbox ID in `cathedral_sandbox_operations` before orchestrator I/O. A replay can therefore + re-enter the existing Redis reservation with the same sandbox ID; a changed body returns 409, + and a completed operation returns its immutable stored response. The authenticated + `/v1/cathedral/operations/{key}` route is the durable recovery lookup. - **Secrets**: `/secrets` is the only public surface for secret management (create, list, get, update, delete). The API authenticates the caller with the customer alternatives above, converts the authenticated team UUID to the project UUID the backend knows, checks the `customer-secrets` @@ -360,7 +365,7 @@ planes today. | Store | Owner packages | What lives there | |---|---|---| -| **PostgreSQL** | `packages/db` (goose migrations, sqlc) | Durable control-plane state: `teams`, `users`, `tiers` (quota defaults), `project_limits` (per-team quota overrides pushed in by the owning service; the `team_limits` view reads it in preference to `tiers`), `envs` (templates), `env_builds` (build rows: vcpu, ram_mb, status, versions), `env_aliases`, `snapshots` (paused sandboxes), `team_api_keys`, `volumes`, `clusters` | +| **PostgreSQL** | `packages/db` (goose migrations, sqlc) | Durable control-plane state: `teams`, `users`, `tiers` (quota defaults), `project_limits` (per-team quota overrides pushed in by the owning service; the `team_limits` view reads it in preference to `tiers`), `envs` (templates), `env_builds` (build rows: vcpu, ram_mb, status, versions), `env_aliases`, `snapshots` (paused sandboxes), `cathedral_sandbox_operations` (team-scoped durable create bindings and terminal responses), `team_api_keys`, `volumes`, `clusters` | | **Redis** | API, client-proxy, orchestrator | Ephemeral runtime state: running-sandbox store (source of truth), sandbox→node routing catalog, team/template/snapshot caches, rate limiting, P2P chunk peer registry | | **ClickHouse** | `packages/clickhouse` | Time-series/analytics: `metrics_gauge`/`metrics_sum` (written by the OTel collector), `sandbox_events`, `sandbox_host_stats` (written by orchestrator), team metrics, and optionally `sandbox_logs` during the log migration. Read by API and dashboard-api | | **Object storage** (GCS/S3/local, `packages/shared/pkg/storage`) | orchestrator, template-manager | Template & snapshot artifacts, keyed by build ID: `{buildID}/memfile`, `{buildID}/rootfs.ext4`, `{buildID}/snapfile`, `{buildID}/metadata.json` + `.header` index files | @@ -382,17 +387,23 @@ are resolved through the `.header` files). ### Sandbox creation ```mermaid +%%{init: {'theme':'base','themeVariables':{'background':'#FAF9F5','primaryColor':'#E8DCCA','primaryTextColor':'#191919','primaryBorderColor':'#191919','lineColor':'#191919','fontSize':'16px'}}}%% sequenceDiagram autonumber participant C as SDK participant API as API + participant PG as Durable create ledger participant R as Redis participant O as Orchestrator (chosen node) participant FC as Firecracker participant E as envd (in VM) C->>API: POST /sandboxes {templateID} - API->>API: auth team, resolve template alias → ready build (Postgres/cache) + API->>API: auth team, validate request, resolve template → ready build + opt Idempotency-Key present + API->>PG: reserve team + key + request digest + sandboxID + PG-->>API: new binding or the existing sandboxID + end API->>API: best-of-K placement → pick node API->>O: gRPC SandboxService.Create(SandboxConfig) O->>O: fetch template (local cache / NFS / object storage) @@ -402,6 +413,9 @@ sequenceDiagram E-->>O: 204 O-->>API: Create OK API->>R: store running sandbox + routing catalog entry + opt Idempotency-Key present + API->>PG: store immutable 201 response, mark ready + end API-->>C: 201 sandbox {sandboxID, domain} ``` @@ -409,6 +423,9 @@ The API blocks on the gRPC `Create`, which itself blocks on envd's `/init` — w gets a response, the sandbox is fully usable. Fresh creates are internally a *resume* of the template's base snapshot (cold boots happen for filesystem-only templates and builds, or when an explicit resume requests one — see pause and resume below; template creates never do). +For a durable create, the binding is reserved only after request validation and before the first +orchestrator call. If the response is lost, replaying the same key and body uses the same sandbox +ID; it never interprets an empty inventory list as proof that no sandbox exists. ### Sandbox traffic diff --git a/packages/api/internal/api/api.gen.go b/packages/api/internal/api/api.gen.go index da9095ae3f..4668cb510a 100644 --- a/packages/api/internal/api/api.gen.go +++ b/packages/api/internal/api/api.gen.go @@ -39,6 +39,45 @@ func (e AWSRegistryType) Valid() bool { } } +// Defines values for CathedralCapabilitiesSchema. +const ( + N1 CathedralCapabilitiesSchema = 1 +) + +// Valid indicates whether the value is a known member of the CathedralCapabilitiesSchema enum. +func (e CathedralCapabilitiesSchema) Valid() bool { + switch e { + case N1: + return true + default: + return false + } +} + +// Defines values for CathedralSandboxOperationState. +const ( + CathedralSandboxOperationStateCreating CathedralSandboxOperationState = "creating" + CathedralSandboxOperationStateFailed CathedralSandboxOperationState = "failed" + CathedralSandboxOperationStateReady CathedralSandboxOperationState = "ready" + CathedralSandboxOperationStateReserved CathedralSandboxOperationState = "reserved" +) + +// Valid indicates whether the value is a known member of the CathedralSandboxOperationState enum. +func (e CathedralSandboxOperationState) Valid() bool { + switch e { + case CathedralSandboxOperationStateCreating: + return true + case CathedralSandboxOperationStateFailed: + return true + case CathedralSandboxOperationStateReady: + return true + case CathedralSandboxOperationStateReserved: + return true + default: + return false + } +} + // Defines values for GCPRegistryType. const ( Gcp GCPRegistryType = "gcp" @@ -413,6 +452,30 @@ type BuildStatusReason struct { // CPUCount CPU cores for the sandbox type CPUCount = int32 +// CathedralCapabilities defines model for CathedralCapabilities. +type CathedralCapabilities struct { + DurableCreateIdempotency bool `json:"durable_create_idempotency"` + OperationLookup bool `json:"operation_lookup"` + SafeFork bool `json:"safe_fork"` + Schema CathedralCapabilitiesSchema `json:"schema"` +} + +// CathedralCapabilitiesSchema defines model for CathedralCapabilities.Schema. +type CathedralCapabilitiesSchema int + +// CathedralSandboxOperation defines model for CathedralSandboxOperation. +type CathedralSandboxOperation struct { + ErrorCode *int `json:"error_code,omitempty"` + ErrorMessage *string `json:"error_message,omitempty"` + IdempotencyKey string `json:"idempotency_key"` + Sandbox *Sandbox `json:"sandbox,omitempty"` + SandboxId string `json:"sandbox_id"` + State CathedralSandboxOperationState `json:"state"` +} + +// CathedralSandboxOperationState defines model for CathedralSandboxOperation.State. +type CathedralSandboxOperationState string + // ConnectSandbox defines model for ConnectSandbox. type ConnectSandbox struct { // Memory Defaults to true. When false and the sandbox is paused, resume from disk state only: the sandbox cold-boots fresh and any memory in the snapshot is ignored, never modified or deleted. Disk state has crash-recovery semantics — writes not flushed before the pause may be lost. A no-op for snapshots that contain no memory. Rejected with an error in environments where this capability is not enabled, never silently downgraded to a memory restore. @@ -1865,6 +1928,9 @@ type ApiKeyID = string // BuildID defines model for buildID. type BuildID = string +// CathedralOperationKey defines model for cathedralOperationKey. +type CathedralOperationKey = string + // ClusterID defines model for clusterID. type ClusterID = openapi_types.UUID @@ -2013,6 +2079,12 @@ type GetSandboxesParams struct { Metadata *string `form:"metadata,omitempty" json:"metadata,omitempty"` } +// PostSandboxesParams defines parameters for PostSandboxes. +type PostSandboxesParams struct { + // IdempotencyKey Durable Cathedral create operation key. Replays with the same authenticated team and request body return the same sandbox; reuse with a different request body is rejected. + IdempotencyKey *string `json:"Idempotency-Key,omitempty"` +} + // GetSandboxesMetricsParams defines parameters for GetSandboxesMetrics. type GetSandboxesMetricsParams struct { // SandboxIds Comma-separated list of sandbox IDs to get metrics for @@ -2726,7 +2798,7 @@ type ClientInterface interface { // Corresponds with POST /sandboxes (the `PostSandboxes` operationId). // // Deprecated: this operation has been marked as deprecated upstream, but no `x-deprecated-reason` was set - PostSandboxesWithBody(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) + PostSandboxesWithBody(ctx context.Context, params *PostSandboxesParams, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) // PostSandboxes Create sandbox // @@ -2737,7 +2809,7 @@ type ClientInterface interface { // Corresponds with POST /sandboxes (the `PostSandboxes` operationId). // // Deprecated: this operation has been marked as deprecated upstream, but no `x-deprecated-reason` was set - PostSandboxes(ctx context.Context, body PostSandboxesJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) + PostSandboxes(ctx context.Context, params *PostSandboxesParams, body PostSandboxesJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) // GetSandboxesMetrics List sandbox metrics // @@ -3125,6 +3197,16 @@ type ClientInterface interface { // Corresponds with GET /templates/{templateID}/tags (the `GetTemplatesTemplateIDTags` operationId). GetTemplatesTemplateIDTags(ctx context.Context, templateID TemplateID, reqEditors ...RequestEditorFn) (*http.Response, error) + // GetV1CathedralCapabilities Get the Cathedral durability contract supported by this control plane + // + // Corresponds with GET /v1/cathedral/capabilities (the `GetV1CathedralCapabilities` operationId). + GetV1CathedralCapabilities(ctx context.Context, reqEditors ...RequestEditorFn) (*http.Response, error) + + // GetV1CathedralOperationsIdempotencyKey Recover a Cathedral create operation by its durable idempotency key + // + // Corresponds with GET /v1/cathedral/operations/{idempotencyKey} (the `GetV1CathedralOperationsIdempotencyKey` operationId). + GetV1CathedralOperationsIdempotencyKey(ctx context.Context, idempotencyKey CathedralOperationKey, reqEditors ...RequestEditorFn) (*http.Response, error) + // GetV2Sandboxes List sandboxes (v2) // // List all sandboxes. @@ -3881,8 +3963,8 @@ func (c *Client) GetSandboxes(ctx context.Context, params *GetSandboxesParams, r // // Corresponds with POST /sandboxes (the `PostSandboxes` operationId). // Deprecated: this operation has been marked as deprecated upstream, but no `x-deprecated-reason` was set -func (c *Client) PostSandboxesWithBody(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) { - req, err := NewPostSandboxesRequestWithBody(c.Server, contentType, body) +func (c *Client) PostSandboxesWithBody(ctx context.Context, params *PostSandboxesParams, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewPostSandboxesRequestWithBody(c.Server, params, contentType, body) if err != nil { return nil, err } @@ -3901,8 +3983,8 @@ func (c *Client) PostSandboxesWithBody(ctx context.Context, contentType string, // // Corresponds with POST /sandboxes (the `PostSandboxes` operationId). // Deprecated: this operation has been marked as deprecated upstream, but no `x-deprecated-reason` was set -func (c *Client) PostSandboxes(ctx context.Context, body PostSandboxesJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) { - req, err := NewPostSandboxesRequest(c.Server, body) +func (c *Client) PostSandboxes(ctx context.Context, params *PostSandboxesParams, body PostSandboxesJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewPostSandboxesRequest(c.Server, params, body) if err != nil { return nil, err } @@ -4751,6 +4833,36 @@ func (c *Client) GetTemplatesTemplateIDTags(ctx context.Context, templateID Temp return c.Client.Do(req) } +// GetV1CathedralCapabilities Get the Cathedral durability contract supported by this control plane +// +// Corresponds with GET /v1/cathedral/capabilities (the `GetV1CathedralCapabilities` operationId). +func (c *Client) GetV1CathedralCapabilities(ctx context.Context, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewGetV1CathedralCapabilitiesRequest(c.Server) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if err := c.applyEditors(ctx, req, reqEditors); err != nil { + return nil, err + } + return c.Client.Do(req) +} + +// GetV1CathedralOperationsIdempotencyKey Recover a Cathedral create operation by its durable idempotency key +// +// Corresponds with GET /v1/cathedral/operations/{idempotencyKey} (the `GetV1CathedralOperationsIdempotencyKey` operationId). +func (c *Client) GetV1CathedralOperationsIdempotencyKey(ctx context.Context, idempotencyKey CathedralOperationKey, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewGetV1CathedralOperationsIdempotencyKeyRequest(c.Server, idempotencyKey) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if err := c.applyEditors(ctx, req, reqEditors); err != nil { + return nil, err + } + return c.Client.Do(req) +} + // GetV2Sandboxes List sandboxes (v2) // // List all sandboxes. @@ -6492,18 +6604,18 @@ func NewGetSandboxesRequest(server string, params *GetSandboxesParams) (*http.Re } // NewPostSandboxesRequest calls the generic PostSandboxes builder with application/json body -func NewPostSandboxesRequest(server string, body PostSandboxesJSONRequestBody) (*http.Request, error) { +func NewPostSandboxesRequest(server string, params *PostSandboxesParams, body PostSandboxesJSONRequestBody) (*http.Request, error) { var bodyReader io.Reader buf, err := json.Marshal(body) if err != nil { return nil, err } bodyReader = bytes.NewReader(buf) - return NewPostSandboxesRequestWithBody(server, "application/json", bodyReader) + return NewPostSandboxesRequestWithBody(server, params, "application/json", bodyReader) } // NewPostSandboxesRequestWithBody constructs an http.Request for the PostSandboxes method, with any body, and a specified content type -func NewPostSandboxesRequestWithBody(server string, contentType string, body io.Reader) (*http.Request, error) { +func NewPostSandboxesRequestWithBody(server string, params *PostSandboxesParams, contentType string, body io.Reader) (*http.Request, error) { var err error serverURL, err := url.Parse(server) @@ -6528,6 +6640,21 @@ func NewPostSandboxesRequestWithBody(server string, contentType string, body io. req.Header.Add("Content-Type", contentType) + if params != nil { + + if params.IdempotencyKey != nil { + var headerParam0 string + + headerParam0, err = runtime.StyleParamWithOptions("simple", false, "Idempotency-Key", *params.IdempotencyKey, runtime.StyleParamOptions{ParamLocation: runtime.ParamLocationHeader, Type: "string", Format: ""}) + if err != nil { + return nil, err + } + + req.Header.Set("Idempotency-Key", headerParam0) + } + + } + return req, nil } @@ -8272,6 +8399,67 @@ func NewGetTemplatesTemplateIDTagsRequest(server string, templateID TemplateID) return req, nil } +// NewGetV1CathedralCapabilitiesRequest constructs an http.Request for the GetV1CathedralCapabilities method +func NewGetV1CathedralCapabilitiesRequest(server string) (*http.Request, error) { + var err error + + serverURL, err := url.Parse(server) + if err != nil { + return nil, err + } + + operationPath := fmt.Sprintf("/v1/cathedral/capabilities") + if operationPath[0] == '/' { + operationPath = "." + operationPath + } + + queryURL, err := serverURL.Parse(operationPath) + if err != nil { + return nil, err + } + + req, err := http.NewRequest(http.MethodGet, queryURL.String(), nil) + if err != nil { + return nil, err + } + + return req, nil +} + +// NewGetV1CathedralOperationsIdempotencyKeyRequest constructs an http.Request for the GetV1CathedralOperationsIdempotencyKey method +func NewGetV1CathedralOperationsIdempotencyKeyRequest(server string, idempotencyKey CathedralOperationKey) (*http.Request, error) { + var err error + + var pathParam0 string + + pathParam0, err = runtime.StyleParamWithOptions("simple", false, "idempotencyKey", idempotencyKey, runtime.StyleParamOptions{ParamLocation: runtime.ParamLocationPath, Type: "string", Format: ""}) + if err != nil { + return nil, err + } + + serverURL, err := url.Parse(server) + if err != nil { + return nil, err + } + + operationPath := fmt.Sprintf("/v1/cathedral/operations/%s", pathParam0) + if operationPath[0] == '/' { + operationPath = "." + operationPath + } + + queryURL, err := serverURL.Parse(operationPath) + if err != nil { + return nil, err + } + + req, err := http.NewRequest(http.MethodGet, queryURL.String(), nil) + if err != nil { + return nil, err + } + + return req, nil +} + // NewGetV2SandboxesRequest constructs an http.Request for the GetV2Sandboxes method func NewGetV2SandboxesRequest(server string, params *GetV2SandboxesParams) (*http.Request, error) { var err error @@ -9295,7 +9483,7 @@ type ClientWithResponsesInterface interface { // Corresponds with POST /sandboxes (the `PostSandboxes` operationId). // // Deprecated: this operation has been marked as deprecated upstream, but no `x-deprecated-reason` was set - PostSandboxesWithBodyWithResponse(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*PostSandboxesResponse, error) + PostSandboxesWithBodyWithResponse(ctx context.Context, params *PostSandboxesParams, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*PostSandboxesResponse, error) // PostSandboxesWithResponse Create sandbox // @@ -9306,7 +9494,7 @@ type ClientWithResponsesInterface interface { // Corresponds with POST /sandboxes (the `PostSandboxes` operationId). // // Deprecated: this operation has been marked as deprecated upstream, but no `x-deprecated-reason` was set - PostSandboxesWithResponse(ctx context.Context, body PostSandboxesJSONRequestBody, reqEditors ...RequestEditorFn) (*PostSandboxesResponse, error) + PostSandboxesWithResponse(ctx context.Context, params *PostSandboxesParams, body PostSandboxesJSONRequestBody, reqEditors ...RequestEditorFn) (*PostSandboxesResponse, error) // GetSandboxesMetricsWithResponse List sandbox metrics // @@ -9734,6 +9922,20 @@ type ClientWithResponsesInterface interface { // Corresponds with GET /templates/{templateID}/tags (the `GetTemplatesTemplateIDTags` operationId). GetTemplatesTemplateIDTagsWithResponse(ctx context.Context, templateID TemplateID, reqEditors ...RequestEditorFn) (*GetTemplatesTemplateIDTagsResponse, error) + // GetV1CathedralCapabilitiesWithResponse Get the Cathedral durability contract supported by this control plane + // + // Returns a wrapper object for the known response body format(s). + // + // Corresponds with GET /v1/cathedral/capabilities (the `GetV1CathedralCapabilities` operationId). + GetV1CathedralCapabilitiesWithResponse(ctx context.Context, reqEditors ...RequestEditorFn) (*GetV1CathedralCapabilitiesResponse, error) + + // GetV1CathedralOperationsIdempotencyKeyWithResponse Recover a Cathedral create operation by its durable idempotency key + // + // Returns a wrapper object for the known response body format(s). + // + // Corresponds with GET /v1/cathedral/operations/{idempotencyKey} (the `GetV1CathedralOperationsIdempotencyKey` operationId). + GetV1CathedralOperationsIdempotencyKeyWithResponse(ctx context.Context, idempotencyKey CathedralOperationKey, reqEditors ...RequestEditorFn) (*GetV1CathedralOperationsIdempotencyKeyResponse, error) + // GetV2SandboxesWithResponse List sandboxes (v2) // // List all sandboxes. @@ -11941,6 +12143,11 @@ func (r GetSandboxesResponse) ContentType() string { return "" } +// PostSandboxesResponse201Headers the declared response headers of an HTTP 201 response for PostSandboxes +type PostSandboxesResponse201Headers struct { + XE2BIdempotencyKey *string +} + // PostSandboxesResponse429Headers the declared response headers of an HTTP 429 response for PostSandboxes type PostSandboxesResponse429Headers struct { RetryAfter *int @@ -11955,6 +12162,8 @@ type PostSandboxesResponse struct { JSON400 *N400 // JSON401 the response for an HTTP 401 `application/json` response JSON401 *N401 + // JSON409 the response for an HTTP 409 `application/json` response + JSON409 *N409 // JSON429 the response for an HTTP 429 `application/json` response JSON429 *N429 // JSON500 the response for an HTTP 500 `application/json` response @@ -11963,6 +12172,8 @@ type PostSandboxesResponse struct { JSON503 *N503 // JSON504 the response for an HTTP 504 `application/json` response JSON504 *N504 + // Headers201 the parsed response headers for an HTTP 201 response + Headers201 *PostSandboxesResponse201Headers // Headers429 the parsed response headers for an HTTP 429 response Headers429 *PostSandboxesResponse429Headers } @@ -11982,6 +12193,11 @@ func (r PostSandboxesResponse) GetJSON401() *N401 { return r.JSON401 } +// GetJSON409 returns the response for an HTTP 409 `application/json` response +func (r PostSandboxesResponse) GetJSON409() *N409 { + return r.JSON409 +} + // GetJSON429 returns the response for an HTTP 429 `application/json` response func (r PostSandboxesResponse) GetJSON429() *N429 { return r.JSON429 @@ -14791,6 +15007,130 @@ func (r GetTemplatesTemplateIDTagsResponse) ContentType() string { return "" } +type GetV1CathedralCapabilitiesResponse struct { + Body []byte + HTTPResponse *http.Response + // JSON200 the response for an HTTP 200 `application/json` response + JSON200 *CathedralCapabilities + // JSON401 the response for an HTTP 401 `application/json` response + JSON401 *N401 + // JSON500 the response for an HTTP 500 `application/json` response + JSON500 *N500 +} + +// GetJSON200 returns the response for an HTTP 200 `application/json` response +func (r GetV1CathedralCapabilitiesResponse) GetJSON200() *CathedralCapabilities { + return r.JSON200 +} + +// GetJSON401 returns the response for an HTTP 401 `application/json` response +func (r GetV1CathedralCapabilitiesResponse) GetJSON401() *N401 { + return r.JSON401 +} + +// GetJSON500 returns the response for an HTTP 500 `application/json` response +func (r GetV1CathedralCapabilitiesResponse) GetJSON500() *N500 { + return r.JSON500 +} + +// GetBody returns the raw response body bytes +func (r GetV1CathedralCapabilitiesResponse) GetBody() []byte { + return r.Body +} + +// Status returns HTTPResponse.Status +func (r GetV1CathedralCapabilitiesResponse) Status() string { + if r.HTTPResponse != nil { + return r.HTTPResponse.Status + } + return http.StatusText(0) +} + +// StatusCode returns HTTPResponse.StatusCode +func (r GetV1CathedralCapabilitiesResponse) StatusCode() int { + if r.HTTPResponse != nil { + return r.HTTPResponse.StatusCode + } + return 0 +} + +// ContentType is a convenience method to retrieve the Content-Type value from the HTTP response headers +func (r GetV1CathedralCapabilitiesResponse) ContentType() string { + if r.HTTPResponse != nil { + return r.HTTPResponse.Header.Get("Content-Type") + } + return "" +} + +type GetV1CathedralOperationsIdempotencyKeyResponse struct { + Body []byte + HTTPResponse *http.Response + // JSON200 the response for an HTTP 200 `application/json` response + JSON200 *CathedralSandboxOperation + // JSON400 the response for an HTTP 400 `application/json` response + JSON400 *N400 + // JSON401 the response for an HTTP 401 `application/json` response + JSON401 *N401 + // JSON404 the response for an HTTP 404 `application/json` response + JSON404 *N404 + // JSON500 the response for an HTTP 500 `application/json` response + JSON500 *N500 +} + +// GetJSON200 returns the response for an HTTP 200 `application/json` response +func (r GetV1CathedralOperationsIdempotencyKeyResponse) GetJSON200() *CathedralSandboxOperation { + return r.JSON200 +} + +// GetJSON400 returns the response for an HTTP 400 `application/json` response +func (r GetV1CathedralOperationsIdempotencyKeyResponse) GetJSON400() *N400 { + return r.JSON400 +} + +// GetJSON401 returns the response for an HTTP 401 `application/json` response +func (r GetV1CathedralOperationsIdempotencyKeyResponse) GetJSON401() *N401 { + return r.JSON401 +} + +// GetJSON404 returns the response for an HTTP 404 `application/json` response +func (r GetV1CathedralOperationsIdempotencyKeyResponse) GetJSON404() *N404 { + return r.JSON404 +} + +// GetJSON500 returns the response for an HTTP 500 `application/json` response +func (r GetV1CathedralOperationsIdempotencyKeyResponse) GetJSON500() *N500 { + return r.JSON500 +} + +// GetBody returns the raw response body bytes +func (r GetV1CathedralOperationsIdempotencyKeyResponse) GetBody() []byte { + return r.Body +} + +// Status returns HTTPResponse.Status +func (r GetV1CathedralOperationsIdempotencyKeyResponse) Status() string { + if r.HTTPResponse != nil { + return r.HTTPResponse.Status + } + return http.StatusText(0) +} + +// StatusCode returns HTTPResponse.StatusCode +func (r GetV1CathedralOperationsIdempotencyKeyResponse) StatusCode() int { + if r.HTTPResponse != nil { + return r.HTTPResponse.StatusCode + } + return 0 +} + +// ContentType is a convenience method to retrieve the Content-Type value from the HTTP response headers +func (r GetV1CathedralOperationsIdempotencyKeyResponse) ContentType() string { + if r.HTTPResponse != nil { + return r.HTTPResponse.Header.Get("Content-Type") + } + return "" +} + // GetV2SandboxesResponse200Headers the declared response headers of an HTTP 200 response for GetV2Sandboxes type GetV2SandboxesResponse200Headers struct { XNextToken *string @@ -16280,8 +16620,8 @@ func (c *ClientWithResponses) GetSandboxesWithResponse(ctx context.Context, para // Corresponds with POST /sandboxes (the `PostSandboxes` operationId). // // Deprecated: this operation has been marked as deprecated upstream, but no `x-deprecated-reason` was set -func (c *ClientWithResponses) PostSandboxesWithBodyWithResponse(ctx context.Context, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*PostSandboxesResponse, error) { - rsp, err := c.PostSandboxesWithBody(ctx, contentType, body, reqEditors...) +func (c *ClientWithResponses) PostSandboxesWithBodyWithResponse(ctx context.Context, params *PostSandboxesParams, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*PostSandboxesResponse, error) { + rsp, err := c.PostSandboxesWithBody(ctx, params, contentType, body, reqEditors...) if err != nil { return nil, err } @@ -16296,8 +16636,8 @@ func (c *ClientWithResponses) PostSandboxesWithBodyWithResponse(ctx context.Cont // // Corresponds with POST /sandboxes (the `PostSandboxes` operationId). // Deprecated: this operation has been marked as deprecated upstream, but no `x-deprecated-reason` was set -func (c *ClientWithResponses) PostSandboxesWithResponse(ctx context.Context, body PostSandboxesJSONRequestBody, reqEditors ...RequestEditorFn) (*PostSandboxesResponse, error) { - rsp, err := c.PostSandboxes(ctx, body, reqEditors...) +func (c *ClientWithResponses) PostSandboxesWithResponse(ctx context.Context, params *PostSandboxesParams, body PostSandboxesJSONRequestBody, reqEditors ...RequestEditorFn) (*PostSandboxesResponse, error) { + rsp, err := c.PostSandboxes(ctx, params, body, reqEditors...) if err != nil { return nil, err } @@ -17003,6 +17343,32 @@ func (c *ClientWithResponses) GetTemplatesTemplateIDTagsWithResponse(ctx context return ParseGetTemplatesTemplateIDTagsResponse(rsp) } +// GetV1CathedralCapabilitiesWithResponse Get the Cathedral durability contract supported by this control plane +// +// Returns a wrapper object for the known response body format(s). +// +// Corresponds with GET /v1/cathedral/capabilities (the `GetV1CathedralCapabilities` operationId). +func (c *ClientWithResponses) GetV1CathedralCapabilitiesWithResponse(ctx context.Context, reqEditors ...RequestEditorFn) (*GetV1CathedralCapabilitiesResponse, error) { + rsp, err := c.GetV1CathedralCapabilities(ctx, reqEditors...) + if err != nil { + return nil, err + } + return ParseGetV1CathedralCapabilitiesResponse(rsp) +} + +// GetV1CathedralOperationsIdempotencyKeyWithResponse Recover a Cathedral create operation by its durable idempotency key +// +// Returns a wrapper object for the known response body format(s). +// +// Corresponds with GET /v1/cathedral/operations/{idempotencyKey} (the `GetV1CathedralOperationsIdempotencyKey` operationId). +func (c *ClientWithResponses) GetV1CathedralOperationsIdempotencyKeyWithResponse(ctx context.Context, idempotencyKey CathedralOperationKey, reqEditors ...RequestEditorFn) (*GetV1CathedralOperationsIdempotencyKeyResponse, error) { + rsp, err := c.GetV1CathedralOperationsIdempotencyKey(ctx, idempotencyKey, reqEditors...) + if err != nil { + return nil, err + } + return ParseGetV1CathedralOperationsIdempotencyKeyResponse(rsp) +} + // GetV2SandboxesWithResponse List sandboxes (v2) // // List all sandboxes. @@ -19070,6 +19436,13 @@ func ParsePostSandboxesResponse(rsp *http.Response) (*PostSandboxesResponse, err } response.JSON401 = &dest + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 409: + var dest N409 + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON409 = &dest + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 429: var dest N429 if err := json.Unmarshal(bodyBytes, &dest); err != nil { @@ -19101,6 +19474,16 @@ func ParsePostSandboxesResponse(rsp *http.Response) (*PostSandboxesResponse, err } switch { + case rsp.StatusCode == 201: + var headers PostSandboxesResponse201Headers + if values := rsp.Header.Values("X-E2B-Idempotency-Key"); len(values) > 0 { + var value string + if err := runtime.BindStyledParameterWithOptions("simple", "X-E2B-Idempotency-Key", values[0], &value, runtime.BindStyledParameterOptions{ParamLocation: runtime.ParamLocationHeader, Explode: false, Required: false, Type: "string", Format: ""}); err != nil { + return nil, err + } + headers.XE2BIdempotencyKey = &value + } + response.Headers201 = &headers case rsp.StatusCode == 429: var headers PostSandboxesResponse429Headers if values := rsp.Header.Values("Retry-After"); len(values) > 0 { @@ -21615,6 +21998,100 @@ func ParseGetTemplatesTemplateIDTagsResponse(rsp *http.Response) (*GetTemplatesT return response, nil } +// ParseGetV1CathedralCapabilitiesResponse parses an HTTP response from a GetV1CathedralCapabilitiesWithResponse call +func ParseGetV1CathedralCapabilitiesResponse(rsp *http.Response) (*GetV1CathedralCapabilitiesResponse, error) { + bodyBytes, err := io.ReadAll(rsp.Body) + defer func() { _ = rsp.Body.Close() }() + if err != nil { + return nil, err + } + + response := &GetV1CathedralCapabilitiesResponse{ + Body: bodyBytes, + HTTPResponse: rsp, + } + + switch { + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 200: + var dest CathedralCapabilities + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON200 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 401: + var dest N401 + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON401 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 500: + var dest N500 + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON500 = &dest + + } + + return response, nil +} + +// ParseGetV1CathedralOperationsIdempotencyKeyResponse parses an HTTP response from a GetV1CathedralOperationsIdempotencyKeyWithResponse call +func ParseGetV1CathedralOperationsIdempotencyKeyResponse(rsp *http.Response) (*GetV1CathedralOperationsIdempotencyKeyResponse, error) { + bodyBytes, err := io.ReadAll(rsp.Body) + defer func() { _ = rsp.Body.Close() }() + if err != nil { + return nil, err + } + + response := &GetV1CathedralOperationsIdempotencyKeyResponse{ + Body: bodyBytes, + HTTPResponse: rsp, + } + + switch { + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 200: + var dest CathedralSandboxOperation + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON200 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 400: + var dest N400 + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON400 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 401: + var dest N401 + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON401 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 404: + var dest N404 + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON404 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 500: + var dest N500 + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON500 = &dest + + } + + return response, nil +} + // ParseGetV2SandboxesResponse parses an HTTP response from a GetV2SandboxesWithResponse call func ParseGetV2SandboxesResponse(rsp *http.Response) (*GetV2SandboxesResponse, error) { bodyBytes, err := io.ReadAll(rsp.Body) @@ -22593,7 +23070,7 @@ type ServerInterface interface { // (POST /sandboxes) // // Deprecated: this operation has been marked as deprecated upstream, but no `x-deprecated-reason` was set - PostSandboxes(c *gin.Context) + PostSandboxes(c *gin.Context, params PostSandboxesParams) // GetSandboxesMetrics List sandbox metrics // (GET /sandboxes/metrics) GetSandboxesMetrics(c *gin.Context, params GetSandboxesMetricsParams) @@ -22703,6 +23180,12 @@ type ServerInterface interface { // GetTemplatesTemplateIDTags List template tags // (GET /templates/{templateID}/tags) GetTemplatesTemplateIDTags(c *gin.Context, templateID TemplateID) + // GetV1CathedralCapabilities Get the Cathedral durability contract supported by this control plane + // (GET /v1/cathedral/capabilities) + GetV1CathedralCapabilities(c *gin.Context) + // GetV1CathedralOperationsIdempotencyKey Recover a Cathedral create operation by its durable idempotency key + // (GET /v1/cathedral/operations/{idempotencyKey}) + GetV1CathedralOperationsIdempotencyKey(c *gin.Context, idempotencyKey CathedralOperationKey) // GetV2Sandboxes List sandboxes (v2) // (GET /v2/sandboxes) GetV2Sandboxes(c *gin.Context, params GetV2SandboxesParams) @@ -23602,6 +24085,33 @@ func (siw *ServerInterfaceWrapper) GetSandboxes(c *gin.Context) { // PostSandboxes operation middleware func (siw *ServerInterfaceWrapper) PostSandboxes(c *gin.Context) { + var err error + _ = err + + // Parameter object where we will unmarshal all parameters from the context + var params PostSandboxesParams + + headers := c.Request.Header + + // ------------- Optional header parameter "Idempotency-Key" ------------- + if valueList, found := headers[http.CanonicalHeaderKey("Idempotency-Key")]; found { + var IdempotencyKey string + n := len(valueList) + if n != 1 { + siw.ErrorHandler(c, fmt.Errorf("Expected one value for Idempotency-Key, got %d", n), http.StatusBadRequest) + return + } + + err = runtime.BindStyledParameterWithOptions("simple", "Idempotency-Key", valueList[0], &IdempotencyKey, runtime.BindStyledParameterOptions{ParamLocation: runtime.ParamLocationHeader, Explode: false, Required: false, Type: "string", Format: ""}) + if err != nil { + siw.ErrorHandler(c, fmt.Errorf("Invalid format for parameter Idempotency-Key: %w", err), http.StatusBadRequest) + return + } + + params.IdempotencyKey = &IdempotencyKey + + } + for _, middleware := range siw.HandlerMiddlewares { middleware(c) if c.IsAborted() { @@ -23609,7 +24119,7 @@ func (siw *ServerInterfaceWrapper) PostSandboxes(c *gin.Context) { } } - siw.Handler.PostSandboxes(c) + siw.Handler.PostSandboxes(c, params) } // GetSandboxesMetrics operation middleware @@ -24629,6 +25139,44 @@ func (siw *ServerInterfaceWrapper) GetTemplatesTemplateIDTags(c *gin.Context) { siw.Handler.GetTemplatesTemplateIDTags(c, templateID) } +// GetV1CathedralCapabilities operation middleware +func (siw *ServerInterfaceWrapper) GetV1CathedralCapabilities(c *gin.Context) { + + for _, middleware := range siw.HandlerMiddlewares { + middleware(c) + if c.IsAborted() { + return + } + } + + siw.Handler.GetV1CathedralCapabilities(c) +} + +// GetV1CathedralOperationsIdempotencyKey operation middleware +func (siw *ServerInterfaceWrapper) GetV1CathedralOperationsIdempotencyKey(c *gin.Context) { + + var err error + _ = err + + // ------------- Path parameter "idempotencyKey" ------------- + var idempotencyKey CathedralOperationKey + + err = runtime.BindStyledParameterWithOptions("simple", "idempotencyKey", c.Param("idempotencyKey"), &idempotencyKey, runtime.BindStyledParameterOptions{ParamLocation: runtime.ParamLocationPath, Explode: false, Required: true, Type: "string", Format: "", ValueIsUnescaped: true}) + if err != nil { + siw.ErrorHandler(c, fmt.Errorf("Invalid format for parameter idempotencyKey: %w", err), http.StatusBadRequest) + return + } + + for _, middleware := range siw.HandlerMiddlewares { + middleware(c) + if c.IsAborted() { + return + } + } + + siw.Handler.GetV1CathedralOperationsIdempotencyKey(c, idempotencyKey) +} + // GetV2Sandboxes operation middleware func (siw *ServerInterfaceWrapper) GetV2Sandboxes(c *gin.Context) { @@ -25032,6 +25580,8 @@ func RegisterHandlersWithOptions(router gin.IRouter, si ServerInterface, options router.GET(options.BaseURL+"/teams", wrapper.GetTeams) router.GET(options.BaseURL+"/teams/:teamID/metrics", wrapper.GetTeamsTeamIDMetrics) router.GET(options.BaseURL+"/teams/:teamID/metrics/max", wrapper.GetTeamsTeamIDMetricsMax) + router.GET(options.BaseURL+"/v1/cathedral/capabilities", wrapper.GetV1CathedralCapabilities) + router.GET(options.BaseURL+"/v1/cathedral/operations/:idempotencyKey", wrapper.GetV1CathedralOperationsIdempotencyKey) router.GET(options.BaseURL+"/sandboxes", wrapper.GetSandboxes) router.POST(options.BaseURL+"/sandboxes", wrapper.PostSandboxes) router.GET(options.BaseURL+"/v2/sandboxes", wrapper.GetV2Sandboxes) @@ -25109,304 +25659,317 @@ func RegisterHandlersWithOptions(router gin.IRouter, si ServerInterface, options // const string: with thousands of chunks the chained `+` fold is several // times slower for the Go compiler than parsing a slice literal. var swaggerSpec = []string{ - "7P3rchs39iiKvwqK/10Ve/+pi+VkauLUfJAlO6Pf+KKS5GT2Hvt4Q90giZ+aAAdAS+K4XHUe4jzheZJT", - "a+HSaDa62aQkWk5Y+RCZjTvWDev6ZZDJ6UwKJowevPgymDCaM4V//vMduzUX8ooJ+FfOdKb4zHApBi8G", - "R6XSUhEjyYiZbELMhBHBbg2Z0TEjckQU02Vh9JDwEZlKxQi75doMhgOdTdiUwohmPmODFwNtFBfjwdev", - "w8E/L6ShxVkpBPzSmPRdOb1kCke3TYimIr+Ut0yTKTXZBH6ClYx4YZjSQ3LJRjD3jI65oDAK4ZrQ2azg", - "LN8l70UxJzPFNBOG3EyYSIx7wxQjiv27ZNqwfPejqG1hJNWUmsGLARfm+cFg6PfEhWFjpgZfYVczquiU", - "GXeqdMb/weYnx/A3h13NqJkMhgNBp9AzfB4OYFauWD54YVTJuk/usuRF3jqo/7ramFlRasPUyXHzJk5y", - "JgwfcXsbcOSu8WCYmr8aqWsF4TDLkueDYWJFQuasdZPu42p7rCDjDZ9y09zpW3rLp+WUiAB73LCpBshX", - "zJRKkBlTCPV+6/8umZpXyypw3HgVORvRsjCDF8/294dNEJraGd3nKRfuXwngitffC1m1ocrgfRVcGzJS", - "ctqybBGG6z5AxccpADnjY8IrIHnCdse75KPf+sfB0zSg2NFWu0KHq61wUX1fcVyWKWbah/Wfu0ZdhjV2", - "EPJEs+wzUKIRv2X50yGRinCjSUaFFDyjBSnkDVM7GdWMwPxIhppLNoxOWxfsPq52CIZNZwU1rGPU0GC1", - "ka9lUU7bxw2fVxv1hl1OpLxqHbb6fhdKBHDP9EwKzZCm/7i/D//LpDBMGEvlZwXPEDH3/ltLRMpq/P+h", - "2GjwYvD/26t47579qvdeKSUd46hDz0uae0Y0+Doc/Lj/7OHnPCzNBGDWjkqYbQeTP3/4yV9LdcnznAk7", - "448PP+M7achIliK3M/788DMeSTEqeGZv9GADE15ISaZUzD0o6cEwlvnOmFHzncORYapJv34HEcnJS0Mr", - "8gWuqFkmRY588YZy4yUvBeN5scxNuTsYDtgtnc4KNnjxfD9GvMDt9lOi1Nfh4KdNYNo5U9dMVdD+0/6z", - "zcAehzOZMmFYTi7nxEy4JjmbFXIOP9qlHGyC0mRXTOTxATzfzKnzjJFS0GvKC3pZMDv3j5vbseFTJktj", - "+b/tBGMe/n5+xsZcGzWHf86UnDFluCX+9EYfZhnTGuT2vIk1h7+fE9uA/IPNyckxGUlFXh2dEVqjrk0+", - "M4SxYWIp0sPab/B0UQwxDEZVbqWEa1LIjBqWtwx9jtJHWHx6Dtso3kH/5dsfFke9mM/cE9EttDEQE0AD", - "/gVrHHxKCToV6/6X/TpcvIbkBuMDrcaVl//NLAU+zKdcvITH0hEVGSvO8BHbvPIMvxYsP5KlMF0PVXx5", - "aaJLXMOoLIo5Cb0T78XhYET5CgObCTXEdgHSa4ceJJ8K8ZktbKA+6yd/EudWcv4HL1pPoudqq/f0woKv", - "eFEkjwE+rDRw7Yht7+XncMWXHsIFo1OnkXDngQ0s6uc5hyXR4rR+KtGb7i8/DrpfcQ2RgGYTlpOCXzO/", - "PcJFzm5JBhOTKzZ37IHRKTk53iV2QWRK5+RScTYq5h8FF1lR5iw+eUWFxuUCP5aliRQpv+BgmtxwM4Ev", - "OB/LP4qqO1WMyCk3QQvSxB6t+VhcuAfBBR3rMyeuNsDG0LFOEAY6RgGC4kDwF9A0/8IYDAf49E4I/mEx", - "VCk6x39TNWYmNQX8HsYkXJCP+DZ4Yej444C4m1tKc+zwQ7uRT2HzLI+339x3pKZZ9jTEpnAUMuNAw/Fu", - "4ItmBGcdLnumDFuO2S8Vh/HTrXHKjTPBRfktwqEgKX0jx69EknMW7JoVy3j2Gzl+g+2+DgdTpjUdJ1jK", - "Gzkm7iPxkkLiPLRhs2bnc8NmAAjVqc+URG6nWIFH7yCxkGPCcCups+ZTpg2dJia48J/8YccDhUvMqWE7", - "MMpy6AtTVUcydKcZjv3cUFPqM0adhLRw9PZS3L+CRupfn4aJk2W25eJxaJyBKDtFBDdd11kHiQTmtt7x", - "W/sh4EF9/iHJSqWYMAU8bWZSGaRyorDyCoqyrseKkBFxrKU34xcPt3B0+qGFfR2dfiCZVEzj0nArlswO", - "UurATtZxJIVgmXGcqXnPUzaVKiHZHdsbR3JrVMl2CT7uRrTQjFCRx6siXJMZLTXLh6jWnzJUHJKc6ys8", - "Uobn/KLWJ5NFvnMppdFkpJie4KDw7rQr8vimBZ3piTQwBx8LqWASweD5NZU5EMScSEVyVjBgPOS4mnNC", - "NckU1ZMdxTJ5zdScaDalwvBMk//3//5/yI3ihmki4FFflBqYqnuVwsy4I8szASG12SWHRMgdOcNb8Qtz", - "EgsQFcoFEdJtYJecMWB8njBTpx2BjTFxzZUU8GbTQTjnmmR0Ri95wQ3K5rAuJuCZE7aseWEhOJc3Yqxo", - "bpGN+kNTTBup2G4Fh5dSFowKT4Dg7ZIkP5aph1c63h+q7S3S4MOHUHj2k5sJzya1u9QTWRY5Ybczrlgn", - "jO4vlbj8KlPidx2YfzvYgvMWnGvs6TlYRL4ZcKctME0oVgy4JLwbDk9P3Lt64f1omxyaJbLC4ekJiPsE", - "29uncR9xYegneIlz06J4Pxq8+Fc3Y4b1ftCwqU/DgSgLq4BB1fjX4YDnfWRWt94+oulVSt9wRm/INS1K", - "1hywMUBBtfmgWWJdb6h2d45g6g/xhmoCWN92iPU9J2acUn21TLypzuQt1VdcjI+ZobzQ0N+aIBpvWTpd", - "vt0FIopHahviotzYwwiwQAI5RjLT60G2fG3RA6HnO8M/5yy1W/9h4fYWnlpANN8yo3iWeGHlDJSIKQ4B", - "vxM/1uICRrxgeq4Nm14kdVavw3cCfa1Bc0jYrflxSG5H+mlq0CmIf6eSp2TAt/CNzOCjP2FgQMnTBaeI", - "l3PDUmcM34ie0QzfspfYKkY/r4Foio6ACy2jAl6tM+iiNFztf+gvpnHU8UJqe/VXfc7/w96+TNwo8k7+", - "H7YoRcOa3/KXqwoqw8Ercf0bVZ2anfoSXlWckVxTxYF8pIT6Jja/Etf5b0zppGrXffBwwcR1HjxTuOge", - "eziwSu4mz5F5Aq6xMcFvw+XeLMMBCgWf02O9pdmEC7ajGM3hJIL84kQJ6LVLwM5BSVZIhDFmfiFccMNp", - "YSm/fuH39hnki4yb+efIIjAMX2cFzdBW8tlJCtUnIT8DaaaGXxbss5B51M1SyM/2PTcksDUlaPFZo83n", - "M650N4nMbe9Se97LSLY74viB+FrJ6cmUjllsW8g5jD3lghp7i1M6mzlvKFCHt/Cd2EIxHIyzWVvDX49O", - "o4YqzNzSmgmmaBF6fB16qJq/c/Z/2PXX4UAK1kPIiJf5ddjdNl7p0raL64TzjQdooIO21qbDDJWq/6VT", - "eOgtUq4R+a/z9+8Qu389Ot2A9QNusa/1I7Gd1Atr8ZwaxzKjWt9IlZCqTt0X4Ojw1PBUTlXQdO8nEMb+", - "lBi81IC5KbHlg/vSf6npQw0zDKtzSZ1qq9DXfL9SfcXy34DQnaK3T+Kc8XeUVAkXxPYg13WWYMAzi0jV", - "JhxH85yXo+Q89vc7zjPr3gS+NYMbmG4MSdxBN8ZFVvCGibGZJOR7/L17iW0iiVtwfYZh4l5SZwhE5Q3X", - "huWt+jZacJpSucPPfSTprOBMGG8hmClm7bfuSbLcCZO3aL6zWRmUkV2ENCgtwSxVE766ekViGsgIovVl", - "a71sY1nthhdF4vHd+bpldeGp0+IfNUUmDoqH5Rt669thH0NzapY6FziYeOubL7omLru8DpEO3TbZKqdK", - "NXGdep8qqqF6bvIc2zb8A5dt0be2Khqri+G6tnL3gk0SBXQJfBusr73sDV6ZWPVdbsiKXRpj79GAnPGN", - "RLgVwVcNezxK+DOuQzBSFW/kSpggUCG5CCKeQ+bsshwPhgMuRnIwHNxQhfwTRdIU03wjx/qYK5aZ5Msj", - "fIosVU5l6PRkl8z5PbM8WsZIqhuq4JdLml3hn43Zh4PbHWi/c02Rq2roWFvP6zBK7eeXYUi3gXNZqtQb", - "3/6+4tLhtqWiKBXM4Eo0Wg/7L9/OehENU/16Gg34dehfSCdwWc0H2qw8VNmEG5aZUrG02YhGLfxGhX1a", - "pGj+azrlxTw91Ai/9RjkrcxZkR5jCp/6DvEuKaxVw4hI25Qea/FNFTYYrXNhvmHjXO1F3IK+02qREkSV", - "0SmZ4kdnbowsrvVbWzD7dnPshiHYzbGKLTiyNH8QKdmrcxLCBYFuuCPyxKvMNRcZI2wms8nTBUVAi/YI", - "5afE5DCfC5moaXJDCI1fjlNkjPk1E/YZfk0jVyDrWdpp+q6fg18SXm8261DiNDxu3h6dkkyKER+Xyjo6", - "N1U4Ldrh6hHwNhItFobHL+toqZ4d/DV19m+5eK0YQz3oZUKLXh21HYiMFGNOn2fNIDEz/kE7rwdt2EwP", - "3bp2yfspN/4FZdvT6Q+aOGPMLjln+HmfYCgMTon2pGjOHbSpjZW8MZNdcmEjs7welWtvJpqoUlwNiXVZ", - "skquCVPcWGMZLRSj+XynoGrMVDSC3iW/4tAw1CVMz0YjqcyQaBlP5AU4klFBCkav7X6CTsmdjC74eALW", - "qktWyJtFoLW72l1dqfiO3XS8Fgp589nqoJj5TNGDLfV6gAV5uDGS2Ia4RN/ZWTy1vRQ0fA4J+mFP6DXz", - "MhbYwTTRM5bxEbpl50zM35f2Infxv719j5qCmRuprhxqpK10tDTyFEyPNTudnb4ZSyCnFF754KGH9sq6", - "6BeBprP9dc74NrIFu3lTNPdICqNkoev22SsucmIoPB0bwjPMsOPWJ4VfDHmCziSKFeya+njBsBgUY1XJ", - "nsaWZ+sjXw1HciVn/tp2nBHUWn+pyImTPrT1WjF1bHlCo3/tYAu/mae/WDs2Yo6JTdVPFMM/ntb2F6zd", - "u+S8zCaEVseSUSEkAI1dNQ7rnA0VHY14hgudltpYacp+ZrfgkM1NMUfU4/E4mZxechHMyqWRZ9hrlyza", - "7MkT8N30tuGwuXa4swP1fAAchg5HCM7uHen1/UvekNgMzKB02nO+EzqFDtNstqwDMKu7PTMdovbs+c62", - "ro5Bsywpb57j74QWBXFAmMnptBQ+GghvtPFqjS5ptcehZ/PdlsaEV8Czn1JyFoAV+tAm+K4Te9ag5t/g", - "DfqpxkSsV8zCRcVvZ7gZH2hDDjvvbsEphhY3dK6JBYf8l0UhAc1QXj+IJECXs5lUxvdwbGl3MNyyuS2b", - "27K5LZt7VGxuQ9yoxUftzuzo2SNjRxhalfLQ7Hm/2L92vUtdkGy0/JCUgv+7tI7gjobPlIR3+C6B5jZW", - "pYqYD36S2khFx5YKeU2g5QLURLH2cAe/+Al9WD7hmiiGXgI53qFdTJTkQNuwmCm99YYrfL5PuQj/Hg5m", - "1BimYGP/17/ozn8Od/73/s7Pn3c+/f//R6sZLPHALwWqb6ZUXTGF2wLuq6ND+kGTEVfaeIZtX9/KdVRM", - "ywLYpX3hU6eqoSZAJRsrx8q7VXDOPOpULwm1yDt20+UJeX8+cTiSA04L2uvMZhFqsNpFJVeSOguZJxa1", - "ZpqVpppVTpMZTI7wdz+AVNmEaaPQvaTVc/S1N18vibtz5hoMj+jrd2a7nNtwPbbKLDr06TdTP6fVNrX1", - "tK6s7+RtUVPseXvu4+YS25M529GZnLE8aB1ZXoll+ZRrlLExcww4TImZ1NwA80A800Shr3aQ962uDn23", - "QSgrxZWQN2gTF9IQKmqXvttPvzutHCy7dg6b8b6Y4AVUGm2oACXe745V1zf//tKRUBS5J7LIrfDp7iG9", - "k1/If5iSJJfO/R38RaQCzz90LYUTSEJE91NOR9Gc/UJMvQ9etODFeVvc5nQwm2Jc0vI5XUNSwVH91tKz", - "WCegEwF3kCUFHe/SxF2biH8twwcXPNUDIGzoWdXnaELFuI/5Gqb2YV1gwi6oNiSzvXubR657ulV2U8KU", - "x3XzfIe11FYhvGxx2wvAVmFXkxrWKXAL5FSbDGS/Tq8+OW5jHZG2PGfLc7Y850/Hc7bc4DvhBst4QIrY", - "BwaSIvtRdEyTDpCqb1NNi84LR6cfuoAztCMhkLknSIaeVqnZEn1yiHEj9ZmqMMtVQlxir8dU3EyVvirs", - "ZA1Ey2blKVMZE6blwGHwEmPXZ7YdHfcdG/SXOhXNZGwKFXeXNsadZhNUeO5Nq+CivnH5cVBVIip/Uo7Z", - "KR0z8GhruTb4hJdGNBfjghHog+lgV7w1P5c+c/qWLmAMOhk/myZPLJZgEq3SIPWeM0NQP8fyp6uuAkFl", - "OQTNFKvAtVpMBzgtm/iD7t56NQkX6KXxBKOnyA76XvTdp8Wsi6UxZsKSjnXQ0Pb60B5v9i4a2zukrx11", - "ViNjLTSnhrTNBSZ8OqMD8ljZgJHFu0uBcgqVPNk+Dzy16QNa6pgj734UOwSlnsv5i/CrD8OmGQhuhT3L", - "ISKBNVb4lLXosUPzOWpCMykMFyUjyNDE2NtArDqzCsWg+Ry2rigX1h8ys+H69h+lmDBamMncMjxYGPw1", - "KQ00+AwR4D19KquTOHNzVr8cV7NXPx7F66h+/hCtqPr1PKwt+s2t8hgXWbsMy7Xv7QG1NCx6dbFqAfrd", - "ALCL9ypnasHZ1xkpcMmDRgY+qQzJQ4fq8mlonHQqtganjqCIuvW020fynuyn62SK2OaD+M7yQWzO8wPQ", - "7IwnctIf1l+6lg7PpCwIeMJbo7IUQANkmROd0QJI7FjJsulL7ANej2zeii7e719sOkp7RI2xqeNcfibF", - "x/2Cev28x0xbMtJEGvxARGJ+Ltab7C29bc+27hsRJkZSZYAlYZofdP0YF97vQhqsA+B6anIpS5Fr8gSC", - "Nt+e/Bpy3FHhGCWa62FIpp5aprfKNnjibfmWi+9kGzxfL3d7IhJQXvM8lTX4CCHff0e0CAmB+Zg8+Qhx", - "zR8HQAc/DsbZrGUCxTSGVKRY7lGw3Vo0820h2aq3Hcdn3VzE4dk7uJvD38+HRLNitFNwcQW//Hp0+rSf", - "PiCcQG2tTewaNvD8kyUuR/7nIHOksdB3BzzXzHTAVIPC5G0YfnipZVEaRvK1UX2FJFB+GW7jIWPBgjji", - "tmIZT/suhwQhlPq0on7BlT9cXjIr8zqwYLcTWmr49FE0zoi2xEYd4u/kksHEM6bgADy2wsrcMjMkyLlD", - "m6OzV4cXJ+9+fZrWnqfSKJw6ONqx/mu19Alu1P/9/t2rz2evzt9/ODt69fn0/fs3n1/98++HH84vXh0P", - "yWs8huSM/mgSUqv7Em3FpvvTxEis4kLFfKW8CH8vp1RUySDsmL71agkUf0+f8d2zJyayMpzx8UnrKR2K", - "Crpidkt7YV9H2qeww0AnXWP80c8Z8whesNo3whdSvO721pHyvB0Kq/FPjsmTV0cHtR88yQy/2RguSzaH", - "hBZaVq/Tk+OGptWlakymZDRMYRqMZCmgi4WdS0E4SKN07jPbwlSW2OMjlwonhF6yTE4ZcW9DQseUi7TA", - "GR9megXhsrh2xEaMh85AAH8p5n8F7ibNhKkbDmJ5aeyv8QUmFpFiM/Vl1Y8J4Pe7C03P5ZSmhKiXVDNi", - "P0YZzoN7sHPU5Nq5CUOulz5ZvMCN2WYEb6mWYz+6vAKojULlKng/13yo7zcy/b5CxTcZkO3uoPM08efK", - "mxKO0t1XVWaLXHMKuHQ7311+g2sEay9GW7f5yjZBofIOTgQI2iSSla2u6fju3r8r++6+cv0WN+vHSzmX", - "tQ7Sy2Pd79LNQEYFHac36R2arcok/VZ3a2nzPrgrJcJ4ghMXEXDYEk3w+4QBvQ2RAw7s0JJYeXKHDUtF", - "cq7d5ut6g13yznpsU4EekzfUaTeqUTQzHaAbncwfIb/Hxgn2BtKJPEKOUPARy+ZZ0df1/01ov/lEJ3eN", - "QNvmSdnmSemTJ8Wt8hV6p5+CxNDGus/fH/3j/CcrVViNd92znbwvDarUyMXRKR5uKQQr8MGlZDme+NfY", - "7dxZHJDr7OVMzF19UVs3ox5DVwp6QxWE/SAd3MGypMhP5Q2mnyWKTaVh5PjdOXlyePG/Tv9mKebTFP9Y", - "YJt5rpKsrrZX14pwQSZSmxcYoWchO6jyrKzlanDtZnL64tn+X/c/Dp4mMx+254R7j3/QgvgVuJbkydnr", - "I/Ls54Ofnw4hHQQ5+Okna9DdrUdIHPz000qZ3RYn9C3vNOGi/dgdc4eQ9eo6qZ4/9yB2bbn0gjAIvx5R", - "w8bBMNXN/31bTxVwAJ959ofAHn4Ykh9cwYofhoSZbDep+sHex468t5wqJjgEmp6qOxK2tTQVBjZ8Qy9Z", - "0Web2LCSuYLSC3cbJ9v9wefwdEQl2vhnKaddm09pOPASfSxRpe/uZaW1C8FaFqmSXx8WB22YojDjRjxV", - "sIi3TfbqlmUljL/ehMx3X2nSteZaaQYIDkrNAr/Hc6QAMjHfkhvzubDXO8PoTdJjg6sUhPGo1TMBUDKt", - "poXntgTXrX6Qtps2qrRJpXzDXjaPhcbDmnK1gqEkEKcupYFai3ASCQGvpbpqzWeeVW5/3lY8bDWmjqS6", - "it+4WEkN5SsbQQ/fwQAnTVVbAIPMvRUd44EX7f0ZnRn0spYiw9yjVOUF0/jCxeXtklfgsQejWzdzlmsi", - "lXVsxxJcMyZytO1aOUOWBnWXcmR9/TAkEfi79ZjiRrskTyGG1daoxWLPaZN371rTdzLAC3bTPOS7WOPb", - "WbOFCl+sbjExEvwOpyNFnNwHlrZLXt3SDN7y8C2KAwjFg7gmmpkX1dvBPybsBVqeVCtDN3Q9sSWPy95h", - "64S+yBvCehSxDJjRU7LvPLUTOm0e17nfqFRXhaS5I41mXteFWccSscOmMzMfQrgAz+1bWZMpnTkNi04M", - "s6hnaZyHHaV/KPqFbb9kp+GVv+illHOWtvq4Lx6vwk5wfUMfgMscAFHtDQP5bjqf/xUT6TIDv9dGRuln", - "eQxuWHg89KflR9CZ6H6FA2+WMISI2jwc047dTM5GXOBcelgVMaQEHL0gUnsiNWwZ2wIX2x20b+FNrBtJ", - "Q22Qj8lMFjybVxHfl/PII2gkm2BXz7qQVivW3uJUxIkk0jpRKS4qAtrjdN+H9s37DsuLh+248TdynK6Z", - "Z9lFPcUh2ssKLljjXPDH5Djwpavw3jcqjocL/lQ7h5ZShCPOirwTIVqksOiwN17O8FudKq6/Wv7Qn179", - "pPXyqoM1vPUSaG4ztza0fauosboKDBYyVazmzX3MuVRlhnMP43NYOLPfDs6c0JY8vWXVGgPxg90EmT6/", - "t9NLbSfawdtIg9yvgorvsVS5W5skmbT1bZzmtC9Ja48QeteMDerpGTgrwfv/NGup/NgVCTQqZFx91idB", - "tVrSzkCUHD2HW0v2tAdnQMd0pAk6I7eGY3SGe2AN48T2KSpDnbvvEwwOyuC3p6tP0XkaHXEqnYOmD+Lt", - "ksiU9iH/nNmBV8jZG9kEIryp7iK66giwIqiNUSOiRHUTUzrx6vtUsUyo7+BDD29YTnKmDbrXSOGi7NB+", - "EIJk7Bvenh7IgZeMUHJ0cnxGLguZXQWV+1938b+95wcfB+AZRS6pYuTkNOjrFxpiK6kI9RZVq+Z2jSLV", - "/cfBkHwc/M/d2k9PUXGBG/DFe112P0gGRwAOWW6fNOATlTPBq6a7K1XaxoM6LS8Lnl3YM1maq+7cJuYj", - "vEbzyYezNzrK1V5ZiW2mOJ9ZLSoVk5a0XbK/9rt1261uCdUu1V2w9E0fVxdhU0wJGXIgOpswTE1UWax6", - "iKwyaPXk0U0TGIQOGjPTp+hQ1/ougtX6vPgQkkb+fnFxek4Udc8aKsisoIDLtwa/7ZLD0YhlRpOJS4ts", - "DU0KdFA+NqcKJOd5ze5vNVczhA+4YcK1m5FxnPGGznfJEaDmCG41OtprpjBCBrVr6JTt6pZL4dQJNqsV", - "GuJhW+TJjz///NfnT6O8fAWWa6ldRjPsNmjB/vLTT89/WqYHm9LbEztWnJ7bXuVwYO0JroErngmVZJym", - "8u9SpwxHDiHAYIeVeJzNF70ZLlnlc4C5WN1FOv1V0ncSQbBLCltFHHS09Ky0rgWLsLvgN8rUjqNY6KkI", - "p20xgtDZrOBW/4VmUXSL9FZYB4VuT7vkH2yuffgSKlbQYmpx7wlip6V3QPzojC+QP0s5C0YxnfgNL/KM", - "qrzRcZFqDm0YF8KfmtKC/8cuFyipyqhG11aMu9olv7tBtd0M0eWl3bcm1GAoWM5mZhJCkAEN6IzdOnr+", - "i2cBHwf/8+MAsIILVJ05RaQ7swVyPSQj6ej65dy968SYaVOdaNisxqJFdkvhK4xKNDMVlk2ZGkPo2UsF", - "eqebsCuNOTM1nhKeDvbwbNEOOgdXG6lZddUsJ5liqOCjBSrSGYauhQVE/HQhoE4YmsHNn+HGc4mrGysq", - "TEjRalnCL1UKE+IZOdFsRhU1EPVqJJkxhdnrJyye0EbFLLzbO3SFMeA3BakadMfA7fiGh+Vq74GXu+Ry", - "CYWnH3Q11Axr6bOdi3iOlj05eSeFsYu7czGHrgKM8zuv72rCaM7UatqVhViCi4tT4oaB1XABe0On6mum", - "FPAcLqo1VmTkUBB2yzU6W9v+sflw6hzWrSGloBkgwm823w1AOzJCYhelf/EpHzF7IlqxyITOZkxo55i3", - "g8zPHYhmmJLGB+Mdnp6sCX4fZiDvt/m5vKtlL/buqSX2sSwjpJAJ7oxndq+6ViTb3ytiXzgjr8heVPlj", - "HIKFaVT+kKxgVGnCTUv6562wfW/C9nedSfvPK5x//zJZoBu0KCqy6igGAFMwe3WKb4hoLkfMVkTbimh3", - "EdHex1a1ZLxmS/Z7VEQBqO9GeS6ueFFg7t9Ss2SmCzcrJrNo9T2Z9svYHyyJ0ruKADNN5Iavp6hIZc3n", - "2mfVd/ceEuW35KyoMuZzU2W7GJJCAl4upu0fBhHCudsxbWFWzpggPg2MFEgiMIyEGytCUBGiJDwdIU9c", - "B4Lhf9D86S+x+XToHr+OwRvFx2OmnLVYXXKjqAqJ+odEsRGmCNEux78XchqJPtLxdW2AdcYwr0frJedO", - "FEqozeMYIbuV+BYqBdcVmxlC0VGm8oWJ1RHP/7K/v7YjzLmDntYdLPGwhc+Vh4YbLLjA7ZKTEaHN373k", - "yLUdwBXvstwCoAZ9gWy9Ma/doNrpdZyoHDhLGJQLbRjN0XPKB1TakaRocZBoPRbvvb9SmU0H/5425F3E", - "wRGk1nM3bQTLdYyAoXI381I6dHb+6K2g1Qx/WSNE3yz3KohjBe6Saj3hcJ4q+3xKzWRZ33R2ehyvYyNM", - "/87NJEqaWN+KjvOq3sFfxk4w+Lq4ymp8NGCEwgrtllLqnqK75MS4sOaMKsVZXALAJm7dXSUKfSFIxg5z", - "Q3UUbdLPCOXA9bdlyTdd7j6Xj4wpp5h2koULJme9s8d+u8oTqUOwnXoGNWHb1Cj2Kb/adWGmVNdxPX+P", - "sPahR6KFK43OexiBVLzeJMbVD78DoeI6GvsHPw6XqYiOSm3klKmq3GPtaOGNjm/ymWKaCTMk6K/oc/5q", - "bGHIVGpDnh/41/kv4Kjm6onwKVakNJI8O/irNQIPfXJk+HH/4Ef/Kz5RqkocYUVGkr8++/nANsNXM9gt", - "Q92Q+ACeH7SenlUI3WfplTvVGQE2/MBVRtrLi3gpJ11S2eUUXNzXaxCy3Vt0lJZvomKkhtGpbW0LqGJF", - "rbHTDsHHHV2U473pfMeP8uL64OlKGgbfsSet6FrshMHqdskHzXS16j0MubbATvFnpBNBu9e1GfeIAQ1Z", - "SGAxokWhMYtSSDZGb6r1nBy7Eell9uzgeRhi+U1HJzF015e6dogGaF43nXFXZGbhJWhLyARpFvaZjBLS", - "x/7B1uX2Cd19OKbb2cKQkb5rOW9pWw383jdclE6XnqwbLtBzd1jxrj+5k20r1rNEcghuJf68ff6l/mKD", - "neDl3KmN348GL/7VTcxgvR80SACfFiPTemfjr0oMLY0jAs6adhJ6AzwX6R6+fvwZAJrhi6HlDJaGv4M5", - "eBlFr7b0lmpIbGYzL+h+ANi3whKeiIMeXFXM+D3s3ENJ9UwKJ2x0VDQARlPlKKu6RAEmC+jew3UvLrZx", - "lnwnplL0+9CPGVPu4dbLpW/rG7bMNywBB4k78pCHVKBBs9jUpSBZdsKvoKHfeAlj9UDOfvTFjbaEuKSw", - "za7e7tDy1pZsKkz32eOhbZrIsdJfVkHdzVKfWbzL2iRICaGz6YeLOE8//ofPnSrwyqmXAP3H/JqJzlwy", - "2Pa8V9ZlfwUvoy5rpl7p/Qavnd7Kr/D7ZqfrZoy5QxIUqs35jN6IlQ/LvoHvxHnXyKHS8uJ4Fz82wjKf", - "LMrndp12Q+FbvtpbwrraLZNg3Qq49q55mGETgtcqr8vLecw+m6KthntZlxIs3kyHD/VaqVPupEpJANJd", - "lCm1RCmervXKh+Ius03DEqP4Iq7U7qdGtuv4OAwMxENvnSjGzAf5R3uAyuZA775gouui3G7i/SPlb+57", - "BVbVyPvQ9tB4UK7i7C9rsJTNc4ARF1xPVtuV79N7W+uQen0XoaE3Kao2dXc6VJGeUJuqla4kaFMDE17z", - "gn2YQYhvEyfu7vPnDGhkElsGfE12zYSpvNRKXIS3MKfihp3Wt5s2gUUd89M5o6Xr5N2bMhd70iRHpUpE", - "In1QRZQGD8eubL92xahOXnptfu2N808rPdegRk01x0KgZmvAIa5j3XBD7NwvVLO2gO5HTrSkJ84Pz6r6", - "7VMV9v3bM4dXYBlE5FJsKq9ZPsQUXdXuVxO9FKN6OX2LKMGZ7XBXYrIJbpigHelg1toaIaT1TgGtDwlf", - "bcGstR04KvTb8840pj2e3YvHDx4qvpsFO+uKbF0ddhOV5O9FRlgrMSQXrxVjWCLucmnHWuOeGnB/JEdU", - "OGMGIxTNLEjjM1lI4f3HZFWSYjrf8X19AYnopxfXz9A592SEI3Hth86H1kHJup8YywIJ1T4gDeeNzSOO", - "ARg61gSBp9f1QPMEe4dBjHReL9584tccSHJ/qhPbCVZNKNtiOVhEAlS2/XaQSB5Vz5kD7ionUzpmcBfw", - "Dz9K4NtWIUNFXqWMaXooYMWT5XmLY9Z9M5GFF5YqLuuKrhhJVCkWUjB1c/SwkwRFxQ0aiThLNTqJahZp", - "mxducspFgJMElISZztiYa5ecogvBXjc6uFFiFeGi8t2t6d5WjQLS0TRVPQa+kGzCsivMjgt3baTLP8cC", - "Kvn5qmpercII6teTcyFc3tssbLaMG3kfdWzbNV8Di/tw9XPDZkkulTB/NuWAJVUDG0vzrm34b+vbdkO5", - "q6Lnq/3ZhFQpZze/BMcfPZNv5ZMbVlA/vOJ4FU3fI1XgdTCoRZ/MO/GnTUqnQWPW1GoZuiDgIcIl8IbN", - "0gjt3Z+bEU5qqSB7qMalrZgXfGph9lUOEh2+/051wjcSfvUniM1CsopopibrWZ3PwlD3wmDTOSwhdZnv", - "m171IhTA1/hOL+j47k/iRLZRrkH866W5623bcWKmx7X+ZlzaUo8GRkwnAF08NtxKpIpesOPbs7SOZq2O", - "zJsiVF8TS2p7Un5r+0vCAblOc8DTGNm2fhyMUrdH4tnvDRFxVbEGd5uafzNG0G9pQdxaA7fWwF5GppS4", - "0qaaX27msxTHksouf74WzQy7qQf/9PUPw+FgZhsYcr8xIfZL8taPV4wKCUMN3ZITLyy7hUORtySLbauB", - "48KQq3f2QnwjHqpdgM/sGAJshx+FzULgHXm9c+n/gQQnH8v9/efZq4OXn4/fvz08eYf/Zv9nl7wHRA3J", - "TT3UfhTeTdWF2/lChRn6yZMnL//X+6OnvjT5L4ReoqkjuPgOCRcfhY/E06y2ILt7wi3i1mI268Rv3ds2", - "LYV4SpDF5BULJzySrsITxQLwVlqrn++GockvPgVVv7PLiZQuK1lrAOFRo9IYoj9sL4TG3diRdFfJsSaN", - "xjTz9VjzpRwkfY1uJ3AHiowUZyIv5q3J50G+paZUrC24yf5uHy1GEqsZBbWem2ZG52AvS+Y6csa3Si5X", - "PK3S9KePdLXPsStUsGGhF4xy6HPoCyG/93sHj/FgkxF30DVstbnMTwu3kUQD16Cq4uvT2a8f0XYnQfLe", - "MYvn7ZtOlSXpSWFXQ03TUn+jvRpGW7zCOuASu5TH8o5bVQAkd/ThnCMAOmYQQa040/BuObXwnIoeHQML", - "Bg7pzie3HbFuPhgitC0XawPN0yV8fHRVr/dPfX3zX2HwJHKzW3NUKp0qgW1/B8ydUa29Mg56YGY7Lzpg", - "8hp4jQRQVyjNC2nbzuiYrV5a0wX3RetrHvu8Hf4Wz7d5mo7WvtWpWueus9sh8Y0xzxMvCl7Fy/dwJkY9", - "9lFBU8ly3tJswgWrKlWDZQoTTkmFKVCwbkOpGMlwgEp1PjFm9hmHHgwHudDhbx887ar1zqQy4ZvbUPh3", - "oI3hl4yKjLkyn8urI0Knt22JxF/FRbcRg91ebIonWRpCgVCA8tuTV3IpUfe/fGrAkRTtqNXBWrG0EvZJ", - "F0eoD+sLzvRxyT9eAMUV1+Su7CWcS3NVTHGXhsZTFtfeH2TbeH+vXKTqQ0IZrh0mMpmz4NQU3KBQwtVM", - "aG74NfMBqIrlNGupB+hG+JB0Ujp74w8lFD7lOuDuYLiEng8HHmzSp3OhSoFqrDp42YLy1W/WBSdj/Jrl", - "fUDPd+x7hG4ad4b3NrsxM2tlO0pW8kfiEWaxfjNYxn/JAhoErWVFzcilLmzsliV0ixWxgTquYUUFnf0M", - "1k25I1u9xYyFMl99sHFJ1YNK2ssTS1+p0qfD55MOOa2edm+VjSRru3uxp5q5IrP1gloVnYw8siKOWqda", - "DZpTowo1/hgfcILf++w2APe6B/MPfBsWybXhmU6w8AU9+DVTjp1VFyVLWy+4EUcX0uX0a81F79YLV+S7", - "DsMCq8kTJ2UFvt7iUU381J2VJH2PdUXRZJK7ipmvxqO7Swd2A30F2jVwjsA8bDVxwH1BkI7HYJk0lgQ3", - "we2yzK7Y+qeJy3iJY6QOti7lrjBuHdO+BvL64ksvPbkvj7BqRKY/DT9EmLe2lbbrcOew/FLwKlyCUowo", - "tfMueSRs7vhiJtPTKrvuiddiYPueuQlhreljhu9bjckfVGMS5ISE6qRVY2IzEpWKm/k5oIyFhMN8ysUh", - "5oIA7Tr8xGE3VlT2U7wY/HMHW+5Ye0h1NdgTNoOf/+v3Cz/KJaOKqdd+b//1+8VgOEBcRUDAr9U48JQO", - "o6DNqnsx0GTn5LgaIFpIr82cnuz8g82T/UszObWev+olLrNlS9Y4+tm4E1m+t2jgu2wRbpK70A/DDTwL", - "Bq8OXoKdLirJ+mKwv/tsdx8mljMm6IwPXgyeQ8JelxENr3+PwoHvhWQGey7T3U4WysePWbKUpimV0IQS", - "PaGK5VVSGpuWFU1SmEGR5c77cgTPVV/xkxx+FG5Sm5kUI4Rc9lmwwLh1YBZIxdDchEtiOSmFQdo2ldc+", - "eRLQNuqLIw9+ZQbhKKQNOLODHdk9VQ843N/B/r7LOmFchBAmxbUVRvb+20V0WC6zjAcF8HUzuhW4ifHm", - "GimW4tzcdo+6Ko+IRryTY7jFH/eftU0f9rMHjaDtwc892h78DG1/2t9f3hYaxRQEA8kbtONfn74OvyxQ", - "gn99glByXU6nVM3RzFIKs5iTnGm/We+nh4XYp1xYwuXAFBrovS/W4/3rHp3xHcjPBcufJQtrWFsPgCke", - "ZJwIyObBpkWoXQLpa7FM/24DoE6lNuFq9QVOb/eN0iRVdMoM6iH+lXzgIhYjmgPiVUgecvJUFN4+8Cto", - "W/aK/NRQT90LIL9jN5HnwELKQpcNYAGLnt3b5PbW8sUFJA62lmloofquxZn9PjizvzJ+7T/v0/a5bftj", - "n7Y/Pna8xSOu4xHVxCLpGki798Xys5PjrxZvC5ayyh7j73fGYDtMCw4fuoV8a1wepm+zWtOeP7KBxfsa", - "/v3Yol3zR2ZPeJM48oeAews5d4d765m1Z407HRwLv9sU+1zszJS0hQ6oyMnMlRRZcMy01Rkwi79loMuZ", - "l/VAtXM9Cg72kPIYbtbu1VWGT7CT8wgriL2kAkQwPKg/KejbM0NQjECOriynVa8LTLDfCvv/4IWD/GYS", - "tzWAPIj//+DFHx/K3W5hrz2hHC4D+IE/pj8plMOJJYCuG8yjl0fyfYw+9DBozDd08qUa3hF3go+eLvmV", - "RN2IXOgGluBg2tzU43ybtmhxIDIrrYiJYKUOQg2tVBqqUs1qgIYwsQgPAbo8RH0aDm534F9jaz8agKMP", - "nEXXC9dVQ4gHT5PICNi2j8bWR+MfApiXvd6SwFejbis90QjtBj/baumL65s8gL5rnvf4CF3iyZQGNrh9", - "kyXiWW3syjKQOoXO9wxR908VG3E4vQjj/hJgdhFBW2B+aGB2oNiPcrpwGr33xf0Fzx/Fx0sERXjgSJVN", - "mDY2wY2QOSMzKQtNnnwcwABY1BicxtzArr5WZXexUStcEZ3RArQEKD/oXfLaVmatYvndCD9owvJxqN39", - "y+LYQhLFx2RKBR2zKROmqumWO3FQY4EvTK4FUootfGFL3mW0CMNRoW+Y0uSn/Wct5pojd25H/tTO+Fiv", - "jMrhzAdfP21CnD7j4/XlaDgmuNgQKm8X//3hMLR91qfts3t6sCHK+KMLQNvyUmvFyD0utKEiY3rvi/9z", - "iaBzwdQUgwgIpqmyfcCbDcPksAyT4uMfdB0FbVFVKAKpiZG2REpGi4Ipkk2k1LZ4ZojGBZzTE8WFLXxq", - "JiyaKqpTnEAky3eTuHTiN3sStnoX9Bo2wiscYa7W2qZL4fEC2vUpCd14I4JZYGG/IcRXwiBQRpFm3MwJ", - "ejBnCqkWVDeJzvTpL9a5xVVThOOt31ZBSwGUmFB/1lObA9Iu+6Pw2/p3aR2k3b7ChMd2NX12V6U4aNKr", - "RLYtf3fEOEDkUmCI9syw/NGo8vf7EJn9n/9ABClNF5A4KT5emTB9UXwM//DgbHNtJIP7Qv1UhP8wNXoR", - "EFe+NUGSLBGy5Zgg6J9qHYsHssx97jtFABuoGLP8F3LNZeGKczumhaP9oAnWC0Z6BTJCwW0wblSVIxAq", - "TXSpRlhISRN0O9YpUnZapmWCMziaI38wd6NgSxrjLTzYq+CMj/02jvB8+z0LDlIWI0f17D1tCcK3Jwge", - "MQNDujMtsJjS/YpQLANU85huu3RQAZdG07qaRsTDxdtqkpfMlmbUslQZFGad0FLDt6dD0DcybciIK21W", - "keoRg1/Z7WwCf4fNeD902Y+qB7mTMtJJ5y0cHisADmI2HoK9D/ZT1VZ9YMJPcRXfZwl/4E29VvDY7/Zk", - "WYSukYeuP7YrwTd65KQx+q7EJLx8lmslQlNCjaHZpBITUgTFJ2TnqoratxWLRR65efqq4lpi3azLefxq", - "WJmWhKfN5sSBDaCq39XdsLW6vi2iPiSitqNKF6Za1/y9Wn3pJDr+ykxsobZhYfUCd4mUhaxVcf4rM69w", - "iLiw2gLupDigHI00a2GB+yvXG/+yKpd91slln+0vY7MtM0qVM3Wos/SkLiVm48XekCte8wKUne5uHJUM", - "KcSroHU4a3Y7KzBk2CoGUqvyTROkY3k9WTPHqAA4rA2RLAdJCFZ3lDBqQP5YqNUdbBpNgvJYrByeKMUx", - "Qg2qtPfF/em0o60UauHilhOdcz/uypw7rGjwdbglVVtStSVVf2ZSFfKyLVGP2GxuVaIWvdtOpH73Y24C", - "IutRvT1A0qfoTe5pC1rrgVYAo0+t3nZn7rw96XRd0hFjCUC6fz1yPanhhn3uFnP4LfWrdx534di2FHOj", - "YJ2gmXtf3F99nfyaJKct2qoO/7/7aVYW9sICW+WDDpDzTn91kNuC0d2oY7uWohd8NLjsQwLHfVI6z6FX", - "ERJvUqmitnB4P1x6iftoExgXryHpU/pQwPlw7D/e1AoOpt8AN7zj6lYGeJQywF4e8ul2P6da0+iuQu6r", - "5L13wa2GfuP9jEIqnsxmzx0pOXUaDnbNZalDXqQfNKmy25IRZwUW3kzpOOxYgyUuaquajX96zFqiE1eI", - "s4IIQg34+PhqelyTkLuq7dww5+IgGSnZWTKjx2Iu2Ugq1nMdTOT3sAqnOIsXMa/nNCtrOjN31ml/wSpt", - "WtmiPVstxWanZq3fVmqpdfvsopa375ur//qnh1tIF96hYumfLnzLyh4ZK9M+QWTrc+VmSa7IVXjZuU8u", - "eW9sDGmexrRZMDZR6N+HFBWqRyM9R/X9wY9kIkulCR3LhybEr25Ti2Iiry9JyJt7JcWbIw0ur2iCJKST", - "WG6fcveEyhNGCzNpxde/42dbWTiFlvb7oFdY6CSEPYH/s534AWN/v8YOI7V94MaFzFmPVAK2WWLn79yH", - "zgQXzeJoVeBMUuANzlGbTHPRy1wC+71bAgN7lH+cpHoIIx4+Uq5G+G3vC/xvmRkf2hDML9kGau9wlJVZ", - "nZ08xee+C9BcBpHr6AaFg+Q/YdqVdxGYLUJsa7KLiZeAXGVzak8wZX27D0h9qOSKMmeubkQI/fjal28h", - "eroTwKTMOMQGA8G/eTjH/eS2sqAUnWYL5Ux7Z3ZX5w0su5HmdJd80Iz8+uqC7F0fVGOj8yijefK90eGk", - "uRDZwAzNqaEE6WUo+jayL/3IYYVpF/vxcVBqpv5GLzMoJ3nwFzqb/W2mZP5x8HSXvKLZxKa7E7mvMzMt", - "oa4NI1A3xpVXaRPzp241nTqzjcgLcB0sd8d4N8GhcaEP+ubfDPo83ofCYjhE4/ArlK1+65FAqRt7Q0ol", - "N2alQPbZ8C0Wn74/743GwJFiPH6g9EsBxjfrB1KbNvHUcucY1RDYXBbSB4syeN6n7XPb9sc+bX/8TnHT", - "4Yu75TRO1lnp3pQZxbMlb17XCMMcxvyaiRrit7PJt27wJdzySE6ndEczaAQQWTiFr4fWk2NUZI1ZbSU9", - "9eJukM88152JANo15VN6e2I/oh2oxrKGA1vtwjVA9H7Q900429+5mfjzvRvjtFEqHhC2XHRjXNTD9zSg", - "SW8eWkfihbiANs8xm18zIg8pX7H7jQn41FsJGHEmnw71j5VQ63sAS4SQbvbR6XLmbxECSfNO1vBA0HXv", - "hHYdXZKu3jdbmH1wmD1fQdqJCOVeJoVgmaknwe5+j/haOx7MbXEvvUtORrUoFMzkAo/+IaScugGkumRE", - "MV1OWb5LLi7eQBMpijlht4YJeLu3PGRSS+75uAkoduR2eldMu/+HklvZSo+l/W/xWKKFYjSfe+EJ0Oob", - "PdscFD3O4hGPIsHM9jkYCjxZcqHXpJAjqa7aawS8luoqpnovrCF1JrmwYfgLDw1Iz4dZ3MgTboD+XSrO", - "RsU8kEqfQDMEznOIrIda+lM2lc7Iz2xqS48FUmArgaWcrxibwYTwy8kxtmO3M+7K3pbCyBKSATzFL1b9", - "4RJ0QcruKOm81TJRE5a0Szzpl8IS8sKQGVO+Jrit2H01JAw0tRlVag4LYTwkEwzaFncYLmEAJhKxc4FP", - "pM2Sx41dAjqKcDHeJYdYCv9g/5m3N0wZFdqesl2BTx7kXPGowMo8V5Z4+bLifRkGXO0j5BZufbC6Mzt8", - "2l7z7MEiVe3UtqbDcuX1IumOAdznZ4VLgug1xapr+sWCERNGgWQ5k8pYwIPGP2giS5PJKdtSck/Jv0vq", - "jPRzXdJcyPEqZrA4KQB0tdJmu6AJbfpZwgLNeCPXyBNczx6w8L5y1K/yKPY+Cbg6S58nsixyK1y7B1iz", - "pPmKPni+VvCyPATLcpl1rXL1vGbP9vdXTpawgecx3vpakfoIwdsX8gZfyPbIV6U0ywwEMWmpFJk9CEar", - "ceAONOOD4LcRvQglqwDbPfUA/FDXtBgCqXBUYohNMXk0tqk28kDEo483bq+tMZGvt7HVlrzJdCAWMO4n", - "H8gmrBpbonQXotRp/OigS4KZG/8+LU1r5CkAg2taDzh1pcUWHqnw0MNHqn1c+aTJzJZ+VGXhzWW1PD31", - "SFbyfsoNSi4Yx0ayglGlCTe7qeTKTcL4zu3s0T7A3ALtCduA136qux+XpAjwEakxBicv70/mVvc94LPD", - "t4VrWxmtURnUrnQ6hc8xfPTUaGC/x4tRuLxOnUYPs6nVo9UUw1bTRUVkfthqDb5rrYFFgHXVBoqNFNMT", - "1lGo/8w2qdFga6TyqlXMI2wkgdCrnuh3FuZ9tCjolrgqEi6IoHabj9AM/F0Cu4fF9cEdSN5qFl7osQaD", - "sR0fIXjbheWRhfGR+J9uDZlbQ2ZvOoBYuS4Z8HaXDq4XPM1nTGmuDSb7d90qr3M35g86PAzRJLlLzv0M", - "XtjyER/Oyli3L4JQ5uYhl2wunSlIKj7mghbRNAUfMWC3fQ13YR2Pl8/6JUaMdqMO8m76EzGSSZWSv/SN", - "u8dv1Uh3cXl317YybQDskqVppwy+gpBrWOmUvcKoxtfA1Qt9Dhi59TrYin6g5OzrP1j43yVHtMDCHZg9", - "Z8rMROZkWhaGzwrbQxN5zdSN4sZppS4u3jhPAxyw1LZ7pa6q1MRUVwpwaOW8MySZMqpLxWpby9tzsSWp", - "zYXt93hpjVvgnWR67e7fX7EHmC16b0RLzEzj5DtQnGWKmR7FdGZK/jfLDJTNsV12yTsZkpGhAw8m2XKf", - "bcxl2hDuplwVB2Z07OpGvmO35kJeMdGn9k3V7Q3aijdkmMFNrmyRKTDaM33gg+FgwmjObCm1f+7AKezY", - "Y2hZjGu+98/owL4+KEN+3qft8+/zbXHQp+3Bn+BtgTTBgWcEnIHEuF96xLKmI1ctAbkE70UJEWXW5oSM", - "e0rVFVPAo9HTkSttyDVTGo1IFygjLBAkH8mNvuMtbDps4aEiWx0x2LDcHs3aI685cuxAtbYkYksi7v7G", - "8Micog6x+LH3xf6xJBrvjF3LKxZBKuoFAN7zsmBIEhwxsIG3YEYW5awtx7vD+3M39epCue/YL2Yvmdp9", - "i3VbrLs3rAt1DjqwriMYUQoPiz9UfHNINCtY5mtdRtmuFBF02iniPwhm7W+aQSpmFGfXW2TdIut9IqsL", - "/+3C1DZzs40JqqAR3ohGKpZ7+fhyTuhs5gzQFHXp9yUl3xdOP4AGCyewTjUbj4vsR0pq3lpbQrIlJPfo", - "RrZc1o5tet3ZWkPTWpneJKdvN591pJZZyH6ZTO5erQE0AbaIf5XeZoWE97WBYB0gupwcD4nEhhQQ09Dx", - "zr9LWoBok4f8ctP5ju/8cTC0P8BB7NU+wHC1ti+un0EOuhZXePzfklIMq6o0h+spTzejCa2ZD9f1UNcR", - "nN2bFnSbh2ftPDzRdQRqE37ryMAD6NODANlmCXpz4T48PNzCTHfLt2g38TghrRWuEpftL8NfNIWmn6rr", - "3PsC/+sb+RRnRuviLHjTFzjwunFPdlnboKc/WNATAMV9RDwBeGwm3GkFOXzLaDztuYivZwXqszelt50U", - "CAHcBR+nqBHgBfxt8zZ6dOlHo97S2y2ZevRkapjIPq14Bl4+XttXgxJ0ZnAZNFvSRQM16kqW6cthZVI4", - "n6PPcUZQn3MTL+OzooYl6mQ9qDb0Lb2NCeuWkP5hCKlPtGB6EFSbI3qtfPFV5ySlrD720BM4kthOP5pl", - "N5wc+U2StPvd3fW94M9o+zq90+s0hkQP5dVvna9T12iPFpzCSXzBP9qLzhxBAi/CR2FOG+tsRQfsS9gt", - "16YbKQ7tbPi/FgSZUTOp8IO6lu38ZiGSxK/Od9wkZ/GT4/bO3CxLUQMoAijjwsGCUQNXvzKHeSA1+Bbr", - "qoIkgAWmAWRN3FtAM9tiaeX+yrmbjr1zeCu3sX0Cbl3Q8UO5dtVngolWCtFIBSjD/rxvyDaO4nvwtghQ", - "b2gtVxD+v92Ke6g1Hwvo9EQ/hYcHjShdyYs8aYTdBFTbla0N1c/ueSEsj5eSDJOkY0Jd2y3SPG6k8WDf", - "jTR1JvHF/7nEPTC4P/n2S1lDGHcNnU3o2j9jfyUj0i4qvxX8706NW6SP4RK7E1Je7RIdtYJRLLrfEwwN", - "v5vwlz6SPlRaeYlHuaoiKX5EcTDZ21G2Rt9H9KyuLqX34xpQw2STPvokn4WsDflOYaAHIeH3L0r51dk9", - "rSRG7fdgId6d7PuuR/Y9O10tYTftcsyexaG9L/j/ZH7ihtmqhn4+NWkfzmRJ8Us701pZh1djVW5Pm85Q", - "nJVKo4nme0pRnMpQbGd1n9exveVcsQz3MOxJqQAqjkOv1oELds2KVQZ9gx0SR3tuPfr63D7oe1rO1voF", - "rrRLO/GGVJ6IczBrb7VnWh6KUH77Tt2M80GK2N6VxrtizStQ+bZa9suovC3R/c3o/InI2a1H7pBKI5xl", - "K6qHRLERY03SITnW70cjzVoI68qJ3/8wpH9tCr0xctiaR2gpGdzSvm9B+xoV5ntSvxEv4KcJ1ZOvnSSP", - "ClLOCkmhjK248soXqqD4O4MsxoZyEdEPOmf2W1/59zW0/TvVk7vSw4QldmKH7WuIhVV4uui3sNwW++xh", - "MBHO5QOefFv2wfhebiZMYRZT9yNiprulrab9sWMx4pK7uQ9nb1ZHZ2+qXeK4jwbaddSnzqJ1n2r4B/T0", - "uaDjuzofxxaQxxLZt0WftMYzaaZq9yOKq0Qtx5nOyvC/HZxHnzuLwr/1QbwoEoaEmiMbBhdlZGM6hLiV", - "mqm/0cvsY7m/f/AXOpv9baZkDuFr5BWky7tiNiM6esBqMi0xASdQEMJEJnNbIa/FJxZXsyzWLR2zFxZ6", - "Oce8BFKRqVT4lrA+H70K2RtLftYrqnJukh59w4E28wJ+AOk9peOSypCggYENuN04x23MTkqO7UMAy/RD", - "f/JEsBusSsiVNq3Rg1LlTPUW799D6wW1Tqo+b3TeuEZwSDRw5nRkQYfrSmO32+VHzvLDkVlYYXjl5NSw", - "HRhnlejNGBIiH4mTY1wfeBq1LShiQPcTbfkd5J17gwnkzqs86uu7oy7UeWH3ZIUbDv65cyENLXbO7ARL", - "O2Nr3/hhE9ht+V89zLPiF9cHT9NpJNfN8uZoYpXp1WHrLjkEnjjXhk1JJqfTUjicqLRDUdVwPNm2Oq11", - "5vlQ2d3sFL8dbDzBW//M8RtPzbzN8L6pNM7ultsRtCGRLq/pv7EK/hdROeZLmc+xiUvL8AsohuSUY/Hf", - "kHA5Epue7+/7wLCl6P/dVfa35GRby39bAmNLIO+llv/aFHKpY4auF4vtesJ/p7Wfv0vPimX2tf1V1xwe", - "0n1OtmXJ9+KZsXCUdhNoV0XrnwtjLpXYJdCbXLJC3thXvG1AFSPsNivKvP1s783T44hqtqOZ0Nzwa0Z0", - "eWkf32QKnnxEClz5lGlNx4x4hpJelWZUZZPasqb09g0TY6AABz/9ZbORbVFJ798O1nPx2Bb3/qbFvXtw", - "hXRQ9Ooh0L8dfLMg6D+Ykuu+w63/WIUVtoieDgZfRPVeEeHXBy32UHwwVz7tK3uwR9Tgj+3D/iCLaOe0", - "Wyf5R+sk34GA3bjWcKTsqD1mTWyTriDaJOrVvCc35Tf5wCiLu8ET6aucPkhHneCCyYQG8+AWgTYjqVpo", - "rnvyLMOi53WJdYkdBnKDt3MqwJbnsdj64ODqGMxvz1cA2HtdhVuA5zF+IW1IYZWS2GWjaslVHIC+tVry", - "e7avRMzreSfayaKcsp6pbYlvnXomhk8P/5Cyc61bpa6xmy2hv+NLpQYZHtL8L+ta2i2FD0OniXwEdA9i", - "JPeQtlkLuZ31UOTRG71HKbTmmW3FnY2S3BhYG2gQk9u9L/aP/plB2vHANnKY8JsbduWXgF/PXYqC0Sbs", - "bZXCm8of0g17w67IvdC1NWzvIaFr/1uRzapA1hZwv1Ga7C5qibtS1x7CSlUMXgwmxsz0i709OuO77OBy", - "l85mCFOu/5dFH1qNeo16FZf6j5hGNv73jO9csXmtjYt1CP+uBMdqbFez5uunr//fAA==", + "7L3pcts42jB6KyidryrJd2TZcdJT3UnND8dOuj2dxWU7nXnfTk5emIQkjEmAA4C2NalUnYs4V3iu5Cs8", + "WAiSIEV5UZy0qn90LGLHs+FZv4wSnhecEabk6NmX0ZzglAj45z/fkit1ys8J03+lRCaCFopyNno22i+F", + "5AIpjqZEJXOk5gQxcqVQgWcE8SkSRJaZkmNEpyjngiByRaUajUcymZMc6xHVoiCjZyOpBGWz0dev49E/", + "T7nC2XHJmP6lNenbMj8jAkY3TZDELD3jV0SiHKtkrn/SK5nSTBEhx+iMTPXcBZ5RhvUoiEqEiyKjJJ2g", + "dyxboEIQSZhCl3PCIuNeEkGQIP8uiVQknXxktS1MucixGj0bUaae7I7Gbk+UKTIjYvRV76rAAudE2VPF", + "Bf2dLA4P9L+p3lWB1Xw0HjGc657+83ikZ6WCpKNnSpSk/+TOSpqlnYO6r6uNmWA1J6nA2buCCDi938ki", + "AgquGUpLgc8yghJBsCKIu27onCxG49jCaErygivCksXv0KZ7fTm+ek3YTM1Hzx7v/jwe5ZS5v38ex1af", + "lVIRYc6kvuLDlDBFp9TAkgYY2zi+yGqkvvV5UChLmo5iK2I8JZ1XZD+udkMVXL+mOVXtnb7BVzQvc8Q8", + "5lBFcqnxVhBVCoYKIgBn3db/XRKxqJaVwbjhKlIyxWWmRs8e7+yM2wiQmxnt55wy+1cENcL1DyI1UmGh", + "4L4yKhWaCp53LJv54foPUNBZDECO6QzRCkgekslsgj66rX8cPYoDihlttSu0lKYTLqrvK45LEkFU97Du", + "c9+oy7DGDIIeSpJ81nR0Sq9I+miMuEBUSZRgxhlNcIYyfknEVoIlQXp+IKLtJSuC884F24+rHYIieZFh", + "RXpG9Q1WG/mCZ2XePa7/vNqol+Rszvl557DV95tQIg33RBacSQIc6enOjv5fwpkiTBkeVWQ0AcTc/pfk", + "gJTV+P9LkOno2ej/2q4kh23zVW6/FIJbtleHnhc4dWx09HU8errz+O7n3CvVXMOsGRUR005P/uTuJ3/F", + "xRlNU8LMjE/vfsa3XKEpL1lqZvzl7mfc52ya0cTc6O4aJjzlHOWYLRwoydE4lFiPiRKLrb2pZuUt+vVB", + "C3hW2hsbgdVzRUkSzlLgi5eYKic3Cj2eEyrtlJPReESucF5kZPTsyU5NRHHcbicmCH4dj35aB6adEHFB", + "RAXtP60D1TTsUX0mOWGKpOhsgdScSpSSIuML/aNZyu46KE1yTlgaHsCT9Zw6TQgqGb7ANNNysJn76fp2", + "rGhOeKkM/zed9Jh7H06OyYxKJUB6L4QWzBU1xB9fyr0kIVLqV0faxpq9DyfINEC/kwU6PEBTLtDL/WOE", + "a9S1zWfGemw9MWfxYc03/fASBDBMjyrsSvVDLeMJViTtGPoEpA+/+PgcplG4g+HLNz80Rz1dFPaBaxfa", + "GogwTQP+1GscfYoJOhXr/tN8HTevIbrB8ECrcfnZv4ihwHtpTtkL/dTbxywh2TE8wdtXnsDXjKT7vGSq", + "75kN70aJZAlrmJZZtkC+d+S1Ox5NMV1hYDXHCpkumvSaoUfRp0J4Zo0N1Gf95E7ixEjOv9Os8yQGrrbS", + "BjQWfE6zLHoM+sNKA9eO2PRefg7hLB2HcEpwbvUp9jyggUH9NKV6STg7qp9K8Kb729NR/yuuJRLgZE5S", + "lNEL4raHKEvJFUr0xOicLCx7IDhHhwcTZBaEcrxAZ4KSabb4yChLsjIl4ckLzCQsV/NjXqpADfQcBpPo", + "kqq5/gLzkfQjq7pjQRDPqfI6nDb2SEln7NQ+CE7xTB5bcbUFNgrPZIQw4BkIEBgG0v/SNM29MPSLUT+9", + "I4K/XwwWAi/gbyxmRMWm0L/7MRFl6CO8DZ4pPPs4QvbmltIcM/zYbOST3zxJw+239x0omZY9DaGpPgqe", + "UE3D4W70F0kQzDpe9kwZdxyzWyoM46a7xim3zgQW5baoDwVI6Ws+e8minDMjFyRbxrNf89lraPd1PMqJ", + "lHgWYSmv+QzZj8hJCpHzkIoU7c4nihQaEKpTLwQHbidIBkdvITHjM0RgK7GzpjmRCueRCU7dJ3fY4UD+", + "ElOsyJYeZTn0+amqIxnb0/THfqKwKuUxwVZCahy9uRT7l9dI/flpHDlZYlo2j0PCDEiYKQK46bvOOkhE", + "MLfzjt/Y+3V4UJ9/jJJSCMJUpp82BRcKqBzLjLwCoqztsSJkBBxr6c24xetb2D9638G+9o/eo4QLImFp", + "sBVDZkcxdWAv6/Cq431c4DOaUXfB9eu2euXPRq/8OVAaB3h+xnlGMLy2veL5c8b5eVnEW0k8JZ+nXJx3", + "fPYyuZXnHn9aypBtn3HfiiPLC9cSE+z8MVkW7vXx7aMCSPmc8BRAkJWZeYlYHVH7Bkz7AGo7ulSwFuzl", + "8zlZRCmtA4gl2GS3E/T4TNP4gEoT9+ouRvoZLy4ApuGMdTt9FzhdeFFoufDd3EttHW7W6I1wxkiiTqp9", + "1q8hJzkXkSfJgSFVICfo850g0EpMcSYJwvoJNw/EJokKXEqSjsGalhPQeKOUynOgBQQIxLNan4Rn6dYZ", + "50qiqSByDoNitkBmRY5RSIYLOddvdonojHGhJ2HkggiU81Rz8hRxgVKSES0xoYNqzjmWKBFYzrcESfgF", + "EQskSY71A0ii/////f/QpaCKSMS4QtOslFoatOoUPTPsyAh7mpNINUF7iPEtXgA5cQuzorbmhpgyxLjd", + "wAQdE30JTqLAVq2nN0bYBRWc5RrG/KuSSpQ44gKPSr0uwjSI+y1LmhnSm/JLNhM4NVwCu0MTRCouyKQi", + "oAGdcI/uKN800qhXL8H9gb3JUHt4sSM8VUSgyzm1dlR3l3LOyyxF5KqggvQS152llMmtcjkw/7G7AecN", + "ONfkqic7O+NvB9xx02EbioHVpvrBu3d0aBVCDcWHabKnlgi5e0eH+p1q7NhGpzNEzh27CV7A3DjL3k1H", + "z/7s54F6ve+l3tSncYP5ArMd8tiy6x3ypjqPKcqO8SW6wFlJ2gO2BsiwVO8liazrNZb2zgFM3SFeYok0", + "1ncd4lKBI8fyfJkkUZ3JGyzPKZsdEIVpJnV/YztrKWFwvny7LXFhZE7QLsqOPQ4AS4vOB0BmBmkSlq8t", + "eNkOfCA7PYShdtd/Edu9eR2BJppviBI0iYno5IImJMYhQCvuxmouYEozIhdSkfw0qmx95b8j3ddY4seI", + "XKmnY3Q1lY9ig+b63XLEaezx8gaUUIX+6E5YM6Do6XKFsxcLRWJnrL8hWeAElDBn0CpEP6c6a0vcGhc6", + "RtV4dZ1Bm8+4av9jdzGtow4XUturu+oT+h/y5kXkRoF30v+Q5vNPr/kNfbGqoDIevWQXf2DRq5KsL+Fl", + "xRnRBRZUk4/Ya7SNzS/ZRfoHETJqk7AfHFwQdpF6hzAnaHSNPR4Z60yb59inWGMHwOTh23i5E9m48axr", + "+vgkc8rIln78gPuVk1+sKKF7TdBbrhBGScYBxoh6jiijiuLMUH75zO3ts5YvEqoWnwNT1th/LTKcgJHv", + "s5UUqk+Mf9akGSuqn76Mp0E3+xA2D7Mx0lsTDGef4RUnPsNKJ1Fk7lKomPNeRrLtEYeajVeC54c5npHQ", + "KJZSPXZOGVbmFnNcFNYJEV/KLr4TmtbGo1lSdDX8df8oaCj8zB2tCSMCZ77H17GDqsVb67iid/11POKM", + "DBAywmV+Hfe3DVe6tG1znfp8wwFa6CCNmXQvAWvAP2QMD50p1TZC/zh59xaw+9f9ozWY7fQtDjXbRbYT", + "e2E1z6l1LAWW8pKLiFR1ZL9ojq6fGo7KiQqabv0E/NifIoOXUmNuTGx5b78MX2r8UP0M4+pcYqfaKfS1", + "369YnpP0D03ojsBNLXLO8DtIqprYmx7oos4SFD8nTL8kO4TjYJ6Tchqdx/x+w3mK/k3AW9P7L8rWkMge", + "dGtcYAXOq7Yl38Pv/UvsEknsguszjCP3EjtDTVReU6lI2qlvwxnFMVuR/nmIJJ1klDDlTFuFIMbxwD5J", + "lnsP0w6TTVKUXoveR0i9tv3rWLOiQPjq6xWIaVpGYJ0vW+PcHspqlzTLIo/v3tctqQtPva4qQVNg4jkX", + "i+UbeuPaQR+FU6zwQCXyG9e86VO71Im1W6QDf2OyyqliiWynwafqVdsDNnkCbVuOrcu26O3FoKIxuhgq", + "64o284KNEgXwZX3j3QYGGcqcMrHqu9wCG/rihm7PHjnDGwlwK4CvGvY4lHBnXIdgoCrOOhuxnYFCsgki", + "jkOm5KycgRP4lI/Go0ssgH+CSBpjmq/5TB5QQRIVfXn4T4GJ1aoMrZ7sjFiHfbgjt4wpF5dY6F/OcHIO", + "/2zNPh5dben2WxcYuKrUHWvreeVHqf38wg9pN3DCSxF745vfV1y6vm0uMEgFhb4SCWbv4cs3s54Gw1S/", + "HgUDfh27F9Khvqz2A60o90Qyp4okqhQkbu/EQQu3UWaeFjGa/wrnNFvEh5rCtwGDvOFpDDL1GLn+NHSI", + "t1FhrRqGBdqm+FjNN5XfYLDOxnzj1rmai7g6JTg3WqQIUSU4Rzl8tHbywFWg4YJT91fo59gtDwY7xypO", + "DIGLxHsWk716J9Ginu5m9KMPncpcUpYQRAqezB81FAEd2iOQnyKT6/lsrE9Nk+sj19xyrCJjRi8IM8/w", + "Cxz4sBmX6F6fjfo5uCXB9SZFjxKn5Sr2Zv8IJZxN6ay0cWJtFU6Hdrh6BLwJRIumxwVYO66hpYL4svbZ", + "v6HslSAE9KBnES16ddRmIDQVhFh9njGDhMz4gbTuOlKRQo7tuiboXU6Ve0GZ9jh/IJE1xkzQCYHPOyaG", + "C6YEe1Iw5xbY1GaCX6r5BJ2agEinR6XSmYnmomTneuIkK1Oj5JoTQZUxluEMrOlbGRYzIoIR5AT9CkPr", + "oc709GQ65UKNkeThRE6AQwlmKCP4wuzH65TsyciMzuYqW6AzkvHLJtCaXU1WVyq+JZc9r4WMX342Oiii", + "PmNwvYy9HvSCHNwojkxDWKLrbC2e0lwKGD7HCAII5viCOBkrJ0hLhgVJ6BTiCVLCFu9Kc5ET+G97x6Em", + "I+qSi3OLGnErHS4VP8KlJDU7nZm+HQTDc6xf+Vm2MPbKuugXgKa1/fXO+CawBdt5YzR3nzMleCbr9tlz", + "ylKksH46toRnPcOWXR9nbjHoIXhBCZKRC+zCdP1iQIwVJXkUWp5NcEc1HEoFL9y1bVkjqLH+YpYiK31I", + "426l6tjyEAd/bUELt5lHz40dGzBHhabqh4LAPx7V9uet3RN0UiZzhKtjSTBjXAONWbUxjxsvWYGnU5rA", + "QvNSKiNNmc/kqshoQlW2ANSj4TgJz88o82blUvFj6DVBTZs9ejgts8zZhv3muuHODDTwAbDnO+wDONt3", + "pNP3L3lDQrOv4xHF+cD5DnEOj0fDiXrfmklxw2emRdSBPd+a1tUxSJJE5c0T+B3hLEMWCBOe5yVzYWxw", + "o61Xa2jCX+lx6Nh8v6Ux4hXw+KeYnKXBCpy/I3zXij3XoObf4A36qcZEjFdM46LCt7O+GRchhvZ6767h", + "FIOzS7yQyIBD+rwpJIAZyukHgQTIsii4UK6HZUuTloy8YXMbNrdhcxs2923Z3Jq4UYeP2o3Z0eN7xo4g", + "JjDmoTnwfqF/7XqXuiCZNA9jVDL679JEMFgaXgiu3+ETpJubIKsq1YP3k5SKCzwzVMhpAg0XwCpIEqHv", + "4Lmb0OWT0JTP+Xobj0oT0Fll55AmnqsnPczj8ajASjOy0bPR//Mn3vrP3tZ/72z98nnr0//9vzrNYJEH", + "fslAfZNjca7fz4oD95XBIT2QaEqFVI5hm9e3sB0FkTzT7NK88LFV1WDloZLMhGXl/So4ax61qpeIWuQt", + "uezzhLw9nzgYyQKnAe3rzGYQarTaRUVXEjsL6zbT0PteLz9QW83K82jqnX343Q3ARTInUglwL+n0HH3l", + "zNdLAkZdhqUptB/md2a6nJg4U7LKLNL3GTbTMKfVLrV1XlfW9/K2oCn0vDpxAZ+R7fGUbMmEFyT1WkeS", + "VmJZmlMJMjakPJqgt5wVXFKlmYdxkUICfLW9vG90deC7rYWykp0zfgk2cS2dYFa79Mkw/W5eOVj27Vxv", + "xvlifh2PeKmkwiylbPbBsur65t+dWRIKIvecZ6kRPu09xHfyHP2HCI5Sbt3fcanmXND/EONaqk8gChH9", + "TzkZhCEPi412PnjBgpvzdrjNSW82hYC65XPahqiCo/qtxWcxTkCHTN9BEhV0nEsTtW3C7FJL8MFG/Q0A", + "CBMzWfXZn2M2G2K+1lO7eMRLLFGGpUKJ6T3YPHIx0K2ynxLGPK7b5zuu5WTzcZHNbTeArcKuNjWsU+AO", + "yKk26cl+nV59stzGOCJteM6G52x4zl+O52y4wXfCDZbxgBix9wwkRvaD6Jg2HUBV37aaFpwX9o/e9wGn", + "b4d8BP5AkPQ9jVKzI/pkD+JG6jNVYZarhLiEXo+xuJkq71qVS2B1REuK8oiIhERRWh+4HryEpAuFaWcy", + "TQwZO6XyXMaimZTJ/WPv0iRnwMkcFJ7beRVcNDShRBhUFUknMS9n5AjPyAn9D+m4Nv0JLg1JymYZQboP", + "ZGFe8dbcXPLYxdb3JWB2Ohk3m0QPDZZA9rdSAfVeEIVAP0fSR6uuAkBlOQQVglTgWi2mB5yWTRyPbqy2", + "Xk1CGXhpPIToKbQFvhdD92kw63RpjBkzpOM6aGh6ve+ON3sbjO0c0q8ddVYjYx00p4a07QVGfDqDA3JY", + "2YKR5t3FQDmGSo5sn3ie2vYBLWXIkScf2RYCqefMBqrD9dgwbJxowS0zZzkGJDDGCpdrGTx2cLoATWjC", + "maKsJAgYGps5G4hRZ1ahGCajRSowZcYfMjHh+uaPks0JztR8YRieXpj+17xUusHnlF+ygT6V1Ukc2zmr", + "Xw6q2asf98N1VD+/D1ZU/Xri1xb8Zld5AIusXYbh2rf2gFoaFr26WNXM/GJ+1rt4J1IiGs6+1kgBSx61", + "UkdyoVDqOwQJBH3jqFOxMTj1BEXUraf9PpK3ZD+9TqaITT6I7ywfxPo8PzSaHdNIKYi9+kvX0OGC8wyd", + "4eTcGJU50zSAlymSCc40iZ0JXrZ9iV3A677JW9HH+92LTQb5urBSJuehTSwm6GxYUK+b94BIQ0baSAMf", + "wuz9fn4L6StP9gZfdZcJcI0QYVMuEo0lfpoHsn6Mjfc74wrKb9ieEp3xkqUSPfx1/wi9OfzVJ2fEzDJK", + "MNfrIYl4ZJjeKtugkbflG+sDe/+3EVNXDSk6EIkE5Bc0jaW73gfId98BLXwmazpDDz+O8KX8ONJ08ONo", + "lhQdEwgiIaQixnL3ve3WoJlriw4PvO04POv2IvaO3+q72ftwMkaSZNOtjLJz/cuv+0ePhukD/AnU1trG", + "rnELzz8Z4rLvfvYyRxwLPVQpDo5Q3TDVojBpF4bvnUmelYqg9NqovkISKLcMu3GfsaAhjtitGMbTvcsx", + "AgjFLh+u12Z5f7i0JEbmtWBBrua4lPoT4ElDWumIjdozgVFnRE9cEKEPwGErpGkwy0yAIKcWbfaPX+6d", + "Hr799VFcex5Lo3Bk4WjL+K/V0ifYUf/73duXn49fnrx7f7z/8vPRu3evP7/85297709OXx6M0Ss4huiM", + "7mgiUqs7tGorJk+llpSgeBJmi5XyIvxW5phVySDMmFWezVUyf36In/HN035GsjIc09lh5yntsQq6QnaL", + "B2FfT9onv0NPJ21j60Zn5gx5BM1I7ZtxzQtyE08G60hjTMBBYTX+4QF6+HJ/t/aDI5n+NxPDZcjmGOFM", + "8up1enjQ0rTaHKPRXKKKCEiDEa3AddrYOWdQ6eUSL1xKZgguAmIPj1zMrBB6RhKeE2TfhgjPMGVxgTM8", + "zPgK/GVp6ddmfxxbA4H+lyDuV83duJoTcUm1WF4q82t4gZFFxNhMfVn1Y9Lw+92Fpqc8xzEh6gWWBJmP", + "QWp+7x5sHTWptG7C9CwblMWLsIvUpLLvKPNkU/SbvAKgjQLlKrtI6z7UtxuZfluh4usMyLZ30Hua8HPl", + "TamP0t5XVd0OXVCscelqMVl+g9cI1m5GW3f5yrZBofIOjgQImiSSla2u7fhu378r++6+tP2am3XjxZzL", + "OgcZ5LHudmlnQNMMz+KbdA7NRmUSf6vbtXR5H9yUEkE8waGNCNjriCb4MCea3vrIARdNcIll4MntN8wF", + "Sqm0m6/rDSborfHYxgw8JvUIoN2oRpFE9YBucDI/Qn6PtRPsNaQTuYccIaNTkiySbKjr/2vffv2JTm4a", + "gbbJk7LJkzIkT4pd5UvwTj/SEkMX6z55t//7yU9GqjAa77pnO3pXKlCpodP9IzjckjEC5XTmgpezuXuN", + "XS2sxQG4znZK2MKW9TUFX+oxdCXDl1iQCToAOrgF1YCBn/JLSD+LBMm5Iujg7Ql6uHf6X0d/NxTzUYx/", + "NNhmmoooq6vt1bZClKE5l+oZROgZyPaqPCNr2eJxk4Tnzx7v/LzzcfQomvmwOyfcu8KkV0BuBS5H3MPj", + "V/vo8S+7vzwaoxxfod2ffjIG3Uk9QmL3p59WyuzWnNC1vNGETfuxPeYeIevlRVQ97+IwyYXh0g1hUP+6", + "jxWZecNUP/93bX0eUD2Ayzz7wLOHB2P0wFZaeTBGRCWTqOoHeh9Y8t5xqpDgEFIlRArm+G0tTYUBDV/j", + "M5c1pn+b0LCSubzSC3YbJtt94HJ4WqISbPwz53nf5mMaDrhEF0tU6bsHWWnNQqAIS6xW3fvmoC1TFGTc", + "CKfyFvGuyV5ekaTU419vQuK6rzTpteZaaYZTgvPYLJCCJ5gjBpCR+ZbcmMuFfb0zDN4kAza4SiUjh1oD", + "EwBF02oaeO5KcN3pB2m6SSVKk1SqcjYcYPNoNB7XlKsVDEWBOHYpLdRqwkkgBLzi4rwzn3lSuf05W/G4", + "05g65eI8fONCCUCQr0wEvf4u0RnnqqotAEHmzooO8cBNe3+CCwVe1pwlkHsUizTTnJlPTQm6CXqJkzmM", + "btzMSSr1OxQc26F2XEFYCrZdI2fwUoHukk+Nrx+EJGr+bjymqJI2yZOPYTXFlaFKedzkPbhI+o0M8Ixc", + "tg/5Jtb4btZsoMJVWWwmRtK/69PhLEzuo5c2QS+vcKLf8vpbEAfgq15RqZ/5z6q3g3tMmAs0PKlWP3Fs", + "e0JLGtZrhNYRfZEzhA2ovrp6caOeUzs0Ad9xcUa/1TKOU0sa1aKuCzOOJWyL5IVajNEFzmhq3soS5biw", + "GhYZGaapZ2knV4NRhoein5r2S3bqX/lNL6WUkrjVx35xeOV3AusbuwBcYgEIS2cYSCfxfP7nhMXLDHyo", + "jQzSz/IYXL/wcOhPy4+gN9H9Cgferr35Fuc23EJvZstsJiVTyPTOmRxX1TcxSnCWEbGVzLnUW4a2motN", + "Rt1beB3qRuJQ6+VjVPCMJosq4vtsEXgETXkkX0kt60JcrVh7i2MWJpKI60Q5O60I6IDTfefbt+/bLy8c", + "tufGX/NZvNijYRf1FIdgL8soI61zgR+j4+gvfRUjv1FVR1jwp9o5dNTQnFKSpb0I0VWzpDrstdfh/Fan", + "CusPa2ba06uftFxeLrPu5Gwl0NRkbm1p+1ZRY/VVxsx4rFjN69uYc6nKDOYeh+fQOLM/do+t0BY9vWVl", + "Rj3x07vxMn16a6cX206wgzeBBnlYBRXXY6lytzZJNGnrmzDN6VCS1h0h9LYdGzTQM7Ao30uSHiUdJUv7", + "IoGmGQ/LJrskqEZL2huIYryVO0v2dAdn6I7xSBNwRu4Mx+gN94Di2zE/PVCGWnffhxAclOjfHq0+Re9p", + "9MSp9A4aP4g3SyJTuof8a2YHXiFnb2ATCPCmuovgqgPACqA2RI2AEtVNTPHEq+9ixTJfU+lDDy9JilIi", + "FbjXcGaj7MB+4INkzBvenJ6WA88Iwmj/8OAYnWU8Ofcq958n8N/2k92Po0djhNEZFgQdHnl9faMhtOIC", + "YWdRNWpu2yhQ3X8cjdHH0f+e1H56BIoL2ICrOm2z+yl8TpCGQ5KaJ80FESgljFZNJyuViIeDOirPMpqc", + "mjNZmqvuxCTmQ7RG89H749cyyNVeWYlNpjiXWS0oFROXtG2yv+67tdutbgnULtVdkPhNH1QXYVJMMe5z", + "IFqbMFiHRJmteoikMmgN5NFtE9jX8WiuVCGPwKGu810E/nY2Lz4RFwT9dnp6dIIEts8azFCRYY3LVwq+", + "TdDedEoSJdHcpkU2hiZBNC66pFM+kJymNbu/0VwVAB/6hhGVdkZCYcZLvJigfY2aU32rwdFeEAERMqBd", + "A6dsW3CfM6tOMFmtwBAPZq6HT3/55ecnj4K8fBmUa6ldRjvs1mvB/vbTT09+WqYHy/HVoRkrTM9trnI8", + "MvYE28AWz8yxdJrK37iMGY4sQsy5VFCJx9p8wZvhjFQ+B5CL1V6k1V9FfScBBPuksFXEQUtLj0vjWtCE", + "3YbfKBFblmKBp6I+bYMRCBdFRo3+C8yi4BbprLAWCu2eJuh3spAufAkUK2AxNbj3ELDT0DtN/HBBG+TP", + "UM6MYEgnfkmzNMEibXVsUs2xCeMC+BM5zuh/zHIhl1yCJbi2QtzVBH2wg0qzGSTLM7NvibCCULCUFGru", + "Q5AhR2ZBriw9f+5YwMfR//44gvgwBqozq4i0Z9Yg12M05Zauny3su47NCECNPVG/WQlFi8yW/Fc9KpJE", + "VViWEzEj6QS9EBynvrdEEnJmSjglOB3o4diiGXSByFXBJamumqQoEQQUfDgDRTqB0DW/gICfNgLqmMKJ", + "vvlj2HjKYXUzgZnyKVoNS3hepTBBjpEjSQossCIZhEUURED2+jkJJzRRMY13e4+uMAT8tiBVg+4QuC3f", + "cLBc7d3zcptcLqLwdIOuhpp+LUO2cxrO0bEnK+/EMLa5OxtzaCvAWL/z+q7mBKdErKZdacQSnJ4eITuM", + "Xg1lkIyGC5BfhOY5UMPMQbknI3sMkSsqwdna9A/Nh7l1WDeGlAwnGhH+MPluNLQDI0RmUfK5S/kI2RPB", + "ioXmuCgIk9YxbwuYn0s2SSAljQvG2zs6vCb4vS+0vN/l5/K2lr3YuaeW0MewDJ9CxrszHpu9ylqRbHev", + "gH3+jJwiu6nyhzgEA9Og/EFJRrCQiKqO9M8bYfvWhO3vOpP2X1c4//5lMk83MDjZWrJqKYYGJm/26hXf", + "ANFsjpiNiLYR0W4ior0LrWrReM2O7PegiNKgPgnyXJzTLIPcv6Uk0UwXdlZIZtHpe5IPy9jvLYncuYpo", + "ZhrJDV9PURHLmk+ly6pv790nyu/IWVFlzKeqynYxRhmXpv5SLW3/2IsQ1t2OSAOzvCAMuTQwnAGJgDAS", + "qqq0ETZKwteIemg7IAj/080fPQ/Np2P7+LUMXgk6mxFhrcXijCqBhU/UP0aCTCFFiLQ5/p2Q00r0EY+v", + "6wKsYwJ5PTovObWiUERtHsYIma2Et1ApuM5JoRAGR5nKFyZURzz5W80rZzVHmBMLPZ07WOJhCyKQ99Bw", + "fk3OBW6CDqdhRQXvsW4lRyrNALZ4l+EWGmrAF8jUG3PaDSytXseKyp6z+EEpk4rgFDynXEClGYmzDgeJ", + "zmNx3vsrldm08O9oQ9pHHCxB6jx31UWwbMcAGCp3Myelg2Le+KN3glY7/OUaIfpquVdBGCtwk1TrEYfz", + "WNnnI1wVfe7qG89OD+P1bITID1TNg6SJjVLxYV7VG/jLWMPl11bKJj8+GDB8YYVuSym2T9EJOlQ2rDnB", + "QlASlgAwiVsnq0ShN4JkzDCXWAbRJsOMUBZc/1iWfNPm7rP5yOBxP8fKSRY2mJwMzh777SpPRH2OodPA", + "oCZz67FgCHjKr3ZdkCnVdryev4df+9ghUeNKg/MeByAVrjeKcfXD70GosI7Gzu7T8TIV0X4pFc+JqMo9", + "1o5Wv9HhTV4IIglTYwT+ii7nr4QWCuVcKvRk173On6Nz/ZCBeiI0h4qUiqPHuz8bI/DYJUfWP+7sPnW/", + "whOlqsThV6Q4+vnxL7umGbyaucKZrxsSHsCT3c7TMwqh2yy9cqM6I5oN33GVke7yIk7KiZdUtjkFm/t6", + "pYVs+xadxuWboBipIjg3rU0BVaioNbPaIf1xS2blbDtfbLlRnl3sPlpJw+A6DqQVfYudE726CXqvZWK/", + "6m0IubbFvAxdvQzcVHs3Yx8xj8ZBAospzjIJWZR8sjF8Wa3n8MCOiM+Sx7tP/BDLbzo4ibG9vti1nxLj", + "sNxQNBbUFplpvARNCRkvzep9RqOE5IF7sPW5fQJA2HBMu7PGkIG+azlv6VqN/n1ouGhshFa0Jgzn6bk9", + "rHDXn+zJdhXrWSI5eLcSd94u/9JwscFM8GJh1cbvpqNnf/YTM73e91JLAJ+akWmDs/FXJYaWxhFpzhp3", + "EnqteS7QPXj9uDPQaFbKbja8NPw9x3JpXHW1pTdYnlM2M5kX5DAAHFphCU7EQg+sKmT8DnZuoaR6wpkV", + "NnoqGmhGU+Uoq7oEASYNdB/guhcW2ziOvhNjKfpd6EdBhH24DXLp2/iGLfMNi8BB5I4c5AEVaNEsktsU", + "JMtO+KVu6DZeSggEXYqcw+iLHW0JcYlhm1m92aGNPYxnU3EosiS3rmkaybEyXFYB3c1Sn1mjXq3lMtCU", + "UHdWw3AR5hnG/+C5UwVeWfWSRv8ZvdCvo55cMtD2ZFDWZXcFL4Iu10y9MvgNXju9lV/ht81Or5sx5gZJ", + "ULBUJwW+ZCsflnkD34jzXiOHSseL42342PDLfNiUz22aBpMA1n1LV3tLGFe7ZRKsezhI55oHGTazReh1", + "ac1XnaKt1PdyXUrQvJkeH+prpU65kSolAkg3UabUEqU4ujYoH4q9zC4NS4jiTVyp3U+NbNfxcewZiIPe", + "OlEMmQ/wj+4AlfWB3m3BRN9F2d2E+wfK3973Cqyqlfeh66Fxp1zF2l+uwVLWzwGmlFE5X21Xrs/gbV2H", + "1MubCA2DSVG1qZvToYr0+NpUnXQlQptamPCKZuR9kXEcwYmb+/xZA1rl9jfHytdkl/qp573USliEszDH", + "4oat1refNk1pBnTJGS1tJ+felNjYkzY5KkUkEum9yII0eDB2Zfs1KwZ18tJrc2tvnX9c6XkNatRWczQC", + "NTsDDmEd1w03hM7DQjVrC+h/5ARLemj98Iyq3zxV9b7/eGzxqsBqDsglSM4vSDqGFF3V7lcTvQTBcjl9", + "CyjBselwU2KyDm4YoR3xYNbaGl/zmbxRQOtdwldXMGttB5YK/fGkN43pgGd38/gn6MB3M2BnXJGNq8Mk", + "Ukn+VmSEayWGpOyVIARKxJ0t7VhrPFAD7o5kHzNrzCAIg5kFaHzCMzAWg/8Yr0pS5Ist19cVkAh+enbx", + "GJxzD6cwEvjvwdDp2DgoGfcTZROtY+kC0mDe0DxiGYDCM4kAeAZdj24eYe96EMWt14szn3ix2JHk4VQn", + "tBOsmlC2w3LQRAJQtv2xG0keVc+ZMxU8P8zxjEAaI8FzN4rn20Yhg1lapYxpeyhAxZPleYtD1n0555kT", + "lioua4uuKI5EyRopmPo5ut9JhKLCBhUHnMUSnEQlCbTNjZvMKfNwEoESP9MxmVFpk1P0IdirVgc7Sqgi", + "bCrf7ZpubdUgIO3nseoxIDolc5KcQ3ZcsJdym3+OeFRy81XVvDqFEdCvR+cCuLy1WUixjBs5H3Vo2zdf", + "C4uHcPUTRYool4qYP9tywJKqga2lOdc2+Nv4tl1iaqvouWp/JiFVzNnNLcHyR8fkO/nkmhXUd684XkXT", + "d08VeD0MqumTeSP+tE7p1GvM2lot2G4o4AHCRfCGFHGEdu7P7QgnsVSQ3ROz0lTM8z61evZVDhIcvn/D", + "MuIbqX91JwjNfLKKYKY261mdz+qhboXBxnNYni6KyucuuuomFECus+BOT/Hs5k/iSLZRKrX4N0hzN9i2", + "Y8VMh2vDzbi4ox6NHjGeALR5bLCVQBXdsOObszSOZp2OzOsiVF8jS+p6Un5r+0vEAblOcz5QNQe2Le8H", + "o5TdkXjme0tEXFWsMUrzyPzrMYJ+Swvixhq4sQYOMjLFxJUu1fxyM5+hOIZU9vnzdWhmyGU9+GeofxgM", + "p2c2gSG3GxNivkRv/WDFqBA/lNVNxRxMzRb2WNqRLLarBo4NQ67e2Y34RjhUswCX2dEH2I4/MpOFwDny", + "OufS/8EFnXwsd3aeJC93X3w+ePdm7/At/E3+Z4LeaUT1yU0d1H5kzk3Vhtu5QoUJ+Mmjhy/+693+I1ea", + "/DnCZ2Dq8C6+Y0TZR+Yi8SSpLcgGE1ODuLWYzTrxu+5tq45CPKWWxUxqXHPCU24rPGEoAG+ktfr5rhma", + "3OJjUPWBnM05t1nJOgMI91uVxgD9IfmGC427NCPJvpJjbRoNaebrseZLOUj8Gu1OwLUNTQUlLM0Wncnn", + "tXyLVSlIV3CT+d08WhRHRjM6J26bqMCLjOM0muvIGt8quVzQuErTnT7Q1SHHLkDBBoVeIMphyKE3Qn5v", + "9w7u48FGI+50V7/V9jI/NW4jigYOxHwVX5fO/voRbTcSJG8ds2LOox6vImVJBlLY1VBTddTf6K6G0RWv", + "cB1wCV3KQ3nHrsoDkj16f84BAB2QjF4QQYnU75YjA8+x6NGZZsGaQ9rzSU1HqJtP8kJJUy7WBJrHS/i4", + "6KpB75/6+ha/QkngGHKTK7VfChkrgW1+h3QJWEqnjNM9ILOdEx0geY1+jXhQFyDNM27aFnhGVi+taYP7", + "gvW1j33RDX/N822fpqW1b2Ss1rnt7LIGuMaQ54lmGa3i5Qc4E4Meez/DsWQ5b3Ayp4xUlaoZZ1uQcIoL", + "SIECdRtKQVACA1Sq87lSxWejIh+PUib9v13wtK3WW3Ch/De7If+3p43+lwSzhNgyn8urI+pOb7oSib8M", + "i24DBtu9mBRPvFQIa0IxLbOqWsgZB93/8qk1jsRoR60O1oqllaBPvDhCfVhXcGaIS/5BAxRXXJO9shf6", + "XCIcVlCbhsZRFgez9iC7xvutcpGqD/mPk3dvtwhLeEq8U5N3gwIJVxKoAX1BXACqIClOOuoB2hHeR52U", + "jl+7Q/GFT6n0uFs7nRg916MbsImfzqkoGaix6uBlCspXvxkXnITQC9jCUtBzHYceoZ3GnuGtza5UYaxs", + "+9FK/kA8/CzGbwbK+C9ZQIugdayoHbnUh439soTssCK2UMe7/zgqaO1net1Q3yZqLlTDynwNwcYlVQ8q", + "aS+NLH2lSp8Wnw975LR62r1VNhKt7e7EnmrmiszWC2pVdDLwyAo4ap1qtWhOjSrU+GN4wBF+77LbaLiX", + "A5i/59t6kfrxmsgIC2/owS+IsOysuihemnrBrTg6ny5nWGuXAmVA68YVua5jv8Bq8shJGYFvsHhUEz9l", + "byVJ1+O6omg0yV3FzFfj0f2lA/uBvgLtGjgHYO63GjngoSCIZzNBZvrZJ6HHuGUdTM7J9U8TlvECxoim", + "4qtJuSuMW8e0r568hjfUoyd35RFWjch0p+GG8PPWttJ1HfYcll8KXIVNUAoRpWbeJY+E9R1fyGQGWmWv", + "e+K1GNihZ658WGv8mPX3jcbkB9WYeDkhojrp1JiYjESloGpxolHGQMJemlO2B7kg9kqTdovq3RhR2U3x", + "bPTPLWi5Zewh1dWYLBJfx2agf3w4daOcESyIeOX29o8Pp5q0w8QaEOBrNY5+SvtRwGbVvxjdZAuU7u2F", + "DNrM0eHW76FNK+hfqvmR8fwVL2CZHVsyxtHPyp7I8r0FA99ki/omqQ39UFTpZ8Ho5e4LtHd0GJRkfTba", + "mTye7EDpuYIwXNDRs9GTyc5kx2ZEg+vfxvrAt30yg22b6W4r8eXjZyRaSlOVgkmEkZxjQdIqKY1Jywom", + "KcigSFLrfTnVz1VX8RPtfWQ+FafA8JrlzGafRXzqM04mmCFBwNwESyIpKpkC2pbzC5c8SdM27Iojj34l", + "CuDIpw04NoPtmz1VDzjY3+7Ojs06oWyEECTFNRVGtv9lIzoMl1nGgzz42hntCuzEcHOtFEthbm6zR1mV", + "RwQj3uGBvsWnO4+7pvf72daNdNvdXwa03f1Ft/3J7L+/rW4UUhAIJG/Rjj8/fR1/aVCCPz99/TQeyTLP", + "sViAmaVkqpmTnEi3WeenB4XYc8oM4bJgqhvI7S/G4/3rNi7o1jlZGE+aaGENY+vRYAoHGSYCMnmwceZr", + "l1xycQ5l+ictgDriUvmrlVCK+MDsG6RJLHBOFOgh/ow+cAGLAc0hFaFHcp+Tp6Lw5oFfQduyV+Snlnrq", + "VgD5LbkMPAcaKQttNoAGFj2+tcnNraXNBUQOtpZpqFF91+DMzhCc2VkZv3aeDGn7xLR9OqTt0/uOt3DE", + "dTzCEhkkvQbSbn8x/Ozw4KvB24zErLIH8PuNMdgM04HDe3Yh3xqXx/HbrNa07Y5sZPC+hn9PO7Rr7sjM", + "Ca8TR34IuDeQc3O4N55Z28a408Ox4LtJsU/ZViG4KXSAWYoKW1Kk4ZhpqjNAFn/DQJczL+OBaua6Fxzs", + "LuUx2KzZq60MH2EnJwFWIHNJmRbBjD/dXxP0zZkBKAYgh1eW06rXBSTY74T932lmIb+dxO0aQO7F/99d", + "Wv8fGcrtbvVeB0K5vgzND3xytb8mlOsTiwBdP5gHL4/o+xh86PWgId+Q0Zeqf0fcCD4GuuRXEnUrcqEf", + "WLyDaXtT9/Nt2qHF+fNTlyImgJU6CLW0UnGoijWrARrARBMePHQ5iPo0Hl1t6b9mxn40yqg0KUx6Xri2", + "GkI4eJxEBsC2eTR2Php/CGBe9nqLAl+Nuq30RGu80LpeYMteXN/kAfRd87z7R+giT6Y4sIEiOonEs5rY", + "lWUgdaQ73zJE3T5VbMXhDCKMO0uA2UYEbYD5roHZguIwymnDaeT2F/sv/fwRdLZEUNQPHC6SOZHKJLhh", + "PCWo4DyT6OHHkR4AihpPEXYRO7a+VmV3MVErVCCZ4IyymXEnkRP0ylRmrWL57QgPJCLpzNfuft4cm3Ek", + "6AzlmOEZyQlTVU231IqDEgp8QXItLaWYwhem5F2CMz8cZvKSCIl+2nncYa7Zt+e2707tmM7kyqjsz3x0", + "4+fWIHH6mM6uL0dD0QQ6q0LlzeK/PxzWbR8Pafv4lh5sgDLu6DzQdrzUOjFymzKpMEuI3P7i/rlE0Dkl", + "IocgAgRpqkwfRJkpxQVlmASdPZB1FDRFVTPOIO+TKZGS4CwjAiVzzqUpnumjcTXOybmgzBQ+NSVb3VRB", + "neIIIhm+G8WlQ7fZQ7/Vm6DXuBVeYQlztdYuXQoNF9CtT4noxlsRzAwK+41RSqQeBCW4wAlVCwQezIkA", + "qkVS9DA400fPjXOLraYIqqXabWW4ZJoSg38unHVuckCaZcOxw7b+XRoHabsvP+GBWc2Q3VUpDtr0KpJt", + "y90dUhYQKWcQol0okt4bVf7OECKz88sPRJDidAGIk6CzlQnTF0Fn+g8HzibXRjS4z9dPBfj3U4MXAbLl", + "WyMkyRAhU46JSogtCsUDXqYu951AGhswm5H0ObqgPLPFuS3TgtEeSAT1goFeaRkhoyYYN6jK4QmVRLIU", + "UyikJBG4HcsYKTsq4zLBsT6afXcwN6NgSxrDLdzZq+CYztw29uF8hz0LdmMWI0v1zD1tCMK3JwgOMT1D", + "ujEtMJjS/4oQJIECOBbTTZceKmDTaBpX04B42HhbidKSmNKMkpciIYhczXEp9bdHY8TIJZEKTamQahWp", + "HjD4pdnOOvB33I73A5f9oHqQPSnYqpbOOzg8VAAchWzcB3vv7sSqrbrAhJ/CKr6PI/7A63qtwLHf7MnS", + "hK6pg64f25XgGz1y4hh9U2LiXz7LtRK+KcJK4WReiQkxguISslNRRe2bisUsDdw8XVVxyaFu1tkifDWs", + "TEv802Z94sAaUNXt6mbYWl3fBlHvElG7UaUPU41r/natvnQUHX8lKrRQm7CweoG7SMpC0qk4/5WolzBE", + "WFitgTsxDsinU0k6WODOyvXGv6zKZR/3ctnHO8vYbMeMXKRE7MkkPqlNidl6sbfkilc0U0S4u7FU0qcQ", + "r4LWIXb2qsggZNgoBmKrck0jpGN5PVm1gKgAfVhrIlkWkgCsbihh1ID8vlCrG9g02gTlvlg5HFEKY4Ra", + "VGn7iwvINNrRTgrVuLjlROfEjbsy5/YrGnUi9YZUbUjVhlT9JUiVz8u2RD1isrlViVoicQKeSH2ocr3d", + "PUTWo3oHgKRL0Rvd0wa0rgdaHow+dXrbHdvzdqTTdolHjEUA6fb1yPWkhmv2uWvm8FvqV2897vyxbSjm", + "WsE6QjO3v7hcMwOd/Nokpyvaqg7/H9w0Kwt7foGd8kEPyDmnvzrIbcDoZtSxW0sxCD5aXPYugeM2KZ3j", + "0KsIiZexVFEbOLwdLr3EfbQNjM1riPqU3hVw3h37r2WUHu5g+g1wwzmubmSAeykDbKc+n27/c6ozje4q", + "5L5K3nsT3GrpN94V+N8lQYnJnjsVPLcaDnJBeSl9XqQHElXZbdGUkgwKb8Z0HGas0RIXtVXNxj/dZy3R", + "oS3EWUEEwgpx4avpUYl87qquc4Oci6NopGRvyYwBizkjUw7VlYesg7D0FlZhFWfhIhb1nGZlTWdmzzru", + "L1ilTSs7tGerpdjs1awN20otte6QXdTy9n1z9d/w9HCNdOE9Kpbh6cI3rOyesTLpEkR2Plcul+SKXIWX", + "nbjkkrfGxoDmSUibBdkSBfj3AUWdoANDz0F9v/sUzXkpJMIzfteE+OVVbFGEpfUlMX55q6R4faTB5hWN", + "kIR4EsvNU+6WUHlOcGYy4kXx9Tf4bCoLx9DSfB8NCgud+7AnRCUyE99h7O/X0GGktg/YOOMpGZBKwDSL", + "7Pyt/dCb4KJdHK0KnIkKvN45ap1pLgaZS/R+b5bAwBzlj5NUD2DEwUfM1Qi+bX/R/1tmxofYQ8gv2QVq", + "b2GUlVmdmTzG574L0FwGkdfRDTILyX/BtCtvAzBrQmxnsou5k4BsZXNsTjBmfbsNSL2r5Io8JbZuhA/9", + "+DqUbwF62hOApMwmOmd9geDfPJzjdnJbGVAKTrODcsa9M/ur83qW3UpzOkHvJUG/vjxF2xe71djgPEpw", + "Gn1v9DhpNiIbiMIpVhgBvfRF36bmpR84rBBpYz8+jkpJxN/xWfKx3NnZ/Rsuir8XgqcfR48m6CVO5ibd", + "HUtdnZm8lAqdEfT++DWy5VW6xPzcrqZXZ7YWeUFfB0ntMd5McGhd6J2++deDPvf3odAMh2gdfoWy1W8D", + "Eij1Y69PqeTcu7wC2WXDN1h89O5kMBprjjQYjw9KAYXI9rGak1TgzDpOVFGUGi0n6JgUGV7Iyt9b4pwg", + "XKq5lqdM2SXjG87SWlEqC9NVH7uD50iQUhKXWyKl0ykRQYla0xmi0P8FJWU97jfTlx+mJC+4IixZ2Dzr", + "Fc7m+Oo1YTN9zY93fwYVt/v753UmOPYEYb1OM7VpI+9SC3RBwYU6bx/bw4aF/XPr5e6LreZpt+vAJXNu", + "62678FCozKOhLAZbvTT7652qOe9DsOiTIW2fmLZPh7R9+p2SXUsKLUzGyW1dStrOiRI0WaLOsI0ggmVG", + "Lwir0fRuCeiNHXwJAd3neY63JNGNNKhnVpfvcOvwAHSUM1JbyUCThx3kM01lb46HbiNIjq8OzUcw8dWk", + "kfHIFDKxDYAY3enT1Z/tB6rm7nxvJhMZ/uEAYSMgrU1AcvCdezQZLB7VkbgR8tHlFGhSpwbkIeYGeLvh", + "Hp8G63cDPuoy3f5YudK+B7AECOlnH73ehO4WzxYIlHrdrOGOoOvWCe111ISyerpuYPbOYfZkBWknIJTb", + "CWeMJKqe37z/qenKKDkwN3Xb5AQdTmsBRpCkp5QkHSOq0KVGqjOCBJFlTtIJOj19rZtwli0QuVKEpfrX", + "+Bs1tuSB71aPYvt2pzfFtNt/1tmVrfS02/kWTzucCYL1S9wITxqtvtEj00LR/awLsnkO3qvnoCUX8poU", + "csrFeXf5h1dcnIdU75mxkRecMpNhofHQQJQhSNCHHlKl6d+ZoGSaLTypdLlRfU4EqiTS8I1yknPrv0HG", + "Vj1msIAzaMWgSvc5IYWeUP9yeADtyFVBrZ6kZIqXyZykj+CL1aOY3GuMXIb1BIwCESu/pAlypJ8zQ8gz", + "hQoinKbNFGM/HyOCkzlKsBALyABDfZ5Irxuyh2FzQUCOGDNXIcA5iqSaZ8ASwAeIstkE7SHG2dbuzmNn", + "SsoJZkZB5HR9Ni+U9bLEDIounRvi5SrGD2UY+mrvIbew69OrOzbDx01xj+8sCNlMbcp1LLdLNEl3COAu", + "9a6+JIkuib4zd03PDRgRpoSWLAsulAE83fiBRLxUCc/JhpI7Sv5dUmegn9clzRmfrWLhDPM96K5G2uwW", + "NHWbYUZOTzNe82ukgK4nhmi8ryz1q5zFnbsJrM7Q5zkvs9QI1/YB1q5Wv6J7pSsDvSzFxLI0dX2rXD1l", + "3eOdnZXzYKzheQy3fq0kDADBmxfyGl/I5shXpTTLDAQhaakUmQMIRqdx4AY04z2jVwG98NXINLY76gF1", + "KC9wNtakwlKJMTSFvODQptrIHRGPIY7Wg7ZGWHq9ja225HVmejGAcTupXtZh1dgQpZsQpV7jRw9dYkRd", + "uvdpLNu2KwQyJ8g2rccS26pxjUeq9dJIrPXd5cMmpqqnKDMS+G/4FEz1IGX0LqcKJBcIUURJRrCQiKpJ", + "LG92mzC+tTu7tw8wu0BzwiaWeZjq7umS7A8u2DjE4Ojl/cU8Jr8HfLb41ri2ldEalEHdSqcj/blmZBmm", + "0YB+9xejYHm9Oo0BZlOjR6spho2mC7PA/LDRGnzXWgODANdVGwgyFUTODVB15cyCJvWEeWCkcqpVSBGt", + "OMroBRmIfsd+3nuLgnaJqyJhQwQ127yHZuDvEtgdLF4f3DXJW83Cq3tcg8GYjvcQvM3C0sDCeE+8ZTeG", + "zI0hczAdAKy8LhlwdpcerueDCAoiJJUK6ji4Wo0+oMCO+UD6hyGYJCfoxM3ghC0XzGOtjHX7ohbK7Dzo", + "jCy4NQVxQWeU4SyYJqNTotntUMOdX8f95bNuiQGjXas7v53+kE15VKXkLr273PRGjXQPXd7tta1MGzR2", + "8VJ1UwZXHMo2rHTKTmFU42s0y4zPAUFXTgcbBCRpydmV9jDwP0H7OIOaLJAYKSdqzlOUl5miRWZ6SMQv", + "iLgUVFmt1Onpa+tpAAOW0nSv1FWVmhjLSgGuW1nvDI5ygmUpSG1raXeavSi1ObVnd29pjV3gjWR6ae/f", + "XbEDmA16r0VLHFiXlAe3ThQniSBqQJ2kQvB/kUQ9kMh2maC33OeZAwceyJ9mP5tw2rgh3E65Kg4UeGZL", + "gr4lV+qUnxM2pKxR1e012IrXZJiBTa5skckgkDd+4M24OH0KW+YYOhZjm2//Mziwu41tezKk7ZPv822x", + "O6Tt7l/gbQE0wYJnAJyexNhfBoQpx4OSDQE5WyCpuNAsGGxOwLhzLM6J0DwaPB2pkApd6HcBZ0asaBEk", + "F6QPvuMdbNpv4a7icC0xWLPcHsw6IGU9cGxPtTYkYkMibv7GcMgcow6h+LH9xfxjSTTeMbng5ySAVNAL", + "aHhPy4wASbDEwATeJhnBrCy60vdbvD+xU68ulLuOw2L2oln7N1i3wbpbwzpfwqIH63qCETlzsPig4ptj", + "JEkGqTBMiGKVyEwghvNeEf9OMGtn3QxSECUoudgg6wZZbxNZbfhvH6Z2mZtNTFAFjfqNqLggqZOPzxYI", + "F4U1QGPQpd+WlHxbOH0HGiyYwDjVrD0uchgpqXlrbQjJhpDcohvZclk7tOn1J+L1TWsVmKOcvtt81pNa", + "ppHYNJq3v1rD2QJJXoqEBOltVqhlUBtIr0OLLocHY8ShIdaIqfBs698lzrRok/rUgfliy3X+OBqbH/RB", + "bNc+6OFqbZ9dPP44etSVNRD+t6TKxqoqzfH1lKfr0YTWzIfX9VCXAZzdmhZ0k4fn2nl4guvw1Mb/1pOB", + "R6PPAAJkmkXozan9cPdwq2e6WSpNs4n7CWmdcBWr/W/P3CeP1U0/Vde5/UX/b2jkU5gZrY+zwE2fwsDX", + "jXsyy9oEPf1gQU8aKG4j4gkyh64l3GkFOXzDaBztOQ2vZwXqs53jq14KBABug49j1Ejjhf63ydvo0GUY", + "jXqDrzZk6t6TqXEksbigCVLca/tqUALODDaDZkcmcE2N+pJlukpnCWfW5+hzmBHU5dyEy/gssCKREmh3", + "qg19g69CwrohpD8MIXWJFtQAgmrSf1+rFEDVOUopq48D9ASWJHbTj3ZFFStHfpP8+253N30vuDPavE5v", + "9DoNIdFBefVb7+vUNtrGGcX6JL7AP7rrCe3PSXKO6NTPaWKdjegAfRG5olL1I8WemQ3+14EgBYbSXxY/", + "sG3ZzW8akSRuda7jOjmLmxy2d2xnWYoamiJQyO9vl362sKtfmcPckRp8g3VVrRmNBaoFZG3ca6CZadHt", + "8WEt2pVzN5455/BObmP6eNw6xbO7cu2qz6QnWilEIxagrPfnfEM2cRTfg7eFh3qFa7mC4P/dVtw9KemM", + "6U4P5SP98MABpStplkaNsOuAarOya0P141teCEnDpUTDJPEMYdt2gzT3G2kc2PcjTZ1JfHH/XOIe6N2f", + "XPulrMGPew2dje86PGN/JSPiPiq/EfxvTo07pI/xErsTUF5pEx11glEout8SDI2/m/CXIZL+B6rmL+Ao", + "V1UkhY8oqqS9kI3R9z49q6tLGfy41qihkvkQfZLLQtaFfEd6oDsh4bcvSrnVmT2tJEbtDGAhzp1sbTLP", + "BiEaTldL2E23HLNtcGj7C/w/mp+4ZbaqoZ9LTTqEMxlS/MLMdK2sw6uxKrundWcoTkohwUTzPaUojmUo", + "NrPaz9exvaVUkAT2MB5IqTRUHPhenQNn5IJkqwz6GjpEjvbEePQNuf2p4HmX7RJGWWmXZuI1qTwB5/Ss", + "g9WecXkoQPnNO3U9zgcxYntTGm/rcK9A5W39+JWp/Imr+P1t6PwhS8mVQ26fSsOfZSeq+0SxAWON0iE+", + "k++mU0k6COvKid9/GNJ/bQq9NnLYmUdoKRnc0L5vQfukIyUrUr8pzfRPcyznX3tJHmaoLDKOU5RRdu6U", + "L1ggPQLSAIgpC+gHXhDzbaj8+0q3/Q3L+U3pYcQSOzfDDjXE6lU4uui2sNwW+/huMFGfy3s4+a7sg+G9", + "XM6JgCym9kfATHtLG037fcdiwCV7c++PX6+Ozs5Uu8RxHwy011GfWovWbarh79DT5xTPbup8HFpA7ktk", + "3wZ94hrPqJmq24/o4vF2gtWcpAJn2wku8BnNqKI1v7oWTvzxeN/12Q+73KFcFp8wAse+IWS8g5YLVNvY", + "ihD8g0OPczOPnxtnSuBEIVkWBRc2tQIkM4RPPENFhhnpSdtWAzAPSXL7C01JXnBFWLL4nSy+DoS3d36E", + "w1r/lelx0hpRj3Knbwu/CZvB0M8cg+MDfQtZeC823Syvet0PSeYHR5BjkvALosWE7quAhCNKGtTJCApg", + "G50DcHaiR1Clb0C0c+CH38aS3ZPgcwMbWkEEJokCPMl9QuOpCUMOMmIS6UOMS0nE3/FZ8rHc2dn9Gy6K", + "vxeCpx9HjyboJU7mep+Q5AgiECTKS0iArCU4RFjCU1OhtCMmAVazLNY4HjPtF3q2gLwwXKCcC2KyN+uT", + "IFdFxlMyejbFmSSdkSWqrpxdpajViYp6VI9HUi0y/cOUizxmY+BCIa8Bh+hxGzZuAmcgOzQ6MIoYqW9I", + "90cPGbmEqrBUSNUZvc1FSsRg9co73bqhVo/VRw/OG9ZIUoSVPnM8NaBDZWUxmfTF8ZB0T3eJG0FSrMiW", + "HmeV6PkQEgIftcMDWF9GsexaUPAAuJ1o9+8g7+drSOB5UtWxuH44QKPOFrklL4jx6J9bp1zhbOvYVmZf", + "1hlau8Z3m0B08/6oh9lX/OJi91Gc4V03y6aliVWmbYutE7SneeJCKpKjhOd5ySxOVNp5X58cktOWoqtO", + "dp153lV2TTPFH7trT7A5vHLH2lPjbypsrCuNvr3lbgRtSaS1bPoJZ4wkqq+6lCmj7yZKicI0kxN0OG3i", + "oilnNkZUmbT6VQmzCTo9fa2bcJYtbHEq+DUoh3/G0wU0sWlxniPMEM8pFF/3Ce8DsenJzo4LzF2K/j4D", + "/r7d7/3LgG9XViMnX+++HvQQ8oEzQXC6cCKBxq5NCaJNCaJ7TiANPt2YQi51jJP1Yt19T/jvtPb+d+nZ", + "tsy/YWfVNfuH9JCT7VjyrXjGNY7SbAL8WsD7wqaRKAWbIN0bnZGMX5pXvGmABUHkKsnKtPtsb83Tbh9L", + "siUJk1TRC4JkeWYe3yjHKpkjzmDlOZESz4y5XTOUDh0DwSKZ15aV46vXhM00Bdj96W/rjSy26KwP+Y/d", + "67nY1YjHxjK9zjrmgL7LuUI8KcXqKSj+2P1mSSh+MCXXbae7+LEK22wQPZ6Mo4nqgzJyhPjfCgQNYopW", + "jiAKqMGPHUN0J4vo5rSbIKV7G6TUg4D9uNZyZO+p/WhMbPO+JAZR1Kt5r6/Lb/2OURZ2AycyVDm9G4/6", + "My58c+zNgxsEWo+kaqC57km5DIue1CXWJXYYRi57OJXGlieh2Hrn4GoZzB9PVgDYW12FXYDjMW4hXUhh", + "lJLQZa1qyVUcML+1WvJ7tq8EzOtJL9rxrMzJwNTiyLWOPRP9p7t/SJm5rlsltLWbDaG/4UulBhkO0twv", + "17W0Gwrvh44T+QDo7sRI7iBtvRZyM+seS4M3+oBSlO0z24g7ayW5IbC20CAkt9tfzD+GZ2bqxgPTyGLC", + "H3bYlV8Cbj03KcqI27C3UQqvK39TP+yN+yKnfdfOsOm7hK6db0U2qwKFG8D9RmUK+qgl7EpcOAgrRTZ6", + "NporVchn29u4oBOyezbBRQEwZft/afrQStBr1Kto1X+ENN7h3wXdOieLWhsba+b/rgTHamxbM+zrp6//", + "JwAA//8=", } // decodeSpec returns the embedded OpenAPI spec as raw JSON bytes, diff --git a/packages/api/internal/handlers/cathedral_sandbox_operations.go b/packages/api/internal/handlers/cathedral_sandbox_operations.go new file mode 100644 index 0000000000..38c548856b --- /dev/null +++ b/packages/api/internal/handlers/cathedral_sandbox_operations.go @@ -0,0 +1,250 @@ +package handlers + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "net/http" + "regexp" + + "github.com/gin-gonic/gin" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + + "github.com/e2b-dev/infra/packages/api/internal/api" + "github.com/e2b-dev/infra/packages/auth/pkg/auth" + "github.com/e2b-dev/infra/packages/db/queries" + "github.com/e2b-dev/infra/packages/shared/pkg/id" +) + +const cathedralIdempotencyAckHeader = "X-E2B-Idempotency-Key" + +var cathedralIdempotencyKeyPattern = regexp.MustCompile(`^[A-Za-z0-9._:-]{8,128}$`) + +type cathedralCreateClaim struct { + key string + requestSHA256 string + sandboxID string +} + +func hashCathedralCreateRequest(body api.PostSandboxesJSONRequestBody) (string, error) { + canonical, err := json.Marshal(body) + if err != nil { + return "", fmt.Errorf("marshal normalized create request: %w", err) + } + + digest := sha256.Sum256(canonical) + return hex.EncodeToString(digest[:]), nil +} + +// inspectCathedralCreate performs the read half of the durable-create protocol. +// It runs before template lookup so a completed replay still works after the +// referenced template changes or disappears. New keys are only inserted after +// the rest of request validation succeeds. +func (a *APIStore) inspectCathedralCreate( + c *gin.Context, + teamID uuid.UUID, + key *string, + body api.PostSandboxesJSONRequestBody, +) (*cathedralCreateClaim, bool) { + if key == nil { + return nil, true + } + if !cathedralIdempotencyKeyPattern.MatchString(*key) { + a.sendAPIStoreError(c, http.StatusBadRequest, "Idempotency-Key must contain 8 to 128 letters, numbers, periods, underscores, colons, or hyphens.") + return nil, false + } + + requestSHA256, err := hashCathedralCreateRequest(body) + if err != nil { + a.sendAPIStoreError(c, http.StatusInternalServerError, "failed to normalize create request") + return nil, false + } + claim := &cathedralCreateClaim{key: *key, requestSHA256: requestSHA256} + + operation, err := a.sqlcDB.GetCathedralSandboxOperation(c.Request.Context(), queries.GetCathedralSandboxOperationParams{ + TeamID: teamID, + IdempotencyKey: claim.key, + }) + if errors.Is(err, pgx.ErrNoRows) { + return claim, true + } + if err != nil { + a.sendAPIStoreError(c, http.StatusInternalServerError, "failed to inspect durable create operation") + return nil, false + } + + return a.resumeCathedralCreate(c, claim, operation) +} + +func (a *APIStore) resumeCathedralCreate( + c *gin.Context, + claim *cathedralCreateClaim, + operation queries.CathedralSandboxOperation, +) (*cathedralCreateClaim, bool) { + if operation.RequestSha256 != claim.requestSHA256 { + a.sendAPIStoreError(c, http.StatusConflict, "Idempotency-Key was already used for a different sandbox create request.") + return nil, false + } + claim.sandboxID = operation.SandboxID + + switch operation.State { + case "ready": + if operation.ResponseJson == nil { + a.sendAPIStoreError(c, http.StatusInternalServerError, "durable create operation has no stored response") + return nil, false + } + c.Header(cathedralIdempotencyAckHeader, claim.key) + c.Data(http.StatusCreated, "application/json", []byte(*operation.ResponseJson)) + return nil, false + case "failed": + code := http.StatusInternalServerError + if operation.ErrorCode != nil && *operation.ErrorCode >= 400 && *operation.ErrorCode <= 599 { + code = int(*operation.ErrorCode) + } + message := "durable create operation failed" + if operation.ErrorMessage != nil && *operation.ErrorMessage != "" { + message = *operation.ErrorMessage + } + a.sendAPIStoreError(c, code, message) + return nil, false + case "reserved", "creating": + return claim, true + default: + a.sendAPIStoreError(c, http.StatusInternalServerError, "durable create operation has an invalid state") + return nil, false + } +} + +// claimCathedralCreate atomically binds a new key to one sandbox ID. When a +// concurrent request wins the insert, the loser adopts the winner's ID. +func (a *APIStore) claimCathedralCreate( + c *gin.Context, + teamID uuid.UUID, + claim *cathedralCreateClaim, +) (string, bool) { + if claim == nil { + return InstanceIDPrefix + id.Generate(), true + } + if claim.sandboxID != "" { + return claim.sandboxID, true + } + + proposedID := InstanceIDPrefix + id.Generate() + operation, err := a.sqlcDB.ReserveCathedralSandboxOperation(c.Request.Context(), queries.ReserveCathedralSandboxOperationParams{ + TeamID: teamID, + IdempotencyKey: claim.key, + RequestSha256: claim.requestSHA256, + SandboxID: proposedID, + }) + if errors.Is(err, pgx.ErrNoRows) { + operation, err = a.sqlcDB.GetCathedralSandboxOperation(c.Request.Context(), queries.GetCathedralSandboxOperationParams{ + TeamID: teamID, + IdempotencyKey: claim.key, + }) + } + if err != nil { + a.sendAPIStoreError(c, http.StatusInternalServerError, "failed to reserve durable create operation") + return "", false + } + + resumed, proceed := a.resumeCathedralCreate(c, claim, operation) + if !proceed { + return "", false + } + return resumed.sandboxID, true +} + +func (a *APIStore) markCathedralCreateStarted(ctx context.Context, teamID uuid.UUID, claim *cathedralCreateClaim, sandboxID string) error { + if claim == nil { + return nil + } + rows, err := a.sqlcDB.MarkCathedralSandboxOperationCreating(ctx, queries.MarkCathedralSandboxOperationCreatingParams{ + TeamID: teamID, + IdempotencyKey: claim.key, + RequestSha256: claim.requestSHA256, + SandboxID: sandboxID, + }) + if err != nil { + return err + } + if rows != 1 { + return fmt.Errorf("durable create operation transition affected %d rows", rows) + } + return nil +} + +func (a *APIStore) completeCathedralCreate(ctx context.Context, teamID uuid.UUID, claim *cathedralCreateClaim, sandboxID string, response []byte) error { + if claim == nil { + return nil + } + rows, err := a.sqlcDB.CompleteCathedralSandboxOperation(ctx, queries.CompleteCathedralSandboxOperationParams{ + ResponseJson: string(response), + TeamID: teamID, + IdempotencyKey: claim.key, + RequestSha256: claim.requestSHA256, + SandboxID: sandboxID, + }) + if err != nil { + return err + } + if rows != 1 { + return fmt.Errorf("durable create operation completion affected %d rows", rows) + } + return nil +} + +func (a *APIStore) GetV1CathedralCapabilities(c *gin.Context) { + c.JSON(http.StatusOK, api.CathedralCapabilities{ + Schema: api.N1, + DurableCreateIdempotency: true, + OperationLookup: true, + SafeFork: false, + }) +} + +func (a *APIStore) GetV1CathedralOperationsIdempotencyKey(c *gin.Context, idempotencyKey api.CathedralOperationKey) { + if !cathedralIdempotencyKeyPattern.MatchString(idempotencyKey) { + a.sendAPIStoreError(c, http.StatusBadRequest, "invalid Cathedral operation key") + return + } + + teamInfo := auth.MustGetTeamInfo(c) + operation, err := a.sqlcDB.GetCathedralSandboxOperation(c.Request.Context(), queries.GetCathedralSandboxOperationParams{ + TeamID: teamInfo.Team.ID, + IdempotencyKey: idempotencyKey, + }) + if errors.Is(err, pgx.ErrNoRows) { + a.sendAPIStoreError(c, http.StatusNotFound, "Cathedral operation not found") + return + } + if err != nil { + a.sendAPIStoreError(c, http.StatusInternalServerError, "failed to read Cathedral operation") + return + } + + result := api.CathedralSandboxOperation{ + IdempotencyKey: operation.IdempotencyKey, + SandboxId: operation.SandboxID, + State: api.CathedralSandboxOperationState(operation.State), + ErrorCode: nil, + ErrorMessage: operation.ErrorMessage, + } + if operation.ErrorCode != nil { + code := int(*operation.ErrorCode) + result.ErrorCode = &code + } + if operation.State == "ready" && operation.ResponseJson != nil { + var sandbox api.Sandbox + if err := json.Unmarshal([]byte(*operation.ResponseJson), &sandbox); err != nil { + a.sendAPIStoreError(c, http.StatusInternalServerError, "durable operation response is invalid") + return + } + result.Sandbox = &sandbox + } + + c.JSON(http.StatusOK, result) +} diff --git a/packages/api/internal/handlers/cathedral_sandbox_operations_test.go b/packages/api/internal/handlers/cathedral_sandbox_operations_test.go new file mode 100644 index 0000000000..3a032ea0b1 --- /dev/null +++ b/packages/api/internal/handlers/cathedral_sandbox_operations_test.go @@ -0,0 +1,111 @@ +package handlers + +import ( + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/e2b-dev/infra/packages/api/internal/api" + "github.com/e2b-dev/infra/packages/db/queries" +) + +func TestHashCathedralCreateRequestIsCanonicalAndBodyBound(t *testing.T) { + t.Parallel() + + firstMetadata := api.SandboxMetadata{"b": "2", "a": "1"} + secondMetadata := api.SandboxMetadata{"a": "1", "b": "2"} + first, err := hashCathedralCreateRequest(api.PostSandboxesJSONRequestBody{ + TemplateID: "base", + Metadata: &firstMetadata, + }) + require.NoError(t, err) + second, err := hashCathedralCreateRequest(api.PostSandboxesJSONRequestBody{ + TemplateID: "base", + Metadata: &secondMetadata, + }) + require.NoError(t, err) + different, err := hashCathedralCreateRequest(api.PostSandboxesJSONRequestBody{ + TemplateID: "other", + Metadata: &secondMetadata, + }) + require.NoError(t, err) + + assert.Len(t, first, 64) + assert.Equal(t, first, second) + assert.NotEqual(t, first, different) +} + +func TestCathedralIdempotencyKeyValidation(t *testing.T) { + t.Parallel() + + assert.True(t, cathedralIdempotencyKeyPattern.MatchString("box-op:create_1")) + assert.False(t, cathedralIdempotencyKeyPattern.MatchString("short")) + assert.False(t, cathedralIdempotencyKeyPattern.MatchString("contains space")) + assert.False(t, cathedralIdempotencyKeyPattern.MatchString(strings.Repeat("x", 129))) +} + +func TestReadyCathedralCreateReplayReturnsExactStoredResponse(t *testing.T) { + gin.SetMode(gin.TestMode) + recorder := httptest.NewRecorder() + c, _ := gin.CreateTestContext(recorder) + stored := `{"sandboxID":"i-bound","templateID":"base","clientID":"","envdVersion":"0.5.0"}` + claim := &cathedralCreateClaim{ + key: "cathedral-replay-1", + requestSHA256: "a", + } + + resumed, proceed := (&APIStore{}).resumeCathedralCreate(c, claim, queries.CathedralSandboxOperation{ + IdempotencyKey: claim.key, + RequestSha256: claim.requestSHA256, + SandboxID: "i-bound", + State: "ready", + ResponseJson: &stored, + }) + + assert.Nil(t, resumed) + assert.False(t, proceed) + assert.Equal(t, http.StatusCreated, recorder.Code) + assert.Equal(t, claim.key, recorder.Header().Get(cathedralIdempotencyAckHeader)) + assert.Equal(t, stored, recorder.Body.String()) +} + +func TestCathedralCreateReplayRejectsDifferentBody(t *testing.T) { + gin.SetMode(gin.TestMode) + recorder := httptest.NewRecorder() + c, _ := gin.CreateTestContext(recorder) + claim := &cathedralCreateClaim{ + key: "cathedral-conflict-1", + requestSHA256: "new", + } + + resumed, proceed := (&APIStore{}).resumeCathedralCreate(c, claim, queries.CathedralSandboxOperation{ + IdempotencyKey: claim.key, + RequestSha256: "old", + SandboxID: "i-bound", + State: "creating", + }) + + assert.Nil(t, resumed) + assert.False(t, proceed) + assert.Equal(t, http.StatusConflict, recorder.Code) +} + +func TestCathedralCapabilitiesFailClosedOnFork(t *testing.T) { + gin.SetMode(gin.TestMode) + recorder := httptest.NewRecorder() + c, _ := gin.CreateTestContext(recorder) + (&APIStore{}).GetV1CathedralCapabilities(c) + + assert.Equal(t, http.StatusOK, recorder.Code) + assert.JSONEq(t, `{ + "schema": 1, + "durable_create_idempotency": true, + "operation_lookup": true, + "safe_fork": false + }`, recorder.Body.String()) +} diff --git a/packages/api/internal/handlers/sandbox_create.go b/packages/api/internal/handlers/sandbox_create.go index 7a369b9101..1ba9facb94 100644 --- a/packages/api/internal/handlers/sandbox_create.go +++ b/packages/api/internal/handlers/sandbox_create.go @@ -2,6 +2,7 @@ package handlers import ( "context" + "encoding/json" "errors" "fmt" "net" @@ -63,7 +64,7 @@ const ( maxIamTokens = 5 ) -func (a *APIStore) PostSandboxes(c *gin.Context) { +func (a *APIStore) PostSandboxes(c *gin.Context, params api.PostSandboxesParams) { ctx := c.Request.Context() body, err := ginutils.ParseBody[api.PostSandboxesJSONRequestBody](ctx, c) @@ -129,6 +130,16 @@ func (a *APIStore) createSandbox(c *gin.Context, body api.NewSandbox, defaultTim telemetry.ReportEvent(ctx, "Parsed body") + cathedralClaim, proceed := a.inspectCathedralCreate( + c, + teamInfo.Team.ID, + params.IdempotencyKey, + body, + ) + if !proceed { + return + } + identifier, tag, err := id.ParseName(body.TemplateID) if err != nil { a.sendAPIStoreError(c, http.StatusBadRequest, fmt.Sprintf("Invalid template reference: %s", err)) @@ -171,19 +182,8 @@ func (a *APIStore) createSandbox(c *gin.Context, body api.NewSandbox, defaultTim c.Set("envID", env.TemplateID) setTemplateNameMetric(ctx, c, a.featureFlags, env.TemplateID, env.Names) - sandboxID := InstanceIDPrefix + id.Generate() - - c.Set("instanceID", sandboxID) - - sbxlogger.E(&sbxlogger.SandboxMetadata{ - SandboxID: sandboxID, - TemplateID: env.TemplateID, - TeamID: teamInfo.Team.ID.String(), - }).Debug(ctx, "Started creating sandbox") - alias := firstAlias(env.Aliases) telemetry.SetAttributes(ctx, - telemetry.WithSandboxID(sandboxID), telemetry.WithTemplateID(env.TemplateID), telemetry.WithBuildID(build.ID.String()), attribute.String("env.alias", alias), @@ -243,22 +243,19 @@ func (a *APIStore) createSandbox(c *gin.Context, body api.NewSandbox, defaultTim return } - var envdAccessToken *string = nil - if body.Secure != nil && *body.Secure == true { - accessToken, tokenErr := a.getEnvdAccessToken(build.EnvdVersion, sandboxID) - if tokenErr != nil { - telemetry.ReportError(ctx, "secure envd access token error", tokenErr.Err, telemetry.WithSandboxID(sandboxID), telemetry.WithBuildID(build.ID.String())) + secureRequested := body.Secure != nil && *body.Secure + if secureRequested { + if tokenErr := validateEnvdAccessTokenVersion(build.EnvdVersion); tokenErr != nil { + telemetry.ReportError(ctx, "secure envd access token error", tokenErr.Err, telemetry.WithBuildID(build.ID.String())) a.sendAPIStoreError(c, tokenErr.Code, tokenErr.ClientMsg) return } - - envdAccessToken = &accessToken } iamCfg, iamErr := buildSandboxIam(body.Iam) if iamErr != nil { - telemetry.ReportError(ctx, "invalid iam config", iamErr.Err, telemetry.WithSandboxID(sandboxID)) + telemetry.ReportError(ctx, "invalid iam config", iamErr.Err) a.sendAPIStoreError(c, iamErr.Code, iamErr.ClientMsg) return @@ -276,7 +273,7 @@ func (a *APIStore) createSandbox(c *gin.Context, body api.NewSandbox, defaultTim if n := body.Network; n != nil { maxDomains := a.featureFlags.IntFlag(ctx, featureflags.MaxNetworkRuleDomains, featureflags.TeamContext(teamInfo.Team.ID.String())) if err := validateNetworkConfig(ctx, a.featureFlags, teamInfo.Team.ID, sharedUtils.DerefOrDefault(build.EnvdVersion, ""), maxDomains, n); err != nil { - telemetry.ReportError(ctx, "invalid network config", err.Err, telemetry.WithSandboxID(sandboxID)) + telemetry.ReportError(ctx, "invalid network config", err.Err) a.sendAPIStoreError(c, err.Code, err.ClientMsg) return @@ -310,7 +307,7 @@ func (a *APIStore) createSandbox(c *gin.Context, body api.NewSandbox, defaultTim Password: sharedUtils.DerefOrDefault(ep.Password, ""), }, nil) if err != nil { - telemetry.ReportError(ctx, "invalid egress proxy config", err, telemetry.WithSandboxID(sandboxID)) + telemetry.ReportError(ctx, "invalid egress proxy config", err) a.sendAPIStoreError(c, http.StatusBadRequest, fmt.Sprintf("Invalid egress proxy config: %s", err)) return @@ -323,7 +320,7 @@ func (a *APIStore) createSandbox(c *gin.Context, body api.NewSandbox, defaultTim // Make sure envd seucre access is enforced when public access is disabled, // This requirement forces users using newer features to secure sandboxes properly. - if !sharedUtils.DerefOrDefault(network.Ingress.AllowPublicAccess, types.AllowPublicAccessDefault) && envdAccessToken == nil { + if !sharedUtils.DerefOrDefault(network.Ingress.AllowPublicAccess, types.AllowPublicAccessDefault) && !secureRequested { a.sendAPIStoreError(c, http.StatusBadRequest, "You cannot create a sandbox without public access unless you enable secure envd access via 'secure' flag.") return @@ -352,12 +349,42 @@ func (a *APIStore) createSandbox(c *gin.Context, body api.NewSandbox, defaultTim return } - telemetry.ReportError(ctx, "failed to convert volume mounts", err, telemetry.WithSandboxID(sandboxID)) + telemetry.ReportError(ctx, "failed to convert volume mounts", err) a.sendAPIStoreError(c, http.StatusInternalServerError, "failed to convert volume mounts") return } + sandboxID, proceed := a.claimCathedralCreate(c, teamInfo.Team.ID, cathedralClaim) + if !proceed { + return + } + + c.Set("instanceID", sandboxID) + sbxlogger.E(&sbxlogger.SandboxMetadata{ + SandboxID: sandboxID, + TemplateID: env.TemplateID, + TeamID: teamInfo.Team.ID.String(), + }).Debug(ctx, "Started creating sandbox") + telemetry.SetAttributes(ctx, telemetry.WithSandboxID(sandboxID)) + + var envdAccessToken *string + if secureRequested { + accessToken, tokenErr := a.getEnvdAccessToken(build.EnvdVersion, sandboxID) + if tokenErr != nil { + telemetry.ReportError(ctx, "secure envd access token error", tokenErr.Err, telemetry.WithSandboxID(sandboxID), telemetry.WithBuildID(build.ID.String())) + a.sendAPIStoreError(c, tokenErr.Code, tokenErr.ClientMsg) + return + } + envdAccessToken = &accessToken + } + + if err := a.markCathedralCreateStarted(ctx, teamInfo.Team.ID, cathedralClaim, sandboxID); err != nil { + telemetry.ReportError(ctx, "failed to mark durable create operation started", err, telemetry.WithSandboxID(sandboxID)) + a.sendAPIStoreError(c, http.StatusInternalServerError, "failed to start durable create operation") + return + } + getSandboxData := func(_ context.Context) (apiorch.SandboxMetadata, *api.APIError) { // The data can't be influenced by action on the same sandbox as other operations, // so it's safe to reuse the data @@ -409,7 +436,21 @@ func (a *APIStore) createSandbox(c *gin.Context, body api.NewSandbox, defaultTim ) } - c.JSON(http.StatusCreated, &sbx) + response, err := json.Marshal(sbx) + if err != nil { + telemetry.ReportError(ctx, "failed to encode sandbox create response", err, telemetry.WithSandboxID(sandboxID)) + a.sendAPIStoreError(c, http.StatusInternalServerError, "failed to encode sandbox create response") + return + } + if err := a.completeCathedralCreate(ctx, teamInfo.Team.ID, cathedralClaim, sandboxID, response); err != nil { + telemetry.ReportError(ctx, "failed to complete durable create operation", err, telemetry.WithSandboxID(sandboxID)) + a.sendAPIStoreError(c, http.StatusInternalServerError, "sandbox creation outcome is pending durable recovery") + return + } + if cathedralClaim != nil { + c.Header(cathedralIdempotencyAckHeader, cathedralClaim.key) + } + c.Data(http.StatusCreated, "application/json", response) } // iamTokenTypeJWTSVID is the only workload token type accepted in this version. @@ -638,9 +679,9 @@ func getDBVolumesMap(ctx context.Context, sqlcDB *sqlcdb.Client, teamID uuid.UUI return dbVolumesMap, nil } -func (a *APIStore) getEnvdAccessToken(envdVersion *string, sandboxID string) (string, *api.APIError) { +func validateEnvdAccessTokenVersion(envdVersion *string) *api.APIError { if envdVersion == nil { - return "", &api.APIError{ + return &api.APIError{ Code: http.StatusBadRequest, ClientMsg: "You need to re-build template to allow using secured access. Please visit https://e2b.dev/docs/sandbox/secured-access for more information.", Err: errors.New("envd version is required during envd access token creation"), @@ -650,20 +691,28 @@ func (a *APIStore) getEnvdAccessToken(envdVersion *string, sandboxID string) (st // check if the envd version is at least 0.2.0 ok, err := sharedUtils.IsGTEVersion(*envdVersion, minEnvdVersionForSecureFlag) if err != nil { - return "", &api.APIError{ + return &api.APIError{ Code: http.StatusInternalServerError, ClientMsg: "error during envd version check", Err: err, } } if !ok { - return "", &api.APIError{ + return &api.APIError{ Code: http.StatusBadRequest, ClientMsg: "Template is not compatible with secured access. Please visit https://e2b.dev/docs/sandbox/secured-access for more information.", Err: errors.New("envd version is not supported for secure flag"), } } + return nil +} + +func (a *APIStore) getEnvdAccessToken(envdVersion *string, sandboxID string) (string, *api.APIError) { + if apiErr := validateEnvdAccessTokenVersion(envdVersion); apiErr != nil { + return "", apiErr + } + key, err := a.accessTokenGenerator.GenerateEnvdAccessToken(sandboxID) if err != nil { return "", &api.APIError{ diff --git a/packages/api/internal/handlers/sandbox_create_fcgate_test.go b/packages/api/internal/handlers/sandbox_create_fcgate_test.go index 6bf1bbc9d6..787ec51dad 100644 --- a/packages/api/internal/handlers/sandbox_create_fcgate_test.go +++ b/packages/api/internal/handlers/sandbox_create_fcgate_test.go @@ -79,7 +79,7 @@ func TestPostSandboxes_FsOnlyAutoPauseVersionGate(t *testing.T) { Limits: &authtypes.TeamLimits{MaxLengthHours: 24}, }) - store.PostSandboxes(ginCtx) + store.PostSandboxes(ginCtx, api.PostSandboxesParams{}) require.Equal(t, http.StatusBadRequest, recorder.Code) diff --git a/packages/api/internal/handlers/sandbox_create_test.go b/packages/api/internal/handlers/sandbox_create_test.go index 95797ec484..0772b559a9 100644 --- a/packages/api/internal/handlers/sandbox_create_test.go +++ b/packages/api/internal/handlers/sandbox_create_test.go @@ -759,7 +759,7 @@ func TestPostSandboxes_MissingBareAliasUsesPromotedFallbackKey(t *testing.T) { Limits: &authtypes.TeamLimits{MaxLengthHours: 24}, }) - store.PostSandboxes(ginCtx) + store.PostSandboxes(ginCtx, api.PostSandboxesParams{}) require.Equal(t, http.StatusNotFound, recorder.Code) @@ -819,7 +819,7 @@ func TestPostSandboxes_PrivateTemplateHidesAccessDenied(t *testing.T) { Limits: &authtypes.TeamLimits{MaxLengthHours: 24}, }) - store.PostSandboxes(ginCtx) + store.PostSandboxes(ginCtx, api.PostSandboxesParams{}) require.Equal(t, http.StatusNotFound, recorder.Code) @@ -873,7 +873,7 @@ func assertMissingTagDisclosure(t *testing.T, public bool, alias string) { Limits: &authtypes.TeamLimits{MaxLengthHours: 24}, }) - store.PostSandboxes(ginCtx) + store.PostSandboxes(ginCtx, api.PostSandboxesParams{}) require.Equal(t, http.StatusNotFound, recorder.Code) @@ -934,7 +934,7 @@ func assertMissingDefaultTagDisclosure(t *testing.T) { Limits: &authtypes.TeamLimits{MaxLengthHours: 24}, }) - store.PostSandboxes(ginCtx) + store.PostSandboxes(ginCtx, api.PostSandboxesParams{}) require.Equal(t, http.StatusNotFound, recorder.Code) diff --git a/packages/api/internal/middleware/cors.go b/packages/api/internal/middleware/cors.go index 658c9ad1fa..7e01ac24ce 100644 --- a/packages/api/internal/middleware/cors.go +++ b/packages/api/internal/middleware/cors.go @@ -18,6 +18,7 @@ var allowedRequestHeaders = []string{ // API Key header "Authorization", "X-API-Key", + "Idempotency-Key", auth.HeaderTeamID, // Custom headers sent from SDK "browser", @@ -42,6 +43,8 @@ var exposedResponseHeaders = []string{ "X-Next-Token", // Running sandbox total, set by GET /v2/sandboxes "X-Total-Running", + // Durable Cathedral sandbox create acknowledgement + "X-E2B-Idempotency-Key", // Rate limiting "RateLimit-Limit", "RateLimit-Remaining", diff --git a/packages/db/migrations/20260916080217_add_cathedral_sandbox_operations.sql b/packages/db/migrations/20260916080217_add_cathedral_sandbox_operations.sql new file mode 100644 index 0000000000..28d3c47d20 --- /dev/null +++ b/packages/db/migrations/20260916080217_add_cathedral_sandbox_operations.sql @@ -0,0 +1,39 @@ +-- +goose Up +CREATE TABLE public.cathedral_sandbox_operations ( + team_id UUID NOT NULL REFERENCES public.teams(id) ON DELETE CASCADE, + idempotency_key VARCHAR(128) NOT NULL, + request_sha256 CHAR(64) NOT NULL, + operation_kind VARCHAR(16) NOT NULL DEFAULT 'create', + sandbox_id TEXT NOT NULL, + state VARCHAR(16) NOT NULL DEFAULT 'reserved', + response_json TEXT, + error_code INTEGER, + error_message TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (team_id, idempotency_key), + UNIQUE (team_id, sandbox_id), + CONSTRAINT cathedral_sandbox_operations_key_nonempty + CHECK (length(idempotency_key) BETWEEN 8 AND 128), + CONSTRAINT cathedral_sandbox_operations_request_sha256 + CHECK (request_sha256 ~ '^[0-9a-f]{64}$'), + CONSTRAINT cathedral_sandbox_operations_kind + CHECK (operation_kind IN ('create')), + CONSTRAINT cathedral_sandbox_operations_state + CHECK (state IN ('reserved', 'creating', 'ready', 'failed')), + CONSTRAINT cathedral_sandbox_operations_ready_response + CHECK (state <> 'ready' OR response_json IS NOT NULL) +); + +CREATE INDEX cathedral_sandbox_operations_state_updated_idx + ON public.cathedral_sandbox_operations (state, updated_at); + +-- +goose Down +DO $$ +BEGIN + IF EXISTS (SELECT 1 FROM public.cathedral_sandbox_operations LIMIT 1) THEN + RAISE EXCEPTION 'cannot drop cathedral_sandbox_operations while rows exist'; + END IF; +END $$; + +DROP TABLE public.cathedral_sandbox_operations; diff --git a/packages/db/pkg/tests/cathedral_sandbox_operations_test.go b/packages/db/pkg/tests/cathedral_sandbox_operations_test.go new file mode 100644 index 0000000000..e9d16a0b2e --- /dev/null +++ b/packages/db/pkg/tests/cathedral_sandbox_operations_test.go @@ -0,0 +1,143 @@ +package tests + +import ( + "database/sql" + "errors" + "fmt" + "sync" + "testing" + + "github.com/jackc/pgx/v5" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/e2b-dev/infra/packages/db/pkg/testutils" + "github.com/e2b-dev/infra/packages/db/queries" +) + +func TestCathedralSandboxOperationConcurrentReservationBindsOneSandbox(t *testing.T) { + t.Parallel() + + db := testutils.SetupDatabase(t) + sqlDB, err := sql.Open("pgx", db.ConnStr()) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, sqlDB.Close()) }) + teamID := seedTeam(t, sqlDB, "cathedral-operation-race") + + const contenders = 16 + const key = "cathedral-create-race" + const digest = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + var wg sync.WaitGroup + winners := make(chan string, contenders) + errorsCh := make(chan error, contenders) + for i := range contenders { + wg.Add(1) + go func() { + defer wg.Done() + op, err := db.SqlcClient.ReserveCathedralSandboxOperation(t.Context(), queries.ReserveCathedralSandboxOperationParams{ + TeamID: teamID, + IdempotencyKey: key, + RequestSha256: digest, + SandboxID: fmt.Sprintf("i-contender-%02d", i), + }) + if err == nil { + winners <- op.SandboxID + return + } + if !errors.Is(err, pgx.ErrNoRows) { + errorsCh <- err + } + }() + } + wg.Wait() + close(winners) + close(errorsCh) + + for err := range errorsCh { + require.NoError(t, err) + } + var winnerIDs []string + for sandboxID := range winners { + winnerIDs = append(winnerIDs, sandboxID) + } + require.Len(t, winnerIDs, 1) + + op, err := db.SqlcClient.GetCathedralSandboxOperation(t.Context(), queries.GetCathedralSandboxOperationParams{ + TeamID: teamID, + IdempotencyKey: key, + }) + require.NoError(t, err) + assert.Equal(t, winnerIDs[0], op.SandboxID) + assert.Equal(t, digest, op.RequestSha256) + assert.Equal(t, "reserved", op.State) +} + +func TestCathedralSandboxOperationSurvivesAmbiguousCreateAndStoresImmutableResponse(t *testing.T) { + t.Parallel() + + db := testutils.SetupDatabase(t) + sqlDB, err := sql.Open("pgx", db.ConnStr()) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, sqlDB.Close()) }) + teamID := seedTeam(t, sqlDB, "cathedral-operation-recovery") + + const key = "cathedral-create-recovery" + const digest = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + const sandboxID = "i-provider-accepted" + op, err := db.SqlcClient.ReserveCathedralSandboxOperation(t.Context(), queries.ReserveCathedralSandboxOperationParams{ + TeamID: teamID, + IdempotencyKey: key, + RequestSha256: digest, + SandboxID: sandboxID, + }) + require.NoError(t, err) + assert.Equal(t, sandboxID, op.SandboxID) + + rows, err := db.SqlcClient.MarkCathedralSandboxOperationCreating(t.Context(), queries.MarkCathedralSandboxOperationCreatingParams{ + TeamID: teamID, + IdempotencyKey: key, + RequestSha256: digest, + SandboxID: sandboxID, + }) + require.NoError(t, err) + require.Equal(t, int64(1), rows) + + // This lookup represents process recovery after the provider accepted the + // sandbox but the HTTP response was lost. The original binding must survive. + recovered, err := db.SqlcClient.GetCathedralSandboxOperation(t.Context(), queries.GetCathedralSandboxOperationParams{ + TeamID: teamID, + IdempotencyKey: key, + }) + require.NoError(t, err) + assert.Equal(t, "creating", recovered.State) + assert.Equal(t, sandboxID, recovered.SandboxID) + + const response = `{"sandboxID":"i-provider-accepted","templateID":"base","clientID":"","envdVersion":"0.5.0"}` + rows, err = db.SqlcClient.CompleteCathedralSandboxOperation(t.Context(), queries.CompleteCathedralSandboxOperationParams{ + ResponseJson: response, + TeamID: teamID, + IdempotencyKey: key, + RequestSha256: digest, + SandboxID: sandboxID, + }) + require.NoError(t, err) + require.Equal(t, int64(1), rows) + + rows, err = db.SqlcClient.CompleteCathedralSandboxOperation(t.Context(), queries.CompleteCathedralSandboxOperationParams{ + ResponseJson: `{"sandboxID":"different"}`, + TeamID: teamID, + IdempotencyKey: key, + RequestSha256: digest, + SandboxID: sandboxID, + }) + require.NoError(t, err) + assert.Zero(t, rows, "a terminal replay response must be immutable") + + ready, err := db.SqlcClient.GetCathedralSandboxOperation(t.Context(), queries.GetCathedralSandboxOperationParams{ + TeamID: teamID, + IdempotencyKey: key, + }) + require.NoError(t, err) + require.NotNil(t, ready.ResponseJson) + assert.Equal(t, response, *ready.ResponseJson) +} diff --git a/packages/db/pkg/testutils/queries/models.go b/packages/db/pkg/testutils/queries/models.go index 3628a5740c..3de0fe61fe 100644 --- a/packages/db/pkg/testutils/queries/models.go +++ b/packages/db/pkg/testutils/queries/models.go @@ -73,6 +73,20 @@ type BillingSandboxLog struct { TeamID uuid.UUID } +type CathedralSandboxOperation struct { + TeamID uuid.UUID + IdempotencyKey string + RequestSha256 string + OperationKind string + SandboxID string + State string + ResponseJson pgtype.Text + ErrorCode pgtype.Int4 + ErrorMessage pgtype.Text + CreatedAt time.Time + UpdatedAt time.Time +} + type Cluster struct { ID uuid.UUID Endpoint string diff --git a/packages/db/queries/cathedral_sandbox_operations.sql.go b/packages/db/queries/cathedral_sandbox_operations.sql.go new file mode 100644 index 0000000000..ebe3a0efc8 --- /dev/null +++ b/packages/db/queries/cathedral_sandbox_operations.sql.go @@ -0,0 +1,204 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.29.0 +// source: cathedral_sandbox_operations.sql + +package queries + +import ( + "context" + + "github.com/google/uuid" +) + +const completeCathedralSandboxOperation = `-- name: CompleteCathedralSandboxOperation :execrows +UPDATE public.cathedral_sandbox_operations +SET state = 'ready', + response_json = CASE + WHEN state = 'ready' THEN response_json + ELSE $1::text + END, + error_code = NULL, + error_message = NULL, + updated_at = now() +WHERE team_id = $2::uuid + AND idempotency_key = $3::text + AND request_sha256 = $4::text + AND sandbox_id = $5::text + AND ( + state IN ('reserved', 'creating') + OR (state = 'ready' AND response_json = $1::text) + ) +` + +type CompleteCathedralSandboxOperationParams struct { + ResponseJson string + TeamID uuid.UUID + IdempotencyKey string + RequestSha256 string + SandboxID string +} + +func (q *Queries) CompleteCathedralSandboxOperation(ctx context.Context, arg CompleteCathedralSandboxOperationParams) (int64, error) { + result, err := q.db.Exec(ctx, completeCathedralSandboxOperation, + arg.ResponseJson, + arg.TeamID, + arg.IdempotencyKey, + arg.RequestSha256, + arg.SandboxID, + ) + if err != nil { + return 0, err + } + return result.RowsAffected(), nil +} + +const failCathedralSandboxOperation = `-- name: FailCathedralSandboxOperation :execrows +UPDATE public.cathedral_sandbox_operations +SET state = 'failed', + error_code = $1::integer, + error_message = $2::text, + updated_at = now() +WHERE team_id = $3::uuid + AND idempotency_key = $4::text + AND request_sha256 = $5::text + AND sandbox_id = $6::text + AND state IN ('reserved', 'creating', 'failed') +` + +type FailCathedralSandboxOperationParams struct { + ErrorCode int32 + ErrorMessage string + TeamID uuid.UUID + IdempotencyKey string + RequestSha256 string + SandboxID string +} + +func (q *Queries) FailCathedralSandboxOperation(ctx context.Context, arg FailCathedralSandboxOperationParams) (int64, error) { + result, err := q.db.Exec(ctx, failCathedralSandboxOperation, + arg.ErrorCode, + arg.ErrorMessage, + arg.TeamID, + arg.IdempotencyKey, + arg.RequestSha256, + arg.SandboxID, + ) + if err != nil { + return 0, err + } + return result.RowsAffected(), nil +} + +const getCathedralSandboxOperation = `-- name: GetCathedralSandboxOperation :one +SELECT team_id, idempotency_key, request_sha256, operation_kind, sandbox_id, state, response_json, error_code, error_message, created_at, updated_at +FROM public.cathedral_sandbox_operations +WHERE team_id = $1::uuid + AND idempotency_key = $2::text +` + +type GetCathedralSandboxOperationParams struct { + TeamID uuid.UUID + IdempotencyKey string +} + +func (q *Queries) GetCathedralSandboxOperation(ctx context.Context, arg GetCathedralSandboxOperationParams) (CathedralSandboxOperation, error) { + row := q.db.QueryRow(ctx, getCathedralSandboxOperation, arg.TeamID, arg.IdempotencyKey) + var i CathedralSandboxOperation + err := row.Scan( + &i.TeamID, + &i.IdempotencyKey, + &i.RequestSha256, + &i.OperationKind, + &i.SandboxID, + &i.State, + &i.ResponseJson, + &i.ErrorCode, + &i.ErrorMessage, + &i.CreatedAt, + &i.UpdatedAt, + ) + return i, err +} + +const markCathedralSandboxOperationCreating = `-- name: MarkCathedralSandboxOperationCreating :execrows +UPDATE public.cathedral_sandbox_operations +SET state = 'creating', + error_code = NULL, + error_message = NULL, + updated_at = now() +WHERE team_id = $1::uuid + AND idempotency_key = $2::text + AND request_sha256 = $3::text + AND sandbox_id = $4::text + AND state IN ('reserved', 'creating') +` + +type MarkCathedralSandboxOperationCreatingParams struct { + TeamID uuid.UUID + IdempotencyKey string + RequestSha256 string + SandboxID string +} + +func (q *Queries) MarkCathedralSandboxOperationCreating(ctx context.Context, arg MarkCathedralSandboxOperationCreatingParams) (int64, error) { + result, err := q.db.Exec(ctx, markCathedralSandboxOperationCreating, + arg.TeamID, + arg.IdempotencyKey, + arg.RequestSha256, + arg.SandboxID, + ) + if err != nil { + return 0, err + } + return result.RowsAffected(), nil +} + +const reserveCathedralSandboxOperation = `-- name: ReserveCathedralSandboxOperation :one +INSERT INTO public.cathedral_sandbox_operations ( + team_id, + idempotency_key, + request_sha256, + sandbox_id, + state +) VALUES ( + $1::uuid, + $2::text, + $3::text, + $4::text, + 'reserved' +) +ON CONFLICT (team_id, idempotency_key) DO NOTHING +RETURNING team_id, idempotency_key, request_sha256, operation_kind, sandbox_id, state, response_json, error_code, error_message, created_at, updated_at +` + +type ReserveCathedralSandboxOperationParams struct { + TeamID uuid.UUID + IdempotencyKey string + RequestSha256 string + SandboxID string +} + +func (q *Queries) ReserveCathedralSandboxOperation(ctx context.Context, arg ReserveCathedralSandboxOperationParams) (CathedralSandboxOperation, error) { + row := q.db.QueryRow(ctx, reserveCathedralSandboxOperation, + arg.TeamID, + arg.IdempotencyKey, + arg.RequestSha256, + arg.SandboxID, + ) + var i CathedralSandboxOperation + err := row.Scan( + &i.TeamID, + &i.IdempotencyKey, + &i.RequestSha256, + &i.OperationKind, + &i.SandboxID, + &i.State, + &i.ResponseJson, + &i.ErrorCode, + &i.ErrorMessage, + &i.CreatedAt, + &i.UpdatedAt, + ) + return i, err +} diff --git a/packages/db/queries/models.go b/packages/db/queries/models.go index 5b4b64ab60..aeda36d91f 100644 --- a/packages/db/queries/models.go +++ b/packages/db/queries/models.go @@ -26,6 +26,20 @@ type ActiveEnv struct { Source string } +type CathedralSandboxOperation struct { + TeamID uuid.UUID + IdempotencyKey string + RequestSha256 string + OperationKind string + SandboxID string + State string + ResponseJson *string + ErrorCode *int32 + ErrorMessage *string + CreatedAt time.Time + UpdatedAt time.Time +} + type Cluster struct { ID uuid.UUID Endpoint string diff --git a/packages/db/queries/sandboxes/cathedral_sandbox_operations.sql b/packages/db/queries/sandboxes/cathedral_sandbox_operations.sql new file mode 100644 index 0000000000..d1f5bcf13f --- /dev/null +++ b/packages/db/queries/sandboxes/cathedral_sandbox_operations.sql @@ -0,0 +1,65 @@ +-- name: ReserveCathedralSandboxOperation :one +INSERT INTO public.cathedral_sandbox_operations ( + team_id, + idempotency_key, + request_sha256, + sandbox_id, + state +) VALUES ( + sqlc.arg(team_id)::uuid, + sqlc.arg(idempotency_key)::text, + sqlc.arg(request_sha256)::text, + sqlc.arg(sandbox_id)::text, + 'reserved' +) +ON CONFLICT (team_id, idempotency_key) DO NOTHING +RETURNING *; + +-- name: GetCathedralSandboxOperation :one +SELECT * +FROM public.cathedral_sandbox_operations +WHERE team_id = sqlc.arg(team_id)::uuid + AND idempotency_key = sqlc.arg(idempotency_key)::text; + +-- name: MarkCathedralSandboxOperationCreating :execrows +UPDATE public.cathedral_sandbox_operations +SET state = 'creating', + error_code = NULL, + error_message = NULL, + updated_at = now() +WHERE team_id = sqlc.arg(team_id)::uuid + AND idempotency_key = sqlc.arg(idempotency_key)::text + AND request_sha256 = sqlc.arg(request_sha256)::text + AND sandbox_id = sqlc.arg(sandbox_id)::text + AND state IN ('reserved', 'creating'); + +-- name: CompleteCathedralSandboxOperation :execrows +UPDATE public.cathedral_sandbox_operations +SET state = 'ready', + response_json = CASE + WHEN state = 'ready' THEN response_json + ELSE sqlc.arg(response_json)::text + END, + error_code = NULL, + error_message = NULL, + updated_at = now() +WHERE team_id = sqlc.arg(team_id)::uuid + AND idempotency_key = sqlc.arg(idempotency_key)::text + AND request_sha256 = sqlc.arg(request_sha256)::text + AND sandbox_id = sqlc.arg(sandbox_id)::text + AND ( + state IN ('reserved', 'creating') + OR (state = 'ready' AND response_json = sqlc.arg(response_json)::text) + ); + +-- name: FailCathedralSandboxOperation :execrows +UPDATE public.cathedral_sandbox_operations +SET state = 'failed', + error_code = sqlc.arg(error_code)::integer, + error_message = sqlc.arg(error_message)::text, + updated_at = now() +WHERE team_id = sqlc.arg(team_id)::uuid + AND idempotency_key = sqlc.arg(idempotency_key)::text + AND request_sha256 = sqlc.arg(request_sha256)::text + AND sandbox_id = sqlc.arg(sandbox_id)::text + AND state IN ('reserved', 'creating', 'failed'); diff --git a/spec/openapi.yml b/spec/openapi.yml index 7dc37ed62f..f86a30b6c1 100644 --- a/spec/openapi.yml +++ b/spec/openapi.yml @@ -136,6 +136,15 @@ components: description: > Identifier of the secret (sec_ prefixed), or its canonical lower-case name + cathedralOperationKey: + name: idempotencyKey + in: path + required: true + schema: + type: string + minLength: 8 + maxLength: 128 + description: Cathedral durable create operation key webhookID: name: webhookID @@ -786,6 +795,47 @@ components: nullable: true description: Base domain where the sandbox traffic is accessible + CathedralCapabilities: + type: object + required: + - schema + - durable_create_idempotency + - operation_lookup + - safe_fork + properties: + schema: + type: integer + enum: [1] + durable_create_idempotency: + type: boolean + operation_lookup: + type: boolean + safe_fork: + type: boolean + + CathedralSandboxOperation: + type: object + required: + - idempotency_key + - sandbox_id + - state + properties: + idempotency_key: + type: string + sandbox_id: + type: string + state: + type: string + enum: [reserved, creating, ready, failed] + sandbox: + $ref: "#/components/schemas/Sandbox" + error_code: + type: integer + nullable: true + error_message: + type: string + nullable: true + SandboxDetail: required: - templateID @@ -2820,6 +2870,60 @@ paths: "500": $ref: "#/components/responses/500" + /v1/cathedral/capabilities: + get: + summary: Get the Cathedral durability contract supported by this control plane + tags: [sandboxes] + security: + - ApiKeyAuth: [] + - AuthProviderBearerAuth: [] + AuthProviderTeamAuth: [] + - AdminApiKeyAuth: [] + AdminTeamAuth: [] + - AdminJWTAuth: [] + AdminTeamAuth: [] + responses: + "200": + description: Cathedral durability capabilities + content: + application/json: + schema: + $ref: "#/components/schemas/CathedralCapabilities" + "401": + $ref: "#/components/responses/401" + "500": + $ref: "#/components/responses/500" + + /v1/cathedral/operations/{idempotencyKey}: + get: + summary: Recover a Cathedral create operation by its durable idempotency key + tags: [sandboxes] + security: + - ApiKeyAuth: [] + - AuthProviderBearerAuth: [] + AuthProviderTeamAuth: [] + - AdminApiKeyAuth: [] + AdminTeamAuth: [] + - AdminJWTAuth: [] + AdminTeamAuth: [] + parameters: + - $ref: "#/components/parameters/cathedralOperationKey" + responses: + "200": + description: Durable Cathedral create operation + content: + application/json: + schema: + $ref: "#/components/schemas/CathedralSandboxOperation" + "400": + $ref: "#/components/responses/400" + "401": + $ref: "#/components/responses/401" + "404": + $ref: "#/components/responses/404" + "500": + $ref: "#/components/responses/500" + /sandboxes: get: summary: List running sandboxes @@ -2872,6 +2976,18 @@ paths: AdminTeamAuth: [] - AdminJWTAuth: [] AdminTeamAuth: [] + parameters: + - name: Idempotency-Key + in: header + required: false + description: >- + Durable Cathedral create operation key. Replays with the same + authenticated team and request body return the same sandbox; reuse + with a different request body is rejected. + schema: + type: string + minLength: 8 + maxLength: 128 requestBody: required: true content: @@ -2881,6 +2997,11 @@ paths: responses: "201": description: The sandbox was created successfully + headers: + X-E2B-Idempotency-Key: + description: Echoes the accepted durable create operation key + schema: + type: string content: application/json: schema: @@ -2889,6 +3010,8 @@ paths: $ref: "#/components/responses/401" "400": $ref: "#/components/responses/400" + "409": + $ref: "#/components/responses/409" "429": $ref: "#/components/responses/429" "500": From 670b301efb17f2a8a444ef980996c67373078ce7 Mon Sep 17 00:00:00 2001 From: Fred E <7602667+wallscaler@users.noreply.github.com> Date: Wed, 16 Sep 2026 16:14:34 -0400 Subject: [PATCH 2/9] feat(runtime): add durable Cathedral lifecycle evidence --- docs/ARCHITECTURE.md | 8 + docs/cathedral-lifecycle-operations.md | 63 + packages/api/internal/api/api.gen.go | 1651 +++++++++++++---- packages/api/internal/api/compat.go | 10 + .../handlers/cathedral_sandbox_lifecycle.go | 331 ++++ .../cathedral_sandbox_lifecycle_test.go | 93 + .../handlers/cathedral_sandbox_operations.go | 13 +- .../cathedral_sandbox_operations_test.go | 7 +- .../api/internal/handlers/sandbox_resume.go | 12 + packages/api/internal/handlers/store.go | 44 +- .../internal/orchestrator/delete_instance.go | 108 +- .../orchestrator/delete_instance_test.go | 85 +- .../internal/orchestrator/pause_instance.go | 58 +- .../orchestrator/pause_instance_test.go | 16 + ...714_add_cathedral_lifecycle_operations.sql | 57 + .../cathedral_sandbox_operations_test.go | 106 ++ packages/db/pkg/testutils/queries/models.go | 21 + packages/db/pkg/types/types.go | 5 + ...hedral_sandbox_lifecycle_operations.sql.go | 282 +++ packages/db/queries/models.go | 21 + ...cathedral_sandbox_lifecycle_operations.sql | 73 + packages/orchestrator/orchestrator.proto | 6 + packages/orchestrator/pkg/server/sandboxes.go | 33 +- .../pkg/grpc/orchestrator/orchestrator.pb.go | 29 +- spec/openapi.yml | 205 ++ 25 files changed, 2931 insertions(+), 406 deletions(-) create mode 100644 docs/cathedral-lifecycle-operations.md create mode 100644 packages/api/internal/api/compat.go create mode 100644 packages/api/internal/handlers/cathedral_sandbox_lifecycle.go create mode 100644 packages/api/internal/handlers/cathedral_sandbox_lifecycle_test.go create mode 100644 packages/db/migrations/20260916192714_add_cathedral_lifecycle_operations.sql create mode 100644 packages/db/queries/cathedral_sandbox_lifecycle_operations.sql.go create mode 100644 packages/db/queries/sandboxes/cathedral_sandbox_lifecycle_operations.sql diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 52ac5ca9ca..a19a6acfa6 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -553,6 +553,14 @@ sequenceDiagram - **Resume**: same path as creation, but placement prefers the **origin node** — if the snapshot is still in its local cache, resume avoids any object-storage reads. `Checkpoint` is a pause+resume in place used to persist state while keeping the sandbox running. +- **Cathedral lifecycle evidence**: the Cathedral-only lifecycle endpoint binds the authenticated + team, sandbox ID, execution ID, request digest, operation kind, and idempotency key in Postgres + before dispatch. Completion is derived from the execution-bound node RPC, never from a missing + Redis/API listing. An already-running transition or transport ambiguity remains `unknown` and is + recovered by operation key without redispatch. Pause completion additionally records the + successful snapshot build; delete records snapshot/storage cleanup separately. The remaining + lifetime is frozen into the paused snapshot and reused by a resume that does not explicitly + override timeout. See `docs/cathedral-lifecycle-operations.md` for the consumer contract. - **Explicit filesystem-only resume**: `memory: false` on resume/connect demands a cold boot (`RebootSandbox`) even when the snapshot includes memory, as a self-serve rescue when the restored memory state is unusable. Gated per team by the `fs-only-resume-api` flag; when off diff --git a/docs/cathedral-lifecycle-operations.md b/docs/cathedral-lifecycle-operations.md new file mode 100644 index 0000000000..f243cb476f --- /dev/null +++ b/docs/cathedral-lifecycle-operations.md @@ -0,0 +1,63 @@ +# Cathedral lifecycle operations + +This contract is local implementation evidence only. It does not qualify a +host, image, deployment, customer billing path, website, or CLI. + +## Contract + +`POST /v1/cathedral/sandboxes/{sandboxID}/lifecycle-operations` accepts an +authenticated `delete` or `pause` request with: + +- `Idempotency-Key`: durable operation identity; +- `execution_id`: the exact sandbox incarnation the caller observed; +- optional `filesystem_only` for pause. + +The server hashes the normalized request and binds team, sandbox, execution, +operation, and key in Postgres before dispatch. Reusing a key with a different +binding returns `409`. Replaying the same key returns the stored operation and +never dispatches again. Recover it with +`GET /v1/cathedral/lifecycle-operations/{idempotencyKey}`. + +Before the first dispatch, an authenticated consumer reads the current +incarnation from `GET /v1/cathedral/sandboxes/{sandboxID}/identity`. That +endpoint enforces team ownership and returns the execution ID that must be +pinned into the lifecycle request. Recovery by operation key happens first, so +a completed delete remains readable after the live sandbox identity is gone. + +`completed` is written only after an execution-bound node RPC confirms that +the execution stopped. For pause, the snapshot build must also have reached a +durable successful state and its build ID is recorded. The Cathedral pause RPC +waits for remote snapshot storage to complete before returning that evidence; +the ordinary runtime pause path remains asynchronous. `404` from ordinary +sandbox GET/list, a registry row disappearing, a legacy delete acknowledgement, +or joining an in-flight removal is never terminal evidence. + +Delete reports snapshot/storage cleanup separately through `cleanup_state`. +`completed` with `cleanup_state=failed` means compute removal is proven but +storage cleanup debt remains; a consumer must retain that debt and must not +represent full cleanup or final settlement as complete. + +`unknown` is durable and non-retryable by POST. Recover it by key and reconcile +with operator/provider evidence; do not blindly replay the lifecycle action. + +Pause persists `remaining_lifetime_ms`. The snapshot stores the same frozen +remaining lifetime, and a resume without an explicit timeout uses it rather +than granting a new default lifetime. Resume remains the existing authenticated +endpoint; the operation protocol prevents stale pre-resume delete/pause work +from acting on the new execution identity. + +## Consumer rules + +1. Generate one operation key per user intent and persist it before calling. +2. Send the current provider `execution_id`; never identify an incarnation by + sandbox ID alone. +3. Treat `reserved`, `dispatching`, and `unknown` as non-terminal. +4. Treat delete as compute-stopped only when `state=completed` and + `execution_removed_at` is present. Close storage/billing only under the + consumer's separately defined settlement rules and cleanup state. +5. Treat pause as complete only when `state=completed`, + `execution_removed_at`, `snapshot_build_id`, and `snapshot_completed_at` + are all present. +6. After disconnection, GET the operation by key. Never infer success from the + sandbox listing and never submit a new key merely because the first response + was lost. diff --git a/packages/api/internal/api/api.gen.go b/packages/api/internal/api/api.gen.go index 4668cb510a..9321bd314c 100644 --- a/packages/api/internal/api/api.gen.go +++ b/packages/api/internal/api/api.gen.go @@ -54,6 +54,117 @@ func (e CathedralCapabilitiesSchema) Valid() bool { } } +// Defines values for CathedralLifecycleOperationCleanupState. +const ( + CathedralLifecycleOperationCleanupStateCompleted CathedralLifecycleOperationCleanupState = "completed" + CathedralLifecycleOperationCleanupStateFailed CathedralLifecycleOperationCleanupState = "failed" + CathedralLifecycleOperationCleanupStateNotRequired CathedralLifecycleOperationCleanupState = "not_required" + CathedralLifecycleOperationCleanupStatePending CathedralLifecycleOperationCleanupState = "pending" +) + +// Valid indicates whether the value is a known member of the CathedralLifecycleOperationCleanupState enum. +func (e CathedralLifecycleOperationCleanupState) Valid() bool { + switch e { + case CathedralLifecycleOperationCleanupStateCompleted: + return true + case CathedralLifecycleOperationCleanupStateFailed: + return true + case CathedralLifecycleOperationCleanupStateNotRequired: + return true + case CathedralLifecycleOperationCleanupStatePending: + return true + default: + return false + } +} + +// Defines values for CathedralLifecycleOperationOperation. +const ( + CathedralLifecycleOperationOperationDelete CathedralLifecycleOperationOperation = "delete" + CathedralLifecycleOperationOperationPause CathedralLifecycleOperationOperation = "pause" +) + +// Valid indicates whether the value is a known member of the CathedralLifecycleOperationOperation enum. +func (e CathedralLifecycleOperationOperation) Valid() bool { + switch e { + case CathedralLifecycleOperationOperationDelete: + return true + case CathedralLifecycleOperationOperationPause: + return true + default: + return false + } +} + +// Defines values for CathedralLifecycleOperationState. +const ( + CathedralLifecycleOperationStateCompleted CathedralLifecycleOperationState = "completed" + CathedralLifecycleOperationStateDispatching CathedralLifecycleOperationState = "dispatching" + CathedralLifecycleOperationStateFailed CathedralLifecycleOperationState = "failed" + CathedralLifecycleOperationStateReserved CathedralLifecycleOperationState = "reserved" + CathedralLifecycleOperationStateUnknown CathedralLifecycleOperationState = "unknown" +) + +// Valid indicates whether the value is a known member of the CathedralLifecycleOperationState enum. +func (e CathedralLifecycleOperationState) Valid() bool { + switch e { + case CathedralLifecycleOperationStateCompleted: + return true + case CathedralLifecycleOperationStateDispatching: + return true + case CathedralLifecycleOperationStateFailed: + return true + case CathedralLifecycleOperationStateReserved: + return true + case CathedralLifecycleOperationStateUnknown: + return true + default: + return false + } +} + +// Defines values for CathedralLifecycleOperationRequestOperation. +const ( + CathedralLifecycleOperationRequestOperationDelete CathedralLifecycleOperationRequestOperation = "delete" + CathedralLifecycleOperationRequestOperationPause CathedralLifecycleOperationRequestOperation = "pause" +) + +// Valid indicates whether the value is a known member of the CathedralLifecycleOperationRequestOperation enum. +func (e CathedralLifecycleOperationRequestOperation) Valid() bool { + switch e { + case CathedralLifecycleOperationRequestOperationDelete: + return true + case CathedralLifecycleOperationRequestOperationPause: + return true + default: + return false + } +} + +// Defines values for CathedralSandboxIdentityState. +const ( + CathedralSandboxIdentityStateKilling CathedralSandboxIdentityState = "killing" + CathedralSandboxIdentityStatePausing CathedralSandboxIdentityState = "pausing" + CathedralSandboxIdentityStateRunning CathedralSandboxIdentityState = "running" + CathedralSandboxIdentityStateSnapshotting CathedralSandboxIdentityState = "snapshotting" +) + +// Valid indicates whether the value is a known member of the CathedralSandboxIdentityState enum. +func (e CathedralSandboxIdentityState) Valid() bool { + switch e { + case CathedralSandboxIdentityStateKilling: + return true + case CathedralSandboxIdentityStatePausing: + return true + case CathedralSandboxIdentityStateRunning: + return true + case CathedralSandboxIdentityStateSnapshotting: + return true + default: + return false + } +} + // Defines values for CathedralSandboxOperationState. const ( CathedralSandboxOperationStateCreating CathedralSandboxOperationState = "creating" @@ -218,16 +329,16 @@ func (e OrderDirection) Valid() bool { // Defines values for SandboxOnTimeout. const ( - Kill SandboxOnTimeout = "kill" - Pause SandboxOnTimeout = "pause" + SandboxOnTimeoutKill SandboxOnTimeout = "kill" + SandboxOnTimeoutPause SandboxOnTimeout = "pause" ) // Valid indicates whether the value is a known member of the SandboxOnTimeout enum. func (e SandboxOnTimeout) Valid() bool { switch e { - case Kill: + case SandboxOnTimeoutKill: return true - case Pause: + case SandboxOnTimeoutPause: return true default: return false @@ -236,16 +347,16 @@ func (e SandboxOnTimeout) Valid() bool { // Defines values for SandboxState. const ( - Paused SandboxState = "paused" - Running SandboxState = "running" + SandboxStatePaused SandboxState = "paused" + SandboxStateRunning SandboxState = "running" ) // Valid indicates whether the value is a known member of the SandboxState enum. func (e SandboxState) Valid() bool { switch e { - case Paused: + case SandboxStatePaused: return true - case Running: + case SandboxStateRunning: return true default: return false @@ -454,15 +565,65 @@ type CPUCount = int32 // CathedralCapabilities defines model for CathedralCapabilities. type CathedralCapabilities struct { - DurableCreateIdempotency bool `json:"durable_create_idempotency"` - OperationLookup bool `json:"operation_lookup"` - SafeFork bool `json:"safe_fork"` - Schema CathedralCapabilitiesSchema `json:"schema"` + DurableCreateIdempotency bool `json:"durable_create_idempotency"` + DurableLifecycleOperations bool `json:"durable_lifecycle_operations"` + ExecutionIdentity bool `json:"execution_identity"` + OperationLookup bool `json:"operation_lookup"` + PreservesRemainingLifetime bool `json:"preserves_remaining_lifetime"` + SafeDelete bool `json:"safe_delete"` + SafeFork bool `json:"safe_fork"` + SafePause bool `json:"safe_pause"` + Schema CathedralCapabilitiesSchema `json:"schema"` } // CathedralCapabilitiesSchema defines model for CathedralCapabilities.Schema. type CathedralCapabilitiesSchema int +// CathedralLifecycleOperation defines model for CathedralLifecycleOperation. +type CathedralLifecycleOperation struct { + CleanupState CathedralLifecycleOperationCleanupState `json:"cleanup_state"` + ErrorCode *int `json:"error_code,omitempty"` + ErrorMessage *string `json:"error_message,omitempty"` + ExecutionId string `json:"execution_id"` + ExecutionRemovedAt *time.Time `json:"execution_removed_at,omitempty"` + Operation CathedralLifecycleOperationOperation `json:"operation"` + OperationKey string `json:"operation_key"` + RemainingLifetimeMs *int64 `json:"remaining_lifetime_ms,omitempty"` + SandboxId string `json:"sandbox_id"` + SnapshotBuildId *string `json:"snapshot_build_id,omitempty"` + SnapshotCompletedAt *time.Time `json:"snapshot_completed_at,omitempty"` + State CathedralLifecycleOperationState `json:"state"` +} + +// CathedralLifecycleOperationCleanupState defines model for CathedralLifecycleOperation.CleanupState. +type CathedralLifecycleOperationCleanupState string + +// CathedralLifecycleOperationOperation defines model for CathedralLifecycleOperation.Operation. +type CathedralLifecycleOperationOperation string + +// CathedralLifecycleOperationState defines model for CathedralLifecycleOperation.State. +type CathedralLifecycleOperationState string + +// CathedralLifecycleOperationRequest defines model for CathedralLifecycleOperationRequest. +type CathedralLifecycleOperationRequest struct { + ExecutionId string `json:"execution_id"` + FilesystemOnly *bool `json:"filesystem_only,omitempty"` + Operation CathedralLifecycleOperationRequestOperation `json:"operation"` +} + +// CathedralLifecycleOperationRequestOperation defines model for CathedralLifecycleOperationRequest.Operation. +type CathedralLifecycleOperationRequestOperation string + +// CathedralSandboxIdentity defines model for CathedralSandboxIdentity. +type CathedralSandboxIdentity struct { + ExecutionId string `json:"execution_id"` + SandboxId string `json:"sandbox_id"` + State CathedralSandboxIdentityState `json:"state"` +} + +// CathedralSandboxIdentityState defines model for CathedralSandboxIdentity.State. +type CathedralSandboxIdentityState string + // CathedralSandboxOperation defines model for CathedralSandboxOperation. type CathedralSandboxOperation struct { ErrorCode *int `json:"error_code,omitempty"` @@ -2188,6 +2349,11 @@ type GetTemplatesTemplateIDBuildsBuildIDStatusParams struct { Level *LogLevel `form:"level,omitempty" json:"level,omitempty"` } +// PostV1CathedralSandboxesSandboxIDLifecycleOperationsParams defines parameters for PostV1CathedralSandboxesSandboxIDLifecycleOperations. +type PostV1CathedralSandboxesSandboxIDLifecycleOperationsParams struct { + IdempotencyKey string `json:"Idempotency-Key"` +} + // GetV2SandboxesParams defines parameters for GetV2Sandboxes. type GetV2SandboxesParams struct { // Metadata Metadata query used to filter the sandboxes (e.g. "user=abc&app=prod"). Each key and values must be URL encoded. @@ -2312,6 +2478,9 @@ type PostTemplatesTagsJSONRequestBody = AssignTemplateTagsRequest // Deprecated: this type has been marked as deprecated upstream, but no `x-deprecated-reason` was set type PatchTemplatesTemplateIDJSONRequestBody = TemplateUpdateRequest +// PostV1CathedralSandboxesSandboxIDLifecycleOperationsJSONRequestBody defines body for PostV1CathedralSandboxesSandboxIDLifecycleOperations for application/json ContentType. +type PostV1CathedralSandboxesSandboxIDLifecycleOperationsJSONRequestBody = CathedralLifecycleOperationRequest + // PostV2SandboxesJSONRequestBody defines body for PostV2Sandboxes for application/json ContentType. type PostV2SandboxesJSONRequestBody = NewSandboxV2 @@ -3202,11 +3371,35 @@ type ClientInterface interface { // Corresponds with GET /v1/cathedral/capabilities (the `GetV1CathedralCapabilities` operationId). GetV1CathedralCapabilities(ctx context.Context, reqEditors ...RequestEditorFn) (*http.Response, error) + // GetV1CathedralLifecycleOperationsIdempotencyKey Recover a Cathedral lifecycle operation by durable key + // + // Corresponds with GET /v1/cathedral/lifecycle-operations/{idempotencyKey} (the `GetV1CathedralLifecycleOperationsIdempotencyKey` operationId). + GetV1CathedralLifecycleOperationsIdempotencyKey(ctx context.Context, idempotencyKey CathedralOperationKey, reqEditors ...RequestEditorFn) (*http.Response, error) + // GetV1CathedralOperationsIdempotencyKey Recover a Cathedral create operation by its durable idempotency key // // Corresponds with GET /v1/cathedral/operations/{idempotencyKey} (the `GetV1CathedralOperationsIdempotencyKey` operationId). GetV1CathedralOperationsIdempotencyKey(ctx context.Context, idempotencyKey CathedralOperationKey, reqEditors ...RequestEditorFn) (*http.Response, error) + // GetV1CathedralSandboxesSandboxIDIdentity Read the authenticated current Cathedral sandbox execution identity + // + // Corresponds with GET /v1/cathedral/sandboxes/{sandboxID}/identity (the `GetV1CathedralSandboxesSandboxIDIdentity` operationId). + GetV1CathedralSandboxesSandboxIDIdentity(ctx context.Context, sandboxID SandboxID, reqEditors ...RequestEditorFn) (*http.Response, error) + + // PostV1CathedralSandboxesSandboxIDLifecycleOperationsWithBody Start an execution-bound Cathedral lifecycle operation + // + // Takes any type of body and a specified content type. + // + // Corresponds with POST /v1/cathedral/sandboxes/{sandboxID}/lifecycle-operations (the `PostV1CathedralSandboxesSandboxIDLifecycleOperations` operationId). + PostV1CathedralSandboxesSandboxIDLifecycleOperationsWithBody(ctx context.Context, sandboxID SandboxID, params *PostV1CathedralSandboxesSandboxIDLifecycleOperationsParams, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) + + // PostV1CathedralSandboxesSandboxIDLifecycleOperations Start an execution-bound Cathedral lifecycle operation + // + // Takes a body of the `application/json` content type. + // + // Corresponds with POST /v1/cathedral/sandboxes/{sandboxID}/lifecycle-operations (the `PostV1CathedralSandboxesSandboxIDLifecycleOperations` operationId). + PostV1CathedralSandboxesSandboxIDLifecycleOperations(ctx context.Context, sandboxID SandboxID, params *PostV1CathedralSandboxesSandboxIDLifecycleOperationsParams, body PostV1CathedralSandboxesSandboxIDLifecycleOperationsJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) + // GetV2Sandboxes List sandboxes (v2) // // List all sandboxes. @@ -4848,6 +5041,21 @@ func (c *Client) GetV1CathedralCapabilities(ctx context.Context, reqEditors ...R return c.Client.Do(req) } +// GetV1CathedralLifecycleOperationsIdempotencyKey Recover a Cathedral lifecycle operation by durable key +// +// Corresponds with GET /v1/cathedral/lifecycle-operations/{idempotencyKey} (the `GetV1CathedralLifecycleOperationsIdempotencyKey` operationId). +func (c *Client) GetV1CathedralLifecycleOperationsIdempotencyKey(ctx context.Context, idempotencyKey CathedralOperationKey, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewGetV1CathedralLifecycleOperationsIdempotencyKeyRequest(c.Server, idempotencyKey) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if err := c.applyEditors(ctx, req, reqEditors); err != nil { + return nil, err + } + return c.Client.Do(req) +} + // GetV1CathedralOperationsIdempotencyKey Recover a Cathedral create operation by its durable idempotency key // // Corresponds with GET /v1/cathedral/operations/{idempotencyKey} (the `GetV1CathedralOperationsIdempotencyKey` operationId). @@ -4863,6 +5071,55 @@ func (c *Client) GetV1CathedralOperationsIdempotencyKey(ctx context.Context, ide return c.Client.Do(req) } +// GetV1CathedralSandboxesSandboxIDIdentity Read the authenticated current Cathedral sandbox execution identity +// +// Corresponds with GET /v1/cathedral/sandboxes/{sandboxID}/identity (the `GetV1CathedralSandboxesSandboxIDIdentity` operationId). +func (c *Client) GetV1CathedralSandboxesSandboxIDIdentity(ctx context.Context, sandboxID SandboxID, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewGetV1CathedralSandboxesSandboxIDIdentityRequest(c.Server, sandboxID) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if err := c.applyEditors(ctx, req, reqEditors); err != nil { + return nil, err + } + return c.Client.Do(req) +} + +// PostV1CathedralSandboxesSandboxIDLifecycleOperationsWithBody Start an execution-bound Cathedral lifecycle operation +// +// Takes any type of body and a specified content type. +// +// Corresponds with POST /v1/cathedral/sandboxes/{sandboxID}/lifecycle-operations (the `PostV1CathedralSandboxesSandboxIDLifecycleOperations` operationId). +func (c *Client) PostV1CathedralSandboxesSandboxIDLifecycleOperationsWithBody(ctx context.Context, sandboxID SandboxID, params *PostV1CathedralSandboxesSandboxIDLifecycleOperationsParams, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewPostV1CathedralSandboxesSandboxIDLifecycleOperationsRequestWithBody(c.Server, sandboxID, params, contentType, body) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if err := c.applyEditors(ctx, req, reqEditors); err != nil { + return nil, err + } + return c.Client.Do(req) +} + +// PostV1CathedralSandboxesSandboxIDLifecycleOperations Start an execution-bound Cathedral lifecycle operation +// +// Takes a body of the `application/json` content type. +// +// Corresponds with POST /v1/cathedral/sandboxes/{sandboxID}/lifecycle-operations (the `PostV1CathedralSandboxesSandboxIDLifecycleOperations` operationId). +func (c *Client) PostV1CathedralSandboxesSandboxIDLifecycleOperations(ctx context.Context, sandboxID SandboxID, params *PostV1CathedralSandboxesSandboxIDLifecycleOperationsParams, body PostV1CathedralSandboxesSandboxIDLifecycleOperationsJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewPostV1CathedralSandboxesSandboxIDLifecycleOperationsRequest(c.Server, sandboxID, params, body) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if err := c.applyEditors(ctx, req, reqEditors); err != nil { + return nil, err + } + return c.Client.Do(req) +} + // GetV2Sandboxes List sandboxes (v2) // // List all sandboxes. @@ -8426,6 +8683,40 @@ func NewGetV1CathedralCapabilitiesRequest(server string) (*http.Request, error) return req, nil } +// NewGetV1CathedralLifecycleOperationsIdempotencyKeyRequest constructs an http.Request for the GetV1CathedralLifecycleOperationsIdempotencyKey method +func NewGetV1CathedralLifecycleOperationsIdempotencyKeyRequest(server string, idempotencyKey CathedralOperationKey) (*http.Request, error) { + var err error + + var pathParam0 string + + pathParam0, err = runtime.StyleParamWithOptions("simple", false, "idempotencyKey", idempotencyKey, runtime.StyleParamOptions{ParamLocation: runtime.ParamLocationPath, Type: "string", Format: ""}) + if err != nil { + return nil, err + } + + serverURL, err := url.Parse(server) + if err != nil { + return nil, err + } + + operationPath := fmt.Sprintf("/v1/cathedral/lifecycle-operations/%s", pathParam0) + if operationPath[0] == '/' { + operationPath = "." + operationPath + } + + queryURL, err := serverURL.Parse(operationPath) + if err != nil { + return nil, err + } + + req, err := http.NewRequest(http.MethodGet, queryURL.String(), nil) + if err != nil { + return nil, err + } + + return req, nil +} + // NewGetV1CathedralOperationsIdempotencyKeyRequest constructs an http.Request for the GetV1CathedralOperationsIdempotencyKey method func NewGetV1CathedralOperationsIdempotencyKeyRequest(server string, idempotencyKey CathedralOperationKey) (*http.Request, error) { var err error @@ -8460,6 +8751,100 @@ func NewGetV1CathedralOperationsIdempotencyKeyRequest(server string, idempotency return req, nil } +// NewGetV1CathedralSandboxesSandboxIDIdentityRequest constructs an http.Request for the GetV1CathedralSandboxesSandboxIDIdentity method +func NewGetV1CathedralSandboxesSandboxIDIdentityRequest(server string, sandboxID SandboxID) (*http.Request, error) { + var err error + + var pathParam0 string + + pathParam0, err = runtime.StyleParamWithOptions("simple", false, "sandboxID", sandboxID, runtime.StyleParamOptions{ParamLocation: runtime.ParamLocationPath, Type: "string", Format: ""}) + if err != nil { + return nil, err + } + + serverURL, err := url.Parse(server) + if err != nil { + return nil, err + } + + operationPath := fmt.Sprintf("/v1/cathedral/sandboxes/%s/identity", pathParam0) + if operationPath[0] == '/' { + operationPath = "." + operationPath + } + + queryURL, err := serverURL.Parse(operationPath) + if err != nil { + return nil, err + } + + req, err := http.NewRequest(http.MethodGet, queryURL.String(), nil) + if err != nil { + return nil, err + } + + return req, nil +} + +// NewPostV1CathedralSandboxesSandboxIDLifecycleOperationsRequest calls the generic PostV1CathedralSandboxesSandboxIDLifecycleOperations builder with application/json body +func NewPostV1CathedralSandboxesSandboxIDLifecycleOperationsRequest(server string, sandboxID SandboxID, params *PostV1CathedralSandboxesSandboxIDLifecycleOperationsParams, body PostV1CathedralSandboxesSandboxIDLifecycleOperationsJSONRequestBody) (*http.Request, error) { + var bodyReader io.Reader + buf, err := json.Marshal(body) + if err != nil { + return nil, err + } + bodyReader = bytes.NewReader(buf) + return NewPostV1CathedralSandboxesSandboxIDLifecycleOperationsRequestWithBody(server, sandboxID, params, "application/json", bodyReader) +} + +// NewPostV1CathedralSandboxesSandboxIDLifecycleOperationsRequestWithBody constructs an http.Request for the PostV1CathedralSandboxesSandboxIDLifecycleOperations method, with any body, and a specified content type +func NewPostV1CathedralSandboxesSandboxIDLifecycleOperationsRequestWithBody(server string, sandboxID SandboxID, params *PostV1CathedralSandboxesSandboxIDLifecycleOperationsParams, contentType string, body io.Reader) (*http.Request, error) { + var err error + + var pathParam0 string + + pathParam0, err = runtime.StyleParamWithOptions("simple", false, "sandboxID", sandboxID, runtime.StyleParamOptions{ParamLocation: runtime.ParamLocationPath, Type: "string", Format: ""}) + if err != nil { + return nil, err + } + + serverURL, err := url.Parse(server) + if err != nil { + return nil, err + } + + operationPath := fmt.Sprintf("/v1/cathedral/sandboxes/%s/lifecycle-operations", pathParam0) + if operationPath[0] == '/' { + operationPath = "." + operationPath + } + + queryURL, err := serverURL.Parse(operationPath) + if err != nil { + return nil, err + } + + req, err := http.NewRequest(http.MethodPost, queryURL.String(), body) + if err != nil { + return nil, err + } + + req.Header.Add("Content-Type", contentType) + + if params != nil { + + var headerParam0 string + + headerParam0, err = runtime.StyleParamWithOptions("simple", false, "Idempotency-Key", params.IdempotencyKey, runtime.StyleParamOptions{ParamLocation: runtime.ParamLocationHeader, Type: "string", Format: ""}) + if err != nil { + return nil, err + } + + req.Header.Set("Idempotency-Key", headerParam0) + + } + + return req, nil +} + // NewGetV2SandboxesRequest constructs an http.Request for the GetV2Sandboxes method func NewGetV2SandboxesRequest(server string, params *GetV2SandboxesParams) (*http.Request, error) { var err error @@ -9929,6 +10314,13 @@ type ClientWithResponsesInterface interface { // Corresponds with GET /v1/cathedral/capabilities (the `GetV1CathedralCapabilities` operationId). GetV1CathedralCapabilitiesWithResponse(ctx context.Context, reqEditors ...RequestEditorFn) (*GetV1CathedralCapabilitiesResponse, error) + // GetV1CathedralLifecycleOperationsIdempotencyKeyWithResponse Recover a Cathedral lifecycle operation by durable key + // + // Returns a wrapper object for the known response body format(s). + // + // Corresponds with GET /v1/cathedral/lifecycle-operations/{idempotencyKey} (the `GetV1CathedralLifecycleOperationsIdempotencyKey` operationId). + GetV1CathedralLifecycleOperationsIdempotencyKeyWithResponse(ctx context.Context, idempotencyKey CathedralOperationKey, reqEditors ...RequestEditorFn) (*GetV1CathedralLifecycleOperationsIdempotencyKeyResponse, error) + // GetV1CathedralOperationsIdempotencyKeyWithResponse Recover a Cathedral create operation by its durable idempotency key // // Returns a wrapper object for the known response body format(s). @@ -9936,9 +10328,30 @@ type ClientWithResponsesInterface interface { // Corresponds with GET /v1/cathedral/operations/{idempotencyKey} (the `GetV1CathedralOperationsIdempotencyKey` operationId). GetV1CathedralOperationsIdempotencyKeyWithResponse(ctx context.Context, idempotencyKey CathedralOperationKey, reqEditors ...RequestEditorFn) (*GetV1CathedralOperationsIdempotencyKeyResponse, error) - // GetV2SandboxesWithResponse List sandboxes (v2) - // - // List all sandboxes. + // GetV1CathedralSandboxesSandboxIDIdentityWithResponse Read the authenticated current Cathedral sandbox execution identity + // + // Returns a wrapper object for the known response body format(s). + // + // Corresponds with GET /v1/cathedral/sandboxes/{sandboxID}/identity (the `GetV1CathedralSandboxesSandboxIDIdentity` operationId). + GetV1CathedralSandboxesSandboxIDIdentityWithResponse(ctx context.Context, sandboxID SandboxID, reqEditors ...RequestEditorFn) (*GetV1CathedralSandboxesSandboxIDIdentityResponse, error) + + // PostV1CathedralSandboxesSandboxIDLifecycleOperationsWithBodyWithResponse Start an execution-bound Cathedral lifecycle operation + // + // Takes any type of body and a specified content type, and returns a wrapper object for the known response body format(s). + // + // Corresponds with POST /v1/cathedral/sandboxes/{sandboxID}/lifecycle-operations (the `PostV1CathedralSandboxesSandboxIDLifecycleOperations` operationId). + PostV1CathedralSandboxesSandboxIDLifecycleOperationsWithBodyWithResponse(ctx context.Context, sandboxID SandboxID, params *PostV1CathedralSandboxesSandboxIDLifecycleOperationsParams, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*PostV1CathedralSandboxesSandboxIDLifecycleOperationsResponse, error) + + // PostV1CathedralSandboxesSandboxIDLifecycleOperationsWithResponse Start an execution-bound Cathedral lifecycle operation + // + // Takes a body of the `application/json` content type, and returns a wrapper object for the known response body format(s). + // + // Corresponds with POST /v1/cathedral/sandboxes/{sandboxID}/lifecycle-operations (the `PostV1CathedralSandboxesSandboxIDLifecycleOperations` operationId). + PostV1CathedralSandboxesSandboxIDLifecycleOperationsWithResponse(ctx context.Context, sandboxID SandboxID, params *PostV1CathedralSandboxesSandboxIDLifecycleOperationsParams, body PostV1CathedralSandboxesSandboxIDLifecycleOperationsJSONRequestBody, reqEditors ...RequestEditorFn) (*PostV1CathedralSandboxesSandboxIDLifecycleOperationsResponse, error) + + // GetV2SandboxesWithResponse List sandboxes (v2) + // + // List all sandboxes. // // Returns a wrapper object for the known response body format(s). // @@ -15062,6 +15475,75 @@ func (r GetV1CathedralCapabilitiesResponse) ContentType() string { return "" } +type GetV1CathedralLifecycleOperationsIdempotencyKeyResponse struct { + Body []byte + HTTPResponse *http.Response + // JSON200 the response for an HTTP 200 `application/json` response + JSON200 *CathedralLifecycleOperation + // JSON400 the response for an HTTP 400 `application/json` response + JSON400 *N400 + // JSON401 the response for an HTTP 401 `application/json` response + JSON401 *N401 + // JSON404 the response for an HTTP 404 `application/json` response + JSON404 *N404 + // JSON500 the response for an HTTP 500 `application/json` response + JSON500 *N500 +} + +// GetJSON200 returns the response for an HTTP 200 `application/json` response +func (r GetV1CathedralLifecycleOperationsIdempotencyKeyResponse) GetJSON200() *CathedralLifecycleOperation { + return r.JSON200 +} + +// GetJSON400 returns the response for an HTTP 400 `application/json` response +func (r GetV1CathedralLifecycleOperationsIdempotencyKeyResponse) GetJSON400() *N400 { + return r.JSON400 +} + +// GetJSON401 returns the response for an HTTP 401 `application/json` response +func (r GetV1CathedralLifecycleOperationsIdempotencyKeyResponse) GetJSON401() *N401 { + return r.JSON401 +} + +// GetJSON404 returns the response for an HTTP 404 `application/json` response +func (r GetV1CathedralLifecycleOperationsIdempotencyKeyResponse) GetJSON404() *N404 { + return r.JSON404 +} + +// GetJSON500 returns the response for an HTTP 500 `application/json` response +func (r GetV1CathedralLifecycleOperationsIdempotencyKeyResponse) GetJSON500() *N500 { + return r.JSON500 +} + +// GetBody returns the raw response body bytes +func (r GetV1CathedralLifecycleOperationsIdempotencyKeyResponse) GetBody() []byte { + return r.Body +} + +// Status returns HTTPResponse.Status +func (r GetV1CathedralLifecycleOperationsIdempotencyKeyResponse) Status() string { + if r.HTTPResponse != nil { + return r.HTTPResponse.Status + } + return http.StatusText(0) +} + +// StatusCode returns HTTPResponse.StatusCode +func (r GetV1CathedralLifecycleOperationsIdempotencyKeyResponse) StatusCode() int { + if r.HTTPResponse != nil { + return r.HTTPResponse.StatusCode + } + return 0 +} + +// ContentType is a convenience method to retrieve the Content-Type value from the HTTP response headers +func (r GetV1CathedralLifecycleOperationsIdempotencyKeyResponse) ContentType() string { + if r.HTTPResponse != nil { + return r.HTTPResponse.Header.Get("Content-Type") + } + return "" +} + type GetV1CathedralOperationsIdempotencyKeyResponse struct { Body []byte HTTPResponse *http.Response @@ -15131,6 +15613,165 @@ func (r GetV1CathedralOperationsIdempotencyKeyResponse) ContentType() string { return "" } +type GetV1CathedralSandboxesSandboxIDIdentityResponse struct { + Body []byte + HTTPResponse *http.Response + // JSON200 the response for an HTTP 200 `application/json` response + JSON200 *CathedralSandboxIdentity + // JSON400 the response for an HTTP 400 `application/json` response + JSON400 *N400 + // JSON401 the response for an HTTP 401 `application/json` response + JSON401 *N401 + // JSON404 the response for an HTTP 404 `application/json` response + JSON404 *N404 + // JSON500 the response for an HTTP 500 `application/json` response + JSON500 *N500 +} + +// GetJSON200 returns the response for an HTTP 200 `application/json` response +func (r GetV1CathedralSandboxesSandboxIDIdentityResponse) GetJSON200() *CathedralSandboxIdentity { + return r.JSON200 +} + +// GetJSON400 returns the response for an HTTP 400 `application/json` response +func (r GetV1CathedralSandboxesSandboxIDIdentityResponse) GetJSON400() *N400 { + return r.JSON400 +} + +// GetJSON401 returns the response for an HTTP 401 `application/json` response +func (r GetV1CathedralSandboxesSandboxIDIdentityResponse) GetJSON401() *N401 { + return r.JSON401 +} + +// GetJSON404 returns the response for an HTTP 404 `application/json` response +func (r GetV1CathedralSandboxesSandboxIDIdentityResponse) GetJSON404() *N404 { + return r.JSON404 +} + +// GetJSON500 returns the response for an HTTP 500 `application/json` response +func (r GetV1CathedralSandboxesSandboxIDIdentityResponse) GetJSON500() *N500 { + return r.JSON500 +} + +// GetBody returns the raw response body bytes +func (r GetV1CathedralSandboxesSandboxIDIdentityResponse) GetBody() []byte { + return r.Body +} + +// Status returns HTTPResponse.Status +func (r GetV1CathedralSandboxesSandboxIDIdentityResponse) Status() string { + if r.HTTPResponse != nil { + return r.HTTPResponse.Status + } + return http.StatusText(0) +} + +// StatusCode returns HTTPResponse.StatusCode +func (r GetV1CathedralSandboxesSandboxIDIdentityResponse) StatusCode() int { + if r.HTTPResponse != nil { + return r.HTTPResponse.StatusCode + } + return 0 +} + +// ContentType is a convenience method to retrieve the Content-Type value from the HTTP response headers +func (r GetV1CathedralSandboxesSandboxIDIdentityResponse) ContentType() string { + if r.HTTPResponse != nil { + return r.HTTPResponse.Header.Get("Content-Type") + } + return "" +} + +type PostV1CathedralSandboxesSandboxIDLifecycleOperationsResponse struct { + Body []byte + HTTPResponse *http.Response + // JSON200 the response for an HTTP 200 `application/json` response + JSON200 *CathedralLifecycleOperation + // JSON201 the response for an HTTP 201 `application/json` response + JSON201 *CathedralLifecycleOperation + // JSON202 the response for an HTTP 202 `application/json` response + JSON202 *CathedralLifecycleOperation + // JSON400 the response for an HTTP 400 `application/json` response + JSON400 *N400 + // JSON401 the response for an HTTP 401 `application/json` response + JSON401 *N401 + // JSON404 the response for an HTTP 404 `application/json` response + JSON404 *N404 + // JSON409 the response for an HTTP 409 `application/json` response + JSON409 *N409 + // JSON500 the response for an HTTP 500 `application/json` response + JSON500 *N500 +} + +// GetJSON200 returns the response for an HTTP 200 `application/json` response +func (r PostV1CathedralSandboxesSandboxIDLifecycleOperationsResponse) GetJSON200() *CathedralLifecycleOperation { + return r.JSON200 +} + +// GetJSON201 returns the response for an HTTP 201 `application/json` response +func (r PostV1CathedralSandboxesSandboxIDLifecycleOperationsResponse) GetJSON201() *CathedralLifecycleOperation { + return r.JSON201 +} + +// GetJSON202 returns the response for an HTTP 202 `application/json` response +func (r PostV1CathedralSandboxesSandboxIDLifecycleOperationsResponse) GetJSON202() *CathedralLifecycleOperation { + return r.JSON202 +} + +// GetJSON400 returns the response for an HTTP 400 `application/json` response +func (r PostV1CathedralSandboxesSandboxIDLifecycleOperationsResponse) GetJSON400() *N400 { + return r.JSON400 +} + +// GetJSON401 returns the response for an HTTP 401 `application/json` response +func (r PostV1CathedralSandboxesSandboxIDLifecycleOperationsResponse) GetJSON401() *N401 { + return r.JSON401 +} + +// GetJSON404 returns the response for an HTTP 404 `application/json` response +func (r PostV1CathedralSandboxesSandboxIDLifecycleOperationsResponse) GetJSON404() *N404 { + return r.JSON404 +} + +// GetJSON409 returns the response for an HTTP 409 `application/json` response +func (r PostV1CathedralSandboxesSandboxIDLifecycleOperationsResponse) GetJSON409() *N409 { + return r.JSON409 +} + +// GetJSON500 returns the response for an HTTP 500 `application/json` response +func (r PostV1CathedralSandboxesSandboxIDLifecycleOperationsResponse) GetJSON500() *N500 { + return r.JSON500 +} + +// GetBody returns the raw response body bytes +func (r PostV1CathedralSandboxesSandboxIDLifecycleOperationsResponse) GetBody() []byte { + return r.Body +} + +// Status returns HTTPResponse.Status +func (r PostV1CathedralSandboxesSandboxIDLifecycleOperationsResponse) Status() string { + if r.HTTPResponse != nil { + return r.HTTPResponse.Status + } + return http.StatusText(0) +} + +// StatusCode returns HTTPResponse.StatusCode +func (r PostV1CathedralSandboxesSandboxIDLifecycleOperationsResponse) StatusCode() int { + if r.HTTPResponse != nil { + return r.HTTPResponse.StatusCode + } + return 0 +} + +// ContentType is a convenience method to retrieve the Content-Type value from the HTTP response headers +func (r PostV1CathedralSandboxesSandboxIDLifecycleOperationsResponse) ContentType() string { + if r.HTTPResponse != nil { + return r.HTTPResponse.Header.Get("Content-Type") + } + return "" +} + // GetV2SandboxesResponse200Headers the declared response headers of an HTTP 200 response for GetV2Sandboxes type GetV2SandboxesResponse200Headers struct { XNextToken *string @@ -17356,6 +17997,19 @@ func (c *ClientWithResponses) GetV1CathedralCapabilitiesWithResponse(ctx context return ParseGetV1CathedralCapabilitiesResponse(rsp) } +// GetV1CathedralLifecycleOperationsIdempotencyKeyWithResponse Recover a Cathedral lifecycle operation by durable key +// +// Returns a wrapper object for the known response body format(s). +// +// Corresponds with GET /v1/cathedral/lifecycle-operations/{idempotencyKey} (the `GetV1CathedralLifecycleOperationsIdempotencyKey` operationId). +func (c *ClientWithResponses) GetV1CathedralLifecycleOperationsIdempotencyKeyWithResponse(ctx context.Context, idempotencyKey CathedralOperationKey, reqEditors ...RequestEditorFn) (*GetV1CathedralLifecycleOperationsIdempotencyKeyResponse, error) { + rsp, err := c.GetV1CathedralLifecycleOperationsIdempotencyKey(ctx, idempotencyKey, reqEditors...) + if err != nil { + return nil, err + } + return ParseGetV1CathedralLifecycleOperationsIdempotencyKeyResponse(rsp) +} + // GetV1CathedralOperationsIdempotencyKeyWithResponse Recover a Cathedral create operation by its durable idempotency key // // Returns a wrapper object for the known response body format(s). @@ -17369,6 +18023,45 @@ func (c *ClientWithResponses) GetV1CathedralOperationsIdempotencyKeyWithResponse return ParseGetV1CathedralOperationsIdempotencyKeyResponse(rsp) } +// GetV1CathedralSandboxesSandboxIDIdentityWithResponse Read the authenticated current Cathedral sandbox execution identity +// +// Returns a wrapper object for the known response body format(s). +// +// Corresponds with GET /v1/cathedral/sandboxes/{sandboxID}/identity (the `GetV1CathedralSandboxesSandboxIDIdentity` operationId). +func (c *ClientWithResponses) GetV1CathedralSandboxesSandboxIDIdentityWithResponse(ctx context.Context, sandboxID SandboxID, reqEditors ...RequestEditorFn) (*GetV1CathedralSandboxesSandboxIDIdentityResponse, error) { + rsp, err := c.GetV1CathedralSandboxesSandboxIDIdentity(ctx, sandboxID, reqEditors...) + if err != nil { + return nil, err + } + return ParseGetV1CathedralSandboxesSandboxIDIdentityResponse(rsp) +} + +// PostV1CathedralSandboxesSandboxIDLifecycleOperationsWithBodyWithResponse Start an execution-bound Cathedral lifecycle operation +// +// Takes any type of body and a specified content type, and returns a wrapper object for the known response body format(s). +// +// Corresponds with POST /v1/cathedral/sandboxes/{sandboxID}/lifecycle-operations (the `PostV1CathedralSandboxesSandboxIDLifecycleOperations` operationId). +func (c *ClientWithResponses) PostV1CathedralSandboxesSandboxIDLifecycleOperationsWithBodyWithResponse(ctx context.Context, sandboxID SandboxID, params *PostV1CathedralSandboxesSandboxIDLifecycleOperationsParams, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*PostV1CathedralSandboxesSandboxIDLifecycleOperationsResponse, error) { + rsp, err := c.PostV1CathedralSandboxesSandboxIDLifecycleOperationsWithBody(ctx, sandboxID, params, contentType, body, reqEditors...) + if err != nil { + return nil, err + } + return ParsePostV1CathedralSandboxesSandboxIDLifecycleOperationsResponse(rsp) +} + +// PostV1CathedralSandboxesSandboxIDLifecycleOperationsWithResponse Start an execution-bound Cathedral lifecycle operation +// +// Takes a body of the `application/json` content type, and returns a wrapper object for the known response body format(s). +// +// Corresponds with POST /v1/cathedral/sandboxes/{sandboxID}/lifecycle-operations (the `PostV1CathedralSandboxesSandboxIDLifecycleOperations` operationId). +func (c *ClientWithResponses) PostV1CathedralSandboxesSandboxIDLifecycleOperationsWithResponse(ctx context.Context, sandboxID SandboxID, params *PostV1CathedralSandboxesSandboxIDLifecycleOperationsParams, body PostV1CathedralSandboxesSandboxIDLifecycleOperationsJSONRequestBody, reqEditors ...RequestEditorFn) (*PostV1CathedralSandboxesSandboxIDLifecycleOperationsResponse, error) { + rsp, err := c.PostV1CathedralSandboxesSandboxIDLifecycleOperations(ctx, sandboxID, params, body, reqEditors...) + if err != nil { + return nil, err + } + return ParsePostV1CathedralSandboxesSandboxIDLifecycleOperationsResponse(rsp) +} + // GetV2SandboxesWithResponse List sandboxes (v2) // // List all sandboxes. @@ -22038,6 +22731,60 @@ func ParseGetV1CathedralCapabilitiesResponse(rsp *http.Response) (*GetV1Cathedra return response, nil } +// ParseGetV1CathedralLifecycleOperationsIdempotencyKeyResponse parses an HTTP response from a GetV1CathedralLifecycleOperationsIdempotencyKeyWithResponse call +func ParseGetV1CathedralLifecycleOperationsIdempotencyKeyResponse(rsp *http.Response) (*GetV1CathedralLifecycleOperationsIdempotencyKeyResponse, error) { + bodyBytes, err := io.ReadAll(rsp.Body) + defer func() { _ = rsp.Body.Close() }() + if err != nil { + return nil, err + } + + response := &GetV1CathedralLifecycleOperationsIdempotencyKeyResponse{ + Body: bodyBytes, + HTTPResponse: rsp, + } + + switch { + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 200: + var dest CathedralLifecycleOperation + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON200 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 400: + var dest N400 + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON400 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 401: + var dest N401 + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON401 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 404: + var dest N404 + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON404 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 500: + var dest N500 + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON500 = &dest + + } + + return response, nil +} + // ParseGetV1CathedralOperationsIdempotencyKeyResponse parses an HTTP response from a GetV1CathedralOperationsIdempotencyKeyWithResponse call func ParseGetV1CathedralOperationsIdempotencyKeyResponse(rsp *http.Response) (*GetV1CathedralOperationsIdempotencyKeyResponse, error) { bodyBytes, err := io.ReadAll(rsp.Body) @@ -22092,6 +22839,135 @@ func ParseGetV1CathedralOperationsIdempotencyKeyResponse(rsp *http.Response) (*G return response, nil } +// ParseGetV1CathedralSandboxesSandboxIDIdentityResponse parses an HTTP response from a GetV1CathedralSandboxesSandboxIDIdentityWithResponse call +func ParseGetV1CathedralSandboxesSandboxIDIdentityResponse(rsp *http.Response) (*GetV1CathedralSandboxesSandboxIDIdentityResponse, error) { + bodyBytes, err := io.ReadAll(rsp.Body) + defer func() { _ = rsp.Body.Close() }() + if err != nil { + return nil, err + } + + response := &GetV1CathedralSandboxesSandboxIDIdentityResponse{ + Body: bodyBytes, + HTTPResponse: rsp, + } + + switch { + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 200: + var dest CathedralSandboxIdentity + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON200 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 400: + var dest N400 + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON400 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 401: + var dest N401 + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON401 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 404: + var dest N404 + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON404 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 500: + var dest N500 + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON500 = &dest + + } + + return response, nil +} + +// ParsePostV1CathedralSandboxesSandboxIDLifecycleOperationsResponse parses an HTTP response from a PostV1CathedralSandboxesSandboxIDLifecycleOperationsWithResponse call +func ParsePostV1CathedralSandboxesSandboxIDLifecycleOperationsResponse(rsp *http.Response) (*PostV1CathedralSandboxesSandboxIDLifecycleOperationsResponse, error) { + bodyBytes, err := io.ReadAll(rsp.Body) + defer func() { _ = rsp.Body.Close() }() + if err != nil { + return nil, err + } + + response := &PostV1CathedralSandboxesSandboxIDLifecycleOperationsResponse{ + Body: bodyBytes, + HTTPResponse: rsp, + } + + switch { + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 200: + var dest CathedralLifecycleOperation + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON200 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 201: + var dest CathedralLifecycleOperation + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON201 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 202: + var dest CathedralLifecycleOperation + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON202 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 400: + var dest N400 + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON400 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 401: + var dest N401 + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON401 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 404: + var dest N404 + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON404 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 409: + var dest N409 + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON409 = &dest + + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 500: + var dest N500 + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON500 = &dest + + } + + return response, nil +} + // ParseGetV2SandboxesResponse parses an HTTP response from a GetV2SandboxesWithResponse call func ParseGetV2SandboxesResponse(rsp *http.Response) (*GetV2SandboxesResponse, error) { bodyBytes, err := io.ReadAll(rsp.Body) @@ -23183,9 +24059,18 @@ type ServerInterface interface { // GetV1CathedralCapabilities Get the Cathedral durability contract supported by this control plane // (GET /v1/cathedral/capabilities) GetV1CathedralCapabilities(c *gin.Context) + // GetV1CathedralLifecycleOperationsIdempotencyKey Recover a Cathedral lifecycle operation by durable key + // (GET /v1/cathedral/lifecycle-operations/{idempotencyKey}) + GetV1CathedralLifecycleOperationsIdempotencyKey(c *gin.Context, idempotencyKey CathedralOperationKey) // GetV1CathedralOperationsIdempotencyKey Recover a Cathedral create operation by its durable idempotency key // (GET /v1/cathedral/operations/{idempotencyKey}) GetV1CathedralOperationsIdempotencyKey(c *gin.Context, idempotencyKey CathedralOperationKey) + // GetV1CathedralSandboxesSandboxIDIdentity Read the authenticated current Cathedral sandbox execution identity + // (GET /v1/cathedral/sandboxes/{sandboxID}/identity) + GetV1CathedralSandboxesSandboxIDIdentity(c *gin.Context, sandboxID SandboxID) + // PostV1CathedralSandboxesSandboxIDLifecycleOperations Start an execution-bound Cathedral lifecycle operation + // (POST /v1/cathedral/sandboxes/{sandboxID}/lifecycle-operations) + PostV1CathedralSandboxesSandboxIDLifecycleOperations(c *gin.Context, sandboxID SandboxID, params PostV1CathedralSandboxesSandboxIDLifecycleOperationsParams) // GetV2Sandboxes List sandboxes (v2) // (GET /v2/sandboxes) GetV2Sandboxes(c *gin.Context, params GetV2SandboxesParams) @@ -25152,6 +26037,31 @@ func (siw *ServerInterfaceWrapper) GetV1CathedralCapabilities(c *gin.Context) { siw.Handler.GetV1CathedralCapabilities(c) } +// GetV1CathedralLifecycleOperationsIdempotencyKey operation middleware +func (siw *ServerInterfaceWrapper) GetV1CathedralLifecycleOperationsIdempotencyKey(c *gin.Context) { + + var err error + _ = err + + // ------------- Path parameter "idempotencyKey" ------------- + var idempotencyKey CathedralOperationKey + + err = runtime.BindStyledParameterWithOptions("simple", "idempotencyKey", c.Param("idempotencyKey"), &idempotencyKey, runtime.BindStyledParameterOptions{ParamLocation: runtime.ParamLocationPath, Explode: false, Required: true, Type: "string", Format: "", ValueIsUnescaped: true}) + if err != nil { + siw.ErrorHandler(c, fmt.Errorf("Invalid format for parameter idempotencyKey: %w", err), http.StatusBadRequest) + return + } + + for _, middleware := range siw.HandlerMiddlewares { + middleware(c) + if c.IsAborted() { + return + } + } + + siw.Handler.GetV1CathedralLifecycleOperationsIdempotencyKey(c, idempotencyKey) +} + // GetV1CathedralOperationsIdempotencyKey operation middleware func (siw *ServerInterfaceWrapper) GetV1CathedralOperationsIdempotencyKey(c *gin.Context) { @@ -25177,6 +26087,83 @@ func (siw *ServerInterfaceWrapper) GetV1CathedralOperationsIdempotencyKey(c *gin siw.Handler.GetV1CathedralOperationsIdempotencyKey(c, idempotencyKey) } +// GetV1CathedralSandboxesSandboxIDIdentity operation middleware +func (siw *ServerInterfaceWrapper) GetV1CathedralSandboxesSandboxIDIdentity(c *gin.Context) { + + var err error + _ = err + + // ------------- Path parameter "sandboxID" ------------- + var sandboxID SandboxID + + err = runtime.BindStyledParameterWithOptions("simple", "sandboxID", c.Param("sandboxID"), &sandboxID, runtime.BindStyledParameterOptions{ParamLocation: runtime.ParamLocationPath, Explode: false, Required: true, Type: "string", Format: "", ValueIsUnescaped: true}) + if err != nil { + siw.ErrorHandler(c, fmt.Errorf("Invalid format for parameter sandboxID: %w", err), http.StatusBadRequest) + return + } + + for _, middleware := range siw.HandlerMiddlewares { + middleware(c) + if c.IsAborted() { + return + } + } + + siw.Handler.GetV1CathedralSandboxesSandboxIDIdentity(c, sandboxID) +} + +// PostV1CathedralSandboxesSandboxIDLifecycleOperations operation middleware +func (siw *ServerInterfaceWrapper) PostV1CathedralSandboxesSandboxIDLifecycleOperations(c *gin.Context) { + + var err error + _ = err + + // ------------- Path parameter "sandboxID" ------------- + var sandboxID SandboxID + + err = runtime.BindStyledParameterWithOptions("simple", "sandboxID", c.Param("sandboxID"), &sandboxID, runtime.BindStyledParameterOptions{ParamLocation: runtime.ParamLocationPath, Explode: false, Required: true, Type: "string", Format: "", ValueIsUnescaped: true}) + if err != nil { + siw.ErrorHandler(c, fmt.Errorf("Invalid format for parameter sandboxID: %w", err), http.StatusBadRequest) + return + } + + // Parameter object where we will unmarshal all parameters from the context + var params PostV1CathedralSandboxesSandboxIDLifecycleOperationsParams + + headers := c.Request.Header + + // ------------- Required header parameter "Idempotency-Key" ------------- + if valueList, found := headers[http.CanonicalHeaderKey("Idempotency-Key")]; found { + var IdempotencyKey string + n := len(valueList) + if n != 1 { + siw.ErrorHandler(c, fmt.Errorf("Expected one value for Idempotency-Key, got %d", n), http.StatusBadRequest) + return + } + + err = runtime.BindStyledParameterWithOptions("simple", "Idempotency-Key", valueList[0], &IdempotencyKey, runtime.BindStyledParameterOptions{ParamLocation: runtime.ParamLocationHeader, Explode: false, Required: true, Type: "string", Format: ""}) + if err != nil { + siw.ErrorHandler(c, fmt.Errorf("Invalid format for parameter Idempotency-Key: %w", err), http.StatusBadRequest) + return + } + + params.IdempotencyKey = IdempotencyKey + + } else { + siw.ErrorHandler(c, fmt.Errorf("Header parameter Idempotency-Key is required, but not found"), http.StatusBadRequest) + return + } + + for _, middleware := range siw.HandlerMiddlewares { + middleware(c) + if c.IsAborted() { + return + } + } + + siw.Handler.PostV1CathedralSandboxesSandboxIDLifecycleOperations(c, sandboxID, params) +} + // GetV2Sandboxes operation middleware func (siw *ServerInterfaceWrapper) GetV2Sandboxes(c *gin.Context) { @@ -25582,6 +26569,9 @@ func RegisterHandlersWithOptions(router gin.IRouter, si ServerInterface, options router.GET(options.BaseURL+"/teams/:teamID/metrics/max", wrapper.GetTeamsTeamIDMetricsMax) router.GET(options.BaseURL+"/v1/cathedral/capabilities", wrapper.GetV1CathedralCapabilities) router.GET(options.BaseURL+"/v1/cathedral/operations/:idempotencyKey", wrapper.GetV1CathedralOperationsIdempotencyKey) + router.POST(options.BaseURL+"/v1/cathedral/sandboxes/:sandboxID/lifecycle-operations", wrapper.PostV1CathedralSandboxesSandboxIDLifecycleOperations) + router.GET(options.BaseURL+"/v1/cathedral/sandboxes/:sandboxID/identity", wrapper.GetV1CathedralSandboxesSandboxIDIdentity) + router.GET(options.BaseURL+"/v1/cathedral/lifecycle-operations/:idempotencyKey", wrapper.GetV1CathedralLifecycleOperationsIdempotencyKey) router.GET(options.BaseURL+"/sandboxes", wrapper.GetSandboxes) router.POST(options.BaseURL+"/sandboxes", wrapper.PostSandboxes) router.GET(options.BaseURL+"/v2/sandboxes", wrapper.GetV2Sandboxes) @@ -25659,317 +26649,326 @@ func RegisterHandlersWithOptions(router gin.IRouter, si ServerInterface, options // const string: with thousands of chunks the chained `+` fold is several // times slower for the Go compiler than parsing a slice literal. var swaggerSpec = []string{ - "7L3pcts42jB6KyidryrJd2TZcdJT3UnND8dOuj2dxWU7nXnfTk5emIQkjEmAA4C2NalUnYs4V3iu5Cs8", - "WAiSIEV5UZy0qn90LGLHs+FZv4wSnhecEabk6NmX0ZzglAj45z/fkit1ys8J03+lRCaCFopyNno22i+F", - "5AIpjqZEJXOk5gQxcqVQgWcE8SkSRJaZkmNEpyjngiByRaUajUcymZMc6xHVoiCjZyOpBGWz0dev49E/", - "T7nC2XHJmP6lNenbMj8jAkY3TZDELD3jV0SiHKtkrn/SK5nSTBEhx+iMTPXcBZ5RhvUoiEqEiyKjJJ2g", - "dyxboEIQSZhCl3PCIuNeEkGQIP8uiVQknXxktS1MucixGj0bUaae7I7Gbk+UKTIjYvRV76rAAudE2VPF", - "Bf2dLA4P9L+p3lWB1Xw0HjGc657+83ikZ6WCpKNnSpSk/+TOSpqlnYO6r6uNmWA1J6nA2buCCDi938ki", - "AgquGUpLgc8yghJBsCKIu27onCxG49jCaErygivCksXv0KZ7fTm+ek3YTM1Hzx7v/jwe5ZS5v38ex1af", - "lVIRYc6kvuLDlDBFp9TAkgYY2zi+yGqkvvV5UChLmo5iK2I8JZ1XZD+udkMVXL+mOVXtnb7BVzQvc8Q8", - "5lBFcqnxVhBVCoYKIgBn3db/XRKxqJaVwbjhKlIyxWWmRs8e7+yM2wiQmxnt55wy+1cENcL1DyI1UmGh", - "4L4yKhWaCp53LJv54foPUNBZDECO6QzRCkgekslsgj66rX8cPYoDihlttSu0lKYTLqrvK45LEkFU97Du", - "c9+oy7DGDIIeSpJ81nR0Sq9I+miMuEBUSZRgxhlNcIYyfknEVoIlQXp+IKLtJSuC884F24+rHYIieZFh", - "RXpG9Q1WG/mCZ2XePa7/vNqol+Rszvl557DV95tQIg33RBacSQIc6enOjv5fwpkiTBkeVWQ0AcTc/pfk", - "gJTV+P9LkOno2ej/2q4kh23zVW6/FIJbtleHnhc4dWx09HU8errz+O7n3CvVXMOsGRUR005P/uTuJ3/F", - "xRlNU8LMjE/vfsa3XKEpL1lqZvzl7mfc52ya0cTc6O4aJjzlHOWYLRwoydE4lFiPiRKLrb2pZuUt+vVB", - "C3hW2hsbgdVzRUkSzlLgi5eYKic3Cj2eEyrtlJPReESucF5kZPTsyU5NRHHcbicmCH4dj35aB6adEHFB", - "RAXtP60D1TTsUX0mOWGKpOhsgdScSpSSIuML/aNZyu46KE1yTlgaHsCT9Zw6TQgqGb7ANNNysJn76fp2", - "rGhOeKkM/zed9Jh7H06OyYxKJUB6L4QWzBU1xB9fyr0kIVLqV0faxpq9DyfINEC/kwU6PEBTLtDL/WOE", - "a9S1zWfGemw9MWfxYc03/fASBDBMjyrsSvVDLeMJViTtGPoEpA+/+PgcplG4g+HLNz80Rz1dFPaBaxfa", - "GogwTQP+1GscfYoJOhXr/tN8HTevIbrB8ECrcfnZv4ihwHtpTtkL/dTbxywh2TE8wdtXnsDXjKT7vGSq", - "75kN70aJZAlrmJZZtkC+d+S1Ox5NMV1hYDXHCpkumvSaoUfRp0J4Zo0N1Gf95E7ixEjOv9Os8yQGrrbS", - "BjQWfE6zLHoM+sNKA9eO2PRefg7hLB2HcEpwbvUp9jyggUH9NKV6STg7qp9K8Kb729NR/yuuJRLgZE5S", - "lNEL4raHKEvJFUr0xOicLCx7IDhHhwcTZBaEcrxAZ4KSabb4yChLsjIl4ckLzCQsV/NjXqpADfQcBpPo", - "kqq5/gLzkfQjq7pjQRDPqfI6nDb2SEln7NQ+CE7xTB5bcbUFNgrPZIQw4BkIEBgG0v/SNM29MPSLUT+9", - "I4K/XwwWAi/gbyxmRMWm0L/7MRFl6CO8DZ4pPPs4QvbmltIcM/zYbOST3zxJw+239x0omZY9DaGpPgqe", - "UE3D4W70F0kQzDpe9kwZdxyzWyoM46a7xim3zgQW5baoDwVI6Ws+e8minDMjFyRbxrNf89lraPd1PMqJ", - "lHgWYSmv+QzZj8hJCpHzkIoU7c4nihQaEKpTLwQHbidIBkdvITHjM0RgK7GzpjmRCueRCU7dJ3fY4UD+", - "ElOsyJYeZTn0+amqIxnb0/THfqKwKuUxwVZCahy9uRT7l9dI/flpHDlZYlo2j0PCDEiYKQK46bvOOkhE", - "MLfzjt/Y+3V4UJ9/jJJSCMJUpp82BRcKqBzLjLwCoqztsSJkBBxr6c24xetb2D9638G+9o/eo4QLImFp", - "sBVDZkcxdWAv6/Cq431c4DOaUXfB9eu2euXPRq/8OVAaB3h+xnlGMLy2veL5c8b5eVnEW0k8JZ+nXJx3", - "fPYyuZXnHn9aypBtn3HfiiPLC9cSE+z8MVkW7vXx7aMCSPmc8BRAkJWZeYlYHVH7Bkz7AGo7ulSwFuzl", - "8zlZRCmtA4gl2GS3E/T4TNP4gEoT9+ouRvoZLy4ApuGMdTt9FzhdeFFoufDd3EttHW7W6I1wxkiiTqp9", - "1q8hJzkXkSfJgSFVICfo850g0EpMcSYJwvoJNw/EJokKXEqSjsGalhPQeKOUynOgBQQIxLNan4Rn6dYZ", - "50qiqSByDoNitkBmRY5RSIYLOddvdonojHGhJ2HkggiU81Rz8hRxgVKSES0xoYNqzjmWKBFYzrcESfgF", - "EQskSY71A0ii/////f/QpaCKSMS4QtOslFoatOoUPTPsyAh7mpNINUF7iPEtXgA5cQuzorbmhpgyxLjd", - "wAQdE30JTqLAVq2nN0bYBRWc5RrG/KuSSpQ44gKPSr0uwjSI+y1LmhnSm/JLNhM4NVwCu0MTRCouyKQi", - "oAGdcI/uKN800qhXL8H9gb3JUHt4sSM8VUSgyzm1dlR3l3LOyyxF5KqggvQS152llMmtcjkw/7G7AecN", - "ONfkqic7O+NvB9xx02EbioHVpvrBu3d0aBVCDcWHabKnlgi5e0eH+p1q7NhGpzNEzh27CV7A3DjL3k1H", - "z/7s54F6ve+l3tSncYP5ArMd8tiy6x3ypjqPKcqO8SW6wFlJ2gO2BsiwVO8liazrNZb2zgFM3SFeYok0", - "1ncd4lKBI8fyfJkkUZ3JGyzPKZsdEIVpJnV/YztrKWFwvny7LXFhZE7QLsqOPQ4AS4vOB0BmBmkSlq8t", - "eNkOfCA7PYShdtd/Edu9eR2BJppviBI0iYno5IImJMYhQCvuxmouYEozIhdSkfw0qmx95b8j3ddY4seI", - "XKmnY3Q1lY9ig+b63XLEaezx8gaUUIX+6E5YM6Do6XKFsxcLRWJnrL8hWeAElDBn0CpEP6c6a0vcGhc6", - "RtV4dZ1Bm8+4av9jdzGtow4XUturu+oT+h/y5kXkRoF30v+Q5vNPr/kNfbGqoDIevWQXf2DRq5KsL+Fl", - "xRnRBRZUk4/Ya7SNzS/ZRfoHETJqk7AfHFwQdpF6hzAnaHSNPR4Z60yb59inWGMHwOTh23i5E9m48axr", - "+vgkc8rIln78gPuVk1+sKKF7TdBbrhBGScYBxoh6jiijiuLMUH75zO3ts5YvEqoWnwNT1th/LTKcgJHv", - "s5UUqk+Mf9akGSuqn76Mp0E3+xA2D7Mx0lsTDGef4RUnPsNKJ1Fk7lKomPNeRrLtEYeajVeC54c5npHQ", - "KJZSPXZOGVbmFnNcFNYJEV/KLr4TmtbGo1lSdDX8df8oaCj8zB2tCSMCZ77H17GDqsVb67iid/11POKM", - "DBAywmV+Hfe3DVe6tG1znfp8wwFa6CCNmXQvAWvAP2QMD50p1TZC/zh59xaw+9f9ozWY7fQtDjXbRbYT", - "e2E1z6l1LAWW8pKLiFR1ZL9ojq6fGo7KiQqabv0E/NifIoOXUmNuTGx5b78MX2r8UP0M4+pcYqfaKfS1", - "369YnpP0D03ojsBNLXLO8DtIqprYmx7oos4SFD8nTL8kO4TjYJ6Tchqdx/x+w3mK/k3AW9P7L8rWkMge", - "dGtcYAXOq7Yl38Pv/UvsEknsguszjCP3EjtDTVReU6lI2qlvwxnFMVuR/nmIJJ1klDDlTFuFIMbxwD5J", - "lnsP0w6TTVKUXoveR0i9tv3rWLOiQPjq6xWIaVpGYJ0vW+PcHspqlzTLIo/v3tctqQtPva4qQVNg4jkX", - "i+UbeuPaQR+FU6zwQCXyG9e86VO71Im1W6QDf2OyyqliiWynwafqVdsDNnkCbVuOrcu26O3FoKIxuhgq", - "64o284KNEgXwZX3j3QYGGcqcMrHqu9wCG/rihm7PHjnDGwlwK4CvGvY4lHBnXIdgoCrOOhuxnYFCsgki", - "jkOm5KycgRP4lI/Go0ssgH+CSBpjmq/5TB5QQRIVfXn4T4GJ1aoMrZ7sjFiHfbgjt4wpF5dY6F/OcHIO", - "/2zNPh5dben2WxcYuKrUHWvreeVHqf38wg9pN3DCSxF745vfV1y6vm0uMEgFhb4SCWbv4cs3s54Gw1S/", - "HgUDfh27F9Khvqz2A60o90Qyp4okqhQkbu/EQQu3UWaeFjGa/wrnNFvEh5rCtwGDvOFpDDL1GLn+NHSI", - "t1FhrRqGBdqm+FjNN5XfYLDOxnzj1rmai7g6JTg3WqQIUSU4Rzl8tHbywFWg4YJT91fo59gtDwY7xypO", - "DIGLxHsWk716J9Ginu5m9KMPncpcUpYQRAqezB81FAEd2iOQnyKT6/lsrE9Nk+sj19xyrCJjRi8IM8/w", - "Cxz4sBmX6F6fjfo5uCXB9SZFjxKn5Sr2Zv8IJZxN6ay0cWJtFU6Hdrh6BLwJRIumxwVYO66hpYL4svbZ", - "v6HslSAE9KBnES16ddRmIDQVhFh9njGDhMz4gbTuOlKRQo7tuiboXU6Ve0GZ9jh/IJE1xkzQCYHPOyaG", - "C6YEe1Iw5xbY1GaCX6r5BJ2agEinR6XSmYnmomTneuIkK1Oj5JoTQZUxluEMrOlbGRYzIoIR5AT9CkPr", - "oc709GQ65UKNkeThRE6AQwlmKCP4wuzH65TsyciMzuYqW6AzkvHLJtCaXU1WVyq+JZc9r4WMX342Oiii", - "PmNwvYy9HvSCHNwojkxDWKLrbC2e0lwKGD7HCAII5viCOBkrJ0hLhgVJ6BTiCVLCFu9Kc5ET+G97x6Em", - "I+qSi3OLGnErHS4VP8KlJDU7nZm+HQTDc6xf+Vm2MPbKuugXgKa1/fXO+CawBdt5YzR3nzMleCbr9tlz", - "ylKksH46toRnPcOWXR9nbjHoIXhBCZKRC+zCdP1iQIwVJXkUWp5NcEc1HEoFL9y1bVkjqLH+YpYiK31I", - "426l6tjyEAd/bUELt5lHz40dGzBHhabqh4LAPx7V9uet3RN0UiZzhKtjSTBjXAONWbUxjxsvWYGnU5rA", - "QvNSKiNNmc/kqshoQlW2ANSj4TgJz88o82blUvFj6DVBTZs9ejgts8zZhv3muuHODDTwAbDnO+wDONt3", - "pNP3L3lDQrOv4xHF+cD5DnEOj0fDiXrfmklxw2emRdSBPd+a1tUxSJJE5c0T+B3hLEMWCBOe5yVzYWxw", - "o61Xa2jCX+lx6Nh8v6Ux4hXw+KeYnKXBCpy/I3zXij3XoObf4A36qcZEjFdM46LCt7O+GRchhvZ6767h", - "FIOzS7yQyIBD+rwpJIAZyukHgQTIsii4UK6HZUuTloy8YXMbNrdhcxs2923Z3Jq4UYeP2o3Z0eN7xo4g", - "JjDmoTnwfqF/7XqXuiCZNA9jVDL679JEMFgaXgiu3+ETpJubIKsq1YP3k5SKCzwzVMhpAg0XwCpIEqHv", - "4Lmb0OWT0JTP+Xobj0oT0Fll55AmnqsnPczj8ajASjOy0bPR//Mn3vrP3tZ/72z98nnr0//9vzrNYJEH", - "fslAfZNjca7fz4oD95XBIT2QaEqFVI5hm9e3sB0FkTzT7NK88LFV1WDloZLMhGXl/So4ax61qpeIWuQt", - "uezzhLw9nzgYyQKnAe3rzGYQarTaRUVXEjsL6zbT0PteLz9QW83K82jqnX343Q3ARTInUglwL+n0HH3l", - "zNdLAkZdhqUptB/md2a6nJg4U7LKLNL3GTbTMKfVLrV1XlfW9/K2oCn0vDpxAZ+R7fGUbMmEFyT1WkeS", - "VmJZmlMJMjakPJqgt5wVXFKlmYdxkUICfLW9vG90deC7rYWykp0zfgk2cS2dYFa79Mkw/W5eOVj27Vxv", - "xvlifh2PeKmkwiylbPbBsur65t+dWRIKIvecZ6kRPu09xHfyHP2HCI5Sbt3fcanmXND/EONaqk8gChH9", - "TzkZhCEPi412PnjBgpvzdrjNSW82hYC65XPahqiCo/qtxWcxTkCHTN9BEhV0nEsTtW3C7FJL8MFG/Q0A", - "CBMzWfXZn2M2G2K+1lO7eMRLLFGGpUKJ6T3YPHIx0K2ynxLGPK7b5zuu5WTzcZHNbTeArcKuNjWsU+AO", - "yKk26cl+nV59stzGOCJteM6G52x4zl+O52y4wXfCDZbxgBix9wwkRvaD6Jg2HUBV37aaFpwX9o/e9wGn", - "b4d8BP5AkPQ9jVKzI/pkD+JG6jNVYZarhLiEXo+xuJkq71qVS2B1REuK8oiIhERRWh+4HryEpAuFaWcy", - "TQwZO6XyXMaimZTJ/WPv0iRnwMkcFJ7beRVcNDShRBhUFUknMS9n5AjPyAn9D+m4Nv0JLg1JymYZQboP", - "ZGFe8dbcXPLYxdb3JWB2Ohk3m0QPDZZA9rdSAfVeEIVAP0fSR6uuAkBlOQQVglTgWi2mB5yWTRyPbqy2", - "Xk1CGXhpPIToKbQFvhdD92kw63RpjBkzpOM6aGh6ve+ON3sbjO0c0q8ddVYjYx00p4a07QVGfDqDA3JY", - "2YKR5t3FQDmGSo5sn3ie2vYBLWXIkScf2RYCqefMBqrD9dgwbJxowS0zZzkGJDDGCpdrGTx2cLoATWjC", - "maKsJAgYGps5G4hRZ1ahGCajRSowZcYfMjHh+uaPks0JztR8YRieXpj+17xUusHnlF+ygT6V1Ukc2zmr", - "Xw6q2asf98N1VD+/D1ZU/Xri1xb8Zld5AIusXYbh2rf2gFoaFr26WNXM/GJ+1rt4J1IiGs6+1kgBSx61", - "UkdyoVDqOwQJBH3jqFOxMTj1BEXUraf9PpK3ZD+9TqaITT6I7ywfxPo8PzSaHdNIKYi9+kvX0OGC8wyd", - "4eTcGJU50zSAlymSCc40iZ0JXrZ9iV3A677JW9HH+92LTQb5urBSJuehTSwm6GxYUK+b94BIQ0baSAMf", - "wuz9fn4L6StP9gZfdZcJcI0QYVMuEo0lfpoHsn6Mjfc74wrKb9ieEp3xkqUSPfx1/wi9OfzVJ2fEzDJK", - "MNfrIYl4ZJjeKtugkbflG+sDe/+3EVNXDSk6EIkE5Bc0jaW73gfId98BLXwmazpDDz+O8KX8ONJ08ONo", - "lhQdEwgiIaQixnL3ve3WoJlriw4PvO04POv2IvaO3+q72ftwMkaSZNOtjLJz/cuv+0ePhukD/AnU1trG", - "rnELzz8Z4rLvfvYyRxwLPVQpDo5Q3TDVojBpF4bvnUmelYqg9NqovkISKLcMu3GfsaAhjtitGMbTvcsx", - "AgjFLh+u12Z5f7i0JEbmtWBBrua4lPoT4ElDWumIjdozgVFnRE9cEKEPwGErpGkwy0yAIKcWbfaPX+6d", - "Hr799VFcex5Lo3Bk4WjL+K/V0ifYUf/73duXn49fnrx7f7z/8vPRu3evP7/85297709OXx6M0Ss4huiM", - "7mgiUqs7tGorJk+llpSgeBJmi5XyIvxW5phVySDMmFWezVUyf36In/HN035GsjIc09lh5yntsQq6QnaL", - "B2FfT9onv0NPJ21j60Zn5gx5BM1I7ZtxzQtyE08G60hjTMBBYTX+4QF6+HJ/t/aDI5n+NxPDZcjmGOFM", - "8up1enjQ0rTaHKPRXKKKCEiDEa3AddrYOWdQ6eUSL1xKZgguAmIPj1zMrBB6RhKeE2TfhgjPMGVxgTM8", - "zPgK/GVp6ddmfxxbA4H+lyDuV83duJoTcUm1WF4q82t4gZFFxNhMfVn1Y9Lw+92Fpqc8xzEh6gWWBJmP", - "QWp+7x5sHTWptG7C9CwblMWLsIvUpLLvKPNkU/SbvAKgjQLlKrtI6z7UtxuZfluh4usMyLZ30Hua8HPl", - "TamP0t5XVd0OXVCscelqMVl+g9cI1m5GW3f5yrZBofIOjgQImiSSla2u7fhu378r++6+tP2am3XjxZzL", - "OgcZ5LHudmlnQNMMz+KbdA7NRmUSf6vbtXR5H9yUEkE8waGNCNjriCb4MCea3vrIARdNcIll4MntN8wF", - "Sqm0m6/rDSborfHYxgw8JvUIoN2oRpFE9YBucDI/Qn6PtRPsNaQTuYccIaNTkiySbKjr/2vffv2JTm4a", - "gbbJk7LJkzIkT4pd5UvwTj/SEkMX6z55t//7yU9GqjAa77pnO3pXKlCpodP9IzjckjEC5XTmgpezuXuN", - "XS2sxQG4znZK2MKW9TUFX+oxdCXDl1iQCToAOrgF1YCBn/JLSD+LBMm5Iujg7Ql6uHf6X0d/NxTzUYx/", - "NNhmmoooq6vt1bZClKE5l+oZROgZyPaqPCNr2eJxk4Tnzx7v/LzzcfQomvmwOyfcu8KkV0BuBS5H3MPj", - "V/vo8S+7vzwaoxxfod2ffjIG3Uk9QmL3p59WyuzWnNC1vNGETfuxPeYeIevlRVQ97+IwyYXh0g1hUP+6", - "jxWZecNUP/93bX0eUD2Ayzz7wLOHB2P0wFZaeTBGRCWTqOoHeh9Y8t5xqpDgEFIlRArm+G0tTYUBDV/j", - "M5c1pn+b0LCSubzSC3YbJtt94HJ4WqISbPwz53nf5mMaDrhEF0tU6bsHWWnNQqAIS6xW3fvmoC1TFGTc", - "CKfyFvGuyV5ekaTU419vQuK6rzTpteZaaYZTgvPYLJCCJ5gjBpCR+ZbcmMuFfb0zDN4kAza4SiUjh1oD", - "EwBF02oaeO5KcN3pB2m6SSVKk1SqcjYcYPNoNB7XlKsVDEWBOHYpLdRqwkkgBLzi4rwzn3lSuf05W/G4", - "05g65eI8fONCCUCQr0wEvf4u0RnnqqotAEHmzooO8cBNe3+CCwVe1pwlkHsUizTTnJlPTQm6CXqJkzmM", - "btzMSSr1OxQc26F2XEFYCrZdI2fwUoHukk+Nrx+EJGr+bjymqJI2yZOPYTXFlaFKedzkPbhI+o0M8Ixc", - "tg/5Jtb4btZsoMJVWWwmRtK/69PhLEzuo5c2QS+vcKLf8vpbEAfgq15RqZ/5z6q3g3tMmAs0PKlWP3Fs", - "e0JLGtZrhNYRfZEzhA2ovrp6caOeUzs0Ad9xcUa/1TKOU0sa1aKuCzOOJWyL5IVajNEFzmhq3soS5biw", - "GhYZGaapZ2knV4NRhoein5r2S3bqX/lNL6WUkrjVx35xeOV3AusbuwBcYgEIS2cYSCfxfP7nhMXLDHyo", - "jQzSz/IYXL/wcOhPy4+gN9H9Cgferr35Fuc23EJvZstsJiVTyPTOmRxX1TcxSnCWEbGVzLnUW4a2motN", - "Rt1beB3qRuJQ6+VjVPCMJosq4vtsEXgETXkkX0kt60JcrVh7i2MWJpKI60Q5O60I6IDTfefbt+/bLy8c", - "tufGX/NZvNijYRf1FIdgL8soI61zgR+j4+gvfRUjv1FVR1jwp9o5dNTQnFKSpb0I0VWzpDrstdfh/Fan", - "CusPa2ba06uftFxeLrPu5Gwl0NRkbm1p+1ZRY/VVxsx4rFjN69uYc6nKDOYeh+fQOLM/do+t0BY9vWVl", - "Rj3x07vxMn16a6cX206wgzeBBnlYBRXXY6lytzZJNGnrmzDN6VCS1h0h9LYdGzTQM7Ao30uSHiUdJUv7", - "IoGmGQ/LJrskqEZL2huIYryVO0v2dAdn6I7xSBNwRu4Mx+gN94Di2zE/PVCGWnffhxAclOjfHq0+Re9p", - "9MSp9A4aP4g3SyJTuof8a2YHXiFnb2ATCPCmuovgqgPACqA2RI2AEtVNTPHEq+9ixTJfU+lDDy9JilIi", - "FbjXcGaj7MB+4INkzBvenJ6WA88Iwmj/8OAYnWU8Ofcq958n8N/2k92Po0djhNEZFgQdHnl9faMhtOIC", - "YWdRNWpu2yhQ3X8cjdHH0f+e1H56BIoL2ICrOm2z+yl8TpCGQ5KaJ80FESgljFZNJyuViIeDOirPMpqc", - "mjNZmqvuxCTmQ7RG89H749cyyNVeWYlNpjiXWS0oFROXtG2yv+67tdutbgnULtVdkPhNH1QXYVJMMe5z", - "IFqbMFiHRJmteoikMmgN5NFtE9jX8WiuVCGPwKGu810E/nY2Lz4RFwT9dnp6dIIEts8azFCRYY3LVwq+", - "TdDedEoSJdHcpkU2hiZBNC66pFM+kJymNbu/0VwVAB/6hhGVdkZCYcZLvJigfY2aU32rwdFeEAERMqBd", - "A6dsW3CfM6tOMFmtwBAPZq6HT3/55ecnj4K8fBmUa6ldRjvs1mvB/vbTT09+WqYHy/HVoRkrTM9trnI8", - "MvYE28AWz8yxdJrK37iMGY4sQsy5VFCJx9p8wZvhjFQ+B5CL1V6k1V9FfScBBPuksFXEQUtLj0vjWtCE", - "3YbfKBFblmKBp6I+bYMRCBdFRo3+C8yi4BbprLAWCu2eJuh3spAufAkUK2AxNbj3ELDT0DtN/HBBG+TP", - "UM6MYEgnfkmzNMEibXVsUs2xCeMC+BM5zuh/zHIhl1yCJbi2QtzVBH2wg0qzGSTLM7NvibCCULCUFGru", - "Q5AhR2ZBriw9f+5YwMfR//44gvgwBqozq4i0Z9Yg12M05Zauny3su47NCECNPVG/WQlFi8yW/Fc9KpJE", - "VViWEzEj6QS9EBynvrdEEnJmSjglOB3o4diiGXSByFXBJamumqQoEQQUfDgDRTqB0DW/gICfNgLqmMKJ", - "vvlj2HjKYXUzgZnyKVoNS3hepTBBjpEjSQossCIZhEUURED2+jkJJzRRMY13e4+uMAT8tiBVg+4QuC3f", - "cLBc7d3zcptcLqLwdIOuhpp+LUO2cxrO0bEnK+/EMLa5OxtzaCvAWL/z+q7mBKdErKZdacQSnJ4eITuM", - "Xg1lkIyGC5BfhOY5UMPMQbknI3sMkSsqwdna9A/Nh7l1WDeGlAwnGhH+MPluNLQDI0RmUfK5S/kI2RPB", - "ioXmuCgIk9YxbwuYn0s2SSAljQvG2zs6vCb4vS+0vN/l5/K2lr3YuaeW0MewDJ9CxrszHpu9ylqRbHev", - "gH3+jJwiu6nyhzgEA9Og/EFJRrCQiKqO9M8bYfvWhO3vOpP2X1c4//5lMk83MDjZWrJqKYYGJm/26hXf", - "ANFsjpiNiLYR0W4ior0LrWrReM2O7PegiNKgPgnyXJzTLIPcv6Uk0UwXdlZIZtHpe5IPy9jvLYncuYpo", - "ZhrJDV9PURHLmk+ly6pv790nyu/IWVFlzKeqynYxRhmXpv5SLW3/2IsQ1t2OSAOzvCAMuTQwnAGJgDAS", - "qqq0ETZKwteIemg7IAj/080fPQ/Np2P7+LUMXgk6mxFhrcXijCqBhU/UP0aCTCFFiLQ5/p2Q00r0EY+v", - "6wKsYwJ5PTovObWiUERtHsYIma2Et1ApuM5JoRAGR5nKFyZURzz5W80rZzVHmBMLPZ07WOJhCyKQ99Bw", - "fk3OBW6CDqdhRQXvsW4lRyrNALZ4l+EWGmrAF8jUG3PaDSytXseKyp6z+EEpk4rgFDynXEClGYmzDgeJ", - "zmNx3vsrldm08O9oQ9pHHCxB6jx31UWwbMcAGCp3Myelg2Le+KN3glY7/OUaIfpquVdBGCtwk1TrEYfz", - "WNnnI1wVfe7qG89OD+P1bITID1TNg6SJjVLxYV7VG/jLWMPl11bKJj8+GDB8YYVuSym2T9EJOlQ2rDnB", - "QlASlgAwiVsnq0ShN4JkzDCXWAbRJsOMUBZc/1iWfNPm7rP5yOBxP8fKSRY2mJwMzh777SpPRH2OodPA", - "oCZz67FgCHjKr3ZdkCnVdryev4df+9ghUeNKg/MeByAVrjeKcfXD70GosI7Gzu7T8TIV0X4pFc+JqMo9", - "1o5Wv9HhTV4IIglTYwT+ii7nr4QWCuVcKvRk173On6Nz/ZCBeiI0h4qUiqPHuz8bI/DYJUfWP+7sPnW/", - "whOlqsThV6Q4+vnxL7umGbyaucKZrxsSHsCT3c7TMwqh2yy9cqM6I5oN33GVke7yIk7KiZdUtjkFm/t6", - "pYVs+xadxuWboBipIjg3rU0BVaioNbPaIf1xS2blbDtfbLlRnl3sPlpJw+A6DqQVfYudE726CXqvZWK/", - "6m0IubbFvAxdvQzcVHs3Yx8xj8ZBAospzjIJWZR8sjF8Wa3n8MCOiM+Sx7tP/BDLbzo4ibG9vti1nxLj", - "sNxQNBbUFplpvARNCRkvzep9RqOE5IF7sPW5fQJA2HBMu7PGkIG+azlv6VqN/n1ouGhshFa0Jgzn6bk9", - "rHDXn+zJdhXrWSI5eLcSd94u/9JwscFM8GJh1cbvpqNnf/YTM73e91JLAJ+akWmDs/FXJYaWxhFpzhp3", - "EnqteS7QPXj9uDPQaFbKbja8NPw9x3JpXHW1pTdYnlM2M5kX5DAAHFphCU7EQg+sKmT8DnZuoaR6wpkV", - "NnoqGmhGU+Uoq7oEASYNdB/guhcW2ziOvhNjKfpd6EdBhH24DXLp2/iGLfMNi8BB5I4c5AEVaNEsktsU", - "JMtO+KVu6DZeSggEXYqcw+iLHW0JcYlhm1m92aGNPYxnU3EosiS3rmkaybEyXFYB3c1Sn1mjXq3lMtCU", - "UHdWw3AR5hnG/+C5UwVeWfWSRv8ZvdCvo55cMtD2ZFDWZXcFL4Iu10y9MvgNXju9lV/ht81Or5sx5gZJ", - "ULBUJwW+ZCsflnkD34jzXiOHSseL42342PDLfNiUz22aBpMA1n1LV3tLGFe7ZRKsezhI55oHGTazReh1", - "ac1XnaKt1PdyXUrQvJkeH+prpU65kSolAkg3UabUEqU4ujYoH4q9zC4NS4jiTVyp3U+NbNfxcewZiIPe", - "OlEMmQ/wj+4AlfWB3m3BRN9F2d2E+wfK3973Cqyqlfeh66Fxp1zF2l+uwVLWzwGmlFE5X21Xrs/gbV2H", - "1MubCA2DSVG1qZvToYr0+NpUnXQlQptamPCKZuR9kXEcwYmb+/xZA1rl9jfHytdkl/qp573USliEszDH", - "4oat1refNk1pBnTJGS1tJ+felNjYkzY5KkUkEum9yII0eDB2Zfs1KwZ18tJrc2tvnX9c6XkNatRWczQC", - "NTsDDmEd1w03hM7DQjVrC+h/5ARLemj98Iyq3zxV9b7/eGzxqsBqDsglSM4vSDqGFF3V7lcTvQTBcjl9", - "CyjBselwU2KyDm4YoR3xYNbaGl/zmbxRQOtdwldXMGttB5YK/fGkN43pgGd38/gn6MB3M2BnXJGNq8Mk", - "Ukn+VmSEayWGpOyVIARKxJ0t7VhrPFAD7o5kHzNrzCAIg5kFaHzCMzAWg/8Yr0pS5Ist19cVkAh+enbx", - "GJxzD6cwEvjvwdDp2DgoGfcTZROtY+kC0mDe0DxiGYDCM4kAeAZdj24eYe96EMWt14szn3ix2JHk4VQn", - "tBOsmlC2w3LQRAJQtv2xG0keVc+ZMxU8P8zxjEAaI8FzN4rn20Yhg1lapYxpeyhAxZPleYtD1n0555kT", - "lioua4uuKI5EyRopmPo5ut9JhKLCBhUHnMUSnEQlCbTNjZvMKfNwEoESP9MxmVFpk1P0IdirVgc7Sqgi", - "bCrf7ZpubdUgIO3nseoxIDolc5KcQ3ZcsJdym3+OeFRy81XVvDqFEdCvR+cCuLy1WUixjBs5H3Vo2zdf", - "C4uHcPUTRYool4qYP9tywJKqga2lOdc2+Nv4tl1iaqvouWp/JiFVzNnNLcHyR8fkO/nkmhXUd684XkXT", - "d08VeD0MqumTeSP+tE7p1GvM2lot2G4o4AHCRfCGFHGEdu7P7QgnsVSQ3ROz0lTM8z61evZVDhIcvn/D", - "MuIbqX91JwjNfLKKYKY261mdz+qhboXBxnNYni6KyucuuuomFECus+BOT/Hs5k/iSLZRKrX4N0hzN9i2", - "Y8VMh2vDzbi4ox6NHjGeALR5bLCVQBXdsOObszSOZp2OzOsiVF8jS+p6Un5r+0vEAblOcz5QNQe2Le8H", - "o5TdkXjme0tEXFWsMUrzyPzrMYJ+Swvixhq4sQYOMjLFxJUu1fxyM5+hOIZU9vnzdWhmyGU9+GeofxgM", - "p2c2gSG3GxNivkRv/WDFqBA/lNVNxRxMzRb2WNqRLLarBo4NQ67e2Y34RjhUswCX2dEH2I4/MpOFwDny", - "OufS/8EFnXwsd3aeJC93X3w+ePdm7/At/E3+Z4LeaUT1yU0d1H5kzk3Vhtu5QoUJ+Mmjhy/+693+I1ea", - "/DnCZ2Dq8C6+Y0TZR+Yi8SSpLcgGE1ODuLWYzTrxu+5tq45CPKWWxUxqXHPCU24rPGEoAG+ktfr5rhma", - "3OJjUPWBnM05t1nJOgMI91uVxgD9IfmGC427NCPJvpJjbRoNaebrseZLOUj8Gu1OwLUNTQUlLM0Wncnn", - "tXyLVSlIV3CT+d08WhRHRjM6J26bqMCLjOM0muvIGt8quVzQuErTnT7Q1SHHLkDBBoVeIMphyKE3Qn5v", - "9w7u48FGI+50V7/V9jI/NW4jigYOxHwVX5fO/voRbTcSJG8ds2LOox6vImVJBlLY1VBTddTf6K6G0RWv", - "cB1wCV3KQ3nHrsoDkj16f84BAB2QjF4QQYnU75YjA8+x6NGZZsGaQ9rzSU1HqJtP8kJJUy7WBJrHS/i4", - "6KpB75/6+ha/QkngGHKTK7VfChkrgW1+h3QJWEqnjNM9ILOdEx0geY1+jXhQFyDNM27aFnhGVi+taYP7", - "gvW1j33RDX/N822fpqW1b2Ss1rnt7LIGuMaQ54lmGa3i5Qc4E4Meez/DsWQ5b3Ayp4xUlaoZZ1uQcIoL", - "SIECdRtKQVACA1Sq87lSxWejIh+PUib9v13wtK3WW3Ch/De7If+3p43+lwSzhNgyn8urI+pOb7oSib8M", - "i24DBtu9mBRPvFQIa0IxLbOqWsgZB93/8qk1jsRoR60O1oqllaBPvDhCfVhXcGaIS/5BAxRXXJO9shf6", - "XCIcVlCbhsZRFgez9iC7xvutcpGqD/mPk3dvtwhLeEq8U5N3gwIJVxKoAX1BXACqIClOOuoB2hHeR52U", - "jl+7Q/GFT6n0uFs7nRg916MbsImfzqkoGaix6uBlCspXvxkXnITQC9jCUtBzHYceoZ3GnuGtza5UYaxs", - "+9FK/kA8/CzGbwbK+C9ZQIugdayoHbnUh439soTssCK2UMe7/zgqaO1net1Q3yZqLlTDynwNwcYlVQ8q", - "aS+NLH2lSp8Wnw975LR62r1VNhKt7e7EnmrmiszWC2pVdDLwyAo4ap1qtWhOjSrU+GN4wBF+77LbaLiX", - "A5i/59t6kfrxmsgIC2/owS+IsOysuihemnrBrTg6ny5nWGuXAmVA68YVua5jv8Bq8shJGYFvsHhUEz9l", - "byVJ1+O6omg0yV3FzFfj0f2lA/uBvgLtGjgHYO63GjngoSCIZzNBZvrZJ6HHuGUdTM7J9U8TlvECxoim", - "4qtJuSuMW8e0r568hjfUoyd35RFWjch0p+GG8PPWttJ1HfYcll8KXIVNUAoRpWbeJY+E9R1fyGQGWmWv", - "e+K1GNihZ658WGv8mPX3jcbkB9WYeDkhojrp1JiYjESloGpxolHGQMJemlO2B7kg9kqTdovq3RhR2U3x", - "bPTPLWi5Zewh1dWYLBJfx2agf3w4daOcESyIeOX29o8Pp5q0w8QaEOBrNY5+SvtRwGbVvxjdZAuU7u2F", - "DNrM0eHW76FNK+hfqvmR8fwVL2CZHVsyxtHPyp7I8r0FA99ki/omqQ39UFTpZ8Ho5e4LtHd0GJRkfTba", - "mTye7EDpuYIwXNDRs9GTyc5kx2ZEg+vfxvrAt30yg22b6W4r8eXjZyRaSlOVgkmEkZxjQdIqKY1Jywom", - "KcigSFLrfTnVz1VX8RPtfWQ+FafA8JrlzGafRXzqM04mmCFBwNwESyIpKpkC2pbzC5c8SdM27Iojj34l", - "CuDIpw04NoPtmz1VDzjY3+7Ojs06oWyEECTFNRVGtv9lIzoMl1nGgzz42hntCuzEcHOtFEthbm6zR1mV", - "RwQj3uGBvsWnO4+7pvf72daNdNvdXwa03f1Ft/3J7L+/rW4UUhAIJG/Rjj8/fR1/aVCCPz99/TQeyTLP", - "sViAmaVkqpmTnEi3WeenB4XYc8oM4bJgqhvI7S/G4/3rNi7o1jlZGE+aaGENY+vRYAoHGSYCMnmwceZr", - "l1xycQ5l+ictgDriUvmrlVCK+MDsG6RJLHBOFOgh/ow+cAGLAc0hFaFHcp+Tp6Lw5oFfQduyV+Snlnrq", - "VgD5LbkMPAcaKQttNoAGFj2+tcnNraXNBUQOtpZpqFF91+DMzhCc2VkZv3aeDGn7xLR9OqTt0/uOt3DE", - "dTzCEhkkvQbSbn8x/Ozw4KvB24zErLIH8PuNMdgM04HDe3Yh3xqXx/HbrNa07Y5sZPC+hn9PO7Rr7sjM", - "Ca8TR34IuDeQc3O4N55Z28a408Ox4LtJsU/ZViG4KXSAWYoKW1Kk4ZhpqjNAFn/DQJczL+OBaua6Fxzs", - "LuUx2KzZq60MH2EnJwFWIHNJmRbBjD/dXxP0zZkBKAYgh1eW06rXBSTY74T932lmIb+dxO0aQO7F/99d", - "Wv8fGcrtbvVeB0K5vgzND3xytb8mlOsTiwBdP5gHL4/o+xh86PWgId+Q0Zeqf0fcCD4GuuRXEnUrcqEf", - "WLyDaXtT9/Nt2qHF+fNTlyImgJU6CLW0UnGoijWrARrARBMePHQ5iPo0Hl1t6b9mxn40yqg0KUx6Xri2", - "GkI4eJxEBsC2eTR2Php/CGBe9nqLAl+Nuq30RGu80LpeYMteXN/kAfRd87z7R+giT6Y4sIEiOonEs5rY", - "lWUgdaQ73zJE3T5VbMXhDCKMO0uA2UYEbYD5roHZguIwymnDaeT2F/sv/fwRdLZEUNQPHC6SOZHKJLhh", - "PCWo4DyT6OHHkR4AihpPEXYRO7a+VmV3MVErVCCZ4IyymXEnkRP0ylRmrWL57QgPJCLpzNfuft4cm3Ek", - "6AzlmOEZyQlTVU231IqDEgp8QXItLaWYwhem5F2CMz8cZvKSCIl+2nncYa7Zt+e2707tmM7kyqjsz3x0", - "4+fWIHH6mM6uL0dD0QQ6q0LlzeK/PxzWbR8Pafv4lh5sgDLu6DzQdrzUOjFymzKpMEuI3P7i/rlE0Dkl", - "IocgAgRpqkwfRJkpxQVlmASdPZB1FDRFVTPOIO+TKZGS4CwjAiVzzqUpnumjcTXOybmgzBQ+NSVb3VRB", - "neIIIhm+G8WlQ7fZQ7/Vm6DXuBVeYQlztdYuXQoNF9CtT4noxlsRzAwK+41RSqQeBCW4wAlVCwQezIkA", - "qkVS9DA400fPjXOLraYIqqXabWW4ZJoSg38unHVuckCaZcOxw7b+XRoHabsvP+GBWc2Q3VUpDtr0KpJt", - "y90dUhYQKWcQol0okt4bVf7OECKz88sPRJDidAGIk6CzlQnTF0Fn+g8HzibXRjS4z9dPBfj3U4MXAbLl", - "WyMkyRAhU46JSogtCsUDXqYu951AGhswm5H0ObqgPLPFuS3TgtEeSAT1goFeaRkhoyYYN6jK4QmVRLIU", - "UyikJBG4HcsYKTsq4zLBsT6afXcwN6NgSxrDLdzZq+CYztw29uF8hz0LdmMWI0v1zD1tCMK3JwgOMT1D", - "ujEtMJjS/4oQJIECOBbTTZceKmDTaBpX04B42HhbidKSmNKMkpciIYhczXEp9bdHY8TIJZEKTamQahWp", - "HjD4pdnOOvB33I73A5f9oHqQPSnYqpbOOzg8VAAchWzcB3vv7sSqrbrAhJ/CKr6PI/7A63qtwLHf7MnS", - "hK6pg64f25XgGz1y4hh9U2LiXz7LtRK+KcJK4WReiQkxguISslNRRe2bisUsDdw8XVVxyaFu1tkifDWs", - "TEv802Z94sAaUNXt6mbYWl3fBlHvElG7UaUPU41r/natvnQUHX8lKrRQm7CweoG7SMpC0qk4/5WolzBE", - "WFitgTsxDsinU0k6WODOyvXGv6zKZR/3ctnHO8vYbMeMXKRE7MkkPqlNidl6sbfkilc0U0S4u7FU0qcQ", - "r4LWIXb2qsggZNgoBmKrck0jpGN5PVm1gKgAfVhrIlkWkgCsbihh1ID8vlCrG9g02gTlvlg5HFEKY4Ra", - "VGn7iwvINNrRTgrVuLjlROfEjbsy5/YrGnUi9YZUbUjVhlT9JUiVz8u2RD1isrlViVoicQKeSH2ocr3d", - "PUTWo3oHgKRL0Rvd0wa0rgdaHow+dXrbHdvzdqTTdolHjEUA6fb1yPWkhmv2uWvm8FvqV2897vyxbSjm", - "WsE6QjO3v7hcMwOd/Nokpyvaqg7/H9w0Kwt7foGd8kEPyDmnvzrIbcDoZtSxW0sxCD5aXPYugeM2KZ3j", - "0KsIiZexVFEbOLwdLr3EfbQNjM1riPqU3hVw3h37r2WUHu5g+g1wwzmubmSAeykDbKc+n27/c6ozje4q", - "5L5K3nsT3GrpN94V+N8lQYnJnjsVPLcaDnJBeSl9XqQHElXZbdGUkgwKb8Z0HGas0RIXtVXNxj/dZy3R", - "oS3EWUEEwgpx4avpUYl87qquc4Oci6NopGRvyYwBizkjUw7VlYesg7D0FlZhFWfhIhb1nGZlTWdmzzru", - "L1ilTSs7tGerpdjs1awN20otte6QXdTy9n1z9d/w9HCNdOE9Kpbh6cI3rOyesTLpEkR2Plcul+SKXIWX", - "nbjkkrfGxoDmSUibBdkSBfj3AUWdoANDz0F9v/sUzXkpJMIzfteE+OVVbFGEpfUlMX55q6R4faTB5hWN", - "kIR4EsvNU+6WUHlOcGYy4kXx9Tf4bCoLx9DSfB8NCgud+7AnRCUyE99h7O/X0GGktg/YOOMpGZBKwDSL", - "7Pyt/dCb4KJdHK0KnIkKvN45ap1pLgaZS/R+b5bAwBzlj5NUD2DEwUfM1Qi+bX/R/1tmxofYQ8gv2QVq", - "b2GUlVmdmTzG574L0FwGkdfRDTILyX/BtCtvAzBrQmxnsou5k4BsZXNsTjBmfbsNSL2r5Io8JbZuhA/9", - "+DqUbwF62hOApMwmOmd9geDfPJzjdnJbGVAKTrODcsa9M/ur83qW3UpzOkHvJUG/vjxF2xe71djgPEpw", - "Gn1v9DhpNiIbiMIpVhgBvfRF36bmpR84rBBpYz8+jkpJxN/xWfKx3NnZ/Rsuir8XgqcfR48m6CVO5ibd", - "HUtdnZm8lAqdEfT++DWy5VW6xPzcrqZXZ7YWeUFfB0ntMd5McGhd6J2++deDPvf3odAMh2gdfoWy1W8D", - "Eij1Y69PqeTcu7wC2WXDN1h89O5kMBprjjQYjw9KAYXI9rGak1TgzDpOVFGUGi0n6JgUGV7Iyt9b4pwg", - "XKq5lqdM2SXjG87SWlEqC9NVH7uD50iQUhKXWyKl0ykRQYla0xmi0P8FJWU97jfTlx+mJC+4IixZ2Dzr", - "Fc7m+Oo1YTN9zY93fwYVt/v753UmOPYEYb1OM7VpI+9SC3RBwYU6bx/bw4aF/XPr5e6LreZpt+vAJXNu", - "62678FCozKOhLAZbvTT7652qOe9DsOiTIW2fmLZPh7R9+p2SXUsKLUzGyW1dStrOiRI0WaLOsI0ggmVG", - "Lwir0fRuCeiNHXwJAd3neY63JNGNNKhnVpfvcOvwAHSUM1JbyUCThx3kM01lb46HbiNIjq8OzUcw8dWk", - "kfHIFDKxDYAY3enT1Z/tB6rm7nxvJhMZ/uEAYSMgrU1AcvCdezQZLB7VkbgR8tHlFGhSpwbkIeYGeLvh", - "Hp8G63cDPuoy3f5YudK+B7AECOlnH73ehO4WzxYIlHrdrOGOoOvWCe111ISyerpuYPbOYfZkBWknIJTb", - "CWeMJKqe37z/qenKKDkwN3Xb5AQdTmsBRpCkp5QkHSOq0KVGqjOCBJFlTtIJOj19rZtwli0QuVKEpfrX", - "+Bs1tuSB71aPYvt2pzfFtNt/1tmVrfS02/kWTzucCYL1S9wITxqtvtEj00LR/awLsnkO3qvnoCUX8poU", - "csrFeXf5h1dcnIdU75mxkRecMpNhofHQQJQhSNCHHlKl6d+ZoGSaLTypdLlRfU4EqiTS8I1yknPrv0HG", - "Vj1msIAzaMWgSvc5IYWeUP9yeADtyFVBrZ6kZIqXyZykj+CL1aOY3GuMXIb1BIwCESu/pAlypJ8zQ8gz", - "hQoinKbNFGM/HyOCkzlKsBALyABDfZ5Irxuyh2FzQUCOGDNXIcA5iqSaZ8ASwAeIstkE7SHG2dbuzmNn", - "SsoJZkZB5HR9Ni+U9bLEDIounRvi5SrGD2UY+mrvIbew69OrOzbDx01xj+8sCNlMbcp1LLdLNEl3COAu", - "9a6+JIkuib4zd03PDRgRpoSWLAsulAE83fiBRLxUCc/JhpI7Sv5dUmegn9clzRmfrWLhDPM96K5G2uwW", - "NHWbYUZOTzNe82ukgK4nhmi8ryz1q5zFnbsJrM7Q5zkvs9QI1/YB1q5Wv6J7pSsDvSzFxLI0dX2rXD1l", - "3eOdnZXzYKzheQy3fq0kDADBmxfyGl/I5shXpTTLDAQhaakUmQMIRqdx4AY04z2jVwG98NXINLY76gF1", - "KC9wNtakwlKJMTSFvODQptrIHRGPIY7Wg7ZGWHq9ja225HVmejGAcTupXtZh1dgQpZsQpV7jRw9dYkRd", - "uvdpLNu2KwQyJ8g2rccS26pxjUeq9dJIrPXd5cMmpqqnKDMS+G/4FEz1IGX0LqcKJBcIUURJRrCQiKpJ", - "LG92mzC+tTu7tw8wu0BzwiaWeZjq7umS7A8u2DjE4Ojl/cU8Jr8HfLb41ri2ldEalEHdSqcj/blmZBmm", - "0YB+9xejYHm9Oo0BZlOjR6spho2mC7PA/LDRGnzXWgODANdVGwgyFUTODVB15cyCJvWEeWCkcqpVSBGt", - "OMroBRmIfsd+3nuLgnaJqyJhQwQ127yHZuDvEtgdLF4f3DXJW83Cq3tcg8GYjvcQvM3C0sDCeE+8ZTeG", - "zI0hczAdAKy8LhlwdpcerueDCAoiJJUK6ji4Wo0+oMCO+UD6hyGYJCfoxM3ghC0XzGOtjHX7ohbK7Dzo", - "jCy4NQVxQWeU4SyYJqNTotntUMOdX8f95bNuiQGjXas7v53+kE15VKXkLr273PRGjXQPXd7tta1MGzR2", - "8VJ1UwZXHMo2rHTKTmFU42s0y4zPAUFXTgcbBCRpydmV9jDwP0H7OIOaLJAYKSdqzlOUl5miRWZ6SMQv", - "iLgUVFmt1Onpa+tpAAOW0nSv1FWVmhjLSgGuW1nvDI5ygmUpSG1raXeavSi1ObVnd29pjV3gjWR6ae/f", - "XbEDmA16r0VLHFiXlAe3ThQniSBqQJ2kQvB/kUQ9kMh2maC33OeZAwceyJ9mP5tw2rgh3E65Kg4UeGZL", - "gr4lV+qUnxM2pKxR1e012IrXZJiBTa5skckgkDd+4M24OH0KW+YYOhZjm2//Mziwu41tezKk7ZPv822x", - "O6Tt7l/gbQE0wYJnAJyexNhfBoQpx4OSDQE5WyCpuNAsGGxOwLhzLM6J0DwaPB2pkApd6HcBZ0asaBEk", - "F6QPvuMdbNpv4a7icC0xWLPcHsw6IGU9cGxPtTYkYkMibv7GcMgcow6h+LH9xfxjSTTeMbng5ySAVNAL", - "aHhPy4wASbDEwATeJhnBrCy60vdbvD+xU68ulLuOw2L2oln7N1i3wbpbwzpfwqIH63qCETlzsPig4ptj", - "JEkGqTBMiGKVyEwghvNeEf9OMGtn3QxSECUoudgg6wZZbxNZbfhvH6Z2mZtNTFAFjfqNqLggqZOPzxYI", - "F4U1QGPQpd+WlHxbOH0HGiyYwDjVrD0uchgpqXlrbQjJhpDcohvZclk7tOn1J+L1TWsVmKOcvtt81pNa", - "ppHYNJq3v1rD2QJJXoqEBOltVqhlUBtIr0OLLocHY8ShIdaIqfBs698lzrRok/rUgfliy3X+OBqbH/RB", - "bNc+6OFqbZ9dPP44etSVNRD+t6TKxqoqzfH1lKfr0YTWzIfX9VCXAZzdmhZ0k4fn2nl4guvw1Mb/1pOB", - "R6PPAAJkmkXozan9cPdwq2e6WSpNs4n7CWmdcBWr/W/P3CeP1U0/Vde5/UX/b2jkU5gZrY+zwE2fwsDX", - "jXsyy9oEPf1gQU8aKG4j4gkyh64l3GkFOXzDaBztOQ2vZwXqs53jq14KBABug49j1Ejjhf63ydvo0GUY", - "jXqDrzZk6t6TqXEksbigCVLca/tqUALODDaDZkcmcE2N+pJlukpnCWfW5+hzmBHU5dyEy/gssCKREmh3", - "qg19g69CwrohpD8MIXWJFtQAgmrSf1+rFEDVOUopq48D9ASWJHbTj3ZFFStHfpP8+253N30vuDPavE5v", - "9DoNIdFBefVb7+vUNtrGGcX6JL7AP7rrCe3PSXKO6NTPaWKdjegAfRG5olL1I8WemQ3+14EgBYbSXxY/", - "sG3ZzW8akSRuda7jOjmLmxy2d2xnWYoamiJQyO9vl362sKtfmcPckRp8g3VVrRmNBaoFZG3ca6CZadHt", - "8WEt2pVzN5455/BObmP6eNw6xbO7cu2qz6QnWilEIxagrPfnfEM2cRTfg7eFh3qFa7mC4P/dVtw9KemM", - "6U4P5SP98MABpStplkaNsOuAarOya0P141teCEnDpUTDJPEMYdt2gzT3G2kc2PcjTZ1JfHH/XOIe6N2f", - "XPulrMGPew2dje86PGN/JSPiPiq/EfxvTo07pI/xErsTUF5pEx11glEout8SDI2/m/CXIZL+B6rmL+Ao", - "V1UkhY8oqqS9kI3R9z49q6tLGfy41qihkvkQfZLLQtaFfEd6oDsh4bcvSrnVmT2tJEbtDGAhzp1sbTLP", - "BiEaTldL2E23HLNtcGj7C/w/mp+4ZbaqoZ9LTTqEMxlS/MLMdK2sw6uxKrundWcoTkohwUTzPaUojmUo", - "NrPaz9exvaVUkAT2MB5IqTRUHPhenQNn5IJkqwz6GjpEjvbEePQNuf2p4HmX7RJGWWmXZuI1qTwB5/Ss", - "g9WecXkoQPnNO3U9zgcxYntTGm/rcK9A5W39+JWp/Imr+P1t6PwhS8mVQ26fSsOfZSeq+0SxAWON0iE+", - "k++mU0k6COvKid9/GNJ/bQq9NnLYmUdoKRnc0L5vQfukIyUrUr8pzfRPcyznX3tJHmaoLDKOU5RRdu6U", - "L1ggPQLSAIgpC+gHXhDzbaj8+0q3/Q3L+U3pYcQSOzfDDjXE6lU4uui2sNwW+/huMFGfy3s4+a7sg+G9", - "XM6JgCym9kfATHtLG037fcdiwCV7c++PX6+Ozs5Uu8RxHwy011GfWovWbarh79DT5xTPbup8HFpA7ktk", - "3wZ94hrPqJmq24/o4vF2gtWcpAJn2wku8BnNqKI1v7oWTvzxeN/12Q+73KFcFp8wAse+IWS8g5YLVNvY", - "ihD8g0OPczOPnxtnSuBEIVkWBRc2tQIkM4RPPENFhhnpSdtWAzAPSXL7C01JXnBFWLL4nSy+DoS3d36E", - "w1r/lelx0hpRj3Knbwu/CZvB0M8cg+MDfQtZeC823Syvet0PSeYHR5BjkvALosWE7quAhCNKGtTJCApg", - "G50DcHaiR1Clb0C0c+CH38aS3ZPgcwMbWkEEJokCPMl9QuOpCUMOMmIS6UOMS0nE3/FZ8rHc2dn9Gy6K", - "vxeCpx9HjyboJU7mep+Q5AgiECTKS0iArCU4RFjCU1OhtCMmAVazLNY4HjPtF3q2gLwwXKCcC2KyN+uT", - "IFdFxlMyejbFmSSdkSWqrpxdpajViYp6VI9HUi0y/cOUizxmY+BCIa8Bh+hxGzZuAmcgOzQ6MIoYqW9I", - "90cPGbmEqrBUSNUZvc1FSsRg9co73bqhVo/VRw/OG9ZIUoSVPnM8NaBDZWUxmfTF8ZB0T3eJG0FSrMiW", - "HmeV6PkQEgIftcMDWF9GsexaUPAAuJ1o9+8g7+drSOB5UtWxuH44QKPOFrklL4jx6J9bp1zhbOvYVmZf", - "1hlau8Z3m0B08/6oh9lX/OJi91Gc4V03y6aliVWmbYutE7SneeJCKpKjhOd5ySxOVNp5X58cktOWoqtO", - "dp153lV2TTPFH7trT7A5vHLH2lPjbypsrCuNvr3lbgRtSaS1bPoJZ4wkqq+6lCmj7yZKicI0kxN0OG3i", - "oilnNkZUmbT6VQmzCTo9fa2bcJYtbHEq+DUoh3/G0wU0sWlxniPMEM8pFF/3Ce8DsenJzo4LzF2K/j4D", - "/r7d7/3LgG9XViMnX+++HvQQ8oEzQXC6cCKBxq5NCaJNCaJ7TiANPt2YQi51jJP1Yt19T/jvtPb+d+nZ", - "tsy/YWfVNfuH9JCT7VjyrXjGNY7SbAL8WsD7wqaRKAWbIN0bnZGMX5pXvGmABUHkKsnKtPtsb83Tbh9L", - "siUJk1TRC4JkeWYe3yjHKpkjzmDlOZESz4y5XTOUDh0DwSKZ15aV46vXhM00Bdj96W/rjSy26KwP+Y/d", - "67nY1YjHxjK9zjrmgL7LuUI8KcXqKSj+2P1mSSh+MCXXbae7+LEK22wQPZ6Mo4nqgzJyhPjfCgQNYopW", - "jiAKqMGPHUN0J4vo5rSbIKV7G6TUg4D9uNZyZO+p/WhMbPO+JAZR1Kt5r6/Lb/2OURZ2AycyVDm9G4/6", - "My58c+zNgxsEWo+kaqC57km5DIue1CXWJXYYRi57OJXGlieh2Hrn4GoZzB9PVgDYW12FXYDjMW4hXUhh", - "lJLQZa1qyVUcML+1WvJ7tq8EzOtJL9rxrMzJwNTiyLWOPRP9p7t/SJm5rlsltLWbDaG/4UulBhkO0twv", - "17W0Gwrvh44T+QDo7sRI7iBtvRZyM+seS4M3+oBSlO0z24g7ayW5IbC20CAkt9tfzD+GZ2bqxgPTyGLC", - "H3bYlV8Cbj03KcqI27C3UQqvK39TP+yN+yKnfdfOsOm7hK6db0U2qwKFG8D9RmUK+qgl7EpcOAgrRTZ6", - "NporVchn29u4oBOyezbBRQEwZft/afrQStBr1Kto1X+ENN7h3wXdOieLWhsba+b/rgTHamxbM+zrp6//", - "JwAA//8=", + "7P3rctw2FigKvwqqv10Ve3+t1sXJVOLU/JAlO6OJLypJjmfv2EcbItHdGJEABwAl9bhcdR7iPOF5klNY", + "uBAkQTZbl7bsdOVH5CauC+uGhXX5PEp4XnBGmJKj559Hc4JTIuDPf70lN+qMXxKm/5USmQhaKMrZ6Pno", + "oBSSC6Q4mhKVzJGaE8TIjUIFnhHEp0gQWWZKjhGdopwLgsgNlWo0HslkTnKsR1SLgoyej6QSlM1GX76M", + "R/864wpnJyVj+pfWpG/L/IIIGN00QRKz9ILfEIlyrJK5/kmvZEozRYQcowsy1XMXeEYZ1qMgKhEuioyS", + "dILesWyBCkEkYQpdzwmLjHtNBEGC/KckUpF08pHVtjDlIsdq9HxEmXq2Nxq7PVGmyIyI0Re9qwILnBNl", + "oYoL+jtZHB3qv6neVYHVfDQeMZzrnv7zeKRnpYKko+dKlKQfchclzdLOQd3X1cZMsJqTVODsXUEEQO93", + "soiggmuG0lLgi4ygRBCsCOKuG7oki9E4tjCakrzgirBk8Tu06V5fjm9eEzZT89Hz3b2fx6OcMvfvn8ex", + "1WelVEQYmNRXfJQSpuiUGlzSCGMbxxdZjdS3Po8KZUnTUWxFjKek84jsx9VOqMLr1zSnqr3TN/iG5mWO", + "mKccqkguNd0KokrBUEEE0Kzb+n9KIhbVsjIYN1xFSqa4zNTo+e7OzrhNALmZ0X7OKbP/ipBGuP5BrEYq", + "LBScV0alQlPB845lMz9cPwAFncUQ5ITOEK2Q5AmZzCboo9v6x9HTOKKY0VY7QstpOvGi+r7iuCQRRHUP", + "6z73jbqMaswg6Ikkybnmo1N6Q9KnY8QFokqiBDPOaIIzlPFrIrYSLAnS8wMTbS9ZEZx3Lth+XA0IiuRF", + "hhXpGdU3WG3kK56Vefe4/vNqo16Tiznnl53DVt/vwok03hNZcCYJSKQfd3b0/xLOFGHKyKgiowkQ5va/", + "JQeirMb/H4JMR89H/7/tSnPYNl/l9kshuBV7dex5gVMnRkdfxqMfd3Yffs79Us01zppRETHt9OTPHn7y", + "V1xc0DQlzMz448PP+JYrNOUlS82Mvzz8jAecTTOamBPdW8OEZ5yjHLOFQyU5Goca6wlRYrG1P9WivMW/", + "PmgFz2p7Y6OweqkoScJZCnLxGlPl9Eahx3NKpZ1yMhqPyA3Oi4yMnj/bqakoTtrtxBTBL+PRT+ugtFMi", + "roiosP2ndZCaxj2qYZITpkiKLhZIzalEKSkyvtA/mqXsrYPTJJeEpSEAnq0H6jQhqGT4CtNM68Fm7h/X", + "t2NFc8JLZeS/6aTH3P9wekJmVCoB2nshtGKuqGH++FruJwmRUt860jbV7H84RaYB+p0s0NEhmnKBXh6c", + "IFzjrm05M9Zj64k5iw9rvumLlyBAYXpUYVeqL2oZT7AiacfQp6B9+MXH5zCNwh0MX775oTnq2aKwF1y7", + "0NZAhGke8Kde4+hTTNGpRPef5uu4eQzRDYYArcblF/8mhgPvpzllL/RV7wCzhGQncAVvH3kCXzOSHvCS", + "qb5rNtwbJZIlrGFaZtkC+d6R2+54NMV0hYHVHCtkumjWa4YeRa8KIcwaG6jP+slB4tRozr/TrBMSA1db", + "WQMaC76kWRYFg/6w0sA1EJvey+EQztIBhDOCc2tPsfCABob005TqJeHsuA6V4E73tx9H/be4lkqAkzlJ", + "UUaviNseoiwlNyjRE6NLsrDigeAcHR1OkFkQyvECXQhKptniI6MsycqUhJAXmElYrpbHvFSBGehXGEyi", + "a6rm+gvMR9KPrOqOBUE8p8rbcNrUIyWdsTN7ITjDM3li1dUW2ig8kxHGgGegQGAYSP+leZq7Yegbo756", + "RxR/vxgsBF7Av7GYERWbQv/ux0SUoY9wN3iu8OzjCNmTW8pzzPBjs5FPfvMkDbff3ndgZFp2NYSmGhQ8", + "oZqHw9noL5IgmHW87Joy7gCzWyoM46a7BZRbMIFFuS1qoAArfc1nL1lUcmbkimTLZPZrPnsN7b6MRzmR", + "Es8iIuU1nyH7ETlNIQIPqUjR7nyqSKERoYJ6IThIO0EyAL3FxIzPEIGtxGBNcyIVziMTnLlPDtjhQP4Q", + "U6zIlh5lOfb5qSqQjC00PdhPFValPCHYakgN0JtDsf/yFqk/P40jkCWmZRMcEmZAwkwR4E3fcdZRIkK5", + "nWf8xp6vo4P6/GOUlEIQpjJ9tSm4UMDlWGb0FVBlbY8VMSOQWEtPxi1en8LB8fsO8XVw/B4lXBAJS4Ot", + "GDY7ipkDe0WHNx0f4AJf0Iy6A64ft7Urnxu78nlgNA7o/ILzjGC4bbv2GZ2SZJFk5NybomW8B7khSam/", + "nxuzn+oY2Y9znnF+WRbxVnDJFFdEnguSY6pFFKwFyCPaQ+IpOU9JRlRfgykXlz2fC1zKru7+gmGV091P", + "S7UL22fcB/4IRMK1LjmK+r5ru1gCxOiJxXRij2Gv3QL8Y0ZEMdbgKotzTWgkANaIcXXuQTMeFYSlmn7G", + "8HamF596/Sui8Y9HQL/nCU9hVFZm5n5oLXdtujDtA17S0SWYIQBHVPhVDQTJ+RVJz7GqKXshA186HQ+B", + "6KDkz9Ec4ae+fueXZBFdZ/uwz/O4VrocjJYvdYFEMlzIOVfnIDZtq6Vb97382d8Jki1cs2ivMSqlsrDP", + "mnFcG49Kdsn4NVt+z6xDPjzBGpwamOTWN27Qxoqk1qlJN9E2eNTbjQBrSjMiF1KR/FzLxprwn+JMVgCO", + "8eyVcLULfE0I9ULCXrmOApnSv/82eixB4Rb22LdzszHzl74mmr8c7mrtYvmel6PFkM338NuHZ4uBrOrk", + "OE57WaL62e3c6lAqkgYZag5DEJwu+uRG4ziae2nQbc+JcMZIok6rfdaPISc5FxH72aEhLbjUavhOEJjQ", + "gdAQZmmo+yEqEZBSOgbXj5zA8yxKqbwExZWANvu81ifhWbp1wbmSaCqInMOgmC2QWZG71Tis1XPQGeNC", + "T8LIFREo56m+dqaIC2RIOp2gw2rOOZYoEVjOtwRJ+BURCyRJjpmiiUT/7//9/6BrQRWRiHGFplkp5yR1", + "tn89M+zIWCb0tUeqCdpHjG/xAnRftzBrF9JXN0wZYtxuYIJOiD4Ed/3F9g1Kb4ywKyo4yzWOeRMolShx", + "mjBYQPW6CNMo7rcsaWbuCSm/ZjOBU3OlwQ5ogkjFBZmMYrzQWYijlzxjOvFvIXB+4BxhriZgXkZ4qohA", + "13NqnX7cWco5L7MUkZuCCtJ7E9hZqnm6VS5H5j/2Nui8QeeaHvBsZ2f89ZA77ufSxmK4SqVnBOf7x0f2", + "9aJxGTFN9tUSi8z+8RG6JAvjdGUUlCFGmbGb4AXMjbPs3XT0/M9+GajX+17qTX1qqrYgbIdYBu16hxgA", + "L2OvOif4Gl3hrCTtAVsDZFiq95JE1vUaS3vmgKYOiNdYIk31XUBcqnDkWF4u0yQqmLzB8pKy2SFRmGZS", + "9zeOHq0XA5wv325LXRgZCNpF2bHHAWJ9+jIeHQKbGWT2Xr62wAw70JrrjOZeH7+l+dbuzRu0NdN8Q5Sg", + "ScyeRK5oQmISAp5w3Vjdt4+z6MvgK/8d6b7GbWyMyI36cYxupvJpbNCcl0wdcxqztL2BF5NCf3QQ1gIo", + "Cl2ucPZioUgMxvobkgVO4MXgAlqF5Odu1G2NW9NCx6iarm4zaNPmWO1/7A6mBepwIbW9uqM+pf8lb15E", + "ThRkJ/0vadoq9Zrf0BerKirj0Ut29QcWve9n9SW8rCQjusKCavYRM522qfklu0r/IEJGH9DtB4cXhF2l", + "3nvZKRpdY49HxpWgLXPsVayxAxDy8G283OO5ae1qOqQmc8rIlr78gK+w01+sKqF7TdBbrhBGScYBx4j6", + "FVFGFcWZ4fzyudvbudYvEqoW54Hfxdh/LTKcgEfKudUUqk+MgwEHK3qRkXPG06CbNXSai9kY6a0JhrNz", + "uMWJc1jpJErMXdZ/A+9lLNuCODTDvxI8P8rxjIQeHCnVY+eUYWVOMcdFYT3m8bXskjuhH8h4NEuKroa/", + "HRwHDYWfuaM1YUTgzPcAAzhg1eKt9bLUu/4yHnFGBigZ4TK/jPvbhitd2ra5Tg3fcIAWOUjj07OfwNP1", + "P2WMDp3fj22E/nn67i1Q928Hx2vwMdGnONTHJLKd2A2rCacWWAos5TUXEa3q2H7REl1fNRyXExU23TsE", + "/Ngxe3MpNeXG1Jb39svwpcaB6mcYV3CJQbVT6WvfX7G8JOkfmtEdg091BM7wO2iqmtmbHuiqLhIUvyRM", + "3yQ7lONgntNyGp3H/H7HeYr+TcBd0zvby9aQyAK6NS6IAmctbun38Hv/ErtUErvg+gzjyLnEYKiZymsq", + "FUk77W04ozjm2KB/HqJJJxklTDk/jEIQ4yVnryTLQ11oh39BUpT+ybePkfqn4S/wRBEoX329AjVN6wis", + "82ZrIrFCXe2aZlnk8t17uyV15anXrzJoCkI852KxfENvXDvoo3CKFR5oRH7jmjcDQJZGXHSrdBAcQ1aB", + "KpbIdhoMVW/aHrDJU2jbisJYtkXv3AQmGmOLobJuaDM32ChTgMCLN97HbZBXhzMmVn2XuwuFgSNhjI4n", + "zvBEAtoK8KtGPY4kqhe3EIOBqzhXooijBxgkmyhSvXVdlDOIWJry0Xh0jQXIT1BJY0LzNZ/JQypIoqI3", + "D/8p8AeyJkNrJ7sgNroMzsgtY8rFNRb6lwucXMKfrdnHo5st3X7rCoNUlbpjbT2v/Ci1n1/4Ie0GTnkp", + "Ynd88/uKS9enzQUGraDQRyLBR2v48s2sZ8Ew1a/HwYBfxu6GdKQPq31BK8p9kcypIokqBYk75+Cghdso", + "M1eLGM9/hXOaLeJDTeHbgEHe8DSGmXqMXH8aOsTbqLJWDcMCa1N8rOadym8wWGdjvnELruYgbs4Izo0V", + "KcJUCc5RDh+tU1fg19bwF6071/VL7Ja7nZ1jFY+7wJ/vPYvpXr2TaFVPdzP20SfOZC4pSwgiBU/mTxuG", + "gA7rEehPkcn1fDYwtWbJ9WHWbjnWkDGjV4SZa/gVDhyuTfxOr4NhHQ5uSXC8SdFjxGn5Nb85OEYJZ1M6", + "K21Qc9uE02Edri4BbwLVoukeCK8dt7BSQTB0G/ZvKHslCAE76EXEil6B2gyEpoIQa88zzyChMP5BWt9S", + "qUghx3ZdE/Qup8rdoEx7nP8gkX2MmaBTAp93TMAxTAnvScGcW/CmNhP8Ws0n6MxE7zs7KpXumWguSnap", + "J06yMjVGrjkRVJnHMpzBa/pWhsWMiGAEOUG/wdB6qAs9PZlOuVBjJHk4kVPgUIIZygi+MvvxNiULGZnR", + "2VxlC3RBMn7dRFqzq8nqRsW35LrntpDx63NjgyLqHEOcQOz2oBfk8EZxZBrCEl1n++IpzaEYhxkE0W5z", + "fEWcjpUTpDXDgiR0CsFvKWGLd6U5yAn8t73jSJMRdc3FpSWN+CsdLhU/dr6JTX+dVsQmz7G+5WfZwrxX", + "1lW/ADXt21/vjG+Ct2A7b4znHnCmBM9k/X32krIUKayvji3lWc+wZdfHmVsMegIuu4Jk5Aq7nBJ+MaDG", + "ipI8DV+eTSRiNRxKBS/csW3ZR1Dz+otZiqz2IY1vsKpTyxMc/GsLWrjNPP3VvGMD5ajwqfqJIPDH09r+", + "/Gv3BJ2WyRzhCiwJZoxrpDGrNs/jJqRD4OmUJrDQvJTKaFPmM7kpMppQlS2A9Gg4TsLzC8r8s3Kp+An0", + "mqDmmz16Mi2zzL0N+811450ZaOAFYN93OAB0tvdIZ+9fcoeEZl/GI4rzgfMd4Rwuj0YS9d41k+KO10xL", + "qAN7vjWtKzBIkkT1zVP4HeEsQxYJE57nJXMx13CirVtr+IS/0uXQifn+l8aIV8DuTzE9S6MVRCpF5K5V", + "e27Bzb/CHfRTTYgYr5jGQYV3Z30yLpwZ7feeXcMpBmfXeCGRQYf016aSAM9Qzj4ILECWRcGFcj2sWJq0", + "dOSNmNuIuY2Y24i5ryvm1iSNOnzU7iyOdh+ZOIIA9piH5sDzhf61413qgmRyEo1Ryeh/ShNuZ3l4Ibi+", + "h0+Qbm4igqu8RN5PUiou8MxwIWcJNFIAqyCjkT6DX92ELvmR5nzO19t4VJrsA1UqKWmCj3tyme2ORwVW", + "WpCNno/+rz/x1n/3t/73ztYv51uf/v//o/MZLHLBLxmYb3IsLvX9WXGQvjIA0g8STamQyglsc/sWtqMg", + "kmdaXJobPramGqw8VpKZsKK83wRnn0et6SViFnlLrvs8Ie/PJw5GsshpUPs2sxmCGq12UNGVxGBh3Waa", + "kWm3SmbXNrPyPJon7gB+dwNwkcyJVALcSzo9R1+55+sl2Q1cOsAptB/md2a6nJqkCGSVWaTvM2ymYU6r", + "XWbrvG6s75VtQVPoeXPqshNEtsdTsiUTXpDUWx1JWqllaU4l6NiQn2+C3nJWcEmVFh7GRQoJ8NX2+r6x", + "1YHvtlbKbNwY4gKcsjGrHfpkmH03rxws+3auN+N8Mb+MR7xUUmGIoPxgRXV98+8uLAsFlXvOs9Qon/Yc", + "4jv5Ff2XCI5Sbt3fcanmXND/EuNaqiEQxYj+q5wMcmYMS+ThfPCCBTfn7XCbk/7ZFKK/l89pG6IKj+qn", + "Fp/FOAEdMX0GSVTRcS5N1LYJUyEuoQcboj4AIUyAf9XnYI7ZbMjztZ7aBc9fY4kyLBVKTO/BzyNXA90q", + "+zlhzOO6Dd9xLYGoD+JvbruBbBV1tblhnQN3YE61Sc/26/zqk5U2xhFpI3M2Mmcjc/5yMmcjDb4RabBM", + "BsSYvRcgMbYfRMe0+QCq+rbNtOC8cHD8vg85fTvk08UMREnf0xg1O6JP9iFupD5TFWa5SohL6PUYi5up", + "koRWiW9WJ7SkKI+JSEiUpDXA9eAlZAgqTDuTFmnI2CmVlzIWzaRMojp7liaTEE7mYPDczqvgoqHZj8Kg", + "qkjuo3k5I8d4Rk7pf0nHselPcGhIUjbLCNJ9oGTAiqfm5pInLra+r1qAs8m42SR6YqgEUpWWCrj3gigE", + "9jmSPl11FYAqyzGoEKRC12oxPei0bOJ4dGO19WoSysBL4wlET6Et8L0Yuk9DWWdLY8yYYR23IUPT6313", + "vNnbYGznkH7rqLMaG+vgOTWibS8w4tMZAMhRZQtHmmcXQ+UYKTm2feplatsHtJShRJ58ZFsItJ4LG6gO", + "x2PDsHGiFbfMwHIMRGAeK1xhAPDYwekCLKEJZ4qykiAQaGzm3kCMObMKxTAZLVJhUviAMIJwffOPks0J", + "ztR8YQSeXpj+a15CQpLzNJrIJupTWUHixM5Z/XJYzV79eBCuo/r5fbCi6tdTv7bgN7vKQ1hk7TCM1L63", + "C9TSsOjV1apmYhfzs97FO5ES0XD2tY8UsORRK88xFwqlvkOQ7dY3jjoVmwennqCI+utpv4/kPb2f3iZT", + "xCYfxDeWD2J9nh+azE5opG7Rfv2ma/hwwXmGLnByaR6VOdM8gJcpkgnONIudCV62fYldwOuByVvRJ/vd", + "jU0GySWxUiZBr82CKehsWFCvm/eQSMNG2kQDH8JSM35+i+krT/YG33TXtHGNEGFTLhJNJX6aH2QdjI37", + "O+MKakXZnhJd8JKlEj357eAYvTn6zWcSxswKSniu10MS8dQIvVW2QSN3yzfWB/bxbyNmrhpSIScSCciv", + "aBqrzXAAmO++A1n4sgt0hp58HOFr+XGk+eDH0SwpOiYQREJIRUzkHvi3W0Nmri06OvRvxyGs24vYP3mr", + "z2b/w+kYSZJNtzLKLvUvvx0cPx1mD/AQqK21TV3jFp1/MszlwP3sdY44FXqsUhwcobpxqsVh0i4K37+Q", + "PCsVQemtSX2FJFBuGXbjPmNBQx2xWzGCp3uXYwQYil3ydm/N8v5waUmMzmvRgtzMcSn1J6CThrbSERu1", + "bwKjLoieuCBCA8BRK6RpMMtMgCGnlmwOTl7unx29/e1p3HoeS6NwbPFoy/iv1dIn2FH/97u3L89PXp6+", + "e39y8PL8+N271+cv//WP/fenZy8Px+gVgCE6owNNRGt1QKu2YpIqa00JKv1htlgpL8I/yhyzKhmEGbNK", + "Cr1KmuoPcRjfPUd1JCvDCZ0ddUJpn1XYFYpbPIj6etI++R16PmkbWzc6M2coI2hGat+Ma16QSH8y2EYa", + "EwIOC6vxjw7Rk5cHe7UfHMv0v5kYLsM2xwhnkle306PDlqXVJsSOJr5WREAajGi5yLPGzjmDsmTXeOHq", + "B0BwETB7uORiZpXQC5LwnCB7N0R4himLK5whMOMr8IeltV+b/XFsHwj0X4K4X7V042pOxDXVanmpzK/h", + "AUYWERMz9WXVwaTx95sLTU95jmNK1AssCTIfgzoy3j3YOmpSad2E6UU2KIsXYVepqbvSUZPQ1pMxeQXA", + "GgXGVXaV1n2o7zcy/b5CxdcZkG3PoBea8HPlTalBac+rKsWKrijWtHSzmCw/wVsEazejrbt8ZduoUHkH", + "RwIETRLJ6q2u7fhu778r++6+tP2am3XjxZzLOgcZ5LHudmlnQNMMz+KbdA7NxmQSv6vbtXR5H9yVE0E8", + "wZGNCNjviCb4MCea3/rIARdNcI1l4MntN8wFSqm0m6/bDSborfHYxgw8JvUIYN2oRpFE9aBuAJnvIb/H", + "2hn2GtKJPEKJ4MsqDGQfPjX7V0h0ctcItE2elE2elCF5UuwqX4J3+rHWGLpE9+m7g99PfzJahbF41z3b", + "0btSgUkNnR0cA3BLxgjUfpsLXs7m7jZ2s7AvDiB1tlPCFrYGvalOVo+hKxm+xoJM0CHwwS0oXQ/ylF9D", + "+lkkSM4VQYdvT9GT/bP/dfx3wzGfxuRHQ2ymqYiKutpebStEGZpzqZ5DhJ7BbG/KM7qWrXQ6SXj+fHfn", + "552Po6fRzIfdOeHeFSa9AnIrcDninpy8OkC7v+z98nSMcnyD9n76yTzoTuoREns//bRSZrfmhK7lnSZs", + "vh9bMPcoWS+vouZ5F4dJroyUbiiD+tcDrMjMP0z1y3/X1ucB1QO4zLM/ePHwwxj9YMuC/TBGRCWTqOkH", + "eh9a9t4BVUhwCKkSItXd/LaWpsKAhq/xhcsa079NaFjpXN7oBbsNk+3+4HJ4WqYSbPyc87xv8zELBxyi", + "iyWq7N2DXmnNQqBiWKyw6vvmoK2nKMi4EU7lX8S7JnvpinTcbkJf42OlSW8110oznBGcx2aBFDzBHDGE", + "jMy35MRcLuzbwTC4kwzY4Cpl9xxpDUwAFE2rafC5K8F1px+k6SaVKE1SqcrZcMCbR6PxuGZcrXAoisSx", + "Q2mRVhNPAiXgFReXnfnMk8rtz70VjzsfU6dcXIZ3XKhXC/qViaDX3yW64FxVtQUgyNy9okM8cPO9P8GF", + "Ai9rzhLIPYpFmmnJzKemXuoEvcTJHEY3buYklfoeCo7tUOi0ICyFt12jZ/BSge2ST42vH4QkavluPKao", + "kjbJk49hlQVnkqCMmtrm7Tcb8946er67s7Ms1vQOD/CMXLeBfJfX+G7RbLDClQRuJkbSv2vocBYm99FL", + "m6CXNzjRd3n9LYgD8CUaqdTX/OfV3cFdJswBGplUK/Y7tj2hJQ2LC0PriL3IPYQNKBW+enGjHqgdmYDv", + "uDqj72oZxyly1fjqtjDjWMK2SF6oxRhd4Yym5q4sUY4La2GRkWGadpZ2cjUYZXgo+plpv2Sn/pbf9FJK", + "KYm/+tgvjq78TmB9YxeASywCYekeBtJJPJ//JWHxMgMfaiOD9rM8BtcvPBz603IQ9Ca6XwHg7ULRb3Fu", + "wy30ZrbMZlIyhUzvnMlxVSoaowRnGRFbyZxLvWVoq6XYZNS9hdehbSSOtV4/RgXPaLKoIr4vFoFH0JRH", + "8pXUsi7EzYq1uzhmYSKJuE2Us7OKgQ6A7jvfvn3efnnhsD0n/prP4pWJjbiopziE97KMMtKCC/wYHUd/", + "6Stv/JVKEMOCP9Xg0FHweUpJlvYSRFfNkgrYay8a/bWgCusPCzxb6NUhLZfXdq47OVsNNDWZW1vWvlXM", + "WH1lnDMeK1bz+j7mXGoyg7nHIRwaMPtj78QqbVHoLauJ7Zmf3o3X6dN7g15sO8EO3gQW5GEVVFyPpcbd", + "2iTRpK1vwjSnQ1lad4TQ23Zs0EDPwKJ8L0l6nHTU1+6LBJpmPKzx75KgGitpbyCK8VbuLNnTHZyhO8Yj", + "TcAZuTMcozfc4wAn81iuX+MwY919n0BwUKJ/e7r6FL3Q6IlT6R00Dog3SyJTuof8a2YHXiFnb/AmENBN", + "dRbBUQeIFWBtSBoBJ6o/McUTr76LFct8TaUPPbwmKUqJVOBew5mNsoP3Ax8kY+7wBnpaD7wgCKODo8MT", + "dJHx5NKb3H+ewH/bz/Y+jp6OEUYXWBB0dOzt9Y2G0IoLhN2LqjFz20aB6f7jaIw+jv7npPbTUzBcwAaI", + "ETQuu5/ClwRpPCSpudJcEYFSwmjVdLJCqTb7GH9cXmQ0OTMwWZqr7tQk5kO0xvPR+5PXMsjVXr0Sm0xx", + "LrNaUComrmnbZH/dZ2u3W50SmF2qsyDxkz6sDsKkmGLc50C0b8LwOiTKbFUgkupBa6CMbj+BfRmP5koV", + "8hgc6jrvReBvZ/PiE3FF0D/Ozo5PkcD2WoMZKjKsaflGwbcJ2p9OSaIkmtu0yOahSRBNiy7plA8kp2nt", + "3d9YrgrAD33CiEo7I6Ew4zVeTNCBJs2pPtUAtFdEQIQMWNfAKVvSGQODmjUnmKxW8BAPz1xPfvzll5+f", + "PQ3y8mVQrqV2GO2wW28F+9tPPz37aZkdLMc3R2asMD23OcrxyLwn2Aa2eGaOpbNU/oPL2MORJYg5lwoq", + "8dg3X/BmuCCVzwHkYrUHae1XUd9JQME+LWwVddDy0pPSuBY0cbfhN0rEluVY4KmooW0oAuGiyKixf+W2", + "9D3i7hXWYqHd0wT9ThbShS+BYQVeTA3tPQHqNPxOMz9c0Ab7M5wzIxjSiV/TLE2wSFsdm1xzbMK4AP9E", + "jjP6X7NcyCWXYAmurRB3NUEf7KDSbAbJ8sLsWyKsIBQsJYWa+xBkyJFZkBvLz391IuDj6H9+HEF8GAPT", + "mTVEWpg12PUYTbnl6xcLe69jMwJYYyHqNyuhaJHZkv+qR0WSqIrKciJmJJ2gF4Lj1PeWSELOTAlQAuhA", + "DycWzaALRG4KLkl11CRFiSBg4MMZGNIJhK75BQTytBFQxxRO9MmfwMZTDqubCcyUT9FqRMKvVQoT5AQ5", + "kqTAAiuSQVhEQQRkr5+TcEITFdO4t/fYCkPEbytSNewOkdvKDYfL1d69LLfJ5SIGTzfoaqTp1zJkO2fh", + "HB17svpOjGKbu7Mxh7YCjPU7r+9qTnBKxGrWlUYswdnZMbLD6NVQBslouAD9RWiZAzXMHJZ7NrLPELmh", + "EpytTf/w+TC3DuvmISXDiSaEP0y+G43tIAiRWZT81aV8hOyJ8IqF5rgoCJPWMW8LhJ9LNkkgJY0Lxts/", + "Prol+r0vtL7f5efytpa92LmnltDHiAyfQsa7M56YvcpakWx3rkB9HkbOkN00+UMcgsFpMP6gJCNYSERV", + "R/rnjbJ9b8r2N51J+6+rnH/7OpnnGxicbC1btRxDI5N/9upV34DQbI6YjYq2UdHuoqK9C1/VovGaHdnv", + "wRClUX0S5Lm4pFkGuX9LSaKZLuyskMyi0/ckH5ax378kcucqooVpJDd8PUVFLGs+lS6rvj13nyi/I2dF", + "lTGfqirbxRhlXJr6S7W0/WOvQlh3OyINzvKCMOTSwHAGLALCSKiq0kbYKAlfI+qJ7YAg/E83f/pr+Hw6", + "tpdfK+CVoLMZEfa1WFxQJbDwifrHSJAppAiRNse/U3JaiT7i8XVdiHVCIK9H5yGnVhWKmM3DGCGzlfAU", + "KgPXJSkUwuAoU/nChOaIZ3+reeWs5ghzarGncwdLPGxBBfIeGs6vybnATdDRNKyo4D3WreZIpRnAFu8y", + "0kJjDfgCmXpjzrqBpbXrWFXZSxY/KGVSEZyC55QLqDQjcdbhINEJFue9v1KZTYv/jjekfczBMqROuKsu", + "hmU7BshQuZs5LR0M88YfvRO12uEvtwjRV8u9CsJYgbukWo84nMfKPh/jquhzV994dnoYr2cjRH6gah4k", + "TWyUig/zqt7BX8Y+XH5ppWzy48MDhi+s0P1Siu1VdIKOlA1rTrAQlIQlAEzi1skqUeiNIBkzzDWWQbTJ", + "sEcoi65/LEu+aXP32XxkcLmfY+U0CxtMTgZnj/16lSeiPsfQaWBQkzn1WDAEXOVXOy7IlGo73s7fw699", + "7IiocaQBvMcBSoXrjVJcHfg9BBXW0djZ+3G8zER0UErFcyKqco810Oo7OtzJC0EkYWqMwF/R5fyV0EKh", + "nEuFnu252/mv6FJfZKCeCM2hIqXiaHfvZ/MIPHbJkfWPO3s/ul/hilJV4vArUhz9vPvLnmkGt2aucObr", + "hoQAeLbXCT1jELrP0it3qjOixfADVxnpLi/itJx4SWWbU7C5r1daybZ30WlcvwmKkSqCc9PaFFCFiloz", + "ax3SH7dkVs6288WWG+X51d7TlSwMruNAXtG32DnRq5ug91on9qvehpBrW8zL8NXrwE21dzP2EvN0HCSw", + "mOIsk5BFyScbw9fVeo4O7Yj4Itnde+aHWH7SASTG9vhix35GjMNyw9BYUFtkpnETNCVkvDar9xmNEpKH", + "7sLW5/YJCGHDMe3OGkMG9q7lsqVrNfr3oeGisRFa0ZownOfnFljhrj9ZyHYV61miOXi3Egdvl39puNpg", + "JnixsGbjd9PR8z/7mZle73upNYBPzci0wdn4qxJDS+OItGSNOwm91jIX+B7cfhwMNJmVslsMLw1/z7Fc", + "GlddbekNlpeUzUzmBTkMAYdWWAKIWOyBVYWC3+HOPZRUTzizykZPRQMtaKocZVWXIMCkQe4DXPfCYhsn", + "0XtiLEW/C/0oiLAXt0EufRvfsGW+YRE8iJyRwzzgAi2eRXKbgmQZhF/qhm7jpYRA0KXEOYy/2NGWMJcY", + "tZnVmx3a2MN4NhVHIkty65qmkRwrw3UVsN0s9Zk15tVaLgPNCXVnNYwWYZ5h8g+uO1XglTUvafKf0St9", + "O+rJJQNtTwdlXXZH8CLocsvUK4Pv4DXorXwLv29xetuMMXdIgoKlOi3wNVsZWOYOfCfJe4scKh03jrfh", + "ZcMv80lTP7dpGkwCWPctXe0uYVztlmmw7uIgnWseZNjMFqHXpX2+6lRtpT6X23KC5sn0+FDfKnXKnUwp", + "EUS6izGllijF8bVB+VDsYXZZWEISb9JK7XxqbLtOj2MvQBz21pliKHxAfnQHqKwP9e4LJ/oOyu4m3D9w", + "/va+VxBVrbwPXReNB5Uq9v3lFiJl/RJgShmV89V25foM3tZtWL28i9IwmBVVm7o7H6pYj69N1clXIryp", + "RQmvaEbeFxnHEZq4u8+ffUCr3P7mWPma7FJf9byXWgmLcC/Msbhha/Xt501TmgFfco+WtpNzb0ps7Emb", + "HZUiEon0XmRBGjwYu3r7NSsGc/LSY3Nrb8E/bvS8BTdqmzkagZqdAYewjtuGG0LnYaGatQX0X3KCJT2x", + "fnjG1G+uqnrff+xauiqwmgNxCZLzK5KOIUVXtfvVVC9BsFzO3wJOcGI63JWZrEMaRnhHPJi1tsbXfCbv", + "FND6kPjVFcxa24HlQn88601jOuDa3QT/BB36bgbtjCuycXWYRCrJ34uOcKvEkJS9EoRAibiLpR1rjQda", + "wB1IDjCzjxkEYXhmAR6f8Awei8F/jFclKfLFluvrCkgEPz2/2gXn3KMpjAT+ezB0OjYOSsb9RNlE61i6", + "gDSYN3wesQJA4ZlEgDyDjkc3j4h3PYji1uvFPZ94tdix5OFcJ3wnWDWhbMfLQZMIwNj2x14keVQ9Z85U", + "8PwoxzMCaYwEz90oXm4bgwxmaZUypu2hABVPluctDkX39ZxnTlmqpKwtuqI4EiVrpGDql+h+JxGOChtU", + "HGgWS3ASlSSwNjdOMqfM40kES/xMJ2RGpU1O0Udgr1od7CihibBpfLdrurdVg4J0kMeqx4DqlMxJcgnZ", + "ceG9lNv8c8STkpuvqubVqYyAfT06F+Dlvc1CimXSyPmoQ9u++VpUPESqnypSRKVU5PmzrQcsqRrYWppz", + "bYN/G9+2a0xtFT1X7c8kpIo5u7klWPnohHynnFyzgfrhDcerWPoeqQGvR0A1fTLvJJ/WqZ16i1nbqgXb", + "DRU8ILgI3ZAiTtDO/bkd4SSWKrL7Ylaainnep1bPvgogweH7H1hGfCP1rw6C0Mwnqwhmaoue1eWsHupe", + "BGw8h+XZoqh87qKrbmIB5DoLzvQMz+5+JY5kG6VSq3+DLHeD33asmulobfgzLu6oR6NHjCcAbYINthKY", + "ohvv+AaWxtGs05F5XYzqS2RJXVfKr/3+EnFArvOcD1TNQWzLxyEoZXcknvneUhFXVWuM0Twy/3oeQb/m", + "C+LmNXDzGjjokSmmrnSZ5pc/8xmOY1hlnz9fh2WGXNeDf4b6h8FwemYTGHK/MSHmS/TUD1eMCvFDWdtU", + "zMHUbGGfpR3JYrtq4Ngw5Oqe3YhvBKCaBbjMjj7AdvyRmSwEzpHXOZf+H1zQycdyZ+dZ8nLvxfnhuzf7", + "R2/h3+T/TNA7Tag+uanD2o/MuanacDtXqDABP3n05MX/enfw1JUm/xXhC3jq8C6+Y0TZR+Yi8SSpLcgG", + "E1NDuLWYzTrzu+1pq45CPKXWxUxqXAPhKbcVnjAUgDfaWh2+a8Ymt/gYVn0gF3PObVayzgDCg1alMSB/", + "SL7hQuOuzUiyr+RYm0dDmvl6rPlSCRI/RrsTcG1DU0EJS7NFZ/J5rd9iVQrSFdxkfjeXFsWRsYzOidsm", + "KvAi4ziN5jqyj2+VXi5o3KTpoA98dQjYBRjYoNALRDkMAXoj5Pd+z+AxAjYacae7+q22l/mpcRpRMnAo", + "5qv4unT2t49ou5Miee+UFXMe9XQVKUsykMOuRpqqo/5GdzWMrniF26BL6FIe6jt2VR6RLOg9nAMEOiQZ", + "vSKCEqnvLccGn2PRozMtgrWEtPBJTUeom0/yQklTLtYEmsdL+LjoqkH3n/r6Fr9BSeAYcZMbdVAKGSuB", + "bX6HdAlYSmeM0z0gs51THSB5jb6NeFQXoM0zbtoWeEZWL61pg/uC9bXBvujGvyZ829C0vPaNjNU6t51d", + "1gDXGPI80SyjVbz8AGdisGMfZDiWLOcNTuaUkapSNeNsCxJOcQEpUKBuQykISmCAynQ+V6o4Nyby8Shl", + "0v/tgqdttd6CC+W/2Q35f3ve6H9JMEuILfO5vDqi7vSmK5H4y7DoNlCw3YtJ8cRLhbBmFNMyq6qFXHCw", + "/S+fWtNIjHfU6mCtWFoJ+sSLI9SHdQVnhrjkHzZQccU12SN7oeESkbCC2jQ0jrM4nLWA7BrvH5WLVH3I", + "f56+e7tFWMJT4p2avBsUaLiSQA3oK+ICUAVJcdJRD9CO8D7qpHTy2gHFFz6l0tNuDToxfq5HN2gTh86Z", + "KBmYseroZQrKV78ZF5yE0CvYwlLUcx2HgtBOY2F4b7MrVZhXtoNoJX9gHn4W4zcDZfyXLKDF0DpW1I5c", + "6qPGfl1CdrwitkjHu/84Lmjfz/S6ob5N9LlQDSvzNYQal1Q9qLS9NLL0lSp9Wno+6tHT6mn3VtlItLa7", + "U3uqmSs2Wy+oVfHJwCMrkKh1rtXiOTWuUJOPIYAj8t5lt9F4LwcIfy+39SL15TWRERHesINfEWHFWXVQ", + "vDT1gltxdD5dzrDWLgXKgNaNI3Jdx36B1eQRSBmFb7B6VFM/ZW8lSdfjtqpoNMldJcxXk9H9pQP7kb5C", + "7Ro6B2jutxoB8FAUxLOZIDN97ZPQY9x6HUwuye2hCct4AWNEU/HVtNwVxq1T2hfPXsMT6rGTu/IIq0Zk", + "Omi4Ify8ta10HYeFw/JDgaOwCUohotTMu+SSsD7whUJm4KvsbSFei4EdCnPlw1rjYNbfNxaT79Ri4vWE", + "iOmk02JiMhKVgqrFqSYZgwn7aU7ZPuSC2C9N2i2qd2NUZTfF89G/tqDllnkPqY7GZJH4MjYD/fPDmRvl", + "gmBBxCu3t39+ONOsHSbWiABfq3H0VdqPAm9W/YvRTbbA6N5eyKDNHB9t/R6+aQX9SzU/Np6/4gUss2NL", + "5nH0XFmILN9bMPBdtqhPktrQD0WVvhaMXu69QPvHR0FJ1uejncnuZAdKzxWE4YKOno+eTXYmOzYjGhz/", + "NtYA3/bJDLZtprutxJePn5FoKU1VCiYRRnKOBUmrpDQmLSs8SUEGRZJa78upvq66ip9o/yPzqTgFhtss", + "Zzb7LOJTn3EywQwJAs9NsCSSopIp4G05v3LJkzRvw6448ug3ogCPfNqAEzPYgdlTdYGD/e3t7NisE8pG", + "CEFSXFNhZPvfNqLDSJllMsijr53RrsBODCfXSrEU5uY2e5RVeUR4xDs61Kf4485u1/R+P9u6kW6798uA", + "tnu/6LY/mf33t9WNQg4CgeQt3vHnpy/jzw1O8OenL5/GI1nmORYLeGYpmWrmJCfSbdb56UEh9pwyw7gs", + "muoGcvuz8Xj/so0LunVJFsaTJlpYw7z1aDQFQIaJgEwebJz52iXXXFxOM34N+ajqCHXMpfJHK6EU8aHZ", + "N2iTWOCcKLBD/Bm94AIVA5lDKkJP5D4nT8XhzQW/wrZlt8hPLfPUvSDyW3IdeA40UhbabAANKtq9t8nN", + "qaXNBUQAW8s01Ki+a2hmZwjN7KxMXzvPhrR9Ztr+OKTtj4+dbgHEdTrCEhkivQXRbn828uzo8Iuh24zE", + "XmUP4fc7U7AZpoOG9+1CvjYtj+OnWa1p24FsZOi+Rn8/dljXHMgMhNdJI98F3hvMuTveG8+sbfO40yOx", + "4LtJsU/ZViG4KXSAWYoKW1Kk4ZhpqjNAFn8jQJcLL+OBauZ6FBLsIfUx2KzZq60MHxEnpwFVIHNImVbB", + "jD/dXxP1DcwAFQOUwyvradXtAhLsd+L+7zSzmN9O4nYLJPfq/+8urf/3jOV2t3qvA7FcH4aWBz652l8T", + "yzXEIkjXj+bBzSN6PwYfej1oKDdk9Kbq7xF3wo+BLvmVRt2KXOhHFu9g2t7U47ybdlhx/vzUZYgJcKWO", + "Qi2rVByrYs1qiAY40cQHj10Ooz6NRzdb+l8z8340yqg0KUx6bri2GkI4eJxFBsi2uTR2Xhq/C2RednuL", + "Il+Nu610RWvc0LpuYMtuXF/lAvRNy7zHx+giV6Y4soEhOonEs5rYlWUodaw73zNG3T9XbMXhDGKMO0uQ", + "2UYEbZD5oZHZouIwzmnDaeT2Z/uXvv4IOluiKOoLDhfJnEhlEtwwnhJUcJ5J9OTjSA8ARY2nCLuIHVtf", + "q3p3MVErVCCZ4IyymXEnkRP0ylRmrWL57Qg/SETSma/d/WtzbMaRoDOUY4ZnJCdMVTXdUqsOSijwBcm1", + "tJZiCl+YkncJzvxwmMlrIiT6aWe347nmwMLtwEHthM7kyqTsYT6683VrkDp9Qme316OhaAKdVaHyZvHf", + "Hg3rtrtD2u7e04UNSMaBziNtx02tkyK3KZMKs4TI7c/uzyWKzhkROQQRIEhTZfogykwpLijDJOjsB1kn", + "QVNUNeMM8j6ZEikJzjIiUDLnXJrimT4aV9OcnAvKTOFTU7LVTRXUKY4QkpG7UVo6cps98lu9C3mNW+EV", + "ljFXa+2ypdBwAd32lIhtvBXBzKCw3xilROpBUIILnFC1QODBnAjgWiRFTwKYPv3VOLfYaopgWqqdVoZL", + "pjkx+OcCrHOTA9IsG8AO2/pPaRyk7b78hIdmNUN2V6U4aPOrSLYtd3ZIWUSknEGIdqFI+mhM+TtDmMzO", + "L98RQ4rzBWBOgs5WZkyfBZ3pfzh0Nrk2osF9vn4q4L+fGrwIkC3fGmFJhgmZckxUQmxRqB7wMnW57wTS", + "1IDZjKS/oivKM1uc2wotGO0HiaBeMPArrSNk1ATjBlU5PKOSSJZiCoWUJAK3YxljZcdlXCc40aA5cIC5", + "Gwdb0hhO4cFuBSd05rZxAPAddi3Yi70YWa5nzmnDEL4+Q3CE6QXSnXmBoZT+W4QgCRTAsZRuuvRwAZtG", + "07iaBszDxttKlJbElGaUvBQJQeRmjkupvz0dI0auiVRoSoVUq2j1QMEvzXbWQb/jdrwfuOwH1YMspGCr", + "WjvvkPBQAXAUinEf7L23E6u26gITfgqr+O5G/IHXdVsBsN/tytLErqnDru/bleArXXLiFH1XZuJvPsut", + "Er4pwkrhZF6pCTGG4hKyU1FF7ZuKxSwN3DxdVXHJoW7WxSK8NazMS/zVZn3qwBpI1e3qbtRaHd+GUB+S", + "ULtJpY9SjWv+dq2+dJQcfyMqfKE2YWH1AneRlIWk03D+G1EvYYiwsFqDdmISkE+nknSIwJ2V641/XlXK", + "7vZK2d2dZWK2Y0YuUiL2ZRKf1KbEbN3YW3rFK5opItzZWC7pU4hXQesQO3tTZBAybAwDsVW5phHWsbye", + "rFpAVIAG1ppYlsUkQKs7ahg1JH8s3OoObxpthvJYXjkcUwpjhFpcafuzC8g01tFODtU4uOVM59SNu7Lk", + "9isadRL1hlVtWNWGVf0lWJXPy7bEPGKyuVWJWiJxAp5JfahyvT08RtajegegpEvRG93TBrVuh1oejT51", + "etudWHg71mm7xCPGIoh0/3bkelLDNfvcNXP4LfWrtx53HmwbjrlWtI7wzO3PLtfMQCe/Nsvpiraq4/8H", + "N83Kyp5fYKd+0INyzumvjnIbNLobd+y2UgzCj5aUfUjkuE9O5yT0KkridSxV1AYP70dKL3EfbSNj8xii", + "PqUPhZwPJ/5rGaWHO5h+BdpwjqsbHeBR6gDbqc+n23+d6kyjuwq7r5L33oW2WvaNdwX+T0lQYrLnTgXP", + "rYWDXFFeSp8X6QeJquy2aEpJBoU3YzYOM9ZoiYvaqs/GPz1mK9GRLcRZYQTCCnHhq+lRiXzuqi64Qc7F", + "UTRSsrdkxoDFXJAph+rKQ9ZBWHoPq7CGs3ARi3pOs7JmM7OwjvsLVmnTyg7r2WopNnsta8O2UkutO2QX", + "tbx9X938Nzw9XCNdeI+JZXi68I0oe2SiTLoEkZ3XlesluSJXkWWnLrnkvYkx4HkS0mZBtkQB/n3AUSfo", + "0PBzMN/v/YjmvBQS4Rl/aEb88ia2KMLS+pIYv75XVrw+1mDzikZYQjyJ5eYqd0+kPCc4MxnxovT6D/hs", + "KgvHyNJ8Hw0KC537sCdEJTITP2Ds75fQYaS2D9g44ykZkErANIvs/K390Jvgol0crQqciSq83jlqnWku", + "Bj2X6P3eLYGBAeX3k1QPcMThR8zVCL5tf9b/W/aMD7GHkF+yC9XewigrizozeUzOfROouQwjb2MbZBaT", + "/4JpV94GaNbE2M5kF3OnAdnK5thAMPb6dh+Y+lDJFXlKbN0IH/rxZajcAvK0EICkzCY6Z32B4F89nON+", + "clsZVAqg2cE5496Z/dV5vchupTmdoPeSoN9enqHtq71qbHAeJTiN3jd6nDQbkQ1E4RQrjIBf+qJvU3PT", + "DxxWiLSxHx9HpSTi7/gi+Vju7Oz9DRfF3wvB04+jpxP0Eidzk+6Opa7OTF5KhS4Ien/yGtnyKl1qfm5X", + "02szW4u+oI+DpBaMd1McWgf6oHf+9ZDP470oNMMhWsCvSLb6bUACpX7q9SmVnHuXNyC7bPiGio/fnQ4m", + "Yy2RBtPxYSmgENkBVnOSCpxZx4kqilKT5QSdkCLDC1n5e0ucE4RLNdf6lCm7ZHzDWVorSmVxuupjd/Ar", + "EqSUxOWWSOl0SkRQotZ0hij0f0NJWU/7zfTlRynJC64ISxY2z3pFszm+eU3YTB/z7t7PYOJ2//55nQmO", + "PUNYr9NMbdrIvdQiXVBwoS7bxxbYsLB/bb3ce7HVhHa7Dlwy57butgsPhco8GstiuNXLs788qJnzMQSL", + "PhvS9plp++OQtj9+o2zXskKLk3F2W9eStnOiBE2WmDNsI4hgmdErwmo8vVsDemMHX8JAD3ie4y1JdCON", + "6pm15TvaOjoEG+WM1FYy8MnDDnJOU9mb46H7ESTHN0fmIzzx1bSR8cgUMrENgBk96NXVw/YDVXMH37vp", + "REZ+OETYKEhrU5AcfueeTAarR3UiboR8dDkFmtSpAXuIuQHeb7jHp8H23UCOuky331eutG8BLQFD+sVH", + "rzehO8WLBQKjXrdoeCDsundGexszoayurhucfXCcPV1B2wkY5XbCGSOJquc3779qujJKDs1N3TY5QUfT", + "WoARJOkpJUnHiCp0rYnqgiBBZJmTdILOzl7rJpxlC0RuFGGp/jV+R40teeC91ZPYgd3pXSnt/q91dmUr", + "Xe12vsbVDmeCYH0TN8qTJquvdMm0WPQ464JsroOP6jpo2YW8JYeccnHZXf7hFReXIdd7bt7IC06ZybDQ", + "uGggyhAk6ENPqNL870JQMs0WnlW63Kg+JwJVEmn8RjnJufXfIGNrHjNUwBm0YlCl+5KQQk+ofzk6hHbk", + "pqDWTlIyxctkTtKn8MXaUUzuNUauw3oCxoCIlV/SBDnWz5lh5JlCBRHO0maKsV+OEcHJHCVYiAVkgKE+", + "T6S3DVlg2FwQkCPGzFUIcI4iqZYZsATwAaJsNkH7iHG2tbez656ScoKZMRA5W5/NC2W9LDGDokuXhnm5", + "ivFDBYY+2kcoLez69OpOzPDxp7jdBwtCNlObch3L3yWarDtEcJd6Vx+SRNdEn5k7pl8NGhGmhNYsCy6U", + "QTzd+AeJeKkSnpMNJ3ec/JvkzsA/b8uaMz5b5YUzzPeguxpts1vR1G2GPXJ6nvGa3yIFdD0xRON+Zblf", + "5Szu3E1gdYY/z3mZpUa5thewdrX6Fd0rXRnoZSkmlqWp61vl6inrdnd2Vs6DsYbrMZz6rZIwAAZvbshr", + "vCEbkK/KaZY9EISspTJkDmAYnY8Dd+AZ7xm9CfiFr0amqd1xD6hDeYWzsWYVlkuMoSnkBYc21UYeiHkM", + "cbQetDXC0tttbLUlrzPTi0GM+0n1so5XjQ1TugtT6n386OFLjKhrdz+NZdt2hUDmBNmm9VhiWzWucUm1", + "XhqJfX13+bCJqeopyowE/hs+BVM9SBm9y6kCzQVCFFGSESwkomoSy5vdZoxv7c4e7QXMLtBA2MQyDzPd", + "/bgk+4MLNg4pOHp4fzGPyW+Bni29NY5tZbIGY1C30elYf649sgyzaEC/x0tRsLxem8aAZ1NjR6sZho2l", + "C7Pg+WFjNfimrQaGAG5rNhBkKoicG6TqypkFTeoJ8+CRyplWIUW04iijV2Qg+Z34eR8tCdolrkqEDRXU", + "bPMRPgN/k8jucPH26K5Z3movvLrHLQSM6fgI0dssLA1eGB+Jt+zmIXPzkDmYDwBV3pYNuHeXHqnngwgK", + "IiSVCuo4uFqNPqDAjvmD9BdDeJKcoFM3g1O2XDCPfWWsvy9qpczOgy7IgtunIC7ojDKcBdNkdEq0uB36", + "cOfX8XjlrFtiIGjX6s5vpz9iUx41KblD7y43vTEjPUKXd3tsK/MGTV28VN2cwRWHsg0rm7IzGNXkGs0y", + "43NA0I2zwQYBSVpzdqU9DP5P0AHOoCYLJEbKiZrzFOVlpmiRmR4S8SsirgVV1ip1dvbaehrAgKU03Stz", + "VWUmxrIygOtW1juDo5xgWQpS21ranWYvym3OLOweLa+xC7yTTi/t+bsjdgizIe+1WImD1yXl0a2TxEki", + "iBpQJ6kQ/N8kUT9IZLtM0Fvu88yBAw/kT7OfTTht/CHcTrkqDRR4ZkuCviU36oxfEjakrFHV7TW8Fa/p", + "YQY2ufKLTAaBvHGAN+PiNBS2DBg6FmObb/8rANjDxrY9G9L22bd5t9gb0nbvL3C3AJ5g0TNATs9i7C8D", + "wpTjQcmGgVwskFRcaBEMb04guHMsLonQMho8HamQCl3pewFnRq1oMSQXpA++4x1i2m/hoeJwLTNYs94e", + "zDogZT1IbM+1NixiwyLufsdwxBzjDqH6sf3Z/LEkGu+EXPFLEmAq2AU0vqdlRoAlWGZgAm+TjGBWFl3p", + "+y3dn9qpV1fKXcdhMXvRrP0bqttQ3b1RnS9h0UN1PcGInDlc/KGSm2MkSQapMEyIYpXITCCG814V/0Eo", + "a2fdAlIQJSi52hDrhljvk1ht+G8fpXY9N5uYoAob9R1RcUFSpx9fLBAuCvsAjcGWfl9a8n3R9ANYsGAC", + "41Sz9rjIYayk5q21YSQbRnKPbmTLde3wTa8/Ea9vWqvAHJX03c9nPallGolNo3n7qzVcLJDkpUhIkN5m", + "hVoGtYH0OrTqcnQ4RhwaYk2YCs+2/lPiTKs2qU8dmC+2XOePo7H5QQNiu/ZBD1dr+/xq9+PoaVfWQPjf", + "kiobq5o0x7cznq7HElp7Pryth7oM8OzerKCbPDy3zsMTHIfnNv63ngw8mnwGMCDTLMJvzuyHh8dbPdPd", + "UmmaTTxOTOvEq1jtfwtznzxWN/1UHef2Z/2/oZFPYWa0PskCJ30GA9827sksaxP09J0FPWmkuI+IJ8gc", + "upZwpxX08I2gcbznLDyeFbjPdo5vejkQILgNPo5xI00X+m+Tt9GRyzAe9QbfbNjUo2dT40hicUETpLi3", + "9tWwBJwZbAbNjkzgmhv1Jct0lc4SzqzP0XmYEdTl3ITDOBdYkUgJtAe1hr7BNyFj3TDS74aRukQLagBD", + "Nem/b1UKoOoc5ZTVxwF2AssSu/lHu6KK1SO/Sv59t7u73hccjDa30zvdTkNMdFhe/dZ7O7WNtnFGsYbE", + "Z/iju57QwZwkl4hO/Zwm1tmoDtAXkRsqVT9R7JvZ4H8dBFJgKP1l6QPblt3yphFJ4lbnOq5TsrjJYXsn", + "dpalpKE5AoX8/nbpFwu7+pUlzAOZwTdUV9Wa0VSgWkjWpr0GmZkW3R4f9kW7cu7GM+cc3iltTB9PW2d4", + "9lCuXfWZ9EQrhWjEApT1/pxvyCaO4lvwtvBYr3AtVxD8v/sVd19KOmO60xP5VF88cMDpSpql0UfYdWC1", + "WdmtsXr3nhdC0nAp0TBJPEPYtt0QzeMmGof2/URTFxKf3Z9L3AO9+5Nrv1Q0+HFvYbPxXYdn7K90RNzH", + "5TeK/925cYf2MV7y7gScV9pER51oFKru94RD428m/GWIpv+BqvkLAOWqhqTwEkWVtAeyefR9TNfq6lAG", + "X641aahkPsSe5LKQdRHfsR7oQVj4/atSbnVmTyupUTsDRIhzJ1ubzrMhiIbT1RJx063HbBsa2v4M/4/m", + "J249W9XIz6UmHSKZDCt+YWa6Vdbh1USV3dO6MxQnpZDwRPMtpSiOZSg2s9rPt3l7S6kgCexhPJBTaaw4", + "9L06B87IFclWGfQ1dIiA9tR49A05/angedfbJYyy0i7NxGsyeQLN6VkHmz3j+lBA8pt76nqcD2LM9q48", + "3tbhXoHL2/rxK3P5U1fx++vw+SOWkhtH3D6VhodlJ6n7RLGBYI3yIT6T76ZTSToY68qJ378b1n9rDr02", + "dtiZR2gpG9zwvq/B+6RjJStyvynN9E9zLOdfelkeZqgsMo5TlFF26YwvWCA9AtIIiCkL+AdeEPNtqP77", + "Srf9B5bzu/LDyEvs3Aw79CFWr8LxRbeF5W+xuw9DiRou7wHyXdkHw3O5nhMBWUztj0CZ9pQ2lvbHTsVA", + "S/bk3p+8Xp2c3VPtEsd9eKC9jfnUvmjdpxn+AT19zvDsrs7H4QvIY4ns25BP3OIZfabq9iO62t1OsJqT", + "VOBsO8EFvqAZVbTmV9eiiT92D1yfg7DLA+pl8QkjeOwbQsY7aLlAtY2tiMHfOfY4N/M43DhTAicKybIo", + "uLCpFSCZIXziGSoyzEhP2rYagmV0SpJFkpEtj1Ny+zNNSV5wRViy+J0svgzEvNdurHd+qKPaQCuzaL9O", + "P6Ie5UGvGz27iSH3oT6ajCAPR8Sr5o9Dr/nOyeWEJPyKaKWhIpjIaWgySe1hXQIqDqKPu1PFd0EMNsPn", + "IFKojsGmY95QxNeniOZRQEIeJT1JBLi9CnnEs/yaTD9qMZBC2qluj9wAd811uya68OuNqT8ud7mv7kGS", + "Eo7Ag2lDFushC2t4wKWaa8iDI4HP4lzRSs9J3YksYrpWmAm77bLZSyQRdeuO1R3BRGW8XyojVSC1tozY", + "6rZX5fjmNWEzDf3dvZ/BCO3+/XOHrer+HSd6NLjVvSi+hk758oZKePNwzLkmQO/Torfiwl5H1Co9qnFG", + "NO8vRORQVYFo8mYJMUve+1pL9h8RrYTdRalMaSW3WFviW7dhXKEFgRy1V4R9ixVSvvcc5RDBjFnFnrcu", + "eMnSfvW/j3MHZbkHpDcKAm/bSs3eafC5wYpbUcMmaxq8wfkKJlOTdyhIgU+kzylUSiL+ji+Sj+XOzt7f", + "cFH8vRA8/Th6OkEvcTLXihtkNYWQY4nyEiqeoPcnrxFhCU9JOukOQobVLEsuFE+S5Bd6sYBEkFygnAti", + "yrVoSJCbIuMpGT2f4kySzlByVffGWKWK7amKhlCOR1ItMv3DlIs85lTEhULe5QXSRVnpbyLloRwMOjQv", + "r1KfkO6PnjByTaQyWaQ70zVxYSTpMM72Trdu+NE0s/epUrAA3rBGkiKsNMzx1KAOlZWL1KQvcJ+k+7pL", + "3OspxYps6XFWSZcVYkIQlHJ0COvLKJZdCwos/veT3uobSPT/GjL2n1aF624f/9sorEvuye15PPrX1hlX", + "ONs6MRMs7QytXeOHrRiweXCo59Wq5MXV3tO4wLttWn3LE6vSOpZaJ2hfy8SFVCTXqmBeMksTlTuO60yh", + "GkUpjBSKXHZqwvOh0umbKf7YW3tG/eGl+tZeC2tTUm9ddbPsKXcTaEsjrVkQEs4YSVRfOVktE6SfKCUK", + "00xO0NG0SYumfvEYUWXqaFU1iyfo7Oy1bsJZtrDVaOHXqo4WuuDpAprYPJi/ao2c51RBilpX4SpQm57t", + "7LhMPEvJ35s4Dux+H1/JK7uyGjt50CTBw9kHzgTB6cKpBPdtM9jUHN0wyAdgkIae7swhl0bCuAm6QmBi", + "bOhWYS910+omlKXfn3mZQ/POqmv2F+khkO1Y8r2EwjRAaTYBjuzgbm3zxpWCTZDujS5Ixq/NLd40wIIg", + "cpNkZdoN23sLrTnAkmxJwiRV9IogWV6YyzfKsUrmiDNYeU6kxDPjX6sFSoeNgWCRzEcdDwR7P/1tvamE", + "LDlrIP+xd7uYmhrz2LiirsXcG4B8gFSIZ6FbPefcH3tfLevcd2bkuu/8dt9XJcsNocez7zVJfVAKvpD+", + "W5lfgiQCK6cMCLjB95004EEW0S1pN1kJHm1Wgh4C7Ke1VuRqT7F388Q278taFiW9WrjqugJVH5hkYTcA", + "kaHG6b14mg8TszPH/nlwQ0BrdExoxGAvo6JndY11yTsMI9c9kkpTy7NQbX1wdLUC5o9nKyDsva7CLsDJ", + "GLeQLqIwRknoslaz5CoRV1/bLPktv68EwutZL9nxrMzJwFpCyLWOXRP9p4e/SJm5Vr5GZeBk0N7NhtHf", + "8aZSwwyHae6X2760Gw7vh44z+QDpHuSR3GHael/Izaz7LA3u6ANqz7dhtlF31spyQ2RtkUHIbrc/mz+G", + "p2LtpgPTyFLCH3bYlW8Cbj13qcKO27i3MQqvK2FrP+6N+1Il+a6deZIeErt2vhbbrCqSbxD3K9Ul6+OW", + "sCtx5TCsFNno+WiuVCGfb2/jgk7I3sUEFwXglO3/uelDK8GuUS+bW/8R6vaE/y7o1iVZ1NrY5BL+35Xi", + "WI1tiwR/+fTl/wsAAP//", } // decodeSpec returns the embedded OpenAPI spec as raw JSON bytes, diff --git a/packages/api/internal/api/compat.go b/packages/api/internal/api/compat.go new file mode 100644 index 0000000000..c397c6b6c8 --- /dev/null +++ b/packages/api/internal/api/compat.go @@ -0,0 +1,10 @@ +package api + +// Keep the long-standing public enum names stable when OpenAPI codegen finds +// duplicate values in newer, independently named schemas. +const ( + Kill SandboxOnTimeout = SandboxOnTimeoutKill + Pause SandboxOnTimeout = SandboxOnTimeoutPause + Paused SandboxState = SandboxStatePaused + Running SandboxState = SandboxStateRunning +) diff --git a/packages/api/internal/handlers/cathedral_sandbox_lifecycle.go b/packages/api/internal/handlers/cathedral_sandbox_lifecycle.go new file mode 100644 index 0000000000..9fea74036e --- /dev/null +++ b/packages/api/internal/handlers/cathedral_sandbox_lifecycle.go @@ -0,0 +1,331 @@ +package handlers + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "net/http" + "time" + + "github.com/gin-gonic/gin" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + + "github.com/e2b-dev/infra/packages/api/internal/api" + "github.com/e2b-dev/infra/packages/api/internal/db" + "github.com/e2b-dev/infra/packages/api/internal/orchestrator" + "github.com/e2b-dev/infra/packages/api/internal/sandbox" + "github.com/e2b-dev/infra/packages/api/internal/utils" + "github.com/e2b-dev/infra/packages/auth/pkg/auth" + "github.com/e2b-dev/infra/packages/db/queries" + "github.com/e2b-dev/infra/packages/shared/pkg/ginutils" +) + +const cathedralLifecycleDispatchTimeout = 2 * time.Minute + +func cathedralLifecycleDispatchContext(parent context.Context) (context.Context, context.CancelFunc) { + return context.WithTimeout(context.WithoutCancel(parent), cathedralLifecycleDispatchTimeout) +} + +type cathedralLifecycleOrchestrator interface { + GetSandbox(context.Context, uuid.UUID, string) (sandbox.Sandbox, error) + RemoveSandboxWithEvidence(context.Context, uuid.UUID, string, sandbox.RemoveOpts) (orchestrator.SandboxRemovalEvidence, error) +} + +func (a *APIStore) cathedralLifecycleBackend() cathedralLifecycleOrchestrator { + if a.lifecycleBackendOverride != nil { + return a.lifecycleBackendOverride + } + + return a.orchestrator +} + +func hashCathedralLifecycleRequest(sandboxID string, body api.CathedralLifecycleOperationRequest) (string, error) { + filesystemOnly := body.FilesystemOnly != nil && *body.FilesystemOnly + canonical, err := json.Marshal(struct { + SandboxID string `json:"sandbox_id"` + Operation api.CathedralLifecycleOperationRequestOperation `json:"operation"` + ExecutionID string `json:"execution_id"` + FilesystemOnly bool `json:"filesystem_only"` + }{ + SandboxID: sandboxID, Operation: body.Operation, + ExecutionID: body.ExecutionId, FilesystemOnly: filesystemOnly, + }) + if err != nil { + return "", fmt.Errorf("marshal lifecycle request: %w", err) + } + + digest := sha256.Sum256(canonical) + + return hex.EncodeToString(digest[:]), nil +} + +func lifecycleOperationToAPI(op queries.CathedralSandboxLifecycleOperation) api.CathedralLifecycleOperation { + result := api.CathedralLifecycleOperation{ + OperationKey: op.OperationKey, + Operation: api.CathedralLifecycleOperationOperation(op.OperationKind), + SandboxId: op.SandboxID, + ExecutionId: op.ExecutionID, + State: api.CathedralLifecycleOperationState(op.State), + CleanupState: api.CathedralLifecycleOperationCleanupState(op.CleanupState), + ExecutionRemovedAt: op.ExecutionRemovedAt, + SnapshotBuildId: op.SnapshotBuildID, + SnapshotCompletedAt: op.SnapshotCompletedAt, + RemainingLifetimeMs: op.RemainingLifetimeMs, + ErrorMessage: op.ErrorMessage, + } + if op.ErrorCode != nil { + code := int(*op.ErrorCode) + result.ErrorCode = &code + } + + return result +} + +func lifecycleHTTPStatus(op queries.CathedralSandboxLifecycleOperation, replay bool) int { + if replay { + return http.StatusOK + } + if op.State == "completed" { + return http.StatusCreated + } + + return http.StatusAccepted +} + +func (a *APIStore) GetV1CathedralLifecycleOperationsIdempotencyKey(c *gin.Context, operationKey api.CathedralOperationKey) { + if !cathedralIdempotencyKeyPattern.MatchString(operationKey) { + a.sendAPIStoreError(c, http.StatusBadRequest, "invalid Cathedral operation key") + return + } + + teamID := auth.MustGetTeamID(c) + op, err := a.sqlcDB.GetCathedralSandboxLifecycleOperation(c.Request.Context(), queries.GetCathedralSandboxLifecycleOperationParams{ + TeamID: teamID, OperationKey: operationKey, + }) + if errors.Is(err, pgx.ErrNoRows) { + a.sendAPIStoreError(c, http.StatusNotFound, "Cathedral lifecycle operation not found") + return + } + if err != nil { + a.sendAPIStoreError(c, http.StatusInternalServerError, "failed to read Cathedral lifecycle operation") + return + } + + c.JSON(http.StatusOK, lifecycleOperationToAPI(op)) +} + +func (a *APIStore) GetV1CathedralSandboxesSandboxIDIdentity(c *gin.Context, sandboxID api.SandboxID) { + teamID := auth.MustGetTeamID(c) + shortID, err := utils.ShortID(sandboxID) + if err != nil { + a.sendAPIStoreError(c, http.StatusBadRequest, "Invalid sandbox ID") + return + } + + current, err := a.cathedralLifecycleBackend().GetSandbox(c.Request.Context(), teamID, shortID) + if err != nil || current.TeamID != teamID { + a.sendAPIStoreError(c, http.StatusNotFound, utils.SandboxNotFoundMsg(shortID)) + return + } + if current.ExecutionID == "" { + a.sendAPIStoreError(c, http.StatusInternalServerError, "sandbox has no execution identity") + return + } + + c.JSON(http.StatusOK, api.CathedralSandboxIdentity{ + SandboxId: shortID, + ExecutionId: current.ExecutionID, + State: api.CathedralSandboxIdentityState(current.State), + }) +} + +func (a *APIStore) PostV1CathedralSandboxesSandboxIDLifecycleOperations( + c *gin.Context, + sandboxID api.SandboxID, + params api.PostV1CathedralSandboxesSandboxIDLifecycleOperationsParams, +) { + teamID := auth.MustGetTeamID(c) + shortID, err := utils.ShortID(sandboxID) + if err != nil { + a.sendAPIStoreError(c, http.StatusBadRequest, "Invalid sandbox ID") + return + } + if !cathedralIdempotencyKeyPattern.MatchString(params.IdempotencyKey) { + a.sendAPIStoreError(c, http.StatusBadRequest, "invalid Cathedral operation key") + return + } + + body, err := ginutils.ParseBody[api.CathedralLifecycleOperationRequest](c.Request.Context(), c) + if err != nil { + a.sendAPIStoreError(c, http.StatusBadRequest, fmt.Sprintf("Error when parsing request: %s", err)) + return + } + if body.ExecutionId == "" || !body.Operation.Valid() { + a.sendAPIStoreError(c, http.StatusBadRequest, "operation and execution_id are required") + return + } + if body.Operation == api.CathedralLifecycleOperationRequestOperationDelete && body.FilesystemOnly != nil && *body.FilesystemOnly { + a.sendAPIStoreError(c, http.StatusBadRequest, "filesystem_only is only valid for pause") + return + } + + digest, err := hashCathedralLifecycleRequest(shortID, body) + if err != nil { + a.sendAPIStoreError(c, http.StatusInternalServerError, "failed to normalize lifecycle request") + return + } + kind := string(body.Operation) + + // Recover before consulting the live registry. A completed delete has no + // live record by definition, and that absence must not erase its receipt. + existing, getErr := a.sqlcDB.GetCathedralSandboxLifecycleOperation(c.Request.Context(), queries.GetCathedralSandboxLifecycleOperationParams{ + TeamID: teamID, OperationKey: params.IdempotencyKey, + }) + if getErr == nil { + if existing.RequestSha256 != digest || existing.OperationKind != kind || existing.SandboxID != shortID || existing.ExecutionID != body.ExecutionId { + a.sendAPIStoreError(c, http.StatusConflict, "Idempotency-Key was already used for a different lifecycle request") + return + } + c.JSON(http.StatusOK, lifecycleOperationToAPI(existing)) + return + } + if !errors.Is(getErr, pgx.ErrNoRows) { + a.sendAPIStoreError(c, http.StatusInternalServerError, "failed to inspect Cathedral lifecycle operation") + return + } + + // Ownership and execution identity are checked before the durable claim; + // the execution pin is checked again atomically when removal starts. + current, err := a.cathedralLifecycleBackend().GetSandbox(c.Request.Context(), teamID, shortID) + if err != nil || current.TeamID != teamID { + a.sendAPIStoreError(c, http.StatusNotFound, utils.SandboxNotFoundMsg(shortID)) + return + } + if current.ExecutionID != body.ExecutionId { + a.sendAPIStoreError(c, http.StatusConflict, "sandbox execution identity changed") + return + } + + remainingMs := max(time.Until(current.EndTime).Milliseconds(), 0) + op, err := a.sqlcDB.ReserveCathedralSandboxLifecycleOperation(c.Request.Context(), queries.ReserveCathedralSandboxLifecycleOperationParams{ + TeamID: teamID, OperationKey: params.IdempotencyKey, RequestSha256: digest, + OperationKind: kind, SandboxID: shortID, ExecutionID: body.ExecutionId, + RemainingLifetimeMs: &remainingMs, + }) + replay := false + if errors.Is(err, pgx.ErrNoRows) { + replay = true + op, err = a.sqlcDB.GetCathedralSandboxLifecycleOperation(c.Request.Context(), queries.GetCathedralSandboxLifecycleOperationParams{ + TeamID: teamID, OperationKey: params.IdempotencyKey, + }) + } + if err != nil { + a.sendAPIStoreError(c, http.StatusInternalServerError, "failed to reserve Cathedral lifecycle operation") + return + } + if op.RequestSha256 != digest || op.OperationKind != kind || op.SandboxID != shortID || op.ExecutionID != body.ExecutionId { + a.sendAPIStoreError(c, http.StatusConflict, "Idempotency-Key was already used for a different lifecycle request") + return + } + if replay || op.State != "reserved" { + c.JSON(lifecycleHTTPStatus(op, true), lifecycleOperationToAPI(op)) + return + } + + rows, err := a.sqlcDB.MarkCathedralSandboxLifecycleDispatching(c.Request.Context(), queries.MarkCathedralSandboxLifecycleDispatchingParams{ + TeamID: teamID, OperationKey: op.OperationKey, RequestSha256: digest, + OperationKind: kind, SandboxID: shortID, ExecutionID: body.ExecutionId, + }) + if err != nil { + a.sendAPIStoreError(c, http.StatusInternalServerError, "failed to start Cathedral lifecycle operation") + return + } + if rows != 1 { + op, err = a.sqlcDB.GetCathedralSandboxLifecycleOperation(c.Request.Context(), queries.GetCathedralSandboxLifecycleOperationParams{TeamID: teamID, OperationKey: op.OperationKey}) + if err != nil { + a.sendAPIStoreError(c, http.StatusInternalServerError, "failed to recover Cathedral lifecycle operation") + return + } + c.JSON(http.StatusOK, lifecycleOperationToAPI(op)) + return + } + + dispatchCtx, cancel := cathedralLifecycleDispatchContext(c.Request.Context()) + defer cancel() + a.dispatchCathedralLifecycle(dispatchCtx, teamID, op, body) + + op, err = a.sqlcDB.GetCathedralSandboxLifecycleOperation(context.WithoutCancel(c.Request.Context()), queries.GetCathedralSandboxLifecycleOperationParams{TeamID: teamID, OperationKey: op.OperationKey}) + if err != nil { + a.sendAPIStoreError(c, http.StatusInternalServerError, "lifecycle outcome is pending durable recovery") + return + } + c.JSON(lifecycleHTTPStatus(op, false), lifecycleOperationToAPI(op)) +} + +func (a *APIStore) dispatchCathedralLifecycle(ctx context.Context, teamID uuid.UUID, op queries.CathedralSandboxLifecycleOperation, body api.CathedralLifecycleOperationRequest) { + action := sandbox.StateActionKill + if body.Operation == api.CathedralLifecycleOperationRequestOperationPause { + action = sandbox.StateActionPause + } + evidence, err := a.cathedralLifecycleBackend().RemoveSandboxWithEvidence(ctx, teamID, op.SandboxID, sandbox.RemoveOpts{ + Action: action, Reason: sandbox.KillReasonRequest, + FilesystemOnly: body.FilesystemOnly != nil && *body.FilesystemOnly, + ExpectExecutionID: op.ExecutionID, + }) + if err != nil || !evidence.Confirmed { + message := "provider lifecycle outcome is not terminally confirmed" + if err != nil { + message = err.Error() + } + _, _ = a.sqlcDB.MarkCathedralSandboxLifecycleUnknown(ctx, queries.MarkCathedralSandboxLifecycleUnknownParams{ + ErrorMessage: message, TeamID: teamID, OperationKey: op.OperationKey, + RequestSha256: op.RequestSha256, OperationKind: op.OperationKind, + SandboxID: op.SandboxID, ExecutionID: op.ExecutionID, + }) + return + } + + now := time.Now().UTC() + cleanupState := "not_required" + if op.OperationKind == "delete" { + cleanupState = "completed" + cleanup := a.deleteSnapshot + if a.lifecycleSnapshotCleanupOverride != nil { + cleanup = a.lifecycleSnapshotCleanupOverride + } + if cleanupErr := cleanup(ctx, op.SandboxID, teamID); cleanupErr != nil && !errors.Is(cleanupErr, db.ErrSnapshotNotFound) { + cleanupState = "failed" + } + } + + var snapshotBuildID *string + var snapshotCompletedAt *time.Time + if op.OperationKind == "pause" { + if evidence.SnapshotBuildID == "" { + _, _ = a.sqlcDB.MarkCathedralSandboxLifecycleUnknown(ctx, queries.MarkCathedralSandboxLifecycleUnknownParams{ + ErrorMessage: "pause node completion lacked durable snapshot identity", TeamID: teamID, + OperationKey: op.OperationKey, RequestSha256: op.RequestSha256, + OperationKind: op.OperationKind, SandboxID: op.SandboxID, ExecutionID: op.ExecutionID, + }) + return + } + snapshotBuildID = &evidence.SnapshotBuildID + snapshotCompletedAt = &now + } + + resultJSON, _ := json.Marshal(map[string]any{ + "evidence_source": "execution_bound_node_rpc", + "cleanup_state": cleanupState, + }) + _, _ = a.sqlcDB.CompleteCathedralSandboxLifecycleOperation(ctx, queries.CompleteCathedralSandboxLifecycleOperationParams{ + ExecutionRemovedAt: now, SnapshotBuildID: snapshotBuildID, + SnapshotCompletedAt: snapshotCompletedAt, CleanupState: cleanupState, + ResultJson: string(resultJSON), TeamID: teamID, OperationKey: op.OperationKey, + RequestSha256: op.RequestSha256, OperationKind: op.OperationKind, + SandboxID: op.SandboxID, ExecutionID: op.ExecutionID, + }) +} diff --git a/packages/api/internal/handlers/cathedral_sandbox_lifecycle_test.go b/packages/api/internal/handlers/cathedral_sandbox_lifecycle_test.go new file mode 100644 index 0000000000..a764ae23a6 --- /dev/null +++ b/packages/api/internal/handlers/cathedral_sandbox_lifecycle_test.go @@ -0,0 +1,93 @@ +package handlers + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/e2b-dev/infra/packages/api/internal/api" + "github.com/e2b-dev/infra/packages/db/queries" +) + +func TestCathedralLifecycleDispatchSurvivesCallerDisconnect(t *testing.T) { + t.Parallel() + + parent, cancelParent := context.WithCancel(context.Background()) + dispatch, cancelDispatch := cathedralLifecycleDispatchContext(parent) + t.Cleanup(cancelDispatch) + cancelParent() + + select { + case <-dispatch.Done(): + t.Fatalf("detached lifecycle dispatch inherited caller cancellation: %v", dispatch.Err()) + default: + } +} + +func TestHashCathedralLifecycleRequestBindsResourceExecutionAndIntent(t *testing.T) { + t.Parallel() + + base := api.CathedralLifecycleOperationRequest{ + Operation: api.CathedralLifecycleOperationRequestOperationDelete, + ExecutionId: "exec-1", + } + first, err := hashCathedralLifecycleRequest("sbx-1", base) + require.NoError(t, err) + second, err := hashCathedralLifecycleRequest("sbx-1", base) + require.NoError(t, err) + differentExecution, err := hashCathedralLifecycleRequest("sbx-1", api.CathedralLifecycleOperationRequest{ + Operation: api.CathedralLifecycleOperationRequestOperationDelete, + ExecutionId: "exec-2", + }) + require.NoError(t, err) + differentSandbox, err := hashCathedralLifecycleRequest("sbx-2", base) + require.NoError(t, err) + explicitFalse := false + equivalentDefault, err := hashCathedralLifecycleRequest("sbx-1", api.CathedralLifecycleOperationRequest{ + Operation: api.CathedralLifecycleOperationRequestOperationDelete, + ExecutionId: "exec-1", + FilesystemOnly: &explicitFalse, + }) + require.NoError(t, err) + explicitTrue := true + differentFilesystemMode, err := hashCathedralLifecycleRequest("sbx-1", api.CathedralLifecycleOperationRequest{ + Operation: api.CathedralLifecycleOperationRequestOperationDelete, + ExecutionId: "exec-1", + FilesystemOnly: &explicitTrue, + }) + require.NoError(t, err) + + assert.Len(t, first, 64) + assert.Equal(t, first, second) + assert.Equal(t, first, equivalentDefault) + assert.NotEqual(t, first, differentExecution) + assert.NotEqual(t, first, differentSandbox) + assert.NotEqual(t, first, differentFilesystemMode) +} + +func TestLifecycleOperationToAPIPreservesEvidenceAndCleanupDebt(t *testing.T) { + t.Parallel() + + now := time.Now().UTC() + errorCode := int32(503) + errorMessage := "storage cleanup unconfirmed" + remaining := int64(45_000) + buildID := "build-1" + got := lifecycleOperationToAPI(queries.CathedralSandboxLifecycleOperation{ + OperationKey: "lifecycle-1", OperationKind: "pause", SandboxID: "sbx-1", + ExecutionID: "exec-1", State: "unknown", CleanupState: "failed", + ExecutionRemovedAt: &now, SnapshotBuildID: &buildID, + SnapshotCompletedAt: &now, RemainingLifetimeMs: &remaining, + ErrorCode: &errorCode, ErrorMessage: &errorMessage, + }) + + assert.Equal(t, api.CathedralLifecycleOperationStateUnknown, got.State) + assert.Equal(t, api.CathedralLifecycleOperationCleanupStateFailed, got.CleanupState) + assert.Equal(t, "exec-1", got.ExecutionId) + assert.Equal(t, &remaining, got.RemainingLifetimeMs) + require.NotNil(t, got.ErrorCode) + assert.Equal(t, 503, *got.ErrorCode) +} diff --git a/packages/api/internal/handlers/cathedral_sandbox_operations.go b/packages/api/internal/handlers/cathedral_sandbox_operations.go index 38c548856b..66f0bf520f 100644 --- a/packages/api/internal/handlers/cathedral_sandbox_operations.go +++ b/packages/api/internal/handlers/cathedral_sandbox_operations.go @@ -199,10 +199,15 @@ func (a *APIStore) completeCathedralCreate(ctx context.Context, teamID uuid.UUID func (a *APIStore) GetV1CathedralCapabilities(c *gin.Context) { c.JSON(http.StatusOK, api.CathedralCapabilities{ - Schema: api.N1, - DurableCreateIdempotency: true, - OperationLookup: true, - SafeFork: false, + Schema: api.N1, + DurableCreateIdempotency: true, + OperationLookup: true, + SafeFork: false, + DurableLifecycleOperations: true, + SafeDelete: true, + SafePause: true, + PreservesRemainingLifetime: true, + ExecutionIdentity: true, }) } diff --git a/packages/api/internal/handlers/cathedral_sandbox_operations_test.go b/packages/api/internal/handlers/cathedral_sandbox_operations_test.go index 3a032ea0b1..eefac26fb7 100644 --- a/packages/api/internal/handlers/cathedral_sandbox_operations_test.go +++ b/packages/api/internal/handlers/cathedral_sandbox_operations_test.go @@ -105,7 +105,12 @@ func TestCathedralCapabilitiesFailClosedOnFork(t *testing.T) { assert.JSONEq(t, `{ "schema": 1, "durable_create_idempotency": true, + "durable_lifecycle_operations": true, "operation_lookup": true, - "safe_fork": false + "safe_fork": false, + "safe_delete": true, + "safe_pause": true, + "preserves_remaining_lifetime": true, + "execution_identity": true }`, recorder.Body.String()) } diff --git a/packages/api/internal/handlers/sandbox_resume.go b/packages/api/internal/handlers/sandbox_resume.go index 799a9dcecc..b1ae4dc43e 100644 --- a/packages/api/internal/handlers/sandbox_resume.go +++ b/packages/api/internal/handlers/sandbox_resume.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "net/http" + "time" "github.com/gin-gonic/gin" "github.com/google/uuid" @@ -176,6 +177,17 @@ func (a *APIStore) PostSandboxesSandboxIDResume(c *gin.Context, sandboxID api.Sa return } + // A Cathedral pause freezes the remaining lifetime in the durable snapshot. + // Preserve it on an implicit resume instead of granting the ordinary fresh + // default. An explicit timeout remains an intentional override. + if body.Timeout == nil && lastSnapshot.Snapshot.Config != nil && lastSnapshot.Snapshot.Config.RemainingLifetimeSeconds > 0 { + remaining := time.Duration(lastSnapshot.Snapshot.Config.RemainingLifetimeSeconds) * time.Second + if limit := time.Duration(teamInfo.Limits.MaxLengthHours) * time.Hour; limit > 0 && remaining > limit { + remaining = limit + } + timeout = remaining + } + // Pre-flight of the fetcher's authoritative gate so a disabled flag answers // 400 even when the start would otherwise join an in-flight one (409). if _, apiErr := resolveFilesystemBoot(ctx, a.featureFlags, body.Memory, lastSnapshot.Snapshot); apiErr != nil { diff --git a/packages/api/internal/handlers/store.go b/packages/api/internal/handlers/store.go index ff04404947..34ec20ece2 100644 --- a/packages/api/internal/handlers/store.go +++ b/packages/api/internal/handlers/store.go @@ -200,27 +200,29 @@ type APIStore struct { // pauseBackendOverride, when non-nil, replaces the orchestrator for the // pause handler's two calls — tests use it to assert the gate's wiring // (refusal before RemoveSandbox) without a real orchestrator. - pauseBackendOverride pauseOrchestrator - resumeBackendOverride resumeWaitOrchestrator - connectBackendOverride connectOrchestrator - teamSandboxCounter teamRunningSandboxCounter - templateManager *template_manager.TemplateManager - sqlcDB *sqlcdb.Client - authDB *authdb.Client - redisClient redis.UniversalClient - templateCache *templatecache.TemplateCache - templateBuildsCache *templatecache.TemplatesBuildCache - snapshotCache *snapshotcache.SnapshotCache - authService sharedauth.Service - templateSpawnCounter *utils.TemplateSpawnCounter - clickhouseStore clickhouse.Clickhouse - sandboxLogsReader *sandboxlogs.Reader - accessTokenGenerator *sandbox.AccessTokenGenerator - featureFlags *featureflags.Client - clusters *clusters.Pool - snapshotUpsertSem *sharedutils.AdjustableSemaphore - sandboxListSem *sharedutils.AdjustableSemaphore - snapshotBuildQuerySem *sharedutils.AdjustableSemaphore + pauseBackendOverride pauseOrchestrator + resumeBackendOverride resumeWaitOrchestrator + connectBackendOverride connectOrchestrator + lifecycleBackendOverride cathedralLifecycleOrchestrator + lifecycleSnapshotCleanupOverride func(context.Context, string, uuid.UUID) error + teamSandboxCounter teamRunningSandboxCounter + templateManager *template_manager.TemplateManager + sqlcDB *sqlcdb.Client + authDB *authdb.Client + redisClient redis.UniversalClient + templateCache *templatecache.TemplateCache + templateBuildsCache *templatecache.TemplatesBuildCache + snapshotCache *snapshotcache.SnapshotCache + authService sharedauth.Service + templateSpawnCounter *utils.TemplateSpawnCounter + clickhouseStore clickhouse.Clickhouse + sandboxLogsReader *sandboxlogs.Reader + accessTokenGenerator *sandbox.AccessTokenGenerator + featureFlags *featureflags.Client + clusters *clusters.Pool + snapshotUpsertSem *sharedutils.AdjustableSemaphore + sandboxListSem *sharedutils.AdjustableSemaphore + snapshotBuildQuerySem *sharedutils.AdjustableSemaphore // secretsConn and secretsManagement are nil when no secrets store backend // address is configured. The routes stay registered either way and answer diff --git a/packages/api/internal/orchestrator/delete_instance.go b/packages/api/internal/orchestrator/delete_instance.go index e44537c2eb..4b774e456b 100644 --- a/packages/api/internal/orchestrator/delete_instance.go +++ b/packages/api/internal/orchestrator/delete_instance.go @@ -29,16 +29,44 @@ const refusalRetryAfter = 10 * time.Second const pauseTimeout = 80 * time.Second +// SandboxRemovalEvidence is returned only by the Cathedral lifecycle path. +// Confirmed means the execution-bound node RPC completed (or the node +// authoritatively reported that exact execution absent). A missing API record +// or an already-in-progress transition never sets Confirmed. +type SandboxRemovalEvidence struct { + SandboxID string + ExecutionID string + Action sandbox.StateAction + Confirmed bool + AlreadyInProgress bool + SnapshotBuildID string + RemainingLifetime time.Duration +} + func (o *Orchestrator) RemoveSandbox(ctx context.Context, teamID uuid.UUID, sandboxID string, opts sandbox.RemoveOpts) error { + _, err := o.removeSandbox(ctx, teamID, sandboxID, opts, false) + + return err +} + +// RemoveSandboxWithEvidence preserves the legacy RemoveSandbox semantics while +// exposing the stronger completion signal Cathedral needs. Cathedral callers +// must pin ExpectExecutionID; legacy callers retain their existing semantics. +func (o *Orchestrator) RemoveSandboxWithEvidence(ctx context.Context, teamID uuid.UUID, sandboxID string, opts sandbox.RemoveOpts) (SandboxRemovalEvidence, error) { + return o.removeSandbox(ctx, teamID, sandboxID, opts, true) +} + +func (o *Orchestrator) removeSandbox(ctx context.Context, teamID uuid.UUID, sandboxID string, opts sandbox.RemoveOpts, waitForStorage bool) (SandboxRemovalEvidence, error) { ctx, span := tracer.Start(ctx, "remove-sandbox") defer span.End() + evidence := SandboxRemovalEvidence{SandboxID: sandboxID, ExecutionID: opts.ExpectExecutionID, Action: opts.Action} // A pause outlives its caller, so it is tracked from the start: a drain // that already stopped waiting must not admit one. if opts.Action == sandbox.StateActionPause { releaseWork, ok := o.TrackWork() if !ok { - return ErrDraining + return evidence, ErrDraining } defer releaseWork() } @@ -48,7 +76,7 @@ func (o *Orchestrator) RemoveSandbox(ctx context.Context, teamID uuid.UUID, sand if err != nil { // For eviction, propagate all errors to the evictor. if opts.Eviction { - return err + return evidence, err } switch opts.Action { @@ -59,7 +87,7 @@ func (o *Orchestrator) RemoveSandbox(ctx context.Context, teamID uuid.UUID, sand zap.String("kill_reason", opts.Reason.String()), ) - return ErrSandboxNotFound + return evidence, ErrSandboxNotFound } switch sbx.State { @@ -69,7 +97,7 @@ func (o *Orchestrator) RemoveSandbox(ctx context.Context, teamID uuid.UUID, sand zap.String("kill_reason", opts.Reason.String()), ) - return nil + return evidence, nil default: // It shouldn't happen the sandbox ended in paused state logger.L().Error(ctx, "Error killing sandbox", zap.Error(err), @@ -77,36 +105,36 @@ func (o *Orchestrator) RemoveSandbox(ctx context.Context, teamID uuid.UUID, sand zap.String("kill_reason", opts.Reason.String()), ) - return ErrSandboxOperationFailed + return evidence, ErrSandboxOperationFailed } case sandbox.StateActionPause: if errors.Is(err, sandbox.ErrNotFound) { logger.L().Info(ctx, "Sandbox not found for pause", logger.WithSandboxID(sandboxID)) - return ErrSandboxNotFound + return evidence, ErrSandboxNotFound } if transErr, ok := errors.AsType[*sandbox.InvalidStateTransitionError](err); ok { if transErr.CurrentState == sandbox.StateKilling { logger.L().Info(ctx, "Sandbox is already killed", logger.WithSandboxID(sandboxID)) - return ErrSandboxNotFound + return evidence, ErrSandboxNotFound } - return fmt.Errorf("sandbox is in '%s' state: %w", transErr.CurrentState, err) + return evidence, fmt.Errorf("sandbox is in '%s' state: %w", transErr.CurrentState, err) } if errors.Is(err, PauseQueueExhaustedError{}) { - return PauseQueueExhaustedError{} + return evidence, PauseQueueExhaustedError{} } logger.L().Error(ctx, "Error pausing sandbox", zap.Error(err), logger.WithSandboxID(sandboxID)) - return ErrSandboxOperationFailed + return evidence, ErrSandboxOperationFailed default: logger.L().Error(ctx, "Invalid state action", logger.WithSandboxID(sandboxID), zap.String("state_action", opts.Action.Name)) - return ErrSandboxOperationFailed + return evidence, ErrSandboxOperationFailed } } defer func() { @@ -114,6 +142,8 @@ func (o *Orchestrator) RemoveSandbox(ctx context.Context, teamID uuid.UUID, sand }() if alreadyDone { + evidence.ExecutionID = sbx.ExecutionID + evidence.AlreadyInProgress = true logger.L().Info(ctx, "Sandbox was already in the process of being removed", logger.WithSandboxID(sandboxID), zap.String("state", string(sbx.State))) if time.Since(sbx.EndTime) > sandbox.StaleCutoff && opts.Action.Effect == sandbox.TransitionExpires { @@ -121,7 +151,14 @@ func (o *Orchestrator) RemoveSandbox(ctx context.Context, teamID uuid.UUID, sand go o.analyticsRemove(context.WithoutCancel(ctx), sbx, opts.Action) } - return nil + return evidence, nil + } + evidence.ExecutionID = sbx.ExecutionID + if transition.OriginalEndTime != nil { + evidence.RemainingLifetime = time.Until(*transition.OriginalEndTime) + if evidence.RemainingLifetime < 0 { + evidence.RemainingLifetime = 0 + } } if opts.Action == sandbox.StateActionPause { @@ -146,7 +183,8 @@ func (o *Orchestrator) RemoveSandbox(ctx context.Context, teamID uuid.UUID, sand o.sandboxStore.Remove(context.WithoutCancel(ctx), teamID, sandboxID) go o.analyticsRemove(context.WithoutCancel(ctx), sbx, opts.Action) }() - err = o.removeSandboxFromNode(ctx, sbx, opts.Action, opts.Reason, opts.FilesystemOnly, restoreOnRefusal) + var snapshotBuildID string + snapshotBuildID, err = o.removeSandboxFromNodeWithEvidence(ctx, sbx, opts.Action, opts.Reason, opts.FilesystemOnly, restoreOnRefusal, evidence.RemainingLifetime, waitForStorage) if err != nil { if errors.Is(err, PauseQueueExhaustedError{}) { if restoreOnRefusal { @@ -160,7 +198,7 @@ func (o *Orchestrator) RemoveSandbox(ctx context.Context, teamID uuid.UUID, sand preserveRecord = true err = sandbox.ErrTransitionRestored - return fmt.Errorf("%w: %w", ErrSandboxNotFound, sandbox.ErrExecutionMismatch) + return evidence, fmt.Errorf("%w: %w", ErrSandboxNotFound, sandbox.ErrExecutionMismatch) } } @@ -176,10 +214,10 @@ func (o *Orchestrator) RemoveSandbox(ctx context.Context, teamID uuid.UUID, sand o.killRefusedSandbox(ctx, sbx) } - return ErrSandboxOperationFailed + return evidence, ErrSandboxOperationFailed } - return PauseQueueExhaustedError{} + return evidence, PauseQueueExhaustedError{} } if errors.Is(err, ErrRefusedRouteLost) { @@ -189,7 +227,7 @@ func (o *Orchestrator) RemoveSandbox(ctx context.Context, teamID uuid.UUID, sand logger.L().Error(ctx, "Pause refused by the node but the edge lost its route; removing the sandbox", logger.WithSandboxID(sbx.SandboxID)) o.killRefusedSandbox(ctx, sbx) - return ErrSandboxOperationFailed + return evidence, ErrSandboxOperationFailed } fields := []zap.Field{ @@ -203,10 +241,13 @@ func (o *Orchestrator) RemoveSandbox(ctx context.Context, teamID uuid.UUID, sand logger.L().Error(ctx, "Error removing sandbox", fields...) - return ErrSandboxOperationFailed + return evidence, ErrSandboxOperationFailed } - return nil + evidence.Confirmed = true + evidence.SnapshotBuildID = snapshotBuildID + + return evidence, nil } type restoreOutcome string @@ -305,6 +346,21 @@ func (o *Orchestrator) removeSandboxFromNode( filesystemOnly bool, restoreOnRefusal bool, ) error { + _, err := o.removeSandboxFromNodeWithEvidence(ctx, sbx, stateAction, reason, filesystemOnly, restoreOnRefusal, 0, false) + + return err +} + +func (o *Orchestrator) removeSandboxFromNodeWithEvidence( + ctx context.Context, + sbx sandbox.Sandbox, + stateAction sandbox.StateAction, + reason sandbox.KillReason, + filesystemOnly bool, + restoreOnRefusal bool, + remainingLifetime time.Duration, + waitForStorage bool, +) (string, error) { ctx, span := tracer.Start(ctx, "remove-sandbox-from-node") defer span.End() @@ -319,7 +375,7 @@ func (o *Orchestrator) removeSandboxFromNode( logger.L().Error(ctx, "failed to get node", fields...) - return fmt.Errorf("node '%s' not found", sbx.NodeID) + return "", fmt.Errorf("node '%s' not found", sbx.NodeID) } // For remote cluster nodes we are using gPRC metadata for routing registration instead @@ -346,7 +402,7 @@ func (o *Orchestrator) removeSandboxFromNode( switch stateAction { case sandbox.StateActionPause: - err := o.pauseSandbox(ctx, node, sbx, filesystemOnly, restoreOnRefusal) + buildID, err := o.pauseSandboxWithEvidence(ctx, node, sbx, filesystemOnly, restoreOnRefusal, remainingLifetime, waitForStorage) if err != nil { if dberrors.IsForeignKeyViolation(err) { killErr := o.killSandboxOnNode(ctx, node, sbx.ToNodeSandbox(), sandbox.KillReasonBaseTemplateMissing) @@ -358,18 +414,18 @@ func (o *Orchestrator) removeSandboxFromNode( zap.NamedError("kill_error", killErr), ) - return fmt.Errorf("failed to pause sandbox '%s': base template no longer exists: %w", sbx.SandboxID, err) + return "", fmt.Errorf("failed to pause sandbox '%s': base template no longer exists: %w", sbx.SandboxID, err) } - return fmt.Errorf("failed to auto pause sandbox '%s': %w", sbx.SandboxID, err) + return "", fmt.Errorf("failed to auto pause sandbox '%s': %w", sbx.SandboxID, err) } - return nil + return buildID, nil case sandbox.StateActionKill: - return o.killSandboxOnNode(ctx, node, sbx.ToNodeSandbox(), reason) + return "", o.killSandboxOnNode(ctx, node, sbx.ToNodeSandbox(), reason) } - return nil + return "", nil } func (o *Orchestrator) killOrphanSandbox(ctx context.Context, sbx sandbox.NodeSandbox) { diff --git a/packages/api/internal/orchestrator/delete_instance_test.go b/packages/api/internal/orchestrator/delete_instance_test.go index c6860b188a..758bceb424 100644 --- a/packages/api/internal/orchestrator/delete_instance_test.go +++ b/packages/api/internal/orchestrator/delete_instance_test.go @@ -43,7 +43,9 @@ import ( type pauseStubClient struct { orchestrator.SandboxServiceClient - err error + err error + deleteErr error + storageDurable *bool // gate, when set, holds the answer until closed. gate <-chan struct{} // onPause, when set, runs before the answer — a test's chance to change @@ -59,7 +61,7 @@ func (c *pauseStubClient) Delete(context.Context, *orchestrator.SandboxDeleteReq defer c.mu.Unlock() c.deletes++ - return &emptypb.Empty{}, nil + return &emptypb.Empty{}, c.deleteErr } func (c *pauseStubClient) deleteCount() int { @@ -69,7 +71,7 @@ func (c *pauseStubClient) deleteCount() int { return c.deletes } -func (c *pauseStubClient) Pause(_ context.Context, _ *orchestrator.SandboxPauseRequest, _ ...grpc.CallOption) (*orchestrator.SandboxPauseResponse, error) { +func (c *pauseStubClient) Pause(_ context.Context, request *orchestrator.SandboxPauseRequest, _ ...grpc.CallOption) (*orchestrator.SandboxPauseResponse, error) { if c.gate != nil { <-c.gate } @@ -80,7 +82,12 @@ func (c *pauseStubClient) Pause(_ context.Context, _ *orchestrator.SandboxPauseR return nil, c.err } - return &orchestrator.SandboxPauseResponse{}, nil + durable := request.GetWaitForStorage() + if c.storageDurable != nil { + durable = *c.storageDurable + } + + return &orchestrator.SandboxPauseResponse{StorageDurable: durable}, nil } // recordingCollector counts InstanceStopped emissions — the stopped-analytics @@ -520,6 +527,76 @@ func TestRemoveSandbox_SuccessRemovesAndEmits(t *testing.T) { 3*time.Second, 10*time.Millisecond) } +func TestRemoveSandboxWithEvidence_ConfirmedPauseCarriesSnapshotAndRemainingLifetime(t *testing.T) { + t.Parallel() + + f := newRefusalFixture(t, true, consts.LocalClusterID, nil) + evidence, err := f.o.RemoveSandboxWithEvidence(t.Context(), f.sbx.TeamID, f.sbx.SandboxID, sandbox.RemoveOpts{ + Action: sandbox.StateActionPause, ExpectExecutionID: f.sbx.ExecutionID, + }) + require.NoError(t, err) + assert.True(t, evidence.Confirmed) + assert.False(t, evidence.AlreadyInProgress) + assert.Equal(t, f.sbx.ExecutionID, evidence.ExecutionID) + assert.NotEmpty(t, evidence.SnapshotBuildID) + assert.InDelta(t, time.Hour.Seconds(), evidence.RemainingLifetime.Seconds(), 5) +} + +func TestRemoveSandboxWithEvidence_InFlightRemovalIsNeverTerminalProof(t *testing.T) { + t.Parallel() + + f := newRefusalFixture(t, true, consts.LocalClusterID, nil) + _, _, finish, err := f.o.sandboxStore.StartRemoving(t.Context(), f.sbx.TeamID, f.sbx.SandboxID, sandbox.RemoveOpts{ + Action: sandbox.StateActionKill, ExpectExecutionID: f.sbx.ExecutionID, + }) + require.NoError(t, err) + go func() { + time.Sleep(50 * time.Millisecond) + finish(context.WithoutCancel(t.Context()), nil) + }() + + evidence, err := f.o.RemoveSandboxWithEvidence(t.Context(), f.sbx.TeamID, f.sbx.SandboxID, sandbox.RemoveOpts{ + Action: sandbox.StateActionKill, ExpectExecutionID: f.sbx.ExecutionID, + }) + require.NoError(t, err) + assert.False(t, evidence.Confirmed) + assert.True(t, evidence.AlreadyInProgress) +} + +func TestRemoveSandboxWithEvidence_NodeFailureAndMissingRegistryStayUnconfirmed(t *testing.T) { + t.Parallel() + + f := newRefusalFixture(t, true, consts.LocalClusterID, nil) + node, ok := f.o.nodes.Get(f.o.scopedNodeID(consts.LocalClusterID, "node-1")) + require.True(t, ok) + node.SetSandboxClient(&pauseStubClient{deleteErr: errors.New("node transport lost")}) + + evidence, err := f.o.RemoveSandboxWithEvidence(t.Context(), f.sbx.TeamID, f.sbx.SandboxID, sandbox.RemoveOpts{ + Action: sandbox.StateActionKill, ExpectExecutionID: f.sbx.ExecutionID, + }) + require.ErrorIs(t, err, ErrSandboxOperationFailed) + assert.False(t, evidence.Confirmed) + _, getErr := f.o.sandboxStore.Get(t.Context(), f.sbx.TeamID, f.sbx.SandboxID) + require.ErrorIs(t, getErr, sandbox.ErrNotFound, "legacy registry absence is not promoted to completion evidence") +} + +func TestRemoveSandboxWithEvidence_PauseWithoutStorageConfirmationStaysUnconfirmed(t *testing.T) { + t.Parallel() + + f := newRefusalFixture(t, true, consts.LocalClusterID, nil) + node, ok := f.o.nodes.Get(f.o.scopedNodeID(consts.LocalClusterID, "node-1")) + require.True(t, ok) + durable := false + node.SetSandboxClient(&pauseStubClient{storageDurable: &durable}) + + evidence, err := f.o.RemoveSandboxWithEvidence(t.Context(), f.sbx.TeamID, f.sbx.SandboxID, sandbox.RemoveOpts{ + Action: sandbox.StateActionPause, ExpectExecutionID: f.sbx.ExecutionID, + }) + require.ErrorIs(t, err, ErrSandboxOperationFailed) + assert.False(t, evidence.Confirmed) + assert.Empty(t, evidence.SnapshotBuildID) +} + // A fatal (non-retryable) pause failure removes and emits exactly as before. func TestRemoveSandbox_FatalFailureRemovesAndEmits(t *testing.T) { t.Parallel() diff --git a/packages/api/internal/orchestrator/pause_instance.go b/packages/api/internal/orchestrator/pause_instance.go index 479d9647c5..935709bfb6 100644 --- a/packages/api/internal/orchestrator/pause_instance.go +++ b/packages/api/internal/orchestrator/pause_instance.go @@ -4,6 +4,8 @@ import ( "context" "errors" "fmt" + "math" + "time" "github.com/gogo/status" "github.com/google/uuid" @@ -27,14 +29,20 @@ import ( type PauseQueueExhaustedError = sandbox.PauseQueueExhaustedError func (o *Orchestrator) pauseSandbox(ctx context.Context, node *nodemanager.Node, sbx sandbox.Sandbox, filesystemOnly bool, restoreOnRefusal bool) error { + _, err := o.pauseSandboxWithEvidence(ctx, node, sbx, filesystemOnly, restoreOnRefusal, 0, false) + + return err +} + +func (o *Orchestrator) pauseSandboxWithEvidence(ctx context.Context, node *nodemanager.Node, sbx sandbox.Sandbox, filesystemOnly bool, restoreOnRefusal bool, remainingLifetime time.Duration, waitForStorage bool) (string, error) { ctx, span := tracer.Start(ctx, "pause-sandbox") defer span.End() - result, err := o.throttledUpsertSnapshot(ctx, buildUpsertSnapshotParams(sbx, node, filesystemOnly)) + result, err := o.throttledUpsertSnapshot(ctx, buildUpsertSnapshotParams(sbx, node, filesystemOnly, remainingLifetime)) if err != nil { telemetry.ReportCriticalError(ctx, "error inserting snapshot for env", err) - return err + return "", err } // The snapshot's CPU info is pinned to the source build (see @@ -52,7 +60,7 @@ func (o *Orchestrator) pauseSandbox(ctx context.Context, node *nodemanager.Node, zap.String("source_build_id", sbx.BuildID.String()), ) - err = snapshotInstance(ctx, node, sbx, result.TemplateID, result.BuildID.String(), filesystemOnly, restoreOnRefusal) + err = snapshotInstance(ctx, node, sbx, result.TemplateID, result.BuildID.String(), filesystemOnly, restoreOnRefusal, waitForStorage) if err != nil { // The build is already committed, and nothing reaps one left non-terminal. o.failSnapshotBuild(ctx, result.BuildID, err) @@ -60,40 +68,44 @@ func (o *Orchestrator) pauseSandbox(ctx context.Context, node *nodemanager.Node, if errors.Is(err, PauseQueueExhaustedError{}) { telemetry.ReportEvent(ctx, "pause refused retryably", telemetry.WithSandboxID(sbx.SandboxID)) - return PauseQueueExhaustedError{} + return "", PauseQueueExhaustedError{} } telemetry.ReportCriticalError(ctx, "error pausing sandbox", err) - return fmt.Errorf("error pausing sandbox: %w", err) + return "", fmt.Errorf("error pausing sandbox: %w", err) } if err := o.finishSnapshotBuild(ctx, result.BuildID, types.BuildStatusSuccess); err != nil { telemetry.ReportCriticalError(ctx, "error pausing sandbox", err) - return fmt.Errorf("error pausing sandbox: %w", err) + return "", fmt.Errorf("error pausing sandbox: %w", err) } o.snapshotCache.Invalidate(context.WithoutCancel(ctx), sbx.SandboxID) - return nil + return result.BuildID.String(), nil } -func snapshotInstance(ctx context.Context, node *nodemanager.Node, sbx sandbox.Sandbox, templateID, buildID string, filesystemOnly bool, restoreOnRefusal bool) error { +func snapshotInstance(ctx context.Context, node *nodemanager.Node, sbx sandbox.Sandbox, templateID, buildID string, filesystemOnly bool, restoreOnRefusal bool, waitForStorage bool) error { childCtx, childSpan := tracer.Start(ctx, "snapshot-instance") defer childSpan.End() client, childCtx := node.GetSandboxDeleteCtx(childCtx, sbx.SandboxID, sbx.ExecutionID, restoreOnRefusal) - _, err := client.Sandbox.Pause( + response, err := client.Sandbox.Pause( childCtx, &orchestrator.SandboxPauseRequest{ SandboxId: sbx.SandboxID, TemplateId: templateID, BuildId: buildID, FilesystemOnly: filesystemOnly, + WaitForStorage: waitForStorage, }, ) if err == nil { + if waitForStorage && (response == nil || !response.GetStorageDurable()) { + return errors.New("pause completed without durable storage confirmation") + } telemetry.ReportEvent(ctx, "Paused sandbox") return nil @@ -125,7 +137,7 @@ func (o *Orchestrator) WaitForStateChange(ctx context.Context, teamID uuid.UUID, return o.sandboxStore.WaitForStateChange(ctx, teamID, sandboxID) } -func buildUpsertSnapshotParams(sbx sandbox.Sandbox, node *nodemanager.Node, filesystemOnly bool) queries.UpsertSnapshotParams { +func buildUpsertSnapshotParams(sbx sandbox.Sandbox, node *nodemanager.Node, filesystemOnly bool, remaining ...time.Duration) queries.UpsertSnapshotParams { metadata := types.JSONBStringMap(sbx.Metadata) if metadata == nil { metadata = types.JSONBStringMap{} @@ -136,6 +148,17 @@ func buildUpsertSnapshotParams(sbx sandbox.Sandbox, node *nodemanager.Node, file clusterID = &sbx.ClusterID } + remainingLifetime := time.Duration(0) + if len(remaining) > 0 { + remainingLifetime = remaining[0] + } + remainingLifetimeSeconds := uint64(0) + if remainingLifetime > 0 { + // Round up so a valid sub-second remainder cannot serialize as the + // legacy zero/unset value and accidentally regain the default lifetime. + remainingLifetimeSeconds = uint64(math.Ceil(remainingLifetime.Seconds())) + } + return queries.UpsertSnapshotParams{ // Used if there's no snapshot for this sandbox yet TemplateID: id.Generate(), @@ -157,13 +180,14 @@ func buildUpsertSnapshotParams(sbx sandbox.Sandbox, node *nodemanager.Node, file AllowInternetAccess: sbx.AllowInternetAccess, AutoPause: sbx.AutoPause, Config: &types.PausedSandboxConfig{ - Version: types.PausedSandboxConfigVersion, - Network: sbx.Network, - AutoResume: sbx.AutoResume, - VolumeMounts: sbx.VolumeMounts, - FilesystemOnly: filesystemOnly, - AutoPauseFilesystemOnly: sbx.AutoPauseFilesystemOnly, - Iam: sbx.Iam, + Version: types.PausedSandboxConfigVersion, + Network: sbx.Network, + AutoResume: sbx.AutoResume, + VolumeMounts: sbx.VolumeMounts, + FilesystemOnly: filesystemOnly, + AutoPauseFilesystemOnly: sbx.AutoPauseFilesystemOnly, + Iam: sbx.Iam, + RemainingLifetimeSeconds: remainingLifetimeSeconds, }, OriginNodeID: node.ID, Status: types.BuildStatusSnapshotting, diff --git a/packages/api/internal/orchestrator/pause_instance_test.go b/packages/api/internal/orchestrator/pause_instance_test.go index bfa2e58915..df190cc00f 100644 --- a/packages/api/internal/orchestrator/pause_instance_test.go +++ b/packages/api/internal/orchestrator/pause_instance_test.go @@ -2,6 +2,7 @@ package orchestrator import ( "testing" + "time" "github.com/google/uuid" "github.com/stretchr/testify/assert" @@ -34,3 +35,18 @@ func TestBuildUpsertSnapshotParams_PreservesIam(t *testing.T) { assert.Equal(t, in, params.Config.Iam) } } + +func TestBuildUpsertSnapshotParams_PreservesRemainingLifetime(t *testing.T) { + t.Parallel() + + sbx := sandbox.Sandbox{ + SandboxID: "sbx-1", BaseTemplateID: "tmpl", BuildID: uuid.New(), + } + node := &nodemanager.Node{ID: "node-1"} + params := buildUpsertSnapshotParams(sbx, node, false, 37*time.Minute) + + assert.Equal(t, uint64((37 * time.Minute).Seconds()), params.Config.RemainingLifetimeSeconds) + + subsecond := buildUpsertSnapshotParams(sbx, node, false, 500*time.Millisecond) + assert.Equal(t, uint64(1), subsecond.Config.RemainingLifetimeSeconds) +} diff --git a/packages/db/migrations/20260916192714_add_cathedral_lifecycle_operations.sql b/packages/db/migrations/20260916192714_add_cathedral_lifecycle_operations.sql new file mode 100644 index 0000000000..a09ce72b86 --- /dev/null +++ b/packages/db/migrations/20260916192714_add_cathedral_lifecycle_operations.sql @@ -0,0 +1,57 @@ +-- +goose Up +CREATE TABLE public.cathedral_sandbox_lifecycle_operations ( + team_id UUID NOT NULL REFERENCES public.teams(id) ON DELETE CASCADE, + operation_key VARCHAR(128) NOT NULL, + request_sha256 CHAR(64) NOT NULL, + operation_kind VARCHAR(16) NOT NULL, + sandbox_id TEXT NOT NULL, + execution_id TEXT NOT NULL, + state VARCHAR(16) NOT NULL DEFAULT 'reserved', + execution_removed_at TIMESTAMPTZ, + snapshot_build_id TEXT, + snapshot_completed_at TIMESTAMPTZ, + remaining_lifetime_ms BIGINT, + cleanup_state VARCHAR(16) NOT NULL DEFAULT 'not_required', + result_json TEXT, + error_code INTEGER, + error_message TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + dispatch_started_at TIMESTAMPTZ, + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (team_id, operation_key), + CONSTRAINT cathedral_lifecycle_key_nonempty + CHECK (length(operation_key) BETWEEN 8 AND 128), + CONSTRAINT cathedral_lifecycle_request_sha256 + CHECK (request_sha256 ~ '^[0-9a-f]{64}$'), + CONSTRAINT cathedral_lifecycle_kind + CHECK (operation_kind IN ('delete', 'pause')), + CONSTRAINT cathedral_lifecycle_execution_nonempty + CHECK (length(execution_id) > 0), + CONSTRAINT cathedral_lifecycle_state + CHECK (state IN ('reserved', 'dispatching', 'completed', 'failed', 'unknown')), + CONSTRAINT cathedral_lifecycle_cleanup_state + CHECK (cleanup_state IN ('not_required', 'pending', 'completed', 'failed')), + CONSTRAINT cathedral_lifecycle_pause_evidence + CHECK (state <> 'completed' OR operation_kind <> 'pause' OR + (execution_removed_at IS NOT NULL AND snapshot_build_id IS NOT NULL AND snapshot_completed_at IS NOT NULL)), + CONSTRAINT cathedral_lifecycle_delete_evidence + CHECK (state <> 'completed' OR operation_kind <> 'delete' OR execution_removed_at IS NOT NULL), + CONSTRAINT cathedral_lifecycle_result + CHECK (state <> 'completed' OR result_json IS NOT NULL) +); + +CREATE INDEX cathedral_lifecycle_sandbox_idx + ON public.cathedral_sandbox_lifecycle_operations (team_id, sandbox_id, created_at DESC); +CREATE INDEX cathedral_lifecycle_recovery_idx + ON public.cathedral_sandbox_lifecycle_operations (state, updated_at) + WHERE state IN ('reserved', 'dispatching', 'unknown'); + +-- +goose Down +DO $$ +BEGIN + IF EXISTS (SELECT 1 FROM public.cathedral_sandbox_lifecycle_operations LIMIT 1) THEN + RAISE EXCEPTION 'cannot drop cathedral_sandbox_lifecycle_operations while rows exist'; + END IF; +END $$; + +DROP TABLE public.cathedral_sandbox_lifecycle_operations; diff --git a/packages/db/pkg/tests/cathedral_sandbox_operations_test.go b/packages/db/pkg/tests/cathedral_sandbox_operations_test.go index e9d16a0b2e..06096dc544 100644 --- a/packages/db/pkg/tests/cathedral_sandbox_operations_test.go +++ b/packages/db/pkg/tests/cathedral_sandbox_operations_test.go @@ -6,6 +6,7 @@ import ( "fmt" "sync" "testing" + "time" "github.com/jackc/pgx/v5" "github.com/stretchr/testify/assert" @@ -72,6 +73,111 @@ func TestCathedralSandboxOperationConcurrentReservationBindsOneSandbox(t *testin assert.Equal(t, "reserved", op.State) } +func TestCathedralLifecycleOperationCannotCompleteWithoutBoundTerminalEvidence(t *testing.T) { + t.Parallel() + + db := testutils.SetupDatabase(t) + sqlDB, err := sql.Open("pgx", db.ConnStr()) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, sqlDB.Close()) }) + teamID := seedTeam(t, sqlDB, "cathedral-lifecycle-evidence") + + const ( + key = "cathedral-delete-1" + digest = "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" + sandboxID = "i-delete-bound" + executionID = "exec-delete-bound" + ) + remaining := int64(60_000) + op, err := db.SqlcClient.ReserveCathedralSandboxLifecycleOperation(t.Context(), queries.ReserveCathedralSandboxLifecycleOperationParams{ + TeamID: teamID, OperationKey: key, RequestSha256: digest, + OperationKind: "delete", SandboxID: sandboxID, ExecutionID: executionID, + RemainingLifetimeMs: &remaining, + }) + require.NoError(t, err) + assert.Equal(t, "reserved", op.State) + assert.Equal(t, "pending", op.CleanupState) + + // Reserved is not dispatched and therefore cannot be promoted by a stale + // observer that merely noticed the registry row disappear. + rows, err := db.SqlcClient.CompleteCathedralSandboxLifecycleOperation(t.Context(), queries.CompleteCathedralSandboxLifecycleOperationParams{ + ExecutionRemovedAt: time.Now(), CleanupState: "completed", ResultJson: `{}`, + TeamID: teamID, OperationKey: key, RequestSha256: digest, + OperationKind: "delete", SandboxID: sandboxID, ExecutionID: executionID, + }) + require.NoError(t, err) + assert.Zero(t, rows) + + rows, err = db.SqlcClient.MarkCathedralSandboxLifecycleDispatching(t.Context(), queries.MarkCathedralSandboxLifecycleDispatchingParams{ + TeamID: teamID, OperationKey: key, RequestSha256: digest, + OperationKind: "delete", SandboxID: sandboxID, ExecutionID: executionID, + }) + require.NoError(t, err) + require.Equal(t, int64(1), rows) + + // A stale execution identity cannot complete the operation. + rows, err = db.SqlcClient.CompleteCathedralSandboxLifecycleOperation(t.Context(), queries.CompleteCathedralSandboxLifecycleOperationParams{ + ExecutionRemovedAt: time.Now(), CleanupState: "completed", ResultJson: `{}`, + TeamID: teamID, OperationKey: key, RequestSha256: digest, + OperationKind: "delete", SandboxID: sandboxID, ExecutionID: "exec-new", + }) + require.NoError(t, err) + assert.Zero(t, rows) + + rows, err = db.SqlcClient.CompleteCathedralSandboxLifecycleOperation(t.Context(), queries.CompleteCathedralSandboxLifecycleOperationParams{ + ExecutionRemovedAt: time.Now(), CleanupState: "completed", ResultJson: `{"evidence_source":"execution_bound_node_rpc"}`, + TeamID: teamID, OperationKey: key, RequestSha256: digest, + OperationKind: "delete", SandboxID: sandboxID, ExecutionID: executionID, + }) + require.NoError(t, err) + require.Equal(t, int64(1), rows) + + ready, err := db.SqlcClient.GetCathedralSandboxLifecycleOperation(t.Context(), queries.GetCathedralSandboxLifecycleOperationParams{TeamID: teamID, OperationKey: key}) + require.NoError(t, err) + assert.Equal(t, "completed", ready.State) + require.NotNil(t, ready.ExecutionRemovedAt) +} + +func TestCathedralLifecycleOperationRepeatedKeyNeverRedispatchesOrRebinds(t *testing.T) { + t.Parallel() + + db := testutils.SetupDatabase(t) + sqlDB, err := sql.Open("pgx", db.ConnStr()) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, sqlDB.Close()) }) + teamID := seedTeam(t, sqlDB, "cathedral-lifecycle-key") + const digest = "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd" + + _, err = db.SqlcClient.ReserveCathedralSandboxLifecycleOperation(t.Context(), queries.ReserveCathedralSandboxLifecycleOperationParams{ + TeamID: teamID, OperationKey: "cathedral-pause-1", RequestSha256: digest, + OperationKind: "pause", SandboxID: "sbx-one", ExecutionID: "exec-one", + }) + require.NoError(t, err) + _, err = db.SqlcClient.ReserveCathedralSandboxLifecycleOperation(t.Context(), queries.ReserveCathedralSandboxLifecycleOperationParams{ + TeamID: teamID, OperationKey: "cathedral-pause-1", RequestSha256: digest, + OperationKind: "pause", SandboxID: "sbx-two", ExecutionID: "exec-two", + }) + require.ErrorIs(t, err, pgx.ErrNoRows) + + op, err := db.SqlcClient.GetCathedralSandboxLifecycleOperation(t.Context(), queries.GetCathedralSandboxLifecycleOperationParams{TeamID: teamID, OperationKey: "cathedral-pause-1"}) + require.NoError(t, err) + assert.Equal(t, "sbx-one", op.SandboxID) + assert.Equal(t, "exec-one", op.ExecutionID) + + rows, err := db.SqlcClient.MarkCathedralSandboxLifecycleDispatching(t.Context(), queries.MarkCathedralSandboxLifecycleDispatchingParams{ + TeamID: teamID, OperationKey: op.OperationKey, RequestSha256: digest, + OperationKind: "pause", SandboxID: op.SandboxID, ExecutionID: op.ExecutionID, + }) + require.NoError(t, err) + require.Equal(t, int64(1), rows) + rows, err = db.SqlcClient.MarkCathedralSandboxLifecycleDispatching(t.Context(), queries.MarkCathedralSandboxLifecycleDispatchingParams{ + TeamID: teamID, OperationKey: op.OperationKey, RequestSha256: digest, + OperationKind: "pause", SandboxID: op.SandboxID, ExecutionID: op.ExecutionID, + }) + require.NoError(t, err) + assert.Zero(t, rows, "a repeated key cannot win dispatch twice") +} + func TestCathedralSandboxOperationSurvivesAmbiguousCreateAndStoresImmutableResponse(t *testing.T) { t.Parallel() diff --git a/packages/db/pkg/testutils/queries/models.go b/packages/db/pkg/testutils/queries/models.go index 3de0fe61fe..1302bde0d4 100644 --- a/packages/db/pkg/testutils/queries/models.go +++ b/packages/db/pkg/testutils/queries/models.go @@ -73,6 +73,27 @@ type BillingSandboxLog struct { TeamID uuid.UUID } +type CathedralSandboxLifecycleOperation struct { + TeamID uuid.UUID + OperationKey string + RequestSha256 string + OperationKind string + SandboxID string + ExecutionID string + State string + ExecutionRemovedAt *time.Time + SnapshotBuildID pgtype.Text + SnapshotCompletedAt *time.Time + RemainingLifetimeMs pgtype.Int8 + CleanupState string + ResultJson pgtype.Text + ErrorCode pgtype.Int4 + ErrorMessage pgtype.Text + CreatedAt time.Time + DispatchStartedAt *time.Time + UpdatedAt time.Time +} + type CathedralSandboxOperation struct { TeamID uuid.UUID IdempotencyKey string diff --git a/packages/db/pkg/types/types.go b/packages/db/pkg/types/types.go index 8c68c735b0..671acbcc41 100644 --- a/packages/db/pkg/types/types.go +++ b/packages/db/pkg/types/types.go @@ -167,6 +167,11 @@ type PausedSandboxConfig struct { // any workload identity is rederived from the current execution rather than a // stored subject. Pre-existing rows omit the key and decode to nil. Iam *SandboxIam `json:"iam,omitempty"` + + // RemainingLifetimeSeconds freezes the unconsumed lifetime at the point a + // pause transition commits. A resume without an explicit timeout restores + // this value instead of silently granting a fresh default lifetime. + RemainingLifetimeSeconds uint64 `json:"remainingLifetimeSeconds,omitempty"` } func (c PausedSandboxConfig) Value() (driver.Value, error) { diff --git a/packages/db/queries/cathedral_sandbox_lifecycle_operations.sql.go b/packages/db/queries/cathedral_sandbox_lifecycle_operations.sql.go new file mode 100644 index 0000000000..4896304971 --- /dev/null +++ b/packages/db/queries/cathedral_sandbox_lifecycle_operations.sql.go @@ -0,0 +1,282 @@ +// Code generated by sqlc. DO NOT EDIT. +// versions: +// sqlc v1.29.0 +// source: cathedral_sandbox_lifecycle_operations.sql + +package queries + +import ( + "context" + "time" + + "github.com/google/uuid" +) + +const completeCathedralSandboxLifecycleOperation = `-- name: CompleteCathedralSandboxLifecycleOperation :execrows +UPDATE public.cathedral_sandbox_lifecycle_operations +SET state = 'completed', + execution_removed_at = $1::timestamptz, + snapshot_build_id = $2::text, + snapshot_completed_at = $3::timestamptz, + cleanup_state = $4::text, + result_json = $5::text, + error_code = NULL, + error_message = NULL, + updated_at = now() +WHERE team_id = $6::uuid + AND operation_key = $7::text + AND request_sha256 = $8::text + AND operation_kind = $9::text + AND sandbox_id = $10::text + AND execution_id = $11::text + AND state = 'dispatching' +` + +type CompleteCathedralSandboxLifecycleOperationParams struct { + ExecutionRemovedAt time.Time + SnapshotBuildID *string + SnapshotCompletedAt *time.Time + CleanupState string + ResultJson string + TeamID uuid.UUID + OperationKey string + RequestSha256 string + OperationKind string + SandboxID string + ExecutionID string +} + +func (q *Queries) CompleteCathedralSandboxLifecycleOperation(ctx context.Context, arg CompleteCathedralSandboxLifecycleOperationParams) (int64, error) { + result, err := q.db.Exec(ctx, completeCathedralSandboxLifecycleOperation, + arg.ExecutionRemovedAt, + arg.SnapshotBuildID, + arg.SnapshotCompletedAt, + arg.CleanupState, + arg.ResultJson, + arg.TeamID, + arg.OperationKey, + arg.RequestSha256, + arg.OperationKind, + arg.SandboxID, + arg.ExecutionID, + ) + if err != nil { + return 0, err + } + return result.RowsAffected(), nil +} + +const failCathedralSandboxLifecycleOperation = `-- name: FailCathedralSandboxLifecycleOperation :execrows +UPDATE public.cathedral_sandbox_lifecycle_operations +SET state = 'failed', error_code = $1::integer, + error_message = $2::text, updated_at = now() +WHERE team_id = $3::uuid + AND operation_key = $4::text + AND request_sha256 = $5::text + AND operation_kind = $6::text + AND sandbox_id = $7::text + AND execution_id = $8::text + AND state IN ('reserved', 'dispatching', 'failed') +` + +type FailCathedralSandboxLifecycleOperationParams struct { + ErrorCode int32 + ErrorMessage string + TeamID uuid.UUID + OperationKey string + RequestSha256 string + OperationKind string + SandboxID string + ExecutionID string +} + +func (q *Queries) FailCathedralSandboxLifecycleOperation(ctx context.Context, arg FailCathedralSandboxLifecycleOperationParams) (int64, error) { + result, err := q.db.Exec(ctx, failCathedralSandboxLifecycleOperation, + arg.ErrorCode, + arg.ErrorMessage, + arg.TeamID, + arg.OperationKey, + arg.RequestSha256, + arg.OperationKind, + arg.SandboxID, + arg.ExecutionID, + ) + if err != nil { + return 0, err + } + return result.RowsAffected(), nil +} + +const getCathedralSandboxLifecycleOperation = `-- name: GetCathedralSandboxLifecycleOperation :one +SELECT team_id, operation_key, request_sha256, operation_kind, sandbox_id, execution_id, state, execution_removed_at, snapshot_build_id, snapshot_completed_at, remaining_lifetime_ms, cleanup_state, result_json, error_code, error_message, created_at, dispatch_started_at, updated_at +FROM public.cathedral_sandbox_lifecycle_operations +WHERE team_id = $1::uuid + AND operation_key = $2::text +` + +type GetCathedralSandboxLifecycleOperationParams struct { + TeamID uuid.UUID + OperationKey string +} + +func (q *Queries) GetCathedralSandboxLifecycleOperation(ctx context.Context, arg GetCathedralSandboxLifecycleOperationParams) (CathedralSandboxLifecycleOperation, error) { + row := q.db.QueryRow(ctx, getCathedralSandboxLifecycleOperation, arg.TeamID, arg.OperationKey) + var i CathedralSandboxLifecycleOperation + err := row.Scan( + &i.TeamID, + &i.OperationKey, + &i.RequestSha256, + &i.OperationKind, + &i.SandboxID, + &i.ExecutionID, + &i.State, + &i.ExecutionRemovedAt, + &i.SnapshotBuildID, + &i.SnapshotCompletedAt, + &i.RemainingLifetimeMs, + &i.CleanupState, + &i.ResultJson, + &i.ErrorCode, + &i.ErrorMessage, + &i.CreatedAt, + &i.DispatchStartedAt, + &i.UpdatedAt, + ) + return i, err +} + +const markCathedralSandboxLifecycleDispatching = `-- name: MarkCathedralSandboxLifecycleDispatching :execrows +UPDATE public.cathedral_sandbox_lifecycle_operations +SET state = 'dispatching', dispatch_started_at = COALESCE(dispatch_started_at, now()), updated_at = now() +WHERE team_id = $1::uuid + AND operation_key = $2::text + AND request_sha256 = $3::text + AND operation_kind = $4::text + AND sandbox_id = $5::text + AND execution_id = $6::text + AND state = 'reserved' +` + +type MarkCathedralSandboxLifecycleDispatchingParams struct { + TeamID uuid.UUID + OperationKey string + RequestSha256 string + OperationKind string + SandboxID string + ExecutionID string +} + +func (q *Queries) MarkCathedralSandboxLifecycleDispatching(ctx context.Context, arg MarkCathedralSandboxLifecycleDispatchingParams) (int64, error) { + result, err := q.db.Exec(ctx, markCathedralSandboxLifecycleDispatching, + arg.TeamID, + arg.OperationKey, + arg.RequestSha256, + arg.OperationKind, + arg.SandboxID, + arg.ExecutionID, + ) + if err != nil { + return 0, err + } + return result.RowsAffected(), nil +} + +const markCathedralSandboxLifecycleUnknown = `-- name: MarkCathedralSandboxLifecycleUnknown :execrows +UPDATE public.cathedral_sandbox_lifecycle_operations +SET state = 'unknown', error_code = $1::integer, + error_message = $2::text, updated_at = now() +WHERE team_id = $3::uuid + AND operation_key = $4::text + AND request_sha256 = $5::text + AND operation_kind = $6::text + AND sandbox_id = $7::text + AND execution_id = $8::text + AND state IN ('reserved', 'dispatching', 'unknown') +` + +type MarkCathedralSandboxLifecycleUnknownParams struct { + ErrorCode *int32 + ErrorMessage string + TeamID uuid.UUID + OperationKey string + RequestSha256 string + OperationKind string + SandboxID string + ExecutionID string +} + +func (q *Queries) MarkCathedralSandboxLifecycleUnknown(ctx context.Context, arg MarkCathedralSandboxLifecycleUnknownParams) (int64, error) { + result, err := q.db.Exec(ctx, markCathedralSandboxLifecycleUnknown, + arg.ErrorCode, + arg.ErrorMessage, + arg.TeamID, + arg.OperationKey, + arg.RequestSha256, + arg.OperationKind, + arg.SandboxID, + arg.ExecutionID, + ) + if err != nil { + return 0, err + } + return result.RowsAffected(), nil +} + +const reserveCathedralSandboxLifecycleOperation = `-- name: ReserveCathedralSandboxLifecycleOperation :one +INSERT INTO public.cathedral_sandbox_lifecycle_operations ( + team_id, operation_key, request_sha256, operation_kind, sandbox_id, + execution_id, remaining_lifetime_ms, cleanup_state +) VALUES ( + $1::uuid, $2::text, + $3::text, $4::text, + $5::text, $6::text, + $7::bigint, + CASE WHEN $4::text = 'delete' THEN 'pending' ELSE 'not_required' END +) +ON CONFLICT (team_id, operation_key) DO NOTHING +RETURNING team_id, operation_key, request_sha256, operation_kind, sandbox_id, execution_id, state, execution_removed_at, snapshot_build_id, snapshot_completed_at, remaining_lifetime_ms, cleanup_state, result_json, error_code, error_message, created_at, dispatch_started_at, updated_at +` + +type ReserveCathedralSandboxLifecycleOperationParams struct { + TeamID uuid.UUID + OperationKey string + RequestSha256 string + OperationKind string + SandboxID string + ExecutionID string + RemainingLifetimeMs *int64 +} + +func (q *Queries) ReserveCathedralSandboxLifecycleOperation(ctx context.Context, arg ReserveCathedralSandboxLifecycleOperationParams) (CathedralSandboxLifecycleOperation, error) { + row := q.db.QueryRow(ctx, reserveCathedralSandboxLifecycleOperation, + arg.TeamID, + arg.OperationKey, + arg.RequestSha256, + arg.OperationKind, + arg.SandboxID, + arg.ExecutionID, + arg.RemainingLifetimeMs, + ) + var i CathedralSandboxLifecycleOperation + err := row.Scan( + &i.TeamID, + &i.OperationKey, + &i.RequestSha256, + &i.OperationKind, + &i.SandboxID, + &i.ExecutionID, + &i.State, + &i.ExecutionRemovedAt, + &i.SnapshotBuildID, + &i.SnapshotCompletedAt, + &i.RemainingLifetimeMs, + &i.CleanupState, + &i.ResultJson, + &i.ErrorCode, + &i.ErrorMessage, + &i.CreatedAt, + &i.DispatchStartedAt, + &i.UpdatedAt, + ) + return i, err +} diff --git a/packages/db/queries/models.go b/packages/db/queries/models.go index aeda36d91f..77bac3d5af 100644 --- a/packages/db/queries/models.go +++ b/packages/db/queries/models.go @@ -26,6 +26,27 @@ type ActiveEnv struct { Source string } +type CathedralSandboxLifecycleOperation struct { + TeamID uuid.UUID + OperationKey string + RequestSha256 string + OperationKind string + SandboxID string + ExecutionID string + State string + ExecutionRemovedAt *time.Time + SnapshotBuildID *string + SnapshotCompletedAt *time.Time + RemainingLifetimeMs *int64 + CleanupState string + ResultJson *string + ErrorCode *int32 + ErrorMessage *string + CreatedAt time.Time + DispatchStartedAt *time.Time + UpdatedAt time.Time +} + type CathedralSandboxOperation struct { TeamID uuid.UUID IdempotencyKey string diff --git a/packages/db/queries/sandboxes/cathedral_sandbox_lifecycle_operations.sql b/packages/db/queries/sandboxes/cathedral_sandbox_lifecycle_operations.sql new file mode 100644 index 0000000000..1bccf38a5f --- /dev/null +++ b/packages/db/queries/sandboxes/cathedral_sandbox_lifecycle_operations.sql @@ -0,0 +1,73 @@ +-- name: ReserveCathedralSandboxLifecycleOperation :one +INSERT INTO public.cathedral_sandbox_lifecycle_operations ( + team_id, operation_key, request_sha256, operation_kind, sandbox_id, + execution_id, remaining_lifetime_ms, cleanup_state +) VALUES ( + sqlc.arg(team_id)::uuid, sqlc.arg(operation_key)::text, + sqlc.arg(request_sha256)::text, sqlc.arg(operation_kind)::text, + sqlc.arg(sandbox_id)::text, sqlc.arg(execution_id)::text, + sqlc.narg(remaining_lifetime_ms)::bigint, + CASE WHEN sqlc.arg(operation_kind)::text = 'delete' THEN 'pending' ELSE 'not_required' END +) +ON CONFLICT (team_id, operation_key) DO NOTHING +RETURNING *; + +-- name: GetCathedralSandboxLifecycleOperation :one +SELECT * +FROM public.cathedral_sandbox_lifecycle_operations +WHERE team_id = sqlc.arg(team_id)::uuid + AND operation_key = sqlc.arg(operation_key)::text; + +-- name: MarkCathedralSandboxLifecycleDispatching :execrows +UPDATE public.cathedral_sandbox_lifecycle_operations +SET state = 'dispatching', dispatch_started_at = COALESCE(dispatch_started_at, now()), updated_at = now() +WHERE team_id = sqlc.arg(team_id)::uuid + AND operation_key = sqlc.arg(operation_key)::text + AND request_sha256 = sqlc.arg(request_sha256)::text + AND operation_kind = sqlc.arg(operation_kind)::text + AND sandbox_id = sqlc.arg(sandbox_id)::text + AND execution_id = sqlc.arg(execution_id)::text + AND state = 'reserved'; + +-- name: CompleteCathedralSandboxLifecycleOperation :execrows +UPDATE public.cathedral_sandbox_lifecycle_operations +SET state = 'completed', + execution_removed_at = sqlc.arg(execution_removed_at)::timestamptz, + snapshot_build_id = sqlc.narg(snapshot_build_id)::text, + snapshot_completed_at = sqlc.narg(snapshot_completed_at)::timestamptz, + cleanup_state = sqlc.arg(cleanup_state)::text, + result_json = sqlc.arg(result_json)::text, + error_code = NULL, + error_message = NULL, + updated_at = now() +WHERE team_id = sqlc.arg(team_id)::uuid + AND operation_key = sqlc.arg(operation_key)::text + AND request_sha256 = sqlc.arg(request_sha256)::text + AND operation_kind = sqlc.arg(operation_kind)::text + AND sandbox_id = sqlc.arg(sandbox_id)::text + AND execution_id = sqlc.arg(execution_id)::text + AND state = 'dispatching'; + +-- name: MarkCathedralSandboxLifecycleUnknown :execrows +UPDATE public.cathedral_sandbox_lifecycle_operations +SET state = 'unknown', error_code = sqlc.narg(error_code)::integer, + error_message = sqlc.arg(error_message)::text, updated_at = now() +WHERE team_id = sqlc.arg(team_id)::uuid + AND operation_key = sqlc.arg(operation_key)::text + AND request_sha256 = sqlc.arg(request_sha256)::text + AND operation_kind = sqlc.arg(operation_kind)::text + AND sandbox_id = sqlc.arg(sandbox_id)::text + AND execution_id = sqlc.arg(execution_id)::text + AND state IN ('reserved', 'dispatching', 'unknown'); + +-- name: FailCathedralSandboxLifecycleOperation :execrows +UPDATE public.cathedral_sandbox_lifecycle_operations +SET state = 'failed', error_code = sqlc.arg(error_code)::integer, + error_message = sqlc.arg(error_message)::text, updated_at = now() +WHERE team_id = sqlc.arg(team_id)::uuid + AND operation_key = sqlc.arg(operation_key)::text + AND request_sha256 = sqlc.arg(request_sha256)::text + AND operation_kind = sqlc.arg(operation_kind)::text + AND sandbox_id = sqlc.arg(sandbox_id)::text + AND execution_id = sqlc.arg(execution_id)::text + AND state IN ('reserved', 'dispatching', 'failed'); diff --git a/packages/orchestrator/orchestrator.proto b/packages/orchestrator/orchestrator.proto index 446e81bf95..1f26652be7 100644 --- a/packages/orchestrator/orchestrator.proto +++ b/packages/orchestrator/orchestrator.proto @@ -188,6 +188,11 @@ message SandboxPauseRequest { // a snapshot cold-boots (reboots) from the rootfs. Default false = full memory // snapshot, so existing callers are unaffected. bool filesystem_only = 4; + + // Cathedral lifecycle operations require the snapshot to land in remote + // storage before pause can be reported terminal. Existing callers keep the + // asynchronous upload path when this is false. + bool wait_for_storage = 5; } message SchedulingMetadata { @@ -213,6 +218,7 @@ message SchedulingMetadata { message SandboxPauseResponse { SchedulingMetadata scheduling_metadata = 1; + bool storage_durable = 2; } message SandboxCheckpointRequest { diff --git a/packages/orchestrator/pkg/server/sandboxes.go b/packages/orchestrator/pkg/server/sandboxes.go index f9b05867c7..951d242fd1 100644 --- a/packages/orchestrator/pkg/server/sandboxes.go +++ b/packages/orchestrator/pkg/server/sandboxes.go @@ -959,10 +959,36 @@ func (s *Server) Pause(ctx context.Context, in *orchestrator.SandboxPauseRequest return nil, status.Errorf(codes.Internal, "error snapshotting sandbox '%s': %s", in.GetSandboxId(), err) } - s.uploadSnapshotAsync(ctx, sbx, res) + storageDurable := false + if in.GetWaitForStorage() { + uploadErr := retry.Do( + ctx, + defaultUploadRetryPolicy(), + isRetryableUploadErr, + res.upload.Run, + func(attempt int, backoff time.Duration, err error) { + sbxlogger.I(sbx).Warn(ctx, "snapshot upload attempt failed while waiting for durability", + zap.Int("attempt", attempt), + zap.Duration("backoff", backoff), + zap.Error(err), + ) + }, + ) + res.completeUpload(ctx, uploadErr) + if uploadErr != nil { + s.uploadFailedCounter.Add(ctx, 1, metric.WithAttributes(attribute.Bool("fs_only", res.filesystemOnly))) + telemetry.ReportCriticalError(ctx, "error durably uploading paused sandbox", uploadErr, telemetry.WithSandboxID(in.GetSandboxId())) + + return nil, status.Errorf(codes.Internal, "error durably uploading paused sandbox '%s': %s", in.GetSandboxId(), uploadErr) + } + storageDurable = true + } else { + s.uploadSnapshotAsync(ctx, sbx, res) + } - // Best-effort: the local snapshot is now in the cache and the remote upload - // has been kicked off above (still in flight). Harvest a resume page-fault + // Best-effort: the local snapshot is now in the cache. For an ordinary pause + // its remote upload is still in flight; the durability path waited above. + // Harvest a resume page-fault // trace from a throwaway warm resume of the local snapshot and (when enabled) // persist it as a prefetch mapping for the next resume. Runs in the // background; never affects the pause result, and waits for the upload before @@ -1001,6 +1027,7 @@ func (s *Server) Pause(ctx context.Context, in *orchestrator.SandboxPauseRequest return &orchestrator.SandboxPauseResponse{ SchedulingMetadata: res.schedulingMetadata, + StorageDurable: storageDurable, }, nil } diff --git a/packages/shared/pkg/grpc/orchestrator/orchestrator.pb.go b/packages/shared/pkg/grpc/orchestrator/orchestrator.pb.go index e4365ee858..91f1cf76d4 100644 --- a/packages/shared/pkg/grpc/orchestrator/orchestrator.pb.go +++ b/packages/shared/pkg/grpc/orchestrator/orchestrator.pb.go @@ -1126,6 +1126,10 @@ type SandboxPauseRequest struct { // a snapshot cold-boots (reboots) from the rootfs. Default false = full memory // snapshot, so existing callers are unaffected. FilesystemOnly bool `protobuf:"varint,4,opt,name=filesystem_only,json=filesystemOnly,proto3" json:"filesystem_only,omitempty"` + // Cathedral lifecycle operations require the snapshot to land in remote + // storage before pause can be reported terminal. Existing callers keep the + // asynchronous upload path when this is false. + WaitForStorage bool `protobuf:"varint,5,opt,name=wait_for_storage,json=waitForStorage,proto3" json:"wait_for_storage,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -1188,6 +1192,13 @@ func (x *SandboxPauseRequest) GetFilesystemOnly() bool { return false } +func (x *SandboxPauseRequest) GetWaitForStorage() bool { + if x != nil { + return x.WaitForStorage + } + return false +} + type SchedulingMetadata struct { state protoimpl.MessageState `protogen:"open.v1"` // memfile_base_build_id / rootfs_base_build_id are each artifact's root layer @@ -1308,6 +1319,7 @@ func (x *SchedulingMetadata) GetRootfsBaseBuildId() string { type SandboxPauseResponse struct { state protoimpl.MessageState `protogen:"open.v1"` SchedulingMetadata *SchedulingMetadata `protobuf:"bytes,1,opt,name=scheduling_metadata,json=schedulingMetadata,proto3" json:"scheduling_metadata,omitempty"` + StorageDurable bool `protobuf:"varint,2,opt,name=storage_durable,json=storageDurable,proto3" json:"storage_durable,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -1349,6 +1361,13 @@ func (x *SandboxPauseResponse) GetSchedulingMetadata() *SchedulingMetadata { return nil } +func (x *SandboxPauseResponse) GetStorageDurable() bool { + if x != nil { + return x.StorageDurable + } + return false +} + type SandboxCheckpointRequest struct { state protoimpl.MessageState `protogen:"open.v1"` SandboxId string `protobuf:"bytes,1,opt,name=sandbox_id,json=sandboxId,proto3" json:"sandbox_id,omitempty"` @@ -1751,14 +1770,15 @@ const file_orchestrator_proto_rawDesc = "" + "sandbox_id\x18\x01 \x01(\tR\tsandboxId\x12$\n" + "\vkill_reason\x18\x02 \x01(\tH\x00R\n" + "killReason\x88\x01\x01B\x0e\n" + - "\f_kill_reason\"\x99\x01\n" + + "\f_kill_reason\"\xc3\x01\n" + "\x13SandboxPauseRequest\x12\x1d\n" + "\n" + "sandbox_id\x18\x01 \x01(\tR\tsandboxId\x12\x1f\n" + "\vtemplate_id\x18\x02 \x01(\tR\n" + "templateId\x12\x19\n" + "\bbuild_id\x18\x03 \x01(\tR\abuildId\x12'\n" + - "\x0ffilesystem_only\x18\x04 \x01(\bR\x0efilesystemOnly\"\xb1\x03\n" + + "\x0ffilesystem_only\x18\x04 \x01(\bR\x0efilesystemOnly\x12(\n" + + "\x10wait_for_storage\x18\x05 \x01(\bR\x0ewaitForStorage\"\xb1\x03\n" + "\x12SchedulingMetadata\x121\n" + "\x15memfile_base_build_id\x18\x01 \x01(\tR\x12memfileBaseBuildId\x12\x19\n" + "\bbuild_id\x18\x02 \x01(\tR\abuildId\x12*\n" + @@ -1768,9 +1788,10 @@ const file_orchestrator_proto_rawDesc = "" + "\x15rootfs_dropped_builds\x18\x06 \x01(\rR\x13rootfsDroppedBuilds\x12.\n" + "\x13memfile_build_bytes\x18\a \x03(\x04R\x11memfileBuildBytes\x12,\n" + "\x12rootfs_build_bytes\x18\b \x03(\x04R\x10rootfsBuildBytes\x12/\n" + - "\x14rootfs_base_build_id\x18\t \x01(\tR\x11rootfsBaseBuildId\"\\\n" + + "\x14rootfs_base_build_id\x18\t \x01(\tR\x11rootfsBaseBuildId\"\x85\x01\n" + "\x14SandboxPauseResponse\x12D\n" + - "\x13scheduling_metadata\x18\x01 \x01(\v2\x13.SchedulingMetadataR\x12schedulingMetadata\"\xd6\x01\n" + + "\x13scheduling_metadata\x18\x01 \x01(\v2\x13.SchedulingMetadataR\x12schedulingMetadata\x12'\n" + + "\x0fstorage_durable\x18\x02 \x01(\bR\x0estorageDurable\"\xd6\x01\n" + "\x18SandboxCheckpointRequest\x12\x1d\n" + "\n" + "sandbox_id\x18\x01 \x01(\tR\tsandboxId\x12\x19\n" + diff --git a/spec/openapi.yml b/spec/openapi.yml index f86a30b6c1..3ff3015414 100644 --- a/spec/openapi.yml +++ b/spec/openapi.yml @@ -802,6 +802,11 @@ components: - durable_create_idempotency - operation_lookup - safe_fork + - durable_lifecycle_operations + - safe_delete + - safe_pause + - preserves_remaining_lifetime + - execution_identity properties: schema: type: integer @@ -812,6 +817,16 @@ components: type: boolean safe_fork: type: boolean + durable_lifecycle_operations: + type: boolean + safe_delete: + type: boolean + safe_pause: + type: boolean + preserves_remaining_lifetime: + type: boolean + execution_identity: + type: boolean CathedralSandboxOperation: type: object @@ -836,6 +851,79 @@ components: type: string nullable: true + CathedralLifecycleOperationRequest: + type: object + required: [operation, execution_id] + properties: + operation: + type: string + enum: [delete, pause] + execution_id: + type: string + minLength: 1 + filesystem_only: + type: boolean + default: false + + CathedralSandboxIdentity: + type: object + required: [sandbox_id, execution_id, state] + properties: + sandbox_id: + type: string + execution_id: + type: string + state: + type: string + enum: [running, pausing, killing, snapshotting] + + CathedralLifecycleOperation: + type: object + required: + - operation_key + - operation + - sandbox_id + - execution_id + - state + - cleanup_state + properties: + operation_key: + type: string + operation: + type: string + enum: [delete, pause] + sandbox_id: + type: string + execution_id: + type: string + state: + type: string + enum: [reserved, dispatching, completed, failed, unknown] + cleanup_state: + type: string + enum: [not_required, pending, completed, failed] + execution_removed_at: + type: string + format: date-time + nullable: true + snapshot_build_id: + type: string + nullable: true + snapshot_completed_at: + type: string + format: date-time + nullable: true + remaining_lifetime_ms: + type: integer + format: int64 + nullable: true + error_code: + type: integer + nullable: true + error_message: + type: string + nullable: true + SandboxDetail: required: - templateID @@ -2924,6 +3012,123 @@ paths: "500": $ref: "#/components/responses/500" + /v1/cathedral/sandboxes/{sandboxID}/lifecycle-operations: + post: + summary: Start an execution-bound Cathedral lifecycle operation + tags: [sandboxes] + security: + - ApiKeyAuth: [] + - AuthProviderBearerAuth: [] + AuthProviderTeamAuth: [] + - AdminApiKeyAuth: [] + AdminTeamAuth: [] + - AdminJWTAuth: [] + AdminTeamAuth: [] + parameters: + - $ref: "#/components/parameters/sandboxID" + - name: Idempotency-Key + in: header + required: true + schema: + type: string + minLength: 8 + maxLength: 128 + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/CathedralLifecycleOperationRequest" + responses: + "200": + description: Existing durable operation + content: + application/json: + schema: + $ref: "#/components/schemas/CathedralLifecycleOperation" + "201": + description: Lifecycle operation completed with terminal evidence + content: + application/json: + schema: + $ref: "#/components/schemas/CathedralLifecycleOperation" + "202": + description: Operation is durable but its terminal outcome is not yet proven + content: + application/json: + schema: + $ref: "#/components/schemas/CathedralLifecycleOperation" + "400": + $ref: "#/components/responses/400" + "401": + $ref: "#/components/responses/401" + "404": + $ref: "#/components/responses/404" + "409": + $ref: "#/components/responses/409" + "500": + $ref: "#/components/responses/500" + + /v1/cathedral/sandboxes/{sandboxID}/identity: + get: + summary: Read the authenticated current Cathedral sandbox execution identity + tags: [sandboxes] + security: + - ApiKeyAuth: [] + - AuthProviderBearerAuth: [] + AuthProviderTeamAuth: [] + - AdminApiKeyAuth: [] + AdminTeamAuth: [] + - AdminJWTAuth: [] + AdminTeamAuth: [] + parameters: + - $ref: "#/components/parameters/sandboxID" + responses: + "200": + description: Current sandbox execution identity + content: + application/json: + schema: + $ref: "#/components/schemas/CathedralSandboxIdentity" + "400": + $ref: "#/components/responses/400" + "401": + $ref: "#/components/responses/401" + "404": + $ref: "#/components/responses/404" + "500": + $ref: "#/components/responses/500" + + /v1/cathedral/lifecycle-operations/{idempotencyKey}: + get: + summary: Recover a Cathedral lifecycle operation by durable key + tags: [sandboxes] + security: + - ApiKeyAuth: [] + - AuthProviderBearerAuth: [] + AuthProviderTeamAuth: [] + - AdminApiKeyAuth: [] + AdminTeamAuth: [] + - AdminJWTAuth: [] + AdminTeamAuth: [] + parameters: + - $ref: "#/components/parameters/cathedralOperationKey" + responses: + "200": + description: Durable lifecycle operation + content: + application/json: + schema: + $ref: "#/components/schemas/CathedralLifecycleOperation" + "400": + $ref: "#/components/responses/400" + "401": + $ref: "#/components/responses/401" + "404": + $ref: "#/components/responses/404" + "500": + $ref: "#/components/responses/500" + /sandboxes: get: summary: List running sandboxes From 2f6c04b4888c2e7d9e03fdcc8f666a639eb635f3 Mon Sep 17 00:00:00 2001 From: Fred E <7602667+wallscaler@users.noreply.github.com> Date: Wed, 16 Sep 2026 18:57:01 -0400 Subject: [PATCH 3/9] fix durable lifecycle recovery and frozen lifetime --- docs/ARCHITECTURE.md | 5 + docs/cathedral-lifecycle-operations.md | 40 ++- .../handlers/cathedral_sandbox_lifecycle.go | 243 ++++++++++++++--- .../cathedral_sandbox_lifecycle_test.go | 10 + packages/api/internal/handlers/proxy_grpc.go | 4 + .../api/internal/handlers/sandbox_connect.go | 6 + .../api/internal/handlers/sandbox_resume.go | 42 ++- .../internal/handlers/sandbox_resume_test.go | 56 ++++ .../internal/orchestrator/pause_instance.go | 2 +- .../orchestrator/pause_instance_test.go | 11 +- ...add_cathedral_lifecycle_dispatch_lease.sql | 26 ++ .../cathedral_sandbox_operations_test.go | 86 +++++- packages/db/pkg/types/types.go | 4 +- packages/db/pkg/types/types_test.go | 22 ++ ...hedral_sandbox_lifecycle_operations.sql.go | 256 +++++++++++++++--- packages/db/queries/models.go | 39 +-- ...cathedral_sandbox_lifecycle_operations.sql | 61 ++++- 17 files changed, 768 insertions(+), 145 deletions(-) create mode 100644 packages/db/migrations/20260916223945_add_cathedral_lifecycle_dispatch_lease.sql diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index a19a6acfa6..31704bb92c 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -153,6 +153,11 @@ The control-plane entry point (Gin, OpenAPI-generated from `spec/openapi.yml`, p re-enter the existing Redis reservation with the same sandbox ID; a changed body returns 409, and a completed operation returns its immutable stored response. The authenticated `/v1/cathedral/operations/{key}` route is the durable recovery lookup. + Execution-bound Cathedral pause/delete operations use Postgres dispatch leases and fenced + attempts. Recovery retries only when the same execution is provably still running; otherwise it + records an honest failed or unknown outcome rather than replaying a possibly committed action. + Frozen pause lifetime is presence-aware, caps connect and traffic auto-resume, and preserves + explicit zero as exhausted. - **Secrets**: `/secrets` is the only public surface for secret management (create, list, get, update, delete). The API authenticates the caller with the customer alternatives above, converts the authenticated team UUID to the project UUID the backend knows, checks the `customer-secrets` diff --git a/docs/cathedral-lifecycle-operations.md b/docs/cathedral-lifecycle-operations.md index f243cb476f..a92cc7c75d 100644 --- a/docs/cathedral-lifecycle-operations.md +++ b/docs/cathedral-lifecycle-operations.md @@ -13,11 +13,23 @@ authenticated `delete` or `pause` request with: - optional `filesystem_only` for pause. The server hashes the normalized request and binds team, sandbox, execution, -operation, and key in Postgres before dispatch. Reusing a key with a different -binding returns `409`. Replaying the same key returns the stored operation and -never dispatches again. Recover it with +operation, pause mode, and key in Postgres before dispatch. Reusing a key with +a different binding returns `409`. Dispatch uses a fenced attempt number and a +bounded lease. A replay or recovery request may dispatch a never-started +`reserved` operation. It may retry an expired attempt only after the live +registry proves the same pinned execution is still `running`, which proves the +previous attempt did not commit a removal transition. A missing, superseded, +or still-transitioning execution becomes `unknown` instead of being blindly +acted on again. Recover with `GET /v1/cathedral/lifecycle-operations/{idempotencyKey}`. +Node refusals that restore the same execution are returned to `reserved` and +remain retryable. A known execution mismatch is terminal `failed`. Other +unconfirmed provider outcomes remain `unknown`. Terminal writes run on a +detached bounded context, are generation-fenced, and are retried; if durability +still cannot be established, the request returns an error and recovery applies +the lease rules above. + Before the first dispatch, an authenticated consumer reads the current incarnation from `GET /v1/cathedral/sandboxes/{sandboxID}/identity`. That endpoint enforces team ownership and returns the execution ID that must be @@ -34,24 +46,30 @@ or joining an in-flight removal is never terminal evidence. Delete reports snapshot/storage cleanup separately through `cleanup_state`. `completed` with `cleanup_state=failed` means compute removal is proven but -storage cleanup debt remains; a consumer must retain that debt and must not -represent full cleanup or final settlement as complete. +storage cleanup debt remains. Replaying or recovering that exact operation key +retries only the idempotent snapshot cleanup and never redispatches compute. +A consumer must retain remaining debt and must not represent full cleanup or +final settlement as complete. `unknown` is durable and non-retryable by POST. Recover it by key and reconcile with operator/provider evidence; do not blindly replay the lifecycle action. -Pause persists `remaining_lifetime_ms`. The snapshot stores the same frozen -remaining lifetime, and a resume without an explicit timeout uses it rather -than granting a new default lifetime. Resume remains the existing authenticated -endpoint; the operation protocol prevents stale pre-resume delete/pause work -from acting on the new execution identity. +Pause persists `remaining_lifetime_ms` from the same transition-owned remaining +lifetime used to write the snapshot, with both values rounded up to seconds. +Presence is explicit: zero means exhausted, while an absent field identifies a +legacy snapshot. Resume without an explicit timeout uses the frozen value; +connect and traffic auto-resume are capped by it and refuse exhausted snapshots. +An explicit resume timeout remains the only override. The operation protocol +prevents stale pre-resume delete/pause work from acting on the new execution +identity. ## Consumer rules 1. Generate one operation key per user intent and persist it before calling. 2. Send the current provider `execution_id`; never identify an incarnation by sandbox ID alone. -3. Treat `reserved`, `dispatching`, and `unknown` as non-terminal. +3. Treat `reserved` as retryable, `dispatching` as leased in-flight work, and + `unknown` as non-terminal for business settlement but not safe to replay. 4. Treat delete as compute-stopped only when `state=completed` and `execution_removed_at` is present. Close storage/billing only under the consumer's separately defined settlement rules and cleanup state. diff --git a/packages/api/internal/handlers/cathedral_sandbox_lifecycle.go b/packages/api/internal/handlers/cathedral_sandbox_lifecycle.go index 9fea74036e..c6dad60295 100644 --- a/packages/api/internal/handlers/cathedral_sandbox_lifecycle.go +++ b/packages/api/internal/handlers/cathedral_sandbox_lifecycle.go @@ -13,6 +13,7 @@ import ( "github.com/gin-gonic/gin" "github.com/google/uuid" "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgtype" "github.com/e2b-dev/infra/packages/api/internal/api" "github.com/e2b-dev/infra/packages/api/internal/db" @@ -24,7 +25,12 @@ import ( "github.com/e2b-dev/infra/packages/shared/pkg/ginutils" ) -const cathedralLifecycleDispatchTimeout = 2 * time.Minute +const ( + cathedralLifecycleDispatchTimeout = 2 * time.Minute + cathedralLifecycleDispatchLease = 3 * time.Minute + cathedralLifecycleWriteTimeout = 5 * time.Second + cathedralLifecycleWriteAttempts = 3 +) func cathedralLifecycleDispatchContext(parent context.Context) (context.Context, context.CancelFunc) { return context.WithTimeout(context.WithoutCancel(parent), cathedralLifecycleDispatchTimeout) @@ -96,6 +102,45 @@ func lifecycleHTTPStatus(op queries.CathedralSandboxLifecycleOperation, replay b return http.StatusAccepted } +func lifecycleRequestFromOperation(op queries.CathedralSandboxLifecycleOperation) api.CathedralLifecycleOperationRequest { + filesystemOnly := op.FilesystemOnly + return api.CathedralLifecycleOperationRequest{ + Operation: api.CathedralLifecycleOperationRequestOperation(op.OperationKind), + ExecutionId: op.ExecutionID, + FilesystemOnly: &filesystemOnly, + } +} + +func lifecycleLeaseInterval() pgtype.Interval { + return pgtype.Interval{Microseconds: cathedralLifecycleDispatchLease.Microseconds(), Valid: true} +} + +func frozenLifetimeMilliseconds(remaining time.Duration) int64 { + if remaining <= 0 { + return 0 + } + + return int64((remaining+time.Second-1)/time.Second) * 1000 +} + +func (a *APIStore) persistCathedralLifecycleState(ctx context.Context, write func(context.Context) (int64, error)) error { + var lastErr error + for attempt := 0; attempt < cathedralLifecycleWriteAttempts; attempt++ { + writeCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), cathedralLifecycleWriteTimeout) + rows, err := write(writeCtx) + cancel() + if err == nil && rows == 1 { + return nil + } + if err == nil { + err = fmt.Errorf("lifecycle state transition affected %d rows", rows) + } + lastErr = err + } + + return fmt.Errorf("persist Cathedral lifecycle terminal state: %w", lastErr) +} + func (a *APIStore) GetV1CathedralLifecycleOperationsIdempotencyKey(c *gin.Context, operationKey api.CathedralOperationKey) { if !cathedralIdempotencyKeyPattern.MatchString(operationKey) { a.sendAPIStoreError(c, http.StatusBadRequest, "invalid Cathedral operation key") @@ -114,6 +159,11 @@ func (a *APIStore) GetV1CathedralLifecycleOperationsIdempotencyKey(c *gin.Contex a.sendAPIStoreError(c, http.StatusInternalServerError, "failed to read Cathedral lifecycle operation") return } + op, err = a.recoverCathedralLifecycleOperation(c.Request.Context(), teamID, op) + if err != nil { + a.sendAPIStoreError(c, http.StatusInternalServerError, "lifecycle outcome is pending durable recovery") + return + } c.JSON(http.StatusOK, lifecycleOperationToAPI(op)) } @@ -190,6 +240,11 @@ func (a *APIStore) PostV1CathedralSandboxesSandboxIDLifecycleOperations( a.sendAPIStoreError(c, http.StatusConflict, "Idempotency-Key was already used for a different lifecycle request") return } + existing, err = a.recoverCathedralLifecycleOperation(c.Request.Context(), teamID, existing) + if err != nil { + a.sendAPIStoreError(c, http.StatusInternalServerError, "lifecycle outcome is pending durable recovery") + return + } c.JSON(http.StatusOK, lifecycleOperationToAPI(existing)) return } @@ -210,15 +265,12 @@ func (a *APIStore) PostV1CathedralSandboxesSandboxIDLifecycleOperations( return } - remainingMs := max(time.Until(current.EndTime).Milliseconds(), 0) op, err := a.sqlcDB.ReserveCathedralSandboxLifecycleOperation(c.Request.Context(), queries.ReserveCathedralSandboxLifecycleOperationParams{ TeamID: teamID, OperationKey: params.IdempotencyKey, RequestSha256: digest, OperationKind: kind, SandboxID: shortID, ExecutionID: body.ExecutionId, - RemainingLifetimeMs: &remainingMs, + FilesystemOnly: body.FilesystemOnly != nil && *body.FilesystemOnly, }) - replay := false if errors.Is(err, pgx.ErrNoRows) { - replay = true op, err = a.sqlcDB.GetCathedralSandboxLifecycleOperation(c.Request.Context(), queries.GetCathedralSandboxLifecycleOperationParams{ TeamID: teamID, OperationKey: params.IdempotencyKey, }) @@ -231,42 +283,109 @@ func (a *APIStore) PostV1CathedralSandboxesSandboxIDLifecycleOperations( a.sendAPIStoreError(c, http.StatusConflict, "Idempotency-Key was already used for a different lifecycle request") return } - if replay || op.State != "reserved" { - c.JSON(lifecycleHTTPStatus(op, true), lifecycleOperationToAPI(op)) + op, err = a.recoverCathedralLifecycleOperation(c.Request.Context(), teamID, op) + if err != nil { + a.sendAPIStoreError(c, http.StatusInternalServerError, "lifecycle outcome is pending durable recovery") return } + c.JSON(lifecycleHTTPStatus(op, false), lifecycleOperationToAPI(op)) +} - rows, err := a.sqlcDB.MarkCathedralSandboxLifecycleDispatching(c.Request.Context(), queries.MarkCathedralSandboxLifecycleDispatchingParams{ - TeamID: teamID, OperationKey: op.OperationKey, RequestSha256: digest, - OperationKind: kind, SandboxID: shortID, ExecutionID: body.ExecutionId, - }) - if err != nil { - a.sendAPIStoreError(c, http.StatusInternalServerError, "failed to start Cathedral lifecycle operation") - return +func (a *APIStore) recoverCathedralLifecycleOperation(ctx context.Context, teamID uuid.UUID, op queries.CathedralSandboxLifecycleOperation) (queries.CathedralSandboxLifecycleOperation, error) { + if op.State == "completed" && op.OperationKind == "delete" && (op.CleanupState == "pending" || op.CleanupState == "failed") { + cleanupState := "completed" + cleanup := a.deleteSnapshot + if a.lifecycleSnapshotCleanupOverride != nil { + cleanup = a.lifecycleSnapshotCleanupOverride + } + if cleanupErr := cleanup(ctx, op.SandboxID, teamID); cleanupErr != nil && !errors.Is(cleanupErr, db.ErrSnapshotNotFound) { + cleanupState = "failed" + } + if err := a.persistCathedralLifecycleState(ctx, func(writeCtx context.Context) (int64, error) { + return a.sqlcDB.UpdateCathedralSandboxLifecycleCleanup(writeCtx, queries.UpdateCathedralSandboxLifecycleCleanupParams{ + CleanupState: cleanupState, TeamID: teamID, OperationKey: op.OperationKey, + RequestSha256: op.RequestSha256, SandboxID: op.SandboxID, ExecutionID: op.ExecutionID, + }) + }); err != nil { + return op, err + } + return a.sqlcDB.GetCathedralSandboxLifecycleOperation(context.WithoutCancel(ctx), queries.GetCathedralSandboxLifecycleOperationParams{TeamID: teamID, OperationKey: op.OperationKey}) } - if rows != 1 { - op, err = a.sqlcDB.GetCathedralSandboxLifecycleOperation(c.Request.Context(), queries.GetCathedralSandboxLifecycleOperationParams{TeamID: teamID, OperationKey: op.OperationKey}) + if op.State == "completed" || op.State == "failed" || op.State == "unknown" { + return op, nil + } + + if op.State == "dispatching" { + if op.DispatchLeaseExpiresAt != nil && time.Now().Before(*op.DispatchLeaseExpiresAt) { + return op, nil + } + + current, getErr := a.cathedralLifecycleBackend().GetSandbox(ctx, teamID, op.SandboxID) + if getErr != nil || current.TeamID != teamID || current.ExecutionID != op.ExecutionID { + message := "dispatch lease expired and the pinned execution can no longer be proven safe to retry" + err := a.persistCathedralLifecycleState(ctx, func(writeCtx context.Context) (int64, error) { + return a.sqlcDB.MarkCathedralSandboxLifecycleUnknown(writeCtx, queries.MarkCathedralSandboxLifecycleUnknownParams{ + ErrorMessage: message, TeamID: teamID, OperationKey: op.OperationKey, + RequestSha256: op.RequestSha256, OperationKind: op.OperationKind, + SandboxID: op.SandboxID, ExecutionID: op.ExecutionID, DispatchAttempt: op.DispatchAttempt, + }) + }) + if err != nil { + return op, err + } + return a.sqlcDB.GetCathedralSandboxLifecycleOperation(context.WithoutCancel(ctx), queries.GetCathedralSandboxLifecycleOperationParams{TeamID: teamID, OperationKey: op.OperationKey}) + } + if current.State != sandbox.StateRunning { + message := fmt.Sprintf("dispatch lease expired while pinned execution remained in %s", current.State) + err := a.persistCathedralLifecycleState(ctx, func(writeCtx context.Context) (int64, error) { + return a.sqlcDB.MarkCathedralSandboxLifecycleUnknown(writeCtx, queries.MarkCathedralSandboxLifecycleUnknownParams{ + ErrorMessage: message, TeamID: teamID, OperationKey: op.OperationKey, + RequestSha256: op.RequestSha256, OperationKind: op.OperationKind, + SandboxID: op.SandboxID, ExecutionID: op.ExecutionID, DispatchAttempt: op.DispatchAttempt, + }) + }) + if err != nil { + return op, err + } + return a.sqlcDB.GetCathedralSandboxLifecycleOperation(context.WithoutCancel(ctx), queries.GetCathedralSandboxLifecycleOperationParams{TeamID: teamID, OperationKey: op.OperationKey}) + } + + rows, err := a.sqlcDB.RequeueExpiredCathedralSandboxLifecycleDispatch(ctx, queries.RequeueExpiredCathedralSandboxLifecycleDispatchParams{ + ErrorMessage: "expired dispatch proved to be a no-op; retrying pinned execution", + TeamID: teamID, OperationKey: op.OperationKey, RequestSha256: op.RequestSha256, + ExecutionID: op.ExecutionID, DispatchAttempt: op.DispatchAttempt, + }) if err != nil { - a.sendAPIStoreError(c, http.StatusInternalServerError, "failed to recover Cathedral lifecycle operation") - return + return op, err } - c.JSON(http.StatusOK, lifecycleOperationToAPI(op)) - return + if rows == 0 { + return a.sqlcDB.GetCathedralSandboxLifecycleOperation(ctx, queries.GetCathedralSandboxLifecycleOperationParams{TeamID: teamID, OperationKey: op.OperationKey}) + } + op.State = "reserved" } - dispatchCtx, cancel := cathedralLifecycleDispatchContext(c.Request.Context()) - defer cancel() - a.dispatchCathedralLifecycle(dispatchCtx, teamID, op, body) - - op, err = a.sqlcDB.GetCathedralSandboxLifecycleOperation(context.WithoutCancel(c.Request.Context()), queries.GetCathedralSandboxLifecycleOperationParams{TeamID: teamID, OperationKey: op.OperationKey}) + dispatching, err := a.sqlcDB.MarkCathedralSandboxLifecycleDispatching(ctx, queries.MarkCathedralSandboxLifecycleDispatchingParams{ + LeaseDuration: lifecycleLeaseInterval(), TeamID: teamID, OperationKey: op.OperationKey, + RequestSha256: op.RequestSha256, OperationKind: op.OperationKind, + SandboxID: op.SandboxID, ExecutionID: op.ExecutionID, + }) + if errors.Is(err, pgx.ErrNoRows) { + return a.sqlcDB.GetCathedralSandboxLifecycleOperation(ctx, queries.GetCathedralSandboxLifecycleOperationParams{TeamID: teamID, OperationKey: op.OperationKey}) + } if err != nil { - a.sendAPIStoreError(c, http.StatusInternalServerError, "lifecycle outcome is pending durable recovery") - return + return op, err } - c.JSON(lifecycleHTTPStatus(op, false), lifecycleOperationToAPI(op)) + + dispatchCtx, cancel := cathedralLifecycleDispatchContext(ctx) + defer cancel() + if err := a.dispatchCathedralLifecycle(dispatchCtx, teamID, dispatching, lifecycleRequestFromOperation(dispatching)); err != nil { + return dispatching, err + } + + return a.sqlcDB.GetCathedralSandboxLifecycleOperation(context.WithoutCancel(ctx), queries.GetCathedralSandboxLifecycleOperationParams{TeamID: teamID, OperationKey: op.OperationKey}) } -func (a *APIStore) dispatchCathedralLifecycle(ctx context.Context, teamID uuid.UUID, op queries.CathedralSandboxLifecycleOperation, body api.CathedralLifecycleOperationRequest) { +func (a *APIStore) dispatchCathedralLifecycle(ctx context.Context, teamID uuid.UUID, op queries.CathedralSandboxLifecycleOperation, body api.CathedralLifecycleOperationRequest) error { action := sandbox.StateActionKill if body.Operation == api.CathedralLifecycleOperationRequestOperationPause { action = sandbox.StateActionPause @@ -281,12 +400,36 @@ func (a *APIStore) dispatchCathedralLifecycle(ctx context.Context, teamID uuid.U if err != nil { message = err.Error() } - _, _ = a.sqlcDB.MarkCathedralSandboxLifecycleUnknown(ctx, queries.MarkCathedralSandboxLifecycleUnknownParams{ - ErrorMessage: message, TeamID: teamID, OperationKey: op.OperationKey, - RequestSha256: op.RequestSha256, OperationKind: op.OperationKind, - SandboxID: op.SandboxID, ExecutionID: op.ExecutionID, + if errors.Is(err, sandbox.ErrExecutionMismatch) { + return a.persistCathedralLifecycleState(ctx, func(writeCtx context.Context) (int64, error) { + return a.sqlcDB.FailCathedralSandboxLifecycleOperation(writeCtx, queries.FailCathedralSandboxLifecycleOperationParams{ + ErrorCode: http.StatusConflict, ErrorMessage: message, TeamID: teamID, + OperationKey: op.OperationKey, RequestSha256: op.RequestSha256, + OperationKind: op.OperationKind, SandboxID: op.SandboxID, + ExecutionID: op.ExecutionID, DispatchAttempt: op.DispatchAttempt, + }) + }) + } + if errors.Is(err, sandbox.ErrTransitionRestored) || errors.Is(err, orchestrator.PauseQueueExhaustedError{}) { + rows, requeueErr := a.sqlcDB.RequeueCathedralSandboxLifecycleDispatch(context.WithoutCancel(ctx), queries.RequeueCathedralSandboxLifecycleDispatchParams{ + ErrorMessage: message, TeamID: teamID, OperationKey: op.OperationKey, + RequestSha256: op.RequestSha256, ExecutionID: op.ExecutionID, DispatchAttempt: op.DispatchAttempt, + }) + if requeueErr != nil || rows != 1 { + return fmt.Errorf("persist retryable Cathedral lifecycle state: rows=%d: %w", rows, requeueErr) + } + return nil + } + if evidence.AlreadyInProgress { + return nil + } + return a.persistCathedralLifecycleState(ctx, func(writeCtx context.Context) (int64, error) { + return a.sqlcDB.MarkCathedralSandboxLifecycleUnknown(writeCtx, queries.MarkCathedralSandboxLifecycleUnknownParams{ + ErrorMessage: message, TeamID: teamID, OperationKey: op.OperationKey, + RequestSha256: op.RequestSha256, OperationKind: op.OperationKind, + SandboxID: op.SandboxID, ExecutionID: op.ExecutionID, DispatchAttempt: op.DispatchAttempt, + }) }) - return } now := time.Now().UTC() @@ -306,12 +449,14 @@ func (a *APIStore) dispatchCathedralLifecycle(ctx context.Context, teamID uuid.U var snapshotCompletedAt *time.Time if op.OperationKind == "pause" { if evidence.SnapshotBuildID == "" { - _, _ = a.sqlcDB.MarkCathedralSandboxLifecycleUnknown(ctx, queries.MarkCathedralSandboxLifecycleUnknownParams{ - ErrorMessage: "pause node completion lacked durable snapshot identity", TeamID: teamID, - OperationKey: op.OperationKey, RequestSha256: op.RequestSha256, - OperationKind: op.OperationKind, SandboxID: op.SandboxID, ExecutionID: op.ExecutionID, + return a.persistCathedralLifecycleState(ctx, func(writeCtx context.Context) (int64, error) { + return a.sqlcDB.MarkCathedralSandboxLifecycleUnknown(writeCtx, queries.MarkCathedralSandboxLifecycleUnknownParams{ + ErrorMessage: "pause node completion lacked durable snapshot identity", TeamID: teamID, + OperationKey: op.OperationKey, RequestSha256: op.RequestSha256, + OperationKind: op.OperationKind, SandboxID: op.SandboxID, + ExecutionID: op.ExecutionID, DispatchAttempt: op.DispatchAttempt, + }) }) - return } snapshotBuildID = &evidence.SnapshotBuildID snapshotCompletedAt = &now @@ -321,11 +466,19 @@ func (a *APIStore) dispatchCathedralLifecycle(ctx context.Context, teamID uuid.U "evidence_source": "execution_bound_node_rpc", "cleanup_state": cleanupState, }) - _, _ = a.sqlcDB.CompleteCathedralSandboxLifecycleOperation(ctx, queries.CompleteCathedralSandboxLifecycleOperationParams{ - ExecutionRemovedAt: now, SnapshotBuildID: snapshotBuildID, - SnapshotCompletedAt: snapshotCompletedAt, CleanupState: cleanupState, - ResultJson: string(resultJSON), TeamID: teamID, OperationKey: op.OperationKey, - RequestSha256: op.RequestSha256, OperationKind: op.OperationKind, - SandboxID: op.SandboxID, ExecutionID: op.ExecutionID, + var remainingLifetimeMs *int64 + if op.OperationKind == "pause" { + remaining := frozenLifetimeMilliseconds(evidence.RemainingLifetime) + remainingLifetimeMs = &remaining + } + return a.persistCathedralLifecycleState(ctx, func(writeCtx context.Context) (int64, error) { + return a.sqlcDB.CompleteCathedralSandboxLifecycleOperation(writeCtx, queries.CompleteCathedralSandboxLifecycleOperationParams{ + ExecutionRemovedAt: now, SnapshotBuildID: snapshotBuildID, + SnapshotCompletedAt: snapshotCompletedAt, CleanupState: cleanupState, + RemainingLifetimeMs: remainingLifetimeMs, ResultJson: string(resultJSON), + TeamID: teamID, OperationKey: op.OperationKey, RequestSha256: op.RequestSha256, + OperationKind: op.OperationKind, SandboxID: op.SandboxID, + ExecutionID: op.ExecutionID, DispatchAttempt: op.DispatchAttempt, + }) }) } diff --git a/packages/api/internal/handlers/cathedral_sandbox_lifecycle_test.go b/packages/api/internal/handlers/cathedral_sandbox_lifecycle_test.go index a764ae23a6..4cba0183e6 100644 --- a/packages/api/internal/handlers/cathedral_sandbox_lifecycle_test.go +++ b/packages/api/internal/handlers/cathedral_sandbox_lifecycle_test.go @@ -91,3 +91,13 @@ func TestLifecycleOperationToAPIPreservesEvidenceAndCleanupDebt(t *testing.T) { require.NotNil(t, got.ErrorCode) assert.Equal(t, 503, *got.ErrorCode) } + +func TestFrozenLifetimeMillisecondsMatchesSnapshotRounding(t *testing.T) { + t.Parallel() + + assert.Equal(t, int64(0), frozenLifetimeMilliseconds(0)) + assert.Equal(t, int64(0), frozenLifetimeMilliseconds(-time.Second)) + assert.Equal(t, int64(1000), frozenLifetimeMilliseconds(time.Millisecond)) + assert.Equal(t, int64(1000), frozenLifetimeMilliseconds(time.Second)) + assert.Equal(t, int64(2000), frozenLifetimeMilliseconds(time.Second+time.Nanosecond)) +} diff --git a/packages/api/internal/handlers/proxy_grpc.go b/packages/api/internal/handlers/proxy_grpc.go index 000c1c7210..5744179f83 100644 --- a/packages/api/internal/handlers/proxy_grpc.go +++ b/packages/api/internal/handlers/proxy_grpc.go @@ -210,6 +210,10 @@ func (s *SandboxService) ResumeSandbox(ctx context.Context, req *proxygrpc.Sandb minAutoResumeTimeout := time.Duration(s.api.featureFlags.IntFlag(ctx, featureflags.MinAutoResumeTimeoutSeconds)) * time.Second timeout := calculateAutoResumeTimeout(autoResume, minAutoResumeTimeout, team) + timeout, exhausted := clampToFrozenSnapshotLifetime(timeout, snap.Snapshot) + if exhausted { + return nil, status.Error(codes.FailedPrecondition, "sandbox lifetime exhausted") + } var envdAccessToken *string if snap.Snapshot.EnvSecure { diff --git a/packages/api/internal/handlers/sandbox_connect.go b/packages/api/internal/handlers/sandbox_connect.go index 8b48306d07..c3ad733646 100644 --- a/packages/api/internal/handlers/sandbox_connect.go +++ b/packages/api/internal/handlers/sandbox_connect.go @@ -189,6 +189,12 @@ func (a *APIStore) connectSandbox(c *gin.Context, sandboxID api.SandboxID, timeo return } + timeout, exhausted := clampToFrozenSnapshotLifetime(timeout, lastSnapshot.Snapshot) + if exhausted { + a.sendAPIStoreError(c, http.StatusConflict, "Sandbox lifetime was exhausted before pause") + return + } + // A paused filesystem-only snapshot resumes by cold-booting (reboot) from its // rootfs; the orchestrator selects reboot-vs-memory-resume from the snapshot // metadata, so the generic resume path below handles it. In-memory state was diff --git a/packages/api/internal/handlers/sandbox_resume.go b/packages/api/internal/handlers/sandbox_resume.go index b1ae4dc43e..967df9bedd 100644 --- a/packages/api/internal/handlers/sandbox_resume.go +++ b/packages/api/internal/handlers/sandbox_resume.go @@ -180,12 +180,18 @@ func (a *APIStore) PostSandboxesSandboxIDResume(c *gin.Context, sandboxID api.Sa // A Cathedral pause freezes the remaining lifetime in the durable snapshot. // Preserve it on an implicit resume instead of granting the ordinary fresh // default. An explicit timeout remains an intentional override. - if body.Timeout == nil && lastSnapshot.Snapshot.Config != nil && lastSnapshot.Snapshot.Config.RemainingLifetimeSeconds > 0 { - remaining := time.Duration(lastSnapshot.Snapshot.Config.RemainingLifetimeSeconds) * time.Second - if limit := time.Duration(teamInfo.Limits.MaxLengthHours) * time.Hour; limit > 0 && remaining > limit { - remaining = limit + if body.Timeout == nil { + remaining, frozen := frozenSnapshotLifetime(lastSnapshot.Snapshot) + if frozen && remaining <= 0 { + a.sendAPIStoreError(c, http.StatusConflict, "Sandbox lifetime was exhausted before pause") + return + } + if frozen { + if limit := time.Duration(teamInfo.Limits.MaxLengthHours) * time.Hour; limit > 0 && remaining > limit { + remaining = limit + } + timeout = remaining } - timeout = remaining } // Pre-flight of the fetcher's authoritative gate so a disabled flag answers @@ -278,6 +284,32 @@ func snapshotIsFilesystemOnly(snap queries.Snapshot) bool { return snap.Config != nil && snap.Config.FilesystemOnly } +// frozenSnapshotLifetime returns the snapshot-authoritative remaining lifetime. +// false distinguishes legacy rows without this field from an explicitly +// exhausted (zero) Cathedral lifetime. +func frozenSnapshotLifetime(snap queries.Snapshot) (time.Duration, bool) { + if snap.Config == nil || snap.Config.RemainingLifetimeSeconds == nil { + return 0, false + } + + return time.Duration(*snap.Config.RemainingLifetimeSeconds) * time.Second, true +} + +func clampToFrozenSnapshotLifetime(requested time.Duration, snap queries.Snapshot) (time.Duration, bool) { + remaining, frozen := frozenSnapshotLifetime(snap) + if !frozen { + return requested, false + } + if remaining <= 0 { + return 0, true + } + if requested > remaining { + return remaining, false + } + + return requested, false +} + // demandsFilesystemBoot reports whether the request explicitly demands a cold // boot that an in-flight start might not honor: memory:false on a snapshot not // already filesystem-only (an fs-only snapshot cold-boots on any start, so a diff --git a/packages/api/internal/handlers/sandbox_resume_test.go b/packages/api/internal/handlers/sandbox_resume_test.go index 3128abcd0e..622991cff0 100644 --- a/packages/api/internal/handlers/sandbox_resume_test.go +++ b/packages/api/internal/handlers/sandbox_resume_test.go @@ -4,6 +4,7 @@ import ( "net/http" "net/http/httptest" "testing" + "time" "github.com/gin-gonic/gin" "github.com/google/uuid" @@ -96,6 +97,61 @@ func TestSnapshotIsFilesystemOnly(t *testing.T) { } } +func TestFrozenSnapshotLifetimeDistinguishesLegacyAndExhausted(t *testing.T) { + t.Parallel() + + zero := uint64(0) + seconds := uint64(37) + tests := []struct { + name string + snap queries.Snapshot + want time.Duration + frozen bool + }{ + {name: "legacy no config", snap: queries.Snapshot{}}, + {name: "legacy no field", snap: queries.Snapshot{Config: &dbtypes.PausedSandboxConfig{}}}, + {name: "exhausted", snap: queries.Snapshot{Config: &dbtypes.PausedSandboxConfig{RemainingLifetimeSeconds: &zero}}, frozen: true}, + {name: "remaining", snap: queries.Snapshot{Config: &dbtypes.PausedSandboxConfig{RemainingLifetimeSeconds: &seconds}}, want: 37 * time.Second, frozen: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + got, frozen := frozenSnapshotLifetime(tt.snap) + assert.Equal(t, tt.want, got) + assert.Equal(t, tt.frozen, frozen) + }) + } +} + +func TestClampToFrozenSnapshotLifetime(t *testing.T) { + t.Parallel() + + zero := uint64(0) + remaining := uint64(30) + tests := []struct { + name string + snap queries.Snapshot + requested time.Duration + want time.Duration + exhausted bool + }{ + {name: "legacy unchanged", snap: queries.Snapshot{}, requested: time.Minute, want: time.Minute}, + {name: "shorter request unchanged", snap: queries.Snapshot{Config: &dbtypes.PausedSandboxConfig{RemainingLifetimeSeconds: &remaining}}, requested: 10 * time.Second, want: 10 * time.Second}, + {name: "implicit request capped", snap: queries.Snapshot{Config: &dbtypes.PausedSandboxConfig{RemainingLifetimeSeconds: &remaining}}, requested: time.Minute, want: 30 * time.Second}, + {name: "zero remains exhausted", snap: queries.Snapshot{Config: &dbtypes.PausedSandboxConfig{RemainingLifetimeSeconds: &zero}}, requested: time.Minute, exhausted: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + got, exhausted := clampToFrozenSnapshotLifetime(tt.requested, tt.snap) + assert.Equal(t, tt.want, got) + assert.Equal(t, tt.exhausted, exhausted) + }) + } +} + func TestSetMemoryOverrideOutcome(t *testing.T) { t.Parallel() diff --git a/packages/api/internal/orchestrator/pause_instance.go b/packages/api/internal/orchestrator/pause_instance.go index 935709bfb6..8287a7cb92 100644 --- a/packages/api/internal/orchestrator/pause_instance.go +++ b/packages/api/internal/orchestrator/pause_instance.go @@ -187,7 +187,7 @@ func buildUpsertSnapshotParams(sbx sandbox.Sandbox, node *nodemanager.Node, file FilesystemOnly: filesystemOnly, AutoPauseFilesystemOnly: sbx.AutoPauseFilesystemOnly, Iam: sbx.Iam, - RemainingLifetimeSeconds: remainingLifetimeSeconds, + RemainingLifetimeSeconds: &remainingLifetimeSeconds, }, OriginNodeID: node.ID, Status: types.BuildStatusSnapshotting, diff --git a/packages/api/internal/orchestrator/pause_instance_test.go b/packages/api/internal/orchestrator/pause_instance_test.go index df190cc00f..174caf1fec 100644 --- a/packages/api/internal/orchestrator/pause_instance_test.go +++ b/packages/api/internal/orchestrator/pause_instance_test.go @@ -6,6 +6,7 @@ import ( "github.com/google/uuid" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/e2b-dev/infra/packages/api/internal/orchestrator/nodemanager" "github.com/e2b-dev/infra/packages/api/internal/sandbox" @@ -45,8 +46,14 @@ func TestBuildUpsertSnapshotParams_PreservesRemainingLifetime(t *testing.T) { node := &nodemanager.Node{ID: "node-1"} params := buildUpsertSnapshotParams(sbx, node, false, 37*time.Minute) - assert.Equal(t, uint64((37 * time.Minute).Seconds()), params.Config.RemainingLifetimeSeconds) + require.NotNil(t, params.Config.RemainingLifetimeSeconds) + assert.Equal(t, uint64((37 * time.Minute).Seconds()), *params.Config.RemainingLifetimeSeconds) subsecond := buildUpsertSnapshotParams(sbx, node, false, 500*time.Millisecond) - assert.Equal(t, uint64(1), subsecond.Config.RemainingLifetimeSeconds) + require.NotNil(t, subsecond.Config.RemainingLifetimeSeconds) + assert.Equal(t, uint64(1), *subsecond.Config.RemainingLifetimeSeconds) + + exhausted := buildUpsertSnapshotParams(sbx, node, false, 0) + require.NotNil(t, exhausted.Config.RemainingLifetimeSeconds) + assert.Zero(t, *exhausted.Config.RemainingLifetimeSeconds) } diff --git a/packages/db/migrations/20260916223945_add_cathedral_lifecycle_dispatch_lease.sql b/packages/db/migrations/20260916223945_add_cathedral_lifecycle_dispatch_lease.sql new file mode 100644 index 0000000000..db501ff850 --- /dev/null +++ b/packages/db/migrations/20260916223945_add_cathedral_lifecycle_dispatch_lease.sql @@ -0,0 +1,26 @@ +-- +goose Up +ALTER TABLE public.cathedral_sandbox_lifecycle_operations + ADD COLUMN filesystem_only BOOLEAN NOT NULL DEFAULT false, + ADD COLUMN dispatch_attempt INTEGER NOT NULL DEFAULT 0, + ADD COLUMN dispatch_lease_expires_at TIMESTAMPTZ; + +UPDATE public.cathedral_sandbox_lifecycle_operations +SET dispatch_lease_expires_at = COALESCE(dispatch_started_at, updated_at) + interval '2 minutes' +WHERE state = 'dispatching'; + +ALTER TABLE public.cathedral_sandbox_lifecycle_operations + ADD CONSTRAINT cathedral_lifecycle_dispatch_attempt_nonnegative + CHECK (dispatch_attempt >= 0), + ADD CONSTRAINT cathedral_lifecycle_dispatch_lease + CHECK ((state = 'dispatching') = (dispatch_lease_expires_at IS NOT NULL)), + ADD CONSTRAINT cathedral_lifecycle_remaining_nonnegative + CHECK (remaining_lifetime_ms IS NULL OR remaining_lifetime_ms >= 0); + +-- +goose Down +ALTER TABLE public.cathedral_sandbox_lifecycle_operations + DROP CONSTRAINT cathedral_lifecycle_remaining_nonnegative, + DROP CONSTRAINT cathedral_lifecycle_dispatch_lease, + DROP CONSTRAINT cathedral_lifecycle_dispatch_attempt_nonnegative, + DROP COLUMN dispatch_lease_expires_at, + DROP COLUMN dispatch_attempt, + DROP COLUMN filesystem_only; diff --git a/packages/db/pkg/tests/cathedral_sandbox_operations_test.go b/packages/db/pkg/tests/cathedral_sandbox_operations_test.go index 06096dc544..8fe893e005 100644 --- a/packages/db/pkg/tests/cathedral_sandbox_operations_test.go +++ b/packages/db/pkg/tests/cathedral_sandbox_operations_test.go @@ -9,6 +9,7 @@ import ( "time" "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgtype" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -108,25 +109,28 @@ func TestCathedralLifecycleOperationCannotCompleteWithoutBoundTerminalEvidence(t require.NoError(t, err) assert.Zero(t, rows) - rows, err = db.SqlcClient.MarkCathedralSandboxLifecycleDispatching(t.Context(), queries.MarkCathedralSandboxLifecycleDispatchingParams{ - TeamID: teamID, OperationKey: key, RequestSha256: digest, + dispatch, err := db.SqlcClient.MarkCathedralSandboxLifecycleDispatching(t.Context(), queries.MarkCathedralSandboxLifecycleDispatchingParams{ + LeaseDuration: pgtype.Interval{Microseconds: time.Minute.Microseconds(), Valid: true}, + TeamID: teamID, OperationKey: key, RequestSha256: digest, OperationKind: "delete", SandboxID: sandboxID, ExecutionID: executionID, }) require.NoError(t, err) - require.Equal(t, int64(1), rows) + require.Equal(t, int32(1), dispatch.DispatchAttempt) // A stale execution identity cannot complete the operation. rows, err = db.SqlcClient.CompleteCathedralSandboxLifecycleOperation(t.Context(), queries.CompleteCathedralSandboxLifecycleOperationParams{ ExecutionRemovedAt: time.Now(), CleanupState: "completed", ResultJson: `{}`, - TeamID: teamID, OperationKey: key, RequestSha256: digest, + DispatchAttempt: dispatch.DispatchAttempt, + TeamID: teamID, OperationKey: key, RequestSha256: digest, OperationKind: "delete", SandboxID: sandboxID, ExecutionID: "exec-new", }) require.NoError(t, err) assert.Zero(t, rows) rows, err = db.SqlcClient.CompleteCathedralSandboxLifecycleOperation(t.Context(), queries.CompleteCathedralSandboxLifecycleOperationParams{ - ExecutionRemovedAt: time.Now(), CleanupState: "completed", ResultJson: `{"evidence_source":"execution_bound_node_rpc"}`, - TeamID: teamID, OperationKey: key, RequestSha256: digest, + ExecutionRemovedAt: time.Now(), CleanupState: "failed", ResultJson: `{"evidence_source":"execution_bound_node_rpc"}`, + DispatchAttempt: dispatch.DispatchAttempt, + TeamID: teamID, OperationKey: key, RequestSha256: digest, OperationKind: "delete", SandboxID: sandboxID, ExecutionID: executionID, }) require.NoError(t, err) @@ -135,7 +139,19 @@ func TestCathedralLifecycleOperationCannotCompleteWithoutBoundTerminalEvidence(t ready, err := db.SqlcClient.GetCathedralSandboxLifecycleOperation(t.Context(), queries.GetCathedralSandboxLifecycleOperationParams{TeamID: teamID, OperationKey: key}) require.NoError(t, err) assert.Equal(t, "completed", ready.State) + assert.Equal(t, "failed", ready.CleanupState) require.NotNil(t, ready.ExecutionRemovedAt) + + rows, err = db.SqlcClient.UpdateCathedralSandboxLifecycleCleanup(t.Context(), queries.UpdateCathedralSandboxLifecycleCleanupParams{ + CleanupState: "completed", TeamID: teamID, OperationKey: key, RequestSha256: digest, + SandboxID: sandboxID, ExecutionID: executionID, + }) + require.NoError(t, err) + require.Equal(t, int64(1), rows) + ready, err = db.SqlcClient.GetCathedralSandboxLifecycleOperation(t.Context(), queries.GetCathedralSandboxLifecycleOperationParams{TeamID: teamID, OperationKey: key}) + require.NoError(t, err) + assert.Equal(t, "completed", ready.State, "cleanup recovery must not redispatch compute") + assert.Equal(t, "completed", ready.CleanupState) } func TestCathedralLifecycleOperationRepeatedKeyNeverRedispatchesOrRebinds(t *testing.T) { @@ -164,18 +180,64 @@ func TestCathedralLifecycleOperationRepeatedKeyNeverRedispatchesOrRebinds(t *tes assert.Equal(t, "sbx-one", op.SandboxID) assert.Equal(t, "exec-one", op.ExecutionID) - rows, err := db.SqlcClient.MarkCathedralSandboxLifecycleDispatching(t.Context(), queries.MarkCathedralSandboxLifecycleDispatchingParams{ - TeamID: teamID, OperationKey: op.OperationKey, RequestSha256: digest, + dispatch, err := db.SqlcClient.MarkCathedralSandboxLifecycleDispatching(t.Context(), queries.MarkCathedralSandboxLifecycleDispatchingParams{ + LeaseDuration: pgtype.Interval{Microseconds: time.Minute.Microseconds(), Valid: true}, + TeamID: teamID, OperationKey: op.OperationKey, RequestSha256: digest, OperationKind: "pause", SandboxID: op.SandboxID, ExecutionID: op.ExecutionID, }) require.NoError(t, err) - require.Equal(t, int64(1), rows) - rows, err = db.SqlcClient.MarkCathedralSandboxLifecycleDispatching(t.Context(), queries.MarkCathedralSandboxLifecycleDispatchingParams{ - TeamID: teamID, OperationKey: op.OperationKey, RequestSha256: digest, + require.Equal(t, int32(1), dispatch.DispatchAttempt) + _, err = db.SqlcClient.MarkCathedralSandboxLifecycleDispatching(t.Context(), queries.MarkCathedralSandboxLifecycleDispatchingParams{ + LeaseDuration: pgtype.Interval{Microseconds: time.Minute.Microseconds(), Valid: true}, + TeamID: teamID, OperationKey: op.OperationKey, RequestSha256: digest, OperationKind: "pause", SandboxID: op.SandboxID, ExecutionID: op.ExecutionID, }) + require.ErrorIs(t, err, pgx.ErrNoRows, "a repeated key cannot win dispatch twice") +} + +func TestCathedralLifecycleExpiredDispatchCanBeRequeuedWithGenerationFence(t *testing.T) { + t.Parallel() + + db := testutils.SetupDatabase(t) + sqlDB, err := sql.Open("pgx", db.ConnStr()) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, sqlDB.Close()) }) + teamID := seedTeam(t, sqlDB, "cathedral-lifecycle-lease") + const digest = "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee" + + op, err := db.SqlcClient.ReserveCathedralSandboxLifecycleOperation(t.Context(), queries.ReserveCathedralSandboxLifecycleOperationParams{ + TeamID: teamID, OperationKey: "cathedral-pause-lease", RequestSha256: digest, + OperationKind: "pause", SandboxID: "sbx-lease", ExecutionID: "exec-lease", FilesystemOnly: true, + }) + require.NoError(t, err) + assert.True(t, op.FilesystemOnly) + + dispatch, err := db.SqlcClient.MarkCathedralSandboxLifecycleDispatching(t.Context(), queries.MarkCathedralSandboxLifecycleDispatchingParams{ + LeaseDuration: pgtype.Interval{Microseconds: (-time.Second).Microseconds(), Valid: true}, + TeamID: teamID, OperationKey: op.OperationKey, RequestSha256: digest, + OperationKind: op.OperationKind, SandboxID: op.SandboxID, ExecutionID: op.ExecutionID, + }) + require.NoError(t, err) + require.Equal(t, int32(1), dispatch.DispatchAttempt) + + rows, err := db.SqlcClient.RequeueExpiredCathedralSandboxLifecycleDispatch(t.Context(), queries.RequeueExpiredCathedralSandboxLifecycleDispatchParams{ + ErrorMessage: "proved no-op", TeamID: teamID, OperationKey: op.OperationKey, + RequestSha256: digest, ExecutionID: op.ExecutionID, DispatchAttempt: dispatch.DispatchAttempt + 1, + }) + require.NoError(t, err) + assert.Zero(t, rows, "a stale recovery generation must not move the active lease") + + rows, err = db.SqlcClient.RequeueExpiredCathedralSandboxLifecycleDispatch(t.Context(), queries.RequeueExpiredCathedralSandboxLifecycleDispatchParams{ + ErrorMessage: "proved no-op", TeamID: teamID, OperationKey: op.OperationKey, + RequestSha256: digest, ExecutionID: op.ExecutionID, DispatchAttempt: dispatch.DispatchAttempt, + }) + require.NoError(t, err) + require.Equal(t, int64(1), rows) + + requeued, err := db.SqlcClient.GetCathedralSandboxLifecycleOperation(t.Context(), queries.GetCathedralSandboxLifecycleOperationParams{TeamID: teamID, OperationKey: op.OperationKey}) require.NoError(t, err) - assert.Zero(t, rows, "a repeated key cannot win dispatch twice") + assert.Equal(t, "reserved", requeued.State) + assert.Nil(t, requeued.DispatchLeaseExpiresAt) } func TestCathedralSandboxOperationSurvivesAmbiguousCreateAndStoresImmutableResponse(t *testing.T) { diff --git a/packages/db/pkg/types/types.go b/packages/db/pkg/types/types.go index 671acbcc41..4355e1b4d1 100644 --- a/packages/db/pkg/types/types.go +++ b/packages/db/pkg/types/types.go @@ -171,7 +171,9 @@ type PausedSandboxConfig struct { // RemainingLifetimeSeconds freezes the unconsumed lifetime at the point a // pause transition commits. A resume without an explicit timeout restores // this value instead of silently granting a fresh default lifetime. - RemainingLifetimeSeconds uint64 `json:"remainingLifetimeSeconds,omitempty"` + // A pointer distinguishes a deliberately exhausted lifetime (zero) from a + // legacy snapshot that predates frozen-lifetime persistence (nil). + RemainingLifetimeSeconds *uint64 `json:"remainingLifetimeSeconds,omitempty"` } func (c PausedSandboxConfig) Value() (driver.Value, error) { diff --git a/packages/db/pkg/types/types_test.go b/packages/db/pkg/types/types_test.go index cd48b95825..5e2a6ff1d3 100644 --- a/packages/db/pkg/types/types_test.go +++ b/packages/db/pkg/types/types_test.go @@ -125,6 +125,28 @@ func TestPausedSandboxConfig_LegacyRowDefaultsToMemoryAutoPause(t *testing.T) { assert.True(t, decoded.FilesystemOnly, "unrelated fields must still decode") } +func TestPausedSandboxConfigDistinguishesExhaustedFromLegacyLifetime(t *testing.T) { + t.Parallel() + + zero := uint64(0) + v, err := PausedSandboxConfig{ + Version: PausedSandboxConfigVersion, RemainingLifetimeSeconds: &zero, + }.Value() + require.NoError(t, err) + raw, ok := v.(string) + require.True(t, ok) + assert.Contains(t, raw, `"remainingLifetimeSeconds":0`) + + var exhausted PausedSandboxConfig + require.NoError(t, json.Unmarshal([]byte(raw), &exhausted)) + require.NotNil(t, exhausted.RemainingLifetimeSeconds) + assert.Zero(t, *exhausted.RemainingLifetimeSeconds) + + var legacy PausedSandboxConfig + require.NoError(t, json.Unmarshal([]byte(`{"version":"v1"}`), &legacy)) + assert.Nil(t, legacy.RemainingLifetimeSeconds) +} + func TestPausedSandboxConfigHTTPSPortsRoundTrip(t *testing.T) { t.Parallel() diff --git a/packages/db/queries/cathedral_sandbox_lifecycle_operations.sql.go b/packages/db/queries/cathedral_sandbox_lifecycle_operations.sql.go index 4896304971..c682ce4304 100644 --- a/packages/db/queries/cathedral_sandbox_lifecycle_operations.sql.go +++ b/packages/db/queries/cathedral_sandbox_lifecycle_operations.sql.go @@ -1,6 +1,6 @@ // Code generated by sqlc. DO NOT EDIT. // versions: -// sqlc v1.29.0 +// sqlc v1.31.1 // source: cathedral_sandbox_lifecycle_operations.sql package queries @@ -10,6 +10,7 @@ import ( "time" "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgtype" ) const completeCathedralSandboxLifecycleOperation = `-- name: CompleteCathedralSandboxLifecycleOperation :execrows @@ -19,16 +20,19 @@ SET state = 'completed', snapshot_build_id = $2::text, snapshot_completed_at = $3::timestamptz, cleanup_state = $4::text, - result_json = $5::text, + remaining_lifetime_ms = $5::bigint, + dispatch_lease_expires_at = NULL, + result_json = $6::text, error_code = NULL, error_message = NULL, updated_at = now() -WHERE team_id = $6::uuid - AND operation_key = $7::text - AND request_sha256 = $8::text - AND operation_kind = $9::text - AND sandbox_id = $10::text - AND execution_id = $11::text +WHERE team_id = $7::uuid + AND operation_key = $8::text + AND request_sha256 = $9::text + AND operation_kind = $10::text + AND sandbox_id = $11::text + AND execution_id = $12::text + AND dispatch_attempt = $13::integer AND state = 'dispatching' ` @@ -37,6 +41,7 @@ type CompleteCathedralSandboxLifecycleOperationParams struct { SnapshotBuildID *string SnapshotCompletedAt *time.Time CleanupState string + RemainingLifetimeMs *int64 ResultJson string TeamID uuid.UUID OperationKey string @@ -44,6 +49,7 @@ type CompleteCathedralSandboxLifecycleOperationParams struct { OperationKind string SandboxID string ExecutionID string + DispatchAttempt int32 } func (q *Queries) CompleteCathedralSandboxLifecycleOperation(ctx context.Context, arg CompleteCathedralSandboxLifecycleOperationParams) (int64, error) { @@ -52,6 +58,7 @@ func (q *Queries) CompleteCathedralSandboxLifecycleOperation(ctx context.Context arg.SnapshotBuildID, arg.SnapshotCompletedAt, arg.CleanupState, + arg.RemainingLifetimeMs, arg.ResultJson, arg.TeamID, arg.OperationKey, @@ -59,6 +66,7 @@ func (q *Queries) CompleteCathedralSandboxLifecycleOperation(ctx context.Context arg.OperationKind, arg.SandboxID, arg.ExecutionID, + arg.DispatchAttempt, ) if err != nil { return 0, err @@ -68,7 +76,8 @@ func (q *Queries) CompleteCathedralSandboxLifecycleOperation(ctx context.Context const failCathedralSandboxLifecycleOperation = `-- name: FailCathedralSandboxLifecycleOperation :execrows UPDATE public.cathedral_sandbox_lifecycle_operations -SET state = 'failed', error_code = $1::integer, +SET state = 'failed', dispatch_lease_expires_at = NULL, + error_code = $1::integer, error_message = $2::text, updated_at = now() WHERE team_id = $3::uuid AND operation_key = $4::text @@ -76,18 +85,20 @@ WHERE team_id = $3::uuid AND operation_kind = $6::text AND sandbox_id = $7::text AND execution_id = $8::text + AND (state <> 'dispatching' OR dispatch_attempt = $9::integer) AND state IN ('reserved', 'dispatching', 'failed') ` type FailCathedralSandboxLifecycleOperationParams struct { - ErrorCode int32 - ErrorMessage string - TeamID uuid.UUID - OperationKey string - RequestSha256 string - OperationKind string - SandboxID string - ExecutionID string + ErrorCode int32 + ErrorMessage string + TeamID uuid.UUID + OperationKey string + RequestSha256 string + OperationKind string + SandboxID string + ExecutionID string + DispatchAttempt int32 } func (q *Queries) FailCathedralSandboxLifecycleOperation(ctx context.Context, arg FailCathedralSandboxLifecycleOperationParams) (int64, error) { @@ -100,6 +111,7 @@ func (q *Queries) FailCathedralSandboxLifecycleOperation(ctx context.Context, ar arg.OperationKind, arg.SandboxID, arg.ExecutionID, + arg.DispatchAttempt, ) if err != nil { return 0, err @@ -108,7 +120,7 @@ func (q *Queries) FailCathedralSandboxLifecycleOperation(ctx context.Context, ar } const getCathedralSandboxLifecycleOperation = `-- name: GetCathedralSandboxLifecycleOperation :one -SELECT team_id, operation_key, request_sha256, operation_kind, sandbox_id, execution_id, state, execution_removed_at, snapshot_build_id, snapshot_completed_at, remaining_lifetime_ms, cleanup_state, result_json, error_code, error_message, created_at, dispatch_started_at, updated_at +SELECT team_id, operation_key, request_sha256, operation_kind, sandbox_id, execution_id, state, execution_removed_at, snapshot_build_id, snapshot_completed_at, remaining_lifetime_ms, cleanup_state, result_json, error_code, error_message, created_at, dispatch_started_at, updated_at, filesystem_only, dispatch_attempt, dispatch_lease_expires_at FROM public.cathedral_sandbox_lifecycle_operations WHERE team_id = $1::uuid AND operation_key = $2::text @@ -141,23 +153,31 @@ func (q *Queries) GetCathedralSandboxLifecycleOperation(ctx context.Context, arg &i.CreatedAt, &i.DispatchStartedAt, &i.UpdatedAt, + &i.FilesystemOnly, + &i.DispatchAttempt, + &i.DispatchLeaseExpiresAt, ) return i, err } -const markCathedralSandboxLifecycleDispatching = `-- name: MarkCathedralSandboxLifecycleDispatching :execrows +const markCathedralSandboxLifecycleDispatching = `-- name: MarkCathedralSandboxLifecycleDispatching :one UPDATE public.cathedral_sandbox_lifecycle_operations -SET state = 'dispatching', dispatch_started_at = COALESCE(dispatch_started_at, now()), updated_at = now() -WHERE team_id = $1::uuid - AND operation_key = $2::text - AND request_sha256 = $3::text - AND operation_kind = $4::text - AND sandbox_id = $5::text - AND execution_id = $6::text +SET state = 'dispatching', dispatch_started_at = now(), + dispatch_attempt = dispatch_attempt + 1, + dispatch_lease_expires_at = now() + $1::interval, + error_code = NULL, error_message = NULL, updated_at = now() +WHERE team_id = $2::uuid + AND operation_key = $3::text + AND request_sha256 = $4::text + AND operation_kind = $5::text + AND sandbox_id = $6::text + AND execution_id = $7::text AND state = 'reserved' +RETURNING team_id, operation_key, request_sha256, operation_kind, sandbox_id, execution_id, state, execution_removed_at, snapshot_build_id, snapshot_completed_at, remaining_lifetime_ms, cleanup_state, result_json, error_code, error_message, created_at, dispatch_started_at, updated_at, filesystem_only, dispatch_attempt, dispatch_lease_expires_at ` type MarkCathedralSandboxLifecycleDispatchingParams struct { + LeaseDuration pgtype.Interval TeamID uuid.UUID OperationKey string RequestSha256 string @@ -166,8 +186,9 @@ type MarkCathedralSandboxLifecycleDispatchingParams struct { ExecutionID string } -func (q *Queries) MarkCathedralSandboxLifecycleDispatching(ctx context.Context, arg MarkCathedralSandboxLifecycleDispatchingParams) (int64, error) { - result, err := q.db.Exec(ctx, markCathedralSandboxLifecycleDispatching, +func (q *Queries) MarkCathedralSandboxLifecycleDispatching(ctx context.Context, arg MarkCathedralSandboxLifecycleDispatchingParams) (CathedralSandboxLifecycleOperation, error) { + row := q.db.QueryRow(ctx, markCathedralSandboxLifecycleDispatching, + arg.LeaseDuration, arg.TeamID, arg.OperationKey, arg.RequestSha256, @@ -175,15 +196,37 @@ func (q *Queries) MarkCathedralSandboxLifecycleDispatching(ctx context.Context, arg.SandboxID, arg.ExecutionID, ) - if err != nil { - return 0, err - } - return result.RowsAffected(), nil + var i CathedralSandboxLifecycleOperation + err := row.Scan( + &i.TeamID, + &i.OperationKey, + &i.RequestSha256, + &i.OperationKind, + &i.SandboxID, + &i.ExecutionID, + &i.State, + &i.ExecutionRemovedAt, + &i.SnapshotBuildID, + &i.SnapshotCompletedAt, + &i.RemainingLifetimeMs, + &i.CleanupState, + &i.ResultJson, + &i.ErrorCode, + &i.ErrorMessage, + &i.CreatedAt, + &i.DispatchStartedAt, + &i.UpdatedAt, + &i.FilesystemOnly, + &i.DispatchAttempt, + &i.DispatchLeaseExpiresAt, + ) + return i, err } const markCathedralSandboxLifecycleUnknown = `-- name: MarkCathedralSandboxLifecycleUnknown :execrows UPDATE public.cathedral_sandbox_lifecycle_operations -SET state = 'unknown', error_code = $1::integer, +SET state = 'unknown', dispatch_lease_expires_at = NULL, + error_code = $1::integer, error_message = $2::text, updated_at = now() WHERE team_id = $3::uuid AND operation_key = $4::text @@ -191,18 +234,20 @@ WHERE team_id = $3::uuid AND operation_kind = $6::text AND sandbox_id = $7::text AND execution_id = $8::text + AND dispatch_attempt = $9::integer AND state IN ('reserved', 'dispatching', 'unknown') ` type MarkCathedralSandboxLifecycleUnknownParams struct { - ErrorCode *int32 - ErrorMessage string - TeamID uuid.UUID - OperationKey string - RequestSha256 string - OperationKind string - SandboxID string - ExecutionID string + ErrorCode *int32 + ErrorMessage string + TeamID uuid.UUID + OperationKey string + RequestSha256 string + OperationKind string + SandboxID string + ExecutionID string + DispatchAttempt int32 } func (q *Queries) MarkCathedralSandboxLifecycleUnknown(ctx context.Context, arg MarkCathedralSandboxLifecycleUnknownParams) (int64, error) { @@ -215,6 +260,86 @@ func (q *Queries) MarkCathedralSandboxLifecycleUnknown(ctx context.Context, arg arg.OperationKind, arg.SandboxID, arg.ExecutionID, + arg.DispatchAttempt, + ) + if err != nil { + return 0, err + } + return result.RowsAffected(), nil +} + +const requeueCathedralSandboxLifecycleDispatch = `-- name: RequeueCathedralSandboxLifecycleDispatch :execrows +UPDATE public.cathedral_sandbox_lifecycle_operations +SET state = 'reserved', dispatch_lease_expires_at = NULL, + error_code = $1::integer, + error_message = $2::text, updated_at = now() +WHERE team_id = $3::uuid + AND operation_key = $4::text + AND request_sha256 = $5::text + AND execution_id = $6::text + AND state = 'dispatching' + AND dispatch_attempt = $7::integer +` + +type RequeueCathedralSandboxLifecycleDispatchParams struct { + ErrorCode *int32 + ErrorMessage string + TeamID uuid.UUID + OperationKey string + RequestSha256 string + ExecutionID string + DispatchAttempt int32 +} + +func (q *Queries) RequeueCathedralSandboxLifecycleDispatch(ctx context.Context, arg RequeueCathedralSandboxLifecycleDispatchParams) (int64, error) { + result, err := q.db.Exec(ctx, requeueCathedralSandboxLifecycleDispatch, + arg.ErrorCode, + arg.ErrorMessage, + arg.TeamID, + arg.OperationKey, + arg.RequestSha256, + arg.ExecutionID, + arg.DispatchAttempt, + ) + if err != nil { + return 0, err + } + return result.RowsAffected(), nil +} + +const requeueExpiredCathedralSandboxLifecycleDispatch = `-- name: RequeueExpiredCathedralSandboxLifecycleDispatch :execrows +UPDATE public.cathedral_sandbox_lifecycle_operations +SET state = 'reserved', dispatch_lease_expires_at = NULL, + error_code = $1::integer, + error_message = $2::text, updated_at = now() +WHERE team_id = $3::uuid + AND operation_key = $4::text + AND request_sha256 = $5::text + AND execution_id = $6::text + AND state = 'dispatching' + AND dispatch_attempt = $7::integer + AND dispatch_lease_expires_at <= now() +` + +type RequeueExpiredCathedralSandboxLifecycleDispatchParams struct { + ErrorCode *int32 + ErrorMessage string + TeamID uuid.UUID + OperationKey string + RequestSha256 string + ExecutionID string + DispatchAttempt int32 +} + +func (q *Queries) RequeueExpiredCathedralSandboxLifecycleDispatch(ctx context.Context, arg RequeueExpiredCathedralSandboxLifecycleDispatchParams) (int64, error) { + result, err := q.db.Exec(ctx, requeueExpiredCathedralSandboxLifecycleDispatch, + arg.ErrorCode, + arg.ErrorMessage, + arg.TeamID, + arg.OperationKey, + arg.RequestSha256, + arg.ExecutionID, + arg.DispatchAttempt, ) if err != nil { return 0, err @@ -225,16 +350,17 @@ func (q *Queries) MarkCathedralSandboxLifecycleUnknown(ctx context.Context, arg const reserveCathedralSandboxLifecycleOperation = `-- name: ReserveCathedralSandboxLifecycleOperation :one INSERT INTO public.cathedral_sandbox_lifecycle_operations ( team_id, operation_key, request_sha256, operation_kind, sandbox_id, - execution_id, remaining_lifetime_ms, cleanup_state + execution_id, filesystem_only, remaining_lifetime_ms, cleanup_state ) VALUES ( $1::uuid, $2::text, $3::text, $4::text, $5::text, $6::text, - $7::bigint, + $7::boolean, + $8::bigint, CASE WHEN $4::text = 'delete' THEN 'pending' ELSE 'not_required' END ) ON CONFLICT (team_id, operation_key) DO NOTHING -RETURNING team_id, operation_key, request_sha256, operation_kind, sandbox_id, execution_id, state, execution_removed_at, snapshot_build_id, snapshot_completed_at, remaining_lifetime_ms, cleanup_state, result_json, error_code, error_message, created_at, dispatch_started_at, updated_at +RETURNING team_id, operation_key, request_sha256, operation_kind, sandbox_id, execution_id, state, execution_removed_at, snapshot_build_id, snapshot_completed_at, remaining_lifetime_ms, cleanup_state, result_json, error_code, error_message, created_at, dispatch_started_at, updated_at, filesystem_only, dispatch_attempt, dispatch_lease_expires_at ` type ReserveCathedralSandboxLifecycleOperationParams struct { @@ -244,6 +370,7 @@ type ReserveCathedralSandboxLifecycleOperationParams struct { OperationKind string SandboxID string ExecutionID string + FilesystemOnly bool RemainingLifetimeMs *int64 } @@ -255,6 +382,7 @@ func (q *Queries) ReserveCathedralSandboxLifecycleOperation(ctx context.Context, arg.OperationKind, arg.SandboxID, arg.ExecutionID, + arg.FilesystemOnly, arg.RemainingLifetimeMs, ) var i CathedralSandboxLifecycleOperation @@ -277,6 +405,46 @@ func (q *Queries) ReserveCathedralSandboxLifecycleOperation(ctx context.Context, &i.CreatedAt, &i.DispatchStartedAt, &i.UpdatedAt, + &i.FilesystemOnly, + &i.DispatchAttempt, + &i.DispatchLeaseExpiresAt, ) return i, err } + +const updateCathedralSandboxLifecycleCleanup = `-- name: UpdateCathedralSandboxLifecycleCleanup :execrows +UPDATE public.cathedral_sandbox_lifecycle_operations +SET cleanup_state = $1::text, updated_at = now() +WHERE team_id = $2::uuid + AND operation_key = $3::text + AND request_sha256 = $4::text + AND operation_kind = 'delete' + AND sandbox_id = $5::text + AND execution_id = $6::text + AND state = 'completed' + AND cleanup_state IN ('pending', 'failed') +` + +type UpdateCathedralSandboxLifecycleCleanupParams struct { + CleanupState string + TeamID uuid.UUID + OperationKey string + RequestSha256 string + SandboxID string + ExecutionID string +} + +func (q *Queries) UpdateCathedralSandboxLifecycleCleanup(ctx context.Context, arg UpdateCathedralSandboxLifecycleCleanupParams) (int64, error) { + result, err := q.db.Exec(ctx, updateCathedralSandboxLifecycleCleanup, + arg.CleanupState, + arg.TeamID, + arg.OperationKey, + arg.RequestSha256, + arg.SandboxID, + arg.ExecutionID, + ) + if err != nil { + return 0, err + } + return result.RowsAffected(), nil +} diff --git a/packages/db/queries/models.go b/packages/db/queries/models.go index 77bac3d5af..8539aea9a3 100644 --- a/packages/db/queries/models.go +++ b/packages/db/queries/models.go @@ -27,24 +27,27 @@ type ActiveEnv struct { } type CathedralSandboxLifecycleOperation struct { - TeamID uuid.UUID - OperationKey string - RequestSha256 string - OperationKind string - SandboxID string - ExecutionID string - State string - ExecutionRemovedAt *time.Time - SnapshotBuildID *string - SnapshotCompletedAt *time.Time - RemainingLifetimeMs *int64 - CleanupState string - ResultJson *string - ErrorCode *int32 - ErrorMessage *string - CreatedAt time.Time - DispatchStartedAt *time.Time - UpdatedAt time.Time + TeamID uuid.UUID + OperationKey string + RequestSha256 string + OperationKind string + SandboxID string + ExecutionID string + State string + ExecutionRemovedAt *time.Time + SnapshotBuildID *string + SnapshotCompletedAt *time.Time + RemainingLifetimeMs *int64 + CleanupState string + ResultJson *string + ErrorCode *int32 + ErrorMessage *string + CreatedAt time.Time + DispatchStartedAt *time.Time + UpdatedAt time.Time + FilesystemOnly bool + DispatchAttempt int32 + DispatchLeaseExpiresAt *time.Time } type CathedralSandboxOperation struct { diff --git a/packages/db/queries/sandboxes/cathedral_sandbox_lifecycle_operations.sql b/packages/db/queries/sandboxes/cathedral_sandbox_lifecycle_operations.sql index 1bccf38a5f..6d3fa45f7f 100644 --- a/packages/db/queries/sandboxes/cathedral_sandbox_lifecycle_operations.sql +++ b/packages/db/queries/sandboxes/cathedral_sandbox_lifecycle_operations.sql @@ -1,11 +1,12 @@ -- name: ReserveCathedralSandboxLifecycleOperation :one INSERT INTO public.cathedral_sandbox_lifecycle_operations ( team_id, operation_key, request_sha256, operation_kind, sandbox_id, - execution_id, remaining_lifetime_ms, cleanup_state + execution_id, filesystem_only, remaining_lifetime_ms, cleanup_state ) VALUES ( sqlc.arg(team_id)::uuid, sqlc.arg(operation_key)::text, sqlc.arg(request_sha256)::text, sqlc.arg(operation_kind)::text, sqlc.arg(sandbox_id)::text, sqlc.arg(execution_id)::text, + sqlc.arg(filesystem_only)::boolean, sqlc.narg(remaining_lifetime_ms)::bigint, CASE WHEN sqlc.arg(operation_kind)::text = 'delete' THEN 'pending' ELSE 'not_required' END ) @@ -18,16 +19,45 @@ FROM public.cathedral_sandbox_lifecycle_operations WHERE team_id = sqlc.arg(team_id)::uuid AND operation_key = sqlc.arg(operation_key)::text; --- name: MarkCathedralSandboxLifecycleDispatching :execrows +-- name: MarkCathedralSandboxLifecycleDispatching :one UPDATE public.cathedral_sandbox_lifecycle_operations -SET state = 'dispatching', dispatch_started_at = COALESCE(dispatch_started_at, now()), updated_at = now() +SET state = 'dispatching', dispatch_started_at = now(), + dispatch_attempt = dispatch_attempt + 1, + dispatch_lease_expires_at = now() + sqlc.arg(lease_duration)::interval, + error_code = NULL, error_message = NULL, updated_at = now() WHERE team_id = sqlc.arg(team_id)::uuid AND operation_key = sqlc.arg(operation_key)::text AND request_sha256 = sqlc.arg(request_sha256)::text AND operation_kind = sqlc.arg(operation_kind)::text AND sandbox_id = sqlc.arg(sandbox_id)::text AND execution_id = sqlc.arg(execution_id)::text - AND state = 'reserved'; + AND state = 'reserved' +RETURNING *; + +-- name: RequeueExpiredCathedralSandboxLifecycleDispatch :execrows +UPDATE public.cathedral_sandbox_lifecycle_operations +SET state = 'reserved', dispatch_lease_expires_at = NULL, + error_code = sqlc.narg(error_code)::integer, + error_message = sqlc.arg(error_message)::text, updated_at = now() +WHERE team_id = sqlc.arg(team_id)::uuid + AND operation_key = sqlc.arg(operation_key)::text + AND request_sha256 = sqlc.arg(request_sha256)::text + AND execution_id = sqlc.arg(execution_id)::text + AND state = 'dispatching' + AND dispatch_attempt = sqlc.arg(dispatch_attempt)::integer + AND dispatch_lease_expires_at <= now(); + +-- name: RequeueCathedralSandboxLifecycleDispatch :execrows +UPDATE public.cathedral_sandbox_lifecycle_operations +SET state = 'reserved', dispatch_lease_expires_at = NULL, + error_code = sqlc.narg(error_code)::integer, + error_message = sqlc.arg(error_message)::text, updated_at = now() +WHERE team_id = sqlc.arg(team_id)::uuid + AND operation_key = sqlc.arg(operation_key)::text + AND request_sha256 = sqlc.arg(request_sha256)::text + AND execution_id = sqlc.arg(execution_id)::text + AND state = 'dispatching' + AND dispatch_attempt = sqlc.arg(dispatch_attempt)::integer; -- name: CompleteCathedralSandboxLifecycleOperation :execrows UPDATE public.cathedral_sandbox_lifecycle_operations @@ -36,6 +66,8 @@ SET state = 'completed', snapshot_build_id = sqlc.narg(snapshot_build_id)::text, snapshot_completed_at = sqlc.narg(snapshot_completed_at)::timestamptz, cleanup_state = sqlc.arg(cleanup_state)::text, + remaining_lifetime_ms = sqlc.narg(remaining_lifetime_ms)::bigint, + dispatch_lease_expires_at = NULL, result_json = sqlc.arg(result_json)::text, error_code = NULL, error_message = NULL, @@ -46,11 +78,13 @@ WHERE team_id = sqlc.arg(team_id)::uuid AND operation_kind = sqlc.arg(operation_kind)::text AND sandbox_id = sqlc.arg(sandbox_id)::text AND execution_id = sqlc.arg(execution_id)::text + AND dispatch_attempt = sqlc.arg(dispatch_attempt)::integer AND state = 'dispatching'; -- name: MarkCathedralSandboxLifecycleUnknown :execrows UPDATE public.cathedral_sandbox_lifecycle_operations -SET state = 'unknown', error_code = sqlc.narg(error_code)::integer, +SET state = 'unknown', dispatch_lease_expires_at = NULL, + error_code = sqlc.narg(error_code)::integer, error_message = sqlc.arg(error_message)::text, updated_at = now() WHERE team_id = sqlc.arg(team_id)::uuid AND operation_key = sqlc.arg(operation_key)::text @@ -58,11 +92,13 @@ WHERE team_id = sqlc.arg(team_id)::uuid AND operation_kind = sqlc.arg(operation_kind)::text AND sandbox_id = sqlc.arg(sandbox_id)::text AND execution_id = sqlc.arg(execution_id)::text + AND dispatch_attempt = sqlc.arg(dispatch_attempt)::integer AND state IN ('reserved', 'dispatching', 'unknown'); -- name: FailCathedralSandboxLifecycleOperation :execrows UPDATE public.cathedral_sandbox_lifecycle_operations -SET state = 'failed', error_code = sqlc.arg(error_code)::integer, +SET state = 'failed', dispatch_lease_expires_at = NULL, + error_code = sqlc.arg(error_code)::integer, error_message = sqlc.arg(error_message)::text, updated_at = now() WHERE team_id = sqlc.arg(team_id)::uuid AND operation_key = sqlc.arg(operation_key)::text @@ -70,4 +106,17 @@ WHERE team_id = sqlc.arg(team_id)::uuid AND operation_kind = sqlc.arg(operation_kind)::text AND sandbox_id = sqlc.arg(sandbox_id)::text AND execution_id = sqlc.arg(execution_id)::text + AND (state <> 'dispatching' OR dispatch_attempt = sqlc.arg(dispatch_attempt)::integer) AND state IN ('reserved', 'dispatching', 'failed'); + +-- name: UpdateCathedralSandboxLifecycleCleanup :execrows +UPDATE public.cathedral_sandbox_lifecycle_operations +SET cleanup_state = sqlc.arg(cleanup_state)::text, updated_at = now() +WHERE team_id = sqlc.arg(team_id)::uuid + AND operation_key = sqlc.arg(operation_key)::text + AND request_sha256 = sqlc.arg(request_sha256)::text + AND operation_kind = 'delete' + AND sandbox_id = sqlc.arg(sandbox_id)::text + AND execution_id = sqlc.arg(execution_id)::text + AND state = 'completed' + AND cleanup_state IN ('pending', 'failed'); From 0689baec324678c611c906946234307ac18e957f Mon Sep 17 00:00:00 2001 From: Fred E <7602667+wallscaler@users.noreply.github.com> Date: Wed, 16 Sep 2026 18:58:17 -0400 Subject: [PATCH 4/9] Fence sandbox teardown by execution --- docs/ARCHITECTURE.md | 10 ++- .../orchestrator/create_instance_test.go | 2 +- .../internal/orchestrator/delete_instance.go | 27 ++++---- .../orchestrator/delete_instance_test.go | 62 +++++++++++++++++-- .../internal/orchestrator/pause_instance.go | 1 + .../orchestrator/restore_routing_test.go | 2 +- .../internal/sandbox/sandboxtypes/storage.go | 2 +- .../storage/redis/execution_pin_test.go | 27 +++++++- .../storage/redis/expiration_index_test.go | 4 +- .../sandbox/storage/redis/operations.go | 30 ++++++--- .../internal/sandbox/storage/redis/scripts.go | 20 ++++-- packages/api/internal/sandbox/store.go | 6 +- packages/orchestrator/orchestrator.proto | 13 ++++ .../orchestrator/pkg/dummyserver/sandbox.go | 24 ++++++- .../pkg/server/delete_stop_test.go | 42 +++++++++++++ .../pkg/server/pause_admission_test.go | 34 +++++++--- .../pkg/server/sandbox_events_work_test.go | 18 +++++- packages/orchestrator/pkg/server/sandboxes.go | 39 ++++++++++-- .../pkg/grpc/orchestrator/orchestrator.pb.go | 52 +++++++++++++--- 19 files changed, 349 insertions(+), 66 deletions(-) create mode 100644 packages/orchestrator/pkg/server/delete_stop_test.go diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 31704bb92c..f1424cf4de 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -212,6 +212,10 @@ under `pkg/`, almost all Linux-only. gRPC services on :5008 (`pkg/server/`, `pkg/service/`, `pkg/template/server/`, `pkg/volumes/`): - **SandboxService** — `Create`, `Update`, `List`, `Delete`, `Pause`, `Checkpoint`. + `Delete` and `Pause` are execution-fenced: callers provide the expected + execution ID and the node rejects a stale operation rather than act on a + replacement incarnation. Delete is asynchronous by default; evidence-bound + callers can request that it wait for the Firecracker stop result. - **TemplateService** — `TemplateCreate`, `TemplateBuildStatus`, `TemplateBuildDelete` (template-manager role only). - **InfoService** — node identity, roles, capacity, health status (used by API node discovery). - **ChunkService / VolumeService** — peer-to-peer template chunk serving; persistent volumes. @@ -561,7 +565,11 @@ sequenceDiagram - **Cathedral lifecycle evidence**: the Cathedral-only lifecycle endpoint binds the authenticated team, sandbox ID, execution ID, request digest, operation kind, and idempotency key in Postgres before dispatch. Completion is derived from the execution-bound node RPC, never from a missing - Redis/API listing. An already-running transition or transport ambiguity remains `unknown` and is + Redis/API listing. Delete completion waits for the exact execution's Firecracker stop to return + successfully; legacy delete callers retain the asynchronous node RPC mode. Final running-sandbox + removal in Redis compares the expected execution atomically, so delayed cleanup for one execution + cannot delete a replacement installed by a lockless resume. An already-running transition or + transport ambiguity remains `unknown` and is recovered by operation key without redispatch. Pause completion additionally records the successful snapshot build; delete records snapshot/storage cleanup separately. The remaining lifetime is frozen into the paused snapshot and reused by a resume that does not explicitly diff --git a/packages/api/internal/orchestrator/create_instance_test.go b/packages/api/internal/orchestrator/create_instance_test.go index 78c0362b23..bd9080f088 100644 --- a/packages/api/internal/orchestrator/create_instance_test.go +++ b/packages/api/internal/orchestrator/create_instance_test.go @@ -198,7 +198,7 @@ func TestCreateSandbox_StaleDataAfterConcurrentPause(t *testing.T) { assert.Equal(t, "base-tpl", sbx1.BaseTemplateID) // Clean up reservation. - o.sandboxStore.Remove(t.Context(), team.Team.ID, sandboxID) + o.sandboxStore.Remove(t.Context(), team.Team.ID, sandboxID, sbx1.ExecutionID) // Snapshot changes to V2. snap.templateID = "tpl-v2" diff --git a/packages/api/internal/orchestrator/delete_instance.go b/packages/api/internal/orchestrator/delete_instance.go index 4b774e456b..d7f9a42a93 100644 --- a/packages/api/internal/orchestrator/delete_instance.go +++ b/packages/api/internal/orchestrator/delete_instance.go @@ -56,7 +56,7 @@ func (o *Orchestrator) RemoveSandboxWithEvidence(ctx context.Context, teamID uui return o.removeSandbox(ctx, teamID, sandboxID, opts, true) } -func (o *Orchestrator) removeSandbox(ctx context.Context, teamID uuid.UUID, sandboxID string, opts sandbox.RemoveOpts, waitForStorage bool) (SandboxRemovalEvidence, error) { +func (o *Orchestrator) removeSandbox(ctx context.Context, teamID uuid.UUID, sandboxID string, opts sandbox.RemoveOpts, waitForCompletion bool) (SandboxRemovalEvidence, error) { ctx, span := tracer.Start(ctx, "remove-sandbox") defer span.End() evidence := SandboxRemovalEvidence{SandboxID: sandboxID, ExecutionID: opts.ExpectExecutionID, Action: opts.Action} @@ -147,7 +147,7 @@ func (o *Orchestrator) removeSandbox(ctx context.Context, teamID uuid.UUID, sand logger.L().Info(ctx, "Sandbox was already in the process of being removed", logger.WithSandboxID(sandboxID), zap.String("state", string(sbx.State))) if time.Since(sbx.EndTime) > sandbox.StaleCutoff && opts.Action.Effect == sandbox.TransitionExpires { - o.sandboxStore.Remove(context.WithoutCancel(ctx), teamID, sandboxID) + o.sandboxStore.Remove(context.WithoutCancel(ctx), teamID, sandboxID, sbx.ExecutionID) go o.analyticsRemove(context.WithoutCancel(ctx), sbx, opts.Action) } @@ -180,11 +180,11 @@ func (o *Orchestrator) removeSandbox(ctx context.Context, teamID uuid.UUID, sand if preserveRecord { return } - o.sandboxStore.Remove(context.WithoutCancel(ctx), teamID, sandboxID) + o.sandboxStore.Remove(context.WithoutCancel(ctx), teamID, sandboxID, sbx.ExecutionID) go o.analyticsRemove(context.WithoutCancel(ctx), sbx, opts.Action) }() var snapshotBuildID string - snapshotBuildID, err = o.removeSandboxFromNodeWithEvidence(ctx, sbx, opts.Action, opts.Reason, opts.FilesystemOnly, restoreOnRefusal, evidence.RemainingLifetime, waitForStorage) + snapshotBuildID, err = o.removeSandboxFromNodeWithEvidence(ctx, sbx, opts.Action, opts.Reason, opts.FilesystemOnly, restoreOnRefusal, evidence.RemainingLifetime, waitForCompletion) if err != nil { if errors.Is(err, PauseQueueExhaustedError{}) { if restoreOnRefusal { @@ -280,7 +280,7 @@ func (o *Orchestrator) killRefusedSandbox(ctx context.Context, sbx sandbox.Sandb return } - if err := o.killSandboxOnNode(ctx, node, sbx.ToNodeSandbox(), sandbox.KillReasonOrphaned); err != nil { + if err := o.killSandboxOnNode(ctx, node, sbx.ToNodeSandbox(), sandbox.KillReasonOrphaned, false); err != nil { logger.L().Error(ctx, "failed to kill a refused sandbox after a failed restore", zap.Error(err), logger.WithSandboxID(sbx.SandboxID)) } } @@ -359,7 +359,7 @@ func (o *Orchestrator) removeSandboxFromNodeWithEvidence( filesystemOnly bool, restoreOnRefusal bool, remainingLifetime time.Duration, - waitForStorage bool, + waitForCompletion bool, ) (string, error) { ctx, span := tracer.Start(ctx, "remove-sandbox-from-node") defer span.End() @@ -402,10 +402,10 @@ func (o *Orchestrator) removeSandboxFromNodeWithEvidence( switch stateAction { case sandbox.StateActionPause: - buildID, err := o.pauseSandboxWithEvidence(ctx, node, sbx, filesystemOnly, restoreOnRefusal, remainingLifetime, waitForStorage) + buildID, err := o.pauseSandboxWithEvidence(ctx, node, sbx, filesystemOnly, restoreOnRefusal, remainingLifetime, waitForCompletion) if err != nil { if dberrors.IsForeignKeyViolation(err) { - killErr := o.killSandboxOnNode(ctx, node, sbx.ToNodeSandbox(), sandbox.KillReasonBaseTemplateMissing) + killErr := o.killSandboxOnNode(ctx, node, sbx.ToNodeSandbox(), sandbox.KillReasonBaseTemplateMissing, false) logger.L().Error(ctx, "Pause failed due to missing base template, killed sandbox as fallback", logger.WithSandboxID(sbx.SandboxID), zap.String("base_template_id", sbx.BaseTemplateID), @@ -422,7 +422,7 @@ func (o *Orchestrator) removeSandboxFromNodeWithEvidence( return buildID, nil case sandbox.StateActionKill: - return "", o.killSandboxOnNode(ctx, node, sbx.ToNodeSandbox(), reason) + return "", o.killSandboxOnNode(ctx, node, sbx.ToNodeSandbox(), reason, waitForCompletion) } return "", nil @@ -440,7 +440,7 @@ func (o *Orchestrator) killOrphanSandbox(ctx context.Context, sbx sandbox.NodeSa return } - err := o.killSandboxOnNode(ctx, node, sbx, sandbox.KillReasonOrphaned) + err := o.killSandboxOnNode(ctx, node, sbx, sandbox.KillReasonOrphaned, false) if err != nil { logger.L().Error(ctx, "Failed to kill orphan sandbox on node", zap.Error(err), @@ -456,11 +456,14 @@ func (o *Orchestrator) killSandboxOnNode( node *nodemanager.Node, sbx sandbox.NodeSandbox, reason sandbox.KillReason, + waitForStop bool, ) error { killReason := reason.String() req := &orchestrator.SandboxDeleteRequest{ - SandboxId: sbx.SandboxID, - KillReason: &killReason, + SandboxId: sbx.SandboxID, + KillReason: &killReason, + ExecutionId: sbx.ExecutionID, + WaitForStop: waitForStop, } client, ctx := node.GetSandboxDeleteCtx(ctx, sbx.SandboxID, sbx.ExecutionID, false) diff --git a/packages/api/internal/orchestrator/delete_instance_test.go b/packages/api/internal/orchestrator/delete_instance_test.go index 758bceb424..44972fdc42 100644 --- a/packages/api/internal/orchestrator/delete_instance_test.go +++ b/packages/api/internal/orchestrator/delete_instance_test.go @@ -52,14 +52,17 @@ type pauseStubClient struct { // the record underneath the restore. onPause func() - mu sync.Mutex - deletes int + mu sync.Mutex + deletes int + lastDelete *orchestrator.SandboxDeleteRequest + lastPause *orchestrator.SandboxPauseRequest } -func (c *pauseStubClient) Delete(context.Context, *orchestrator.SandboxDeleteRequest, ...grpc.CallOption) (*emptypb.Empty, error) { +func (c *pauseStubClient) Delete(_ context.Context, request *orchestrator.SandboxDeleteRequest, _ ...grpc.CallOption) (*emptypb.Empty, error) { c.mu.Lock() defer c.mu.Unlock() c.deletes++ + c.lastDelete = request return &emptypb.Empty{}, c.deleteErr } @@ -71,7 +74,24 @@ func (c *pauseStubClient) deleteCount() int { return c.deletes } +func (c *pauseStubClient) lastDeleteRequest() *orchestrator.SandboxDeleteRequest { + c.mu.Lock() + defer c.mu.Unlock() + + return c.lastDelete +} + +func (c *pauseStubClient) lastPauseRequest() *orchestrator.SandboxPauseRequest { + c.mu.Lock() + defer c.mu.Unlock() + + return c.lastPause +} + func (c *pauseStubClient) Pause(_ context.Context, request *orchestrator.SandboxPauseRequest, _ ...grpc.CallOption) (*orchestrator.SandboxPauseResponse, error) { + c.mu.Lock() + c.lastPause = request + c.mu.Unlock() if c.gate != nil { <-c.gate } @@ -119,6 +139,7 @@ type refusalFixture struct { recorder *recordingCollector sbx sandbox.Sandbox reader *sdkmetric.ManualReader + client *pauseStubClient } // restoreOutcomes returns the pause-refusal-restore counter by (outcome, caller). @@ -189,7 +210,8 @@ func newRefusalFixture(t *testing.T, restoreFlag bool, clusterID uuid.UUID, paus node := nodemanager.NewTestNode("node-1", api.NodeStatusReady, 0, 8) node.ClusterID = clusterID - node.SetSandboxClient(&pauseStubClient{err: pauseErr}) + client := &pauseStubClient{err: pauseErr} + node.SetSandboxClient(client) recorder := &recordingCollector{} reader := sdkmetric.NewManualReader() @@ -235,7 +257,7 @@ func newRefusalFixture(t *testing.T, restoreFlag bool, clusterID uuid.UUID, paus } require.NoError(t, o.sandboxStore.Add(t.Context(), sbx, nil)) - return refusalFixture{o: o, recorder: recorder, sbx: sbx, reader: reader} + return refusalFixture{o: o, recorder: recorder, sbx: sbx, reader: reader, client: client} } func (f refusalFixture) removePause(t *testing.T) error { @@ -425,7 +447,7 @@ func TestRemoveSandbox_SupersededRefusalLeavesNewIncarnationAlone(t *testing.T) stub := &pauseStubClient{err: refusedPauseErr()} stub.onPause = func() { - f.o.sandboxStore.Remove(t.Context(), f.sbx.TeamID, f.sbx.SandboxID) + f.o.sandboxStore.Remove(t.Context(), f.sbx.TeamID, f.sbx.SandboxID, f.sbx.ExecutionID) require.NoError(t, f.o.sandboxStore.Add(t.Context(), resumed, nil)) } node.SetSandboxClient(stub) @@ -540,6 +562,34 @@ func TestRemoveSandboxWithEvidence_ConfirmedPauseCarriesSnapshotAndRemainingLife assert.Equal(t, f.sbx.ExecutionID, evidence.ExecutionID) assert.NotEmpty(t, evidence.SnapshotBuildID) assert.InDelta(t, time.Hour.Seconds(), evidence.RemainingLifetime.Seconds(), 5) + require.Equal(t, f.sbx.ExecutionID, f.client.lastPauseRequest().GetExecutionId()) + require.True(t, f.client.lastPauseRequest().GetWaitForStorage()) +} + +func TestRemoveSandboxWithEvidence_DeleteWaitsForExecutionStop(t *testing.T) { + t.Parallel() + + f := newRefusalFixture(t, true, consts.LocalClusterID, nil) + evidence, err := f.o.RemoveSandboxWithEvidence(t.Context(), f.sbx.TeamID, f.sbx.SandboxID, sandbox.RemoveOpts{ + Action: sandbox.StateActionKill, ExpectExecutionID: f.sbx.ExecutionID, + }) + require.NoError(t, err) + require.True(t, evidence.Confirmed) + req := f.client.lastDeleteRequest() + require.Equal(t, f.sbx.ExecutionID, req.GetExecutionId()) + require.True(t, req.GetWaitForStop()) +} + +func TestRemoveSandbox_LegacyDeleteRemainsAsync(t *testing.T) { + t.Parallel() + + f := newRefusalFixture(t, true, consts.LocalClusterID, nil) + require.NoError(t, f.o.RemoveSandbox(t.Context(), f.sbx.TeamID, f.sbx.SandboxID, sandbox.RemoveOpts{ + Action: sandbox.StateActionKill, + })) + req := f.client.lastDeleteRequest() + require.Equal(t, f.sbx.ExecutionID, req.GetExecutionId()) + require.False(t, req.GetWaitForStop()) } func TestRemoveSandboxWithEvidence_InFlightRemovalIsNeverTerminalProof(t *testing.T) { diff --git a/packages/api/internal/orchestrator/pause_instance.go b/packages/api/internal/orchestrator/pause_instance.go index 8287a7cb92..ee23171538 100644 --- a/packages/api/internal/orchestrator/pause_instance.go +++ b/packages/api/internal/orchestrator/pause_instance.go @@ -99,6 +99,7 @@ func snapshotInstance(ctx context.Context, node *nodemanager.Node, sbx sandbox.S BuildId: buildID, FilesystemOnly: filesystemOnly, WaitForStorage: waitForStorage, + ExecutionId: sbx.ExecutionID, }, ) diff --git a/packages/api/internal/orchestrator/restore_routing_test.go b/packages/api/internal/orchestrator/restore_routing_test.go index 905a613c03..d5f80acbc0 100644 --- a/packages/api/internal/orchestrator/restore_routing_test.go +++ b/packages/api/internal/orchestrator/restore_routing_test.go @@ -75,7 +75,7 @@ func TestRemoveSandbox_RefusalRouteRestorePreservesSuccessor(t *testing.T) { require.Equal(t, f.sbx.ExecutionID, stored.ExecutionID) if tc.removeOnly { - f.o.sandboxStore.Remove(t.Context(), f.sbx.TeamID, f.sbx.SandboxID) + f.o.sandboxStore.Remove(t.Context(), f.sbx.TeamID, f.sbx.SandboxID, f.sbx.ExecutionID) return } diff --git a/packages/api/internal/sandbox/sandboxtypes/storage.go b/packages/api/internal/sandbox/sandboxtypes/storage.go index 072c8afbdb..1d100ddf05 100644 --- a/packages/api/internal/sandbox/sandboxtypes/storage.go +++ b/packages/api/internal/sandbox/sandboxtypes/storage.go @@ -15,7 +15,7 @@ const ( type Storage interface { Add(ctx context.Context, sandbox Sandbox) error Get(ctx context.Context, teamID uuid.UUID, sandboxID string) (Sandbox, error) - Remove(ctx context.Context, teamID uuid.UUID, sandboxID string) error + Remove(ctx context.Context, teamID uuid.UUID, sandboxID string, executionID string) error TeamItems(ctx context.Context, teamID uuid.UUID, states []State) ([]Sandbox, error) ExpiredItems(ctx context.Context) ([]Sandbox, error) diff --git a/packages/api/internal/sandbox/storage/redis/execution_pin_test.go b/packages/api/internal/sandbox/storage/redis/execution_pin_test.go index 35675a7e57..90981ba67e 100644 --- a/packages/api/internal/sandbox/storage/redis/execution_pin_test.go +++ b/packages/api/internal/sandbox/storage/redis/execution_pin_test.go @@ -231,7 +231,7 @@ func TestStartTransitionScript_RefusesADeletedRecord(t *testing.T) { sbx := createTestSandbox("sbx-cas-deleted") require.NoError(t, storage.Add(ctx, sbx)) - require.NoError(t, storage.Remove(ctx, sbx.TeamID, sbx.SandboxID)) + require.NoError(t, storage.Remove(ctx, sbx.TeamID, sbx.SandboxID, sbx.ExecutionID)) transitionID := uuid.NewString() keys := transitionKeysFor(sbx, transitionID) @@ -271,6 +271,31 @@ func TestStartTransitionScript_UnpinnedWritesUnconditionally(t *testing.T) { assert.Equal(t, int64(1), written) } +func TestRemove_StaleCleanupCannotDeleteReplacementExecution(t *testing.T) { + t.Parallel() + + storage, client := setupTestStorage(t) + ctx := t.Context() + + old := createTestSandbox("sbx-final-remove-race") + replacement := old + replacement.ExecutionID = uuid.NewString() + replacement.EndTime = replacement.EndTime.Add(time.Hour) + + require.NoError(t, storage.Add(ctx, old)) + // Add is intentionally lockless. This models E2 landing after E1's node + // cleanup began but before E1's deferred Redis removal runs. + require.NoError(t, storage.Add(ctx, replacement)) + + err := storage.Remove(ctx, old.TeamID, old.SandboxID, old.ExecutionID) + require.ErrorIs(t, err, sandboxtypes.ErrExecutionMismatch) + + stored, err := storage.Get(ctx, replacement.TeamID, replacement.SandboxID) + require.NoError(t, err) + require.Equal(t, replacement.ExecutionID, stored.ExecutionID) + require.True(t, client.SIsMember(ctx, GetSandboxStorageTeamIndexKey(replacement.TeamID.String()), replacement.SandboxID).Val()) +} + // TestStartRemoving_NoExecutionPinRemovesWhateverIsStored keeps the guard // opt-in: callers acting on user intent or a fresh read must not be forced to // supply an execution ID. diff --git a/packages/api/internal/sandbox/storage/redis/expiration_index_test.go b/packages/api/internal/sandbox/storage/redis/expiration_index_test.go index cc8e057c0c..a869639c4a 100644 --- a/packages/api/internal/sandbox/storage/redis/expiration_index_test.go +++ b/packages/api/internal/sandbox/storage/redis/expiration_index_test.go @@ -91,7 +91,7 @@ func TestAddRemove_ExecutionScopedMember(t *testing.T) { member := expirationMember(teamID.String(), sbx.SandboxID, sbx.ExecutionID) requireMemberScore(t, client, member, float64(sbx.EndTime.UnixMilli())) - require.NoError(t, storage.Remove(t.Context(), teamID, sbx.SandboxID)) + require.NoError(t, storage.Remove(t.Context(), teamID, sbx.SandboxID, sbx.ExecutionID)) requireMemberAbsent(t, client, member) err := client.Get(t.Context(), getSandboxKey(teamID.String(), sbx.SandboxID)).Err() @@ -121,7 +121,7 @@ func TestRemove_DoesNotUnindexFreshExecution(t *testing.T) { Member: freshMember, }).Err()) - require.NoError(t, storage.Remove(t.Context(), teamID, sandboxID)) + require.NoError(t, storage.Remove(t.Context(), teamID, sandboxID, old.ExecutionID)) // Old execution's member removed, fresh execution's member intact. requireMemberAbsent(t, client, expirationMember(teamID.String(), sandboxID, old.ExecutionID)) diff --git a/packages/api/internal/sandbox/storage/redis/operations.go b/packages/api/internal/sandbox/storage/redis/operations.go index 210a20c43d..da4f2c662b 100644 --- a/packages/api/internal/sandbox/storage/redis/operations.go +++ b/packages/api/internal/sandbox/storage/redis/operations.go @@ -75,7 +75,11 @@ func (s *Storage) Get(ctx context.Context, teamID uuid.UUID, sandboxID string) ( } // Remove deletes a sandbox from Redis atomically with its team index entry. -func (s *Storage) Remove(ctx context.Context, teamID uuid.UUID, sandboxID string) error { +func (s *Storage) Remove(ctx context.Context, teamID uuid.UUID, sandboxID string, executionID string) error { + if executionID == "" { + return errors.New("expected execution ID is required to remove sandbox") + } + key := getSandboxKey(teamID.String(), sandboxID) teamKey := GetSandboxStorageTeamIndexKey(teamID.String()) @@ -92,13 +96,25 @@ func (s *Storage) Remove(ctx context.Context, teamID uuid.UUID, sandboxID string } }() - // Execute Lua script for atomic DEL + SREM; it returns the deleted JSON - // so the expiration-index cleanup below is scoped to the execution we - // actually removed. - raw, err := removeSandboxScript.Run(ctx, s.redisClient, []string{key, teamKey}, sandboxID).Text() - if err != nil && !errors.Is(err, redis.Nil) { + // Execute the execution compare plus DEL + SREM atomically. The script + // returns the deleted JSON so expiration cleanup is scoped to the execution + // it actually removed. + result, err := removeSandboxScript.Run(ctx, s.redisClient, []string{key, teamKey}, sandboxID, executionID).Slice() + if err != nil { return fmt.Errorf("failed to remove sandbox from Redis: %w", err) } + if len(result) != 2 { + return fmt.Errorf("failed to remove sandbox from Redis: unexpected script response %v", result) + } + outcome, ok := result[0].(int64) + if !ok { + return fmt.Errorf("failed to remove sandbox from Redis: unexpected script outcome %T", result[0]) + } + if outcome == 2 { + return fmt.Errorf("sandbox %q: %w", sandboxID, sandboxtypes.ErrExecutionMismatch) + } + + raw, _ := result[1].(string) // Clean up from the global expiration index. // Do it after the removal to prevent leaking expired sandboxes. @@ -106,7 +122,7 @@ func (s *Storage) Remove(ctx context.Context, teamID uuid.UUID, sandboxID string // Add for a newer execution wrote a different member, so it can never be // unindexed here. If the key was already gone, any leftover execution // member is swept by ExpiredItems once its score passes. - if raw != "" { + if outcome == 1 && raw != "" { var sbx sandboxtypes.Sandbox if unmarshalErr := json.Unmarshal([]byte(raw), &sbx); unmarshalErr == nil && sbx.ExecutionID != "" { member := expirationMember(teamID.String(), sandboxID, sbx.ExecutionID) diff --git a/packages/api/internal/sandbox/storage/redis/scripts.go b/packages/api/internal/sandbox/storage/redis/scripts.go index b1ed3f46bb..41ff400b36 100644 --- a/packages/api/internal/sandbox/storage/redis/scripts.go +++ b/packages/api/internal/sandbox/storage/redis/scripts.go @@ -35,17 +35,25 @@ var ( return 1 `) - // removeSandboxScript atomically removes a sandbox and its team index entry. - // It returns the stored JSON (or nil if the key was already gone) so the - // caller knows exactly which execution it removed and can scope the - // expiration-index cleanup to that execution. + // removeSandboxScript atomically compares the stored execution, then removes + // the sandbox and its team index entry. It returns {outcome, stored JSON}, + // where outcome is 0 for already absent, 1 for deleted, and 2 for an + // execution mismatch or unreadable record. // KEYS[1] = sandbox key, KEYS[2] = team index key - // ARGV[1] = sandbox ID + // ARGV[1] = sandbox ID, ARGV[2] = expected execution ID removeSandboxScript = redis.NewScript(` local data = redis.call('GET', KEYS[1]) + if not data then + redis.call('SREM', KEYS[2], ARGV[1]) + return {0, false} + end + local ok, decoded = pcall(cjson.decode, data) + if not ok or type(decoded) ~= 'table' or decoded['executionID'] ~= ARGV[2] then + return {2, data} + end redis.call('DEL', KEYS[1]) redis.call('SREM', KEYS[2], ARGV[1]) - return data + return {1, data} `) // startTransitionScript atomically updates sandbox and sets transition key with UUID. diff --git a/packages/api/internal/sandbox/store.go b/packages/api/internal/sandbox/store.go index f4cb375f0b..7ce2841a3b 100644 --- a/packages/api/internal/sandbox/store.go +++ b/packages/api/internal/sandbox/store.go @@ -101,10 +101,12 @@ func (s *Store) Get(ctx context.Context, teamID uuid.UUID, sandboxID string) (Sa return s.storage.Get(ctx, teamID, sandboxID) } -func (s *Store) Remove(ctx context.Context, teamID uuid.UUID, sandboxID string) { - err := s.storage.Remove(ctx, teamID, sandboxID) +func (s *Store) Remove(ctx context.Context, teamID uuid.UUID, sandboxID string, executionID string) { + err := s.storage.Remove(ctx, teamID, sandboxID, executionID) if err != nil { logger.L().Error(ctx, "Failed to remove sandbox from storage", zap.Error(err), logger.WithSandboxID(sandboxID)) + + return } err = s.reservations.Release(ctx, teamID, sandboxID) diff --git a/packages/orchestrator/orchestrator.proto b/packages/orchestrator/orchestrator.proto index 1f26652be7..5279d291b1 100644 --- a/packages/orchestrator/orchestrator.proto +++ b/packages/orchestrator/orchestrator.proto @@ -177,6 +177,15 @@ message SandboxUpdateRequest { message SandboxDeleteRequest { string sandbox_id = 1; optional string kill_reason = 2; + + // The exact sandbox execution the caller intends to stop. The node rejects + // the operation when this does not match the live incarnation, preventing a + // delayed delete from killing a replacement execution with the same ID. + string execution_id = 3; + + // Wait until the Firecracker stop has completed and surface any stop error. + // False preserves the legacy asynchronous delete behavior. + bool wait_for_stop = 4; } message SandboxPauseRequest { @@ -193,6 +202,10 @@ message SandboxPauseRequest { // storage before pause can be reported terminal. Existing callers keep the // asynchronous upload path when this is false. bool wait_for_storage = 5; + + // The exact sandbox execution the caller intends to pause. The node rejects + // the operation when this does not match the live incarnation. + string execution_id = 6; } message SchedulingMetadata { diff --git a/packages/orchestrator/pkg/dummyserver/sandbox.go b/packages/orchestrator/pkg/dummyserver/sandbox.go index f50d2fc1f9..28e66f5a3b 100644 --- a/packages/orchestrator/pkg/dummyserver/sandbox.go +++ b/packages/orchestrator/pkg/dummyserver/sandbox.go @@ -126,10 +126,20 @@ func (s *SandboxServer) Delete(_ context.Context, req *orchestrator.SandboxDelet if req.GetSandboxId() == "" { return nil, status.Error(codes.InvalidArgument, "sandbox_id is required") } + if req.GetExecutionId() == "" { + return nil, status.Error(codes.InvalidArgument, "execution_id is required") + } s.mu.Lock() + defer s.mu.Unlock() + sbx, ok := s.sandboxes[req.GetSandboxId()] + if !ok { + return nil, status.Errorf(codes.NotFound, "sandbox %q not found", req.GetSandboxId()) + } + if sbx.GetExecutionId() != req.GetExecutionId() { + return nil, status.Errorf(codes.FailedPrecondition, "sandbox %q execution changed", req.GetSandboxId()) + } delete(s.sandboxes, req.GetSandboxId()) - s.mu.Unlock() return &emptypb.Empty{}, nil } @@ -138,11 +148,21 @@ func (s *SandboxServer) Pause(_ context.Context, req *orchestrator.SandboxPauseR if req.GetSandboxId() == "" { return nil, status.Error(codes.InvalidArgument, "sandbox_id is required") } + if req.GetExecutionId() == "" { + return nil, status.Error(codes.InvalidArgument, "execution_id is required") + } // Pause is treated as a delete in the dummy: no real snapshotting happens. s.mu.Lock() + defer s.mu.Unlock() + sbx, ok := s.sandboxes[req.GetSandboxId()] + if !ok { + return nil, status.Errorf(codes.NotFound, "sandbox %q not found", req.GetSandboxId()) + } + if sbx.GetExecutionId() != req.GetExecutionId() { + return nil, status.Errorf(codes.FailedPrecondition, "sandbox %q execution changed", req.GetSandboxId()) + } delete(s.sandboxes, req.GetSandboxId()) - s.mu.Unlock() return &orchestrator.SandboxPauseResponse{}, nil } diff --git a/packages/orchestrator/pkg/server/delete_stop_test.go b/packages/orchestrator/pkg/server/delete_stop_test.go new file mode 100644 index 0000000000..f513761bd4 --- /dev/null +++ b/packages/orchestrator/pkg/server/delete_stop_test.go @@ -0,0 +1,42 @@ +//go:build linux + +package server + +import ( + "context" + "errors" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestRunDeleteStop_WaitReturnsExactStopError(t *testing.T) { + t.Parallel() + + want := errors.New("firecracker stop failed") + err := runDeleteStop(t.Context(), true, func(context.Context) error { return want }) + require.ErrorIs(t, err, want) +} + +func TestRunDeleteStop_LegacyReturnsBeforeStopCompletes(t *testing.T) { + t.Parallel() + + entered := make(chan context.Context, 1) + release := make(chan struct{}) + done := make(chan struct{}) + ctx, cancel := context.WithCancel(t.Context()) + cancel() + + require.NoError(t, runDeleteStop(ctx, false, func(stopCtx context.Context) error { + entered <- stopCtx + <-release + close(done) + + return nil + })) + + stopCtx := <-entered + require.NoError(t, stopCtx.Err(), "legacy stop must outlive caller cancellation") + close(release) + <-done +} diff --git a/packages/orchestrator/pkg/server/pause_admission_test.go b/packages/orchestrator/pkg/server/pause_admission_test.go index 220c1982b3..d9418a8ed7 100644 --- a/packages/orchestrator/pkg/server/pause_admission_test.go +++ b/packages/orchestrator/pkg/server/pause_admission_test.go @@ -140,7 +140,7 @@ func admissionTestSandbox(t *testing.T, sandboxID string, slotIdx int, durable * Envd: sandbox.EnvdMetadata{Version: "9.9.9"}, FirecrackerConfig: fc.Config{FirecrackerVersion: "v1.14.1", KernelVersion: "vmlinux-6.1"}, }), - Runtime: sandboxtypes.RuntimeMetadata{SandboxID: sandboxID}, + Runtime: sandboxtypes.RuntimeMetadata{SandboxID: sandboxID, ExecutionID: sandboxID}, }, Resources: &sandbox.Resources{Slot: slot}, Template: admissionTestTemplate{memfile: &admissionRODevice{durable: durable, waiting: make(chan struct{})}}, @@ -158,7 +158,7 @@ func TestPause_AdmissionRefusesBeforeMarkStopping(t *testing.T) { s.sandboxFactory.Sandboxes.MarkRunning(t.Context(), sbx) start := time.Now() - _, pauseErr := s.Pause(t.Context(), &orchestrator.SandboxPauseRequest{SandboxId: "sbx-admission-refuse"}) + _, pauseErr := s.Pause(t.Context(), &orchestrator.SandboxPauseRequest{SandboxId: "sbx-admission-refuse", ExecutionId: "sbx-admission-refuse"}) elapsed := time.Since(start) require.Error(t, pauseErr) @@ -176,6 +176,22 @@ func TestPause_AdmissionRefusesBeforeMarkStopping(t *testing.T) { "a refused pause must leave the sandbox unmarked") } +func TestPause_StaleExecutionCannotPauseReplacement(t *testing.T) { + t.Parallel() + + s := admissionTestServer(t, new(0)) + sbx := admissionTestSandbox(t, "sbx-stale-pause", 31, utils.NewSetOnce[*header.Header]()) + require.NoError(t, s.sandboxFactory.Sandboxes.MarkRunning(t.Context(), sbx)) + + _, err := s.Pause(t.Context(), &orchestrator.SandboxPauseRequest{ + SandboxId: sbx.Runtime.SandboxID, ExecutionId: "stale-execution", + }) + require.Equal(t, codes.FailedPrecondition, status.Code(err)) + got, live := s.sandboxFactory.Sandboxes.Get(sbx.Runtime.SandboxID) + require.True(t, live) + require.Same(t, sbx, got) +} + // The swap resolving mid-grace admits the pause, which // then proceeds into the destructive path (MarkStopping crossed). func TestPause_AdmissionAdmitsWhenSwapResolvesMidGrace(t *testing.T) { @@ -196,7 +212,7 @@ func TestPause_AdmissionAdmitsWhenSwapResolvesMidGrace(t *testing.T) { // Pause parks forever in the fake template's Metadata once admitted. go func() { - _, _ = s.Pause(context.WithoutCancel(t.Context()), &orchestrator.SandboxPauseRequest{SandboxId: "sbx-admission-midgrace"}) + _, _ = s.Pause(context.WithoutCancel(t.Context()), &orchestrator.SandboxPauseRequest{SandboxId: "sbx-admission-midgrace", ExecutionId: "sbx-admission-midgrace"}) }() require.Eventually(t, func() bool { @@ -234,7 +250,7 @@ func TestPause_FlagOffRunsTodaysOrder(t *testing.T) { done := make(chan struct{}) go func() { defer close(done) - _, _ = s.Pause(context.WithoutCancel(t.Context()), &orchestrator.SandboxPauseRequest{SandboxId: sandboxID}) + _, _ = s.Pause(context.WithoutCancel(t.Context()), &orchestrator.SandboxPauseRequest{SandboxId: sandboxID, ExecutionId: sandboxID}) }() // Today's order: MarkStopping happens promptly — no admission @@ -268,7 +284,7 @@ func TestPause_AdmissionInstantProbeRefuses(t *testing.T) { sbx := admissionTestSandbox(t, "sbx-admission-instant", 17, utils.NewSetOnce[*header.Header]()) s.sandboxFactory.Sandboxes.MarkRunning(t.Context(), sbx) - _, pauseErr := s.Pause(t.Context(), &orchestrator.SandboxPauseRequest{SandboxId: "sbx-admission-instant"}) + _, pauseErr := s.Pause(t.Context(), &orchestrator.SandboxPauseRequest{SandboxId: "sbx-admission-instant", ExecutionId: "sbx-admission-instant"}) require.Error(t, pauseErr) st, ok := status.FromError(pauseErr) @@ -319,7 +335,7 @@ func TestPause_AdmissionCallerCancelIsNotARefusal(t *testing.T) { cancel() }() - _, pauseErr := s.Pause(ctx, &orchestrator.SandboxPauseRequest{SandboxId: "sbx-admission-cancel"}) + _, pauseErr := s.Pause(ctx, &orchestrator.SandboxPauseRequest{SandboxId: "sbx-admission-cancel", ExecutionId: "sbx-admission-cancel"}) assert.Zero(t, s.info.OutstandingWork()) require.Error(t, pauseErr) @@ -532,7 +548,7 @@ func TestPauseAdmissionMetrics_RefusedPause(t *testing.T) { sbx := admissionTestSandbox(t, "sbx-metrics-refused", 21, utils.NewSetOnce[*header.Header]()) s.sandboxFactory.Sandboxes.MarkRunning(t.Context(), sbx) - _, pauseErr := s.Pause(t.Context(), &orchestrator.SandboxPauseRequest{SandboxId: "sbx-metrics-refused"}) + _, pauseErr := s.Pause(t.Context(), &orchestrator.SandboxPauseRequest{SandboxId: "sbx-metrics-refused", ExecutionId: "sbx-metrics-refused"}) require.Error(t, pauseErr) points := admissionCounterPoints(t, reader) @@ -614,7 +630,7 @@ func TestPauseAdmissionMetrics_ReadyOutcomes(t *testing.T) { } }() go func() { - _, _ = s.Pause(context.WithoutCancel(t.Context()), &orchestrator.SandboxPauseRequest{SandboxId: "sbx-metrics-raw"}) + _, _ = s.Pause(context.WithoutCancel(t.Context()), &orchestrator.SandboxPauseRequest{SandboxId: "sbx-metrics-raw", ExecutionId: "sbx-metrics-raw"}) }() // The paused state follows completion of admission metric recording. @@ -645,7 +661,7 @@ func TestPauseAdmissionMetrics_ReadyOutcomes(t *testing.T) { s.sandboxFactory.Sandboxes.MarkRunning(t.Context(), sbx) go func() { - _, _ = s.Pause(context.WithoutCancel(t.Context()), &orchestrator.SandboxPauseRequest{SandboxId: "sbx-metrics-ready"}) + _, _ = s.Pause(context.WithoutCancel(t.Context()), &orchestrator.SandboxPauseRequest{SandboxId: "sbx-metrics-ready", ExecutionId: "sbx-metrics-ready"}) }() require.Eventually(t, func() bool { diff --git a/packages/orchestrator/pkg/server/sandbox_events_work_test.go b/packages/orchestrator/pkg/server/sandbox_events_work_test.go index 834c69b87a..6d85e42d16 100644 --- a/packages/orchestrator/pkg/server/sandbox_events_work_test.go +++ b/packages/orchestrator/pkg/server/sandbox_events_work_test.go @@ -211,7 +211,23 @@ func TestUpdateDeleteNotFoundReleaseWork(t *testing.T) { _, err := s.Update(t.Context(), &orchestrator.SandboxUpdateRequest{SandboxId: "missing"}) require.Equal(t, codes.NotFound, status.Code(err)) require.Zero(t, s.info.OutstandingWork()) - _, err = s.Delete(t.Context(), &orchestrator.SandboxDeleteRequest{SandboxId: "missing"}) + _, err = s.Delete(t.Context(), &orchestrator.SandboxDeleteRequest{SandboxId: "missing", ExecutionId: "execution-missing"}) require.Equal(t, codes.NotFound, status.Code(err)) require.Zero(t, s.info.OutstandingWork()) } + +func TestDelete_StaleExecutionCannotStopReplacement(t *testing.T) { + t.Parallel() + + sbx := eventWorkSandbox() + s := &Server{info: &service.ServiceInfo{}, sandboxFactory: &sandbox.Factory{Sandboxes: sandbox.NewSandboxesMap()}} + require.NoError(t, s.sandboxFactory.Sandboxes.MarkRunning(t.Context(), sbx)) + + _, err := s.Delete(t.Context(), &orchestrator.SandboxDeleteRequest{ + SandboxId: sbx.Runtime.SandboxID, ExecutionId: "stale-execution", WaitForStop: true, + }) + require.Equal(t, codes.FailedPrecondition, status.Code(err)) + got, live := s.sandboxFactory.Sandboxes.Get(sbx.Runtime.SandboxID) + require.True(t, live) + require.Same(t, sbx, got) +} diff --git a/packages/orchestrator/pkg/server/sandboxes.go b/packages/orchestrator/pkg/server/sandboxes.go index 951d242fd1..8944bd47e1 100644 --- a/packages/orchestrator/pkg/server/sandboxes.go +++ b/packages/orchestrator/pkg/server/sandboxes.go @@ -686,6 +686,9 @@ func (s *Server) Delete(ctxConn context.Context, in *orchestrator.SandboxDeleteR childSpan.SetAttributes( telemetry.WithSandboxID(in.GetSandboxId()), ) + if in.GetExecutionId() == "" { + return nil, status.Error(codes.InvalidArgument, "execution_id is required") + } sbx, ok := s.sandboxFactory.Sandboxes.Get(in.GetSandboxId()) if !ok { @@ -693,6 +696,9 @@ func (s *Server) Delete(ctxConn context.Context, in *orchestrator.SandboxDeleteR return nil, status.Errorf(codes.NotFound, "sandbox '%s' not found", in.GetSandboxId()) } + if sbx.Runtime.ExecutionID != in.GetExecutionId() { + return nil, status.Errorf(codes.FailedPrecondition, "sandbox '%s' execution changed", in.GetSandboxId()) + } childSpan.SetAttributes( telemetry.WithTeamID(sbx.Runtime.TeamID), @@ -726,10 +732,8 @@ func (s *Server) Delete(ctxConn context.Context, in *orchestrator.SandboxDeleteR // Check health metrics before stopping the sandbox sbx.Checks.Healthcheck(ctx, true) - // Start the cleanup in a goroutine—the initial kill request should be send as the first thing in stop, and at this point you cannot route to the sandbox anymore. - // We don't wait for the whole cleanup to finish here. - go func() { - err := sbx.Stop(context.WithoutCancel(ctx)) + stop := func(stopCtx context.Context) error { + err := sbx.Stop(stopCtx) if err != nil { sbxlogger.I(sbx).Error(ctx, "error stopping sandbox", logger.WithSandboxID(in.GetSandboxId()), @@ -737,13 +741,32 @@ func (s *Server) Delete(ctxConn context.Context, in *orchestrator.SandboxDeleteR zap.Error(err), ) } - }() + + return err + } + if err := runDeleteStop(ctx, in.GetWaitForStop(), stop); err != nil { + return nil, status.Errorf(codes.Internal, "failed to stop sandbox '%s': %s", in.GetSandboxId(), err) + } s.emitSandboxKilled(ctx, sbx, killReason) return &emptypb.Empty{}, nil } +// runDeleteStop preserves the legacy fire-and-forget delete while allowing an +// execution-evidence caller to wait for the exact Firecracker stop result. +func runDeleteStop(ctx context.Context, wait bool, stop func(context.Context) error) error { + if wait { + return stop(ctx) + } + + go func() { + _ = stop(context.WithoutCancel(ctx)) + }() + + return nil +} + // emitSandboxKilled publishes the terminal surfaces of a sandbox kill — the // killed lifecycle event and the kill counter, with a bounded reason. Callers // are the Delete RPC and the paths that destroy a live-registered sandbox @@ -851,6 +874,9 @@ func (s *Server) Pause(ctx context.Context, in *orchestrator.SandboxPauseRequest telemetry.WithTemplateID(in.GetTemplateId()), telemetry.WithBuildID(in.GetBuildId()), ) + if in.GetExecutionId() == "" { + return nil, status.Error(codes.InvalidArgument, "execution_id is required") + } sbx, ok := s.sandboxFactory.Sandboxes.Get(in.GetSandboxId()) if !ok { @@ -858,6 +884,9 @@ func (s *Server) Pause(ctx context.Context, in *orchestrator.SandboxPauseRequest return nil, status.Error(codes.NotFound, "sandbox not found") } + if sbx.Runtime.ExecutionID != in.GetExecutionId() { + return nil, status.Errorf(codes.FailedPrecondition, "sandbox '%s' execution changed", in.GetSandboxId()) + } ctx = featureflags.AddToContext( ctx, diff --git a/packages/shared/pkg/grpc/orchestrator/orchestrator.pb.go b/packages/shared/pkg/grpc/orchestrator/orchestrator.pb.go index 91f1cf76d4..a2f17ee74b 100644 --- a/packages/shared/pkg/grpc/orchestrator/orchestrator.pb.go +++ b/packages/shared/pkg/grpc/orchestrator/orchestrator.pb.go @@ -1066,9 +1066,16 @@ func (x *SandboxUpdateRequest) GetEgress() *SandboxNetworkEgressConfig { } type SandboxDeleteRequest struct { - state protoimpl.MessageState `protogen:"open.v1"` - SandboxId string `protobuf:"bytes,1,opt,name=sandbox_id,json=sandboxId,proto3" json:"sandbox_id,omitempty"` - KillReason *string `protobuf:"bytes,2,opt,name=kill_reason,json=killReason,proto3,oneof" json:"kill_reason,omitempty"` + state protoimpl.MessageState `protogen:"open.v1"` + SandboxId string `protobuf:"bytes,1,opt,name=sandbox_id,json=sandboxId,proto3" json:"sandbox_id,omitempty"` + KillReason *string `protobuf:"bytes,2,opt,name=kill_reason,json=killReason,proto3,oneof" json:"kill_reason,omitempty"` + // The exact sandbox execution the caller intends to stop. The node rejects + // the operation when this does not match the live incarnation, preventing a + // delayed delete from killing a replacement execution with the same ID. + ExecutionId string `protobuf:"bytes,3,opt,name=execution_id,json=executionId,proto3" json:"execution_id,omitempty"` + // Wait until the Firecracker stop has completed and surface any stop error. + // False preserves the legacy asynchronous delete behavior. + WaitForStop bool `protobuf:"varint,4,opt,name=wait_for_stop,json=waitForStop,proto3" json:"wait_for_stop,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -1117,6 +1124,20 @@ func (x *SandboxDeleteRequest) GetKillReason() string { return "" } +func (x *SandboxDeleteRequest) GetExecutionId() string { + if x != nil { + return x.ExecutionId + } + return "" +} + +func (x *SandboxDeleteRequest) GetWaitForStop() bool { + if x != nil { + return x.WaitForStop + } + return false +} + type SandboxPauseRequest struct { state protoimpl.MessageState `protogen:"open.v1"` SandboxId string `protobuf:"bytes,1,opt,name=sandbox_id,json=sandboxId,proto3" json:"sandbox_id,omitempty"` @@ -1130,8 +1151,11 @@ type SandboxPauseRequest struct { // storage before pause can be reported terminal. Existing callers keep the // asynchronous upload path when this is false. WaitForStorage bool `protobuf:"varint,5,opt,name=wait_for_storage,json=waitForStorage,proto3" json:"wait_for_storage,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // The exact sandbox execution the caller intends to pause. The node rejects + // the operation when this does not match the live incarnation. + ExecutionId string `protobuf:"bytes,6,opt,name=execution_id,json=executionId,proto3" json:"execution_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *SandboxPauseRequest) Reset() { @@ -1199,6 +1223,13 @@ func (x *SandboxPauseRequest) GetWaitForStorage() bool { return false } +func (x *SandboxPauseRequest) GetExecutionId() string { + if x != nil { + return x.ExecutionId + } + return "" +} + type SchedulingMetadata struct { state protoimpl.MessageState `protogen:"open.v1"` // memfile_base_build_id / rootfs_base_build_id are each artifact's root layer @@ -1764,13 +1795,15 @@ const file_orchestrator_proto_rawDesc = "" + "\bend_time\x18\x02 \x01(\v2\x1a.google.protobuf.TimestampH\x00R\aendTime\x88\x01\x01\x128\n" + "\x06egress\x18\x03 \x01(\v2\x1b.SandboxNetworkEgressConfigH\x01R\x06egress\x88\x01\x01B\v\n" + "\t_end_timeB\t\n" + - "\a_egress\"k\n" + + "\a_egress\"\xb2\x01\n" + "\x14SandboxDeleteRequest\x12\x1d\n" + "\n" + "sandbox_id\x18\x01 \x01(\tR\tsandboxId\x12$\n" + "\vkill_reason\x18\x02 \x01(\tH\x00R\n" + - "killReason\x88\x01\x01B\x0e\n" + - "\f_kill_reason\"\xc3\x01\n" + + "killReason\x88\x01\x01\x12!\n" + + "\fexecution_id\x18\x03 \x01(\tR\vexecutionId\x12\"\n" + + "\rwait_for_stop\x18\x04 \x01(\bR\vwaitForStopB\x0e\n" + + "\f_kill_reason\"\xe6\x01\n" + "\x13SandboxPauseRequest\x12\x1d\n" + "\n" + "sandbox_id\x18\x01 \x01(\tR\tsandboxId\x12\x1f\n" + @@ -1778,7 +1811,8 @@ const file_orchestrator_proto_rawDesc = "" + "templateId\x12\x19\n" + "\bbuild_id\x18\x03 \x01(\tR\abuildId\x12'\n" + "\x0ffilesystem_only\x18\x04 \x01(\bR\x0efilesystemOnly\x12(\n" + - "\x10wait_for_storage\x18\x05 \x01(\bR\x0ewaitForStorage\"\xb1\x03\n" + + "\x10wait_for_storage\x18\x05 \x01(\bR\x0ewaitForStorage\x12!\n" + + "\fexecution_id\x18\x06 \x01(\tR\vexecutionId\"\xb1\x03\n" + "\x12SchedulingMetadata\x121\n" + "\x15memfile_base_build_id\x18\x01 \x01(\tR\x12memfileBaseBuildId\x12\x19\n" + "\bbuild_id\x18\x02 \x01(\tR\abuildId\x12*\n" + From dd410d91ad509fb38487a86ea99582cb218d6cb1 Mon Sep 17 00:00:00 2001 From: Fred E <7602667+wallscaler@users.noreply.github.com> Date: Wed, 16 Sep 2026 20:24:52 -0400 Subject: [PATCH 5/9] fix(runtime): require stop acknowledgement for evidence deletes --- docs/ARCHITECTURE.md | 3 +- .../internal/orchestrator/delete_instance.go | 5 +- .../orchestrator/delete_instance_test.go | 26 ++- .../internal/orchestrator/nodemanager/mock.go | 4 +- .../api/internal/orchestrator/work_test.go | 5 +- packages/orchestrator/orchestrator.proto | 9 +- .../orchestrator/pkg/dummyserver/sandbox.go | 4 +- packages/orchestrator/pkg/server/sandboxes.go | 4 +- .../pkg/grpc/orchestrator/orchestrator.pb.go | 176 +++++++++++------- .../grpc/orchestrator/orchestrator_grpc.pb.go | 10 +- 10 files changed, 164 insertions(+), 82 deletions(-) diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index f1424cf4de..b1cdc1603c 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -215,7 +215,8 @@ gRPC services on :5008 (`pkg/server/`, `pkg/service/`, `pkg/template/server/`, ` `Delete` and `Pause` are execution-fenced: callers provide the expected execution ID and the node rejects a stale operation rather than act on a replacement incarnation. Delete is asynchronous by default; evidence-bound - callers can request that it wait for the Firecracker stop result. + callers can request that it wait for the Firecracker stop result and require + the response's explicit completion acknowledgement (absent from older nodes). - **TemplateService** — `TemplateCreate`, `TemplateBuildStatus`, `TemplateBuildDelete` (template-manager role only). - **InfoService** — node identity, roles, capacity, health status (used by API node discovery). - **ChunkService / VolumeService** — peer-to-peer template chunk serving; persistent volumes. diff --git a/packages/api/internal/orchestrator/delete_instance.go b/packages/api/internal/orchestrator/delete_instance.go index d7f9a42a93..ad5a5195c9 100644 --- a/packages/api/internal/orchestrator/delete_instance.go +++ b/packages/api/internal/orchestrator/delete_instance.go @@ -467,7 +467,7 @@ func (o *Orchestrator) killSandboxOnNode( } client, ctx := node.GetSandboxDeleteCtx(ctx, sbx.SandboxID, sbx.ExecutionID, false) - _, err := client.Sandbox.Delete(ctx, req) + response, err := client.Sandbox.Delete(ctx, req) st, ok := status.FromError(err) if ok && st.Code() == codes.NotFound { logger.L().Info(ctx, "Sandbox not found during kill", @@ -478,6 +478,9 @@ func (o *Orchestrator) killSandboxOnNode( } else if err != nil { return fmt.Errorf("failed to delete sandbox: %w", err) } + if waitForStop && (response == nil || !response.GetStopCompleted()) { + return errors.New("delete completed without Firecracker stop confirmation") + } node.OptimisticRemove(ctx, nodemanager.SandboxResources{ CPUs: sbx.VCpu, diff --git a/packages/api/internal/orchestrator/delete_instance_test.go b/packages/api/internal/orchestrator/delete_instance_test.go index 44972fdc42..daa99e459d 100644 --- a/packages/api/internal/orchestrator/delete_instance_test.go +++ b/packages/api/internal/orchestrator/delete_instance_test.go @@ -46,6 +46,7 @@ type pauseStubClient struct { err error deleteErr error storageDurable *bool + stopCompleted *bool // gate, when set, holds the answer until closed. gate <-chan struct{} // onPause, when set, runs before the answer — a test's chance to change @@ -58,13 +59,18 @@ type pauseStubClient struct { lastPause *orchestrator.SandboxPauseRequest } -func (c *pauseStubClient) Delete(_ context.Context, request *orchestrator.SandboxDeleteRequest, _ ...grpc.CallOption) (*emptypb.Empty, error) { +func (c *pauseStubClient) Delete(_ context.Context, request *orchestrator.SandboxDeleteRequest, _ ...grpc.CallOption) (*orchestrator.SandboxDeleteResponse, error) { c.mu.Lock() defer c.mu.Unlock() c.deletes++ c.lastDelete = request - return &emptypb.Empty{}, c.deleteErr + completed := request.GetWaitForStop() + if c.stopCompleted != nil { + completed = *c.stopCompleted + } + + return &orchestrator.SandboxDeleteResponse{StopCompleted: completed}, c.deleteErr } func (c *pauseStubClient) deleteCount() int { @@ -592,6 +598,22 @@ func TestRemoveSandbox_LegacyDeleteRemainsAsync(t *testing.T) { require.False(t, req.GetWaitForStop()) } +func TestRemoveSandboxWithEvidence_OlderNodeWithoutStopAcknowledgementStaysUnconfirmed(t *testing.T) { + t.Parallel() + + f := newRefusalFixture(t, true, consts.LocalClusterID, nil) + node := f.o.GetNode(f.sbx.ClusterID, f.sbx.NodeID) + require.NotNil(t, node) + completed := false + node.SetSandboxClient(&pauseStubClient{stopCompleted: &completed}) + + evidence, err := f.o.RemoveSandboxWithEvidence(t.Context(), f.sbx.TeamID, f.sbx.SandboxID, sandbox.RemoveOpts{ + Action: sandbox.StateActionKill, ExpectExecutionID: f.sbx.ExecutionID, + }) + require.ErrorIs(t, err, ErrSandboxOperationFailed) + require.False(t, evidence.Confirmed) +} + func TestRemoveSandboxWithEvidence_InFlightRemovalIsNeverTerminalProof(t *testing.T) { t.Parallel() diff --git a/packages/api/internal/orchestrator/nodemanager/mock.go b/packages/api/internal/orchestrator/nodemanager/mock.go index 7d23f5f696..5bbf8829ce 100644 --- a/packages/api/internal/orchestrator/nodemanager/mock.go +++ b/packages/api/internal/orchestrator/nodemanager/mock.go @@ -104,8 +104,8 @@ func (n *mockLegacySandboxClient) Create(_ context.Context, _ *orchestrator.Sand return &orchestrator.SandboxCreateResponse{}, nil } -func (n *mockLegacySandboxClient) Delete(_ context.Context, _ *orchestrator.SandboxDeleteRequest, _ ...grpc.CallOption) (*emptypb.Empty, error) { - return &emptypb.Empty{}, nil +func (n *mockLegacySandboxClient) Delete(_ context.Context, request *orchestrator.SandboxDeleteRequest, _ ...grpc.CallOption) (*orchestrator.SandboxDeleteResponse, error) { + return &orchestrator.SandboxDeleteResponse{StopCompleted: request.GetWaitForStop()}, nil } // mockTemplateClient implements templatemanager.TemplateServiceClient diff --git a/packages/api/internal/orchestrator/work_test.go b/packages/api/internal/orchestrator/work_test.go index 99e03e4e56..6eddc49bb9 100644 --- a/packages/api/internal/orchestrator/work_test.go +++ b/packages/api/internal/orchestrator/work_test.go @@ -9,7 +9,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "google.golang.org/grpc" - "google.golang.org/protobuf/types/known/emptypb" "github.com/e2b-dev/infra/packages/api/internal/sandbox" sandboxredis "github.com/e2b-dev/infra/packages/api/internal/sandbox/storage/redis" @@ -97,11 +96,11 @@ type gatedKillClient struct { entered chan struct{} } -func (c *gatedKillClient) Delete(context.Context, *orchestrator.SandboxDeleteRequest, ...grpc.CallOption) (*emptypb.Empty, error) { +func (c *gatedKillClient) Delete(_ context.Context, request *orchestrator.SandboxDeleteRequest, _ ...grpc.CallOption) (*orchestrator.SandboxDeleteResponse, error) { close(c.entered) <-c.gate - return &emptypb.Empty{}, nil + return &orchestrator.SandboxDeleteResponse{StopCompleted: request.GetWaitForStop()}, nil } // gatedPauseFixture holds a pause inside its node RPC until the returned diff --git a/packages/orchestrator/orchestrator.proto b/packages/orchestrator/orchestrator.proto index 5279d291b1..78979ead1d 100644 --- a/packages/orchestrator/orchestrator.proto +++ b/packages/orchestrator/orchestrator.proto @@ -188,6 +188,13 @@ message SandboxDeleteRequest { bool wait_for_stop = 4; } +message SandboxDeleteResponse { + // True only when a wait_for_stop request was honored and the exact + // execution's Firecracker stop completed without error. An older server + // decodes to false, so evidence callers fail closed during rolling upgrades. + bool stop_completed = 1; +} + message SandboxPauseRequest { string sandbox_id = 1; string template_id = 2; @@ -278,7 +285,7 @@ service SandboxService { rpc Create(SandboxCreateRequest) returns (SandboxCreateResponse); rpc Update(SandboxUpdateRequest) returns (google.protobuf.Empty); rpc List(google.protobuf.Empty) returns (SandboxListResponse); - rpc Delete(SandboxDeleteRequest) returns (google.protobuf.Empty); + rpc Delete(SandboxDeleteRequest) returns (SandboxDeleteResponse); rpc Pause(SandboxPauseRequest) returns (SandboxPauseResponse); rpc Checkpoint(SandboxCheckpointRequest) returns (SandboxCheckpointResponse); } diff --git a/packages/orchestrator/pkg/dummyserver/sandbox.go b/packages/orchestrator/pkg/dummyserver/sandbox.go index 28e66f5a3b..dfd76c74b5 100644 --- a/packages/orchestrator/pkg/dummyserver/sandbox.go +++ b/packages/orchestrator/pkg/dummyserver/sandbox.go @@ -122,7 +122,7 @@ func (s *SandboxServer) List(_ context.Context, _ *emptypb.Empty) (*orchestrator return &orchestrator.SandboxListResponse{Sandboxes: out}, nil } -func (s *SandboxServer) Delete(_ context.Context, req *orchestrator.SandboxDeleteRequest) (*emptypb.Empty, error) { +func (s *SandboxServer) Delete(_ context.Context, req *orchestrator.SandboxDeleteRequest) (*orchestrator.SandboxDeleteResponse, error) { if req.GetSandboxId() == "" { return nil, status.Error(codes.InvalidArgument, "sandbox_id is required") } @@ -141,7 +141,7 @@ func (s *SandboxServer) Delete(_ context.Context, req *orchestrator.SandboxDelet } delete(s.sandboxes, req.GetSandboxId()) - return &emptypb.Empty{}, nil + return &orchestrator.SandboxDeleteResponse{StopCompleted: req.GetWaitForStop()}, nil } func (s *SandboxServer) Pause(_ context.Context, req *orchestrator.SandboxPauseRequest) (*orchestrator.SandboxPauseResponse, error) { diff --git a/packages/orchestrator/pkg/server/sandboxes.go b/packages/orchestrator/pkg/server/sandboxes.go index 8944bd47e1..804edb3449 100644 --- a/packages/orchestrator/pkg/server/sandboxes.go +++ b/packages/orchestrator/pkg/server/sandboxes.go @@ -673,7 +673,7 @@ func (s *Server) List(ctx context.Context, _ *emptypb.Empty) (*orchestrator.Sand }, nil } -func (s *Server) Delete(ctxConn context.Context, in *orchestrator.SandboxDeleteRequest) (*emptypb.Empty, error) { +func (s *Server) Delete(ctxConn context.Context, in *orchestrator.SandboxDeleteRequest) (*orchestrator.SandboxDeleteResponse, error) { releaseWork := s.info.TrackWork() defer releaseWork() @@ -750,7 +750,7 @@ func (s *Server) Delete(ctxConn context.Context, in *orchestrator.SandboxDeleteR s.emitSandboxKilled(ctx, sbx, killReason) - return &emptypb.Empty{}, nil + return &orchestrator.SandboxDeleteResponse{StopCompleted: in.GetWaitForStop()}, nil } // runDeleteStop preserves the legacy fire-and-forget delete while allowing an diff --git a/packages/shared/pkg/grpc/orchestrator/orchestrator.pb.go b/packages/shared/pkg/grpc/orchestrator/orchestrator.pb.go index a2f17ee74b..2bf8c9e30a 100644 --- a/packages/shared/pkg/grpc/orchestrator/orchestrator.pb.go +++ b/packages/shared/pkg/grpc/orchestrator/orchestrator.pb.go @@ -1138,6 +1138,53 @@ func (x *SandboxDeleteRequest) GetWaitForStop() bool { return false } +type SandboxDeleteResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + // True only when a wait_for_stop request was honored and the exact + // execution's Firecracker stop completed without error. An older server + // decodes to false, so evidence callers fail closed during rolling upgrades. + StopCompleted bool `protobuf:"varint,1,opt,name=stop_completed,json=stopCompleted,proto3" json:"stop_completed,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SandboxDeleteResponse) Reset() { + *x = SandboxDeleteResponse{} + mi := &file_orchestrator_proto_msgTypes[15] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SandboxDeleteResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SandboxDeleteResponse) ProtoMessage() {} + +func (x *SandboxDeleteResponse) ProtoReflect() protoreflect.Message { + mi := &file_orchestrator_proto_msgTypes[15] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SandboxDeleteResponse.ProtoReflect.Descriptor instead. +func (*SandboxDeleteResponse) Descriptor() ([]byte, []int) { + return file_orchestrator_proto_rawDescGZIP(), []int{15} +} + +func (x *SandboxDeleteResponse) GetStopCompleted() bool { + if x != nil { + return x.StopCompleted + } + return false +} + type SandboxPauseRequest struct { state protoimpl.MessageState `protogen:"open.v1"` SandboxId string `protobuf:"bytes,1,opt,name=sandbox_id,json=sandboxId,proto3" json:"sandbox_id,omitempty"` @@ -1160,7 +1207,7 @@ type SandboxPauseRequest struct { func (x *SandboxPauseRequest) Reset() { *x = SandboxPauseRequest{} - mi := &file_orchestrator_proto_msgTypes[15] + mi := &file_orchestrator_proto_msgTypes[16] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1172,7 +1219,7 @@ func (x *SandboxPauseRequest) String() string { func (*SandboxPauseRequest) ProtoMessage() {} func (x *SandboxPauseRequest) ProtoReflect() protoreflect.Message { - mi := &file_orchestrator_proto_msgTypes[15] + mi := &file_orchestrator_proto_msgTypes[16] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1185,7 +1232,7 @@ func (x *SandboxPauseRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use SandboxPauseRequest.ProtoReflect.Descriptor instead. func (*SandboxPauseRequest) Descriptor() ([]byte, []int) { - return file_orchestrator_proto_rawDescGZIP(), []int{15} + return file_orchestrator_proto_rawDescGZIP(), []int{16} } func (x *SandboxPauseRequest) GetSandboxId() string { @@ -1256,7 +1303,7 @@ type SchedulingMetadata struct { func (x *SchedulingMetadata) Reset() { *x = SchedulingMetadata{} - mi := &file_orchestrator_proto_msgTypes[16] + mi := &file_orchestrator_proto_msgTypes[17] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1268,7 +1315,7 @@ func (x *SchedulingMetadata) String() string { func (*SchedulingMetadata) ProtoMessage() {} func (x *SchedulingMetadata) ProtoReflect() protoreflect.Message { - mi := &file_orchestrator_proto_msgTypes[16] + mi := &file_orchestrator_proto_msgTypes[17] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1281,7 +1328,7 @@ func (x *SchedulingMetadata) ProtoReflect() protoreflect.Message { // Deprecated: Use SchedulingMetadata.ProtoReflect.Descriptor instead. func (*SchedulingMetadata) Descriptor() ([]byte, []int) { - return file_orchestrator_proto_rawDescGZIP(), []int{16} + return file_orchestrator_proto_rawDescGZIP(), []int{17} } func (x *SchedulingMetadata) GetMemfileBaseBuildId() string { @@ -1357,7 +1404,7 @@ type SandboxPauseResponse struct { func (x *SandboxPauseResponse) Reset() { *x = SandboxPauseResponse{} - mi := &file_orchestrator_proto_msgTypes[17] + mi := &file_orchestrator_proto_msgTypes[18] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1369,7 +1416,7 @@ func (x *SandboxPauseResponse) String() string { func (*SandboxPauseResponse) ProtoMessage() {} func (x *SandboxPauseResponse) ProtoReflect() protoreflect.Message { - mi := &file_orchestrator_proto_msgTypes[17] + mi := &file_orchestrator_proto_msgTypes[18] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1382,7 +1429,7 @@ func (x *SandboxPauseResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use SandboxPauseResponse.ProtoReflect.Descriptor instead. func (*SandboxPauseResponse) Descriptor() ([]byte, []int) { - return file_orchestrator_proto_rawDescGZIP(), []int{17} + return file_orchestrator_proto_rawDescGZIP(), []int{18} } func (x *SandboxPauseResponse) GetSchedulingMetadata() *SchedulingMetadata { @@ -1413,7 +1460,7 @@ type SandboxCheckpointRequest struct { func (x *SandboxCheckpointRequest) Reset() { *x = SandboxCheckpointRequest{} - mi := &file_orchestrator_proto_msgTypes[18] + mi := &file_orchestrator_proto_msgTypes[19] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1425,7 +1472,7 @@ func (x *SandboxCheckpointRequest) String() string { func (*SandboxCheckpointRequest) ProtoMessage() {} func (x *SandboxCheckpointRequest) ProtoReflect() protoreflect.Message { - mi := &file_orchestrator_proto_msgTypes[18] + mi := &file_orchestrator_proto_msgTypes[19] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1438,7 +1485,7 @@ func (x *SandboxCheckpointRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use SandboxCheckpointRequest.ProtoReflect.Descriptor instead. func (*SandboxCheckpointRequest) Descriptor() ([]byte, []int) { - return file_orchestrator_proto_rawDescGZIP(), []int{18} + return file_orchestrator_proto_rawDescGZIP(), []int{19} } func (x *SandboxCheckpointRequest) GetSandboxId() string { @@ -1471,7 +1518,7 @@ type SandboxCheckpointResponse struct { func (x *SandboxCheckpointResponse) Reset() { *x = SandboxCheckpointResponse{} - mi := &file_orchestrator_proto_msgTypes[19] + mi := &file_orchestrator_proto_msgTypes[20] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1483,7 +1530,7 @@ func (x *SandboxCheckpointResponse) String() string { func (*SandboxCheckpointResponse) ProtoMessage() {} func (x *SandboxCheckpointResponse) ProtoReflect() protoreflect.Message { - mi := &file_orchestrator_proto_msgTypes[19] + mi := &file_orchestrator_proto_msgTypes[20] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1496,7 +1543,7 @@ func (x *SandboxCheckpointResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use SandboxCheckpointResponse.ProtoReflect.Descriptor instead. func (*SandboxCheckpointResponse) Descriptor() ([]byte, []int) { - return file_orchestrator_proto_rawDescGZIP(), []int{19} + return file_orchestrator_proto_rawDescGZIP(), []int{20} } func (x *SandboxCheckpointResponse) GetSchedulingMetadata() *SchedulingMetadata { @@ -1534,7 +1581,7 @@ type RunningSandbox struct { func (x *RunningSandbox) Reset() { *x = RunningSandbox{} - mi := &file_orchestrator_proto_msgTypes[20] + mi := &file_orchestrator_proto_msgTypes[21] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1546,7 +1593,7 @@ func (x *RunningSandbox) String() string { func (*RunningSandbox) ProtoMessage() {} func (x *RunningSandbox) ProtoReflect() protoreflect.Message { - mi := &file_orchestrator_proto_msgTypes[20] + mi := &file_orchestrator_proto_msgTypes[21] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1559,7 +1606,7 @@ func (x *RunningSandbox) ProtoReflect() protoreflect.Message { // Deprecated: Use RunningSandbox.ProtoReflect.Descriptor instead. func (*RunningSandbox) Descriptor() ([]byte, []int) { - return file_orchestrator_proto_rawDescGZIP(), []int{20} + return file_orchestrator_proto_rawDescGZIP(), []int{21} } // Deprecated: Marked as deprecated in orchestrator.proto. @@ -1635,7 +1682,7 @@ type SandboxListResponse struct { func (x *SandboxListResponse) Reset() { *x = SandboxListResponse{} - mi := &file_orchestrator_proto_msgTypes[21] + mi := &file_orchestrator_proto_msgTypes[22] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1647,7 +1694,7 @@ func (x *SandboxListResponse) String() string { func (*SandboxListResponse) ProtoMessage() {} func (x *SandboxListResponse) ProtoReflect() protoreflect.Message { - mi := &file_orchestrator_proto_msgTypes[21] + mi := &file_orchestrator_proto_msgTypes[22] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1660,7 +1707,7 @@ func (x *SandboxListResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use SandboxListResponse.ProtoReflect.Descriptor instead. func (*SandboxListResponse) Descriptor() ([]byte, []int) { - return file_orchestrator_proto_rawDescGZIP(), []int{21} + return file_orchestrator_proto_rawDescGZIP(), []int{22} } func (x *SandboxListResponse) GetSandboxes() []*RunningSandbox { @@ -1803,7 +1850,9 @@ const file_orchestrator_proto_rawDesc = "" + "killReason\x88\x01\x01\x12!\n" + "\fexecution_id\x18\x03 \x01(\tR\vexecutionId\x12\"\n" + "\rwait_for_stop\x18\x04 \x01(\bR\vwaitForStopB\x0e\n" + - "\f_kill_reason\"\xe6\x01\n" + + "\f_kill_reason\">\n" + + "\x15SandboxDeleteResponse\x12%\n" + + "\x0estop_completed\x18\x01 \x01(\bR\rstopCompleted\"\xe6\x01\n" + "\x13SandboxPauseRequest\x12\x1d\n" + "\n" + "sandbox_id\x18\x01 \x01(\tR\tsandboxId\x12\x1f\n" + @@ -1854,7 +1903,7 @@ const file_orchestrator_proto_rawDesc = "" + "\x06Create\x12\x15.SandboxCreateRequest\x1a\x16.SandboxCreateResponse\x127\n" + "\x06Update\x12\x15.SandboxUpdateRequest\x1a\x16.google.protobuf.Empty\x124\n" + "\x04List\x12\x16.google.protobuf.Empty\x1a\x14.SandboxListResponse\x127\n" + - "\x06Delete\x12\x15.SandboxDeleteRequest\x1a\x16.google.protobuf.Empty\x124\n" + + "\x06Delete\x12\x15.SandboxDeleteRequest\x1a\x16.SandboxDeleteResponse\x124\n" + "\x05Pause\x12\x14.SandboxPauseRequest\x1a\x15.SandboxPauseResponse\x12C\n" + "\n" + "Checkpoint\x12\x19.SandboxCheckpointRequest\x1a\x1a.SandboxCheckpointResponseB/Z-https://github.com/e2b-dev/infra/orchestratorb\x06proto3" @@ -1871,7 +1920,7 @@ func file_orchestrator_proto_rawDescGZIP() []byte { return file_orchestrator_proto_rawDescData } -var file_orchestrator_proto_msgTypes = make([]protoimpl.MessageInfo, 28) +var file_orchestrator_proto_msgTypes = make([]protoimpl.MessageInfo, 29) var file_orchestrator_proto_goTypes = []any{ (*SandboxConfig)(nil), // 0: SandboxConfig (*SandboxIam)(nil), // 1: SandboxIam @@ -1888,63 +1937,64 @@ var file_orchestrator_proto_goTypes = []any{ (*SandboxCreateResponse)(nil), // 12: SandboxCreateResponse (*SandboxUpdateRequest)(nil), // 13: SandboxUpdateRequest (*SandboxDeleteRequest)(nil), // 14: SandboxDeleteRequest - (*SandboxPauseRequest)(nil), // 15: SandboxPauseRequest - (*SchedulingMetadata)(nil), // 16: SchedulingMetadata - (*SandboxPauseResponse)(nil), // 17: SandboxPauseResponse - (*SandboxCheckpointRequest)(nil), // 18: SandboxCheckpointRequest - (*SandboxCheckpointResponse)(nil), // 19: SandboxCheckpointResponse - (*RunningSandbox)(nil), // 20: RunningSandbox - (*SandboxListResponse)(nil), // 21: SandboxListResponse - nil, // 22: SandboxConfig.EnvVarsEntry - nil, // 23: SandboxConfig.MetadataEntry - nil, // 24: SandboxIam.TokensEntry - nil, // 25: SandboxNetworkTransform.HeadersEntry - nil, // 26: SandboxNetworkEgressConfig.RulesEntry - nil, // 27: SandboxCheckpointRequest.MetadataEntry - (*timestamppb.Timestamp)(nil), // 28: google.protobuf.Timestamp - (*emptypb.Empty)(nil), // 29: google.protobuf.Empty + (*SandboxDeleteResponse)(nil), // 15: SandboxDeleteResponse + (*SandboxPauseRequest)(nil), // 16: SandboxPauseRequest + (*SchedulingMetadata)(nil), // 17: SchedulingMetadata + (*SandboxPauseResponse)(nil), // 18: SandboxPauseResponse + (*SandboxCheckpointRequest)(nil), // 19: SandboxCheckpointRequest + (*SandboxCheckpointResponse)(nil), // 20: SandboxCheckpointResponse + (*RunningSandbox)(nil), // 21: RunningSandbox + (*SandboxListResponse)(nil), // 22: SandboxListResponse + nil, // 23: SandboxConfig.EnvVarsEntry + nil, // 24: SandboxConfig.MetadataEntry + nil, // 25: SandboxIam.TokensEntry + nil, // 26: SandboxNetworkTransform.HeadersEntry + nil, // 27: SandboxNetworkEgressConfig.RulesEntry + nil, // 28: SandboxCheckpointRequest.MetadataEntry + (*timestamppb.Timestamp)(nil), // 29: google.protobuf.Timestamp + (*emptypb.Empty)(nil), // 30: google.protobuf.Empty } var file_orchestrator_proto_depIdxs = []int32{ - 22, // 0: SandboxConfig.env_vars:type_name -> SandboxConfig.EnvVarsEntry - 23, // 1: SandboxConfig.metadata:type_name -> SandboxConfig.MetadataEntry + 23, // 0: SandboxConfig.env_vars:type_name -> SandboxConfig.EnvVarsEntry + 24, // 1: SandboxConfig.metadata:type_name -> SandboxConfig.MetadataEntry 5, // 2: SandboxConfig.network:type_name -> SandboxNetworkConfig 4, // 3: SandboxConfig.volumeMounts:type_name -> SandboxVolumeMount 3, // 4: SandboxConfig.auto_resume:type_name -> SandboxAutoResumeConfig 1, // 5: SandboxConfig.iam:type_name -> SandboxIam - 24, // 6: SandboxIam.tokens:type_name -> SandboxIam.TokensEntry + 25, // 6: SandboxIam.tokens:type_name -> SandboxIam.TokensEntry 9, // 7: SandboxNetworkConfig.egress:type_name -> SandboxNetworkEgressConfig 10, // 8: SandboxNetworkConfig.ingress:type_name -> SandboxNetworkIngressConfig - 25, // 9: SandboxNetworkTransform.headers:type_name -> SandboxNetworkTransform.HeadersEntry + 26, // 9: SandboxNetworkTransform.headers:type_name -> SandboxNetworkTransform.HeadersEntry 6, // 10: SandboxNetworkRule.transform:type_name -> SandboxNetworkTransform 7, // 11: SandboxNetworkDomainRules.rules:type_name -> SandboxNetworkRule - 26, // 12: SandboxNetworkEgressConfig.rules:type_name -> SandboxNetworkEgressConfig.RulesEntry + 27, // 12: SandboxNetworkEgressConfig.rules:type_name -> SandboxNetworkEgressConfig.RulesEntry 0, // 13: SandboxCreateRequest.sandbox:type_name -> SandboxConfig - 28, // 14: SandboxCreateRequest.start_time:type_name -> google.protobuf.Timestamp - 28, // 15: SandboxCreateRequest.end_time:type_name -> google.protobuf.Timestamp - 16, // 16: SandboxCreateResponse.scheduling_metadata:type_name -> SchedulingMetadata - 28, // 17: SandboxUpdateRequest.end_time:type_name -> google.protobuf.Timestamp + 29, // 14: SandboxCreateRequest.start_time:type_name -> google.protobuf.Timestamp + 29, // 15: SandboxCreateRequest.end_time:type_name -> google.protobuf.Timestamp + 17, // 16: SandboxCreateResponse.scheduling_metadata:type_name -> SchedulingMetadata + 29, // 17: SandboxUpdateRequest.end_time:type_name -> google.protobuf.Timestamp 9, // 18: SandboxUpdateRequest.egress:type_name -> SandboxNetworkEgressConfig - 16, // 19: SandboxPauseResponse.scheduling_metadata:type_name -> SchedulingMetadata - 27, // 20: SandboxCheckpointRequest.metadata:type_name -> SandboxCheckpointRequest.MetadataEntry - 16, // 21: SandboxCheckpointResponse.scheduling_metadata:type_name -> SchedulingMetadata + 17, // 19: SandboxPauseResponse.scheduling_metadata:type_name -> SchedulingMetadata + 28, // 20: SandboxCheckpointRequest.metadata:type_name -> SandboxCheckpointRequest.MetadataEntry + 17, // 21: SandboxCheckpointResponse.scheduling_metadata:type_name -> SchedulingMetadata 0, // 22: RunningSandbox.config:type_name -> SandboxConfig - 28, // 23: RunningSandbox.start_time:type_name -> google.protobuf.Timestamp - 28, // 24: RunningSandbox.end_time:type_name -> google.protobuf.Timestamp - 20, // 25: SandboxListResponse.sandboxes:type_name -> RunningSandbox + 29, // 23: RunningSandbox.start_time:type_name -> google.protobuf.Timestamp + 29, // 24: RunningSandbox.end_time:type_name -> google.protobuf.Timestamp + 21, // 25: SandboxListResponse.sandboxes:type_name -> RunningSandbox 2, // 26: SandboxIam.TokensEntry.value:type_name -> SandboxIamToken 8, // 27: SandboxNetworkEgressConfig.RulesEntry.value:type_name -> SandboxNetworkDomainRules 11, // 28: SandboxService.Create:input_type -> SandboxCreateRequest 13, // 29: SandboxService.Update:input_type -> SandboxUpdateRequest - 29, // 30: SandboxService.List:input_type -> google.protobuf.Empty + 30, // 30: SandboxService.List:input_type -> google.protobuf.Empty 14, // 31: SandboxService.Delete:input_type -> SandboxDeleteRequest - 15, // 32: SandboxService.Pause:input_type -> SandboxPauseRequest - 18, // 33: SandboxService.Checkpoint:input_type -> SandboxCheckpointRequest + 16, // 32: SandboxService.Pause:input_type -> SandboxPauseRequest + 19, // 33: SandboxService.Checkpoint:input_type -> SandboxCheckpointRequest 12, // 34: SandboxService.Create:output_type -> SandboxCreateResponse - 29, // 35: SandboxService.Update:output_type -> google.protobuf.Empty - 21, // 36: SandboxService.List:output_type -> SandboxListResponse - 29, // 37: SandboxService.Delete:output_type -> google.protobuf.Empty - 17, // 38: SandboxService.Pause:output_type -> SandboxPauseResponse - 19, // 39: SandboxService.Checkpoint:output_type -> SandboxCheckpointResponse + 30, // 35: SandboxService.Update:output_type -> google.protobuf.Empty + 22, // 36: SandboxService.List:output_type -> SandboxListResponse + 15, // 37: SandboxService.Delete:output_type -> SandboxDeleteResponse + 18, // 38: SandboxService.Pause:output_type -> SandboxPauseResponse + 20, // 39: SandboxService.Checkpoint:output_type -> SandboxCheckpointResponse 34, // [34:40] is the sub-list for method output_type 28, // [28:34] is the sub-list for method input_type 28, // [28:28] is the sub-list for extension type_name @@ -1970,7 +2020,7 @@ func file_orchestrator_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_orchestrator_proto_rawDesc), len(file_orchestrator_proto_rawDesc)), NumEnums: 0, - NumMessages: 28, + NumMessages: 29, NumExtensions: 0, NumServices: 1, }, diff --git a/packages/shared/pkg/grpc/orchestrator/orchestrator_grpc.pb.go b/packages/shared/pkg/grpc/orchestrator/orchestrator_grpc.pb.go index 8ff0d06dee..8d4e928c8f 100644 --- a/packages/shared/pkg/grpc/orchestrator/orchestrator_grpc.pb.go +++ b/packages/shared/pkg/grpc/orchestrator/orchestrator_grpc.pb.go @@ -35,7 +35,7 @@ type SandboxServiceClient interface { Create(ctx context.Context, in *SandboxCreateRequest, opts ...grpc.CallOption) (*SandboxCreateResponse, error) Update(ctx context.Context, in *SandboxUpdateRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) List(ctx context.Context, in *emptypb.Empty, opts ...grpc.CallOption) (*SandboxListResponse, error) - Delete(ctx context.Context, in *SandboxDeleteRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) + Delete(ctx context.Context, in *SandboxDeleteRequest, opts ...grpc.CallOption) (*SandboxDeleteResponse, error) Pause(ctx context.Context, in *SandboxPauseRequest, opts ...grpc.CallOption) (*SandboxPauseResponse, error) Checkpoint(ctx context.Context, in *SandboxCheckpointRequest, opts ...grpc.CallOption) (*SandboxCheckpointResponse, error) } @@ -78,9 +78,9 @@ func (c *sandboxServiceClient) List(ctx context.Context, in *emptypb.Empty, opts return out, nil } -func (c *sandboxServiceClient) Delete(ctx context.Context, in *SandboxDeleteRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) { +func (c *sandboxServiceClient) Delete(ctx context.Context, in *SandboxDeleteRequest, opts ...grpc.CallOption) (*SandboxDeleteResponse, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) - out := new(emptypb.Empty) + out := new(SandboxDeleteResponse) err := c.cc.Invoke(ctx, SandboxService_Delete_FullMethodName, in, out, cOpts...) if err != nil { return nil, err @@ -115,7 +115,7 @@ type SandboxServiceServer interface { Create(context.Context, *SandboxCreateRequest) (*SandboxCreateResponse, error) Update(context.Context, *SandboxUpdateRequest) (*emptypb.Empty, error) List(context.Context, *emptypb.Empty) (*SandboxListResponse, error) - Delete(context.Context, *SandboxDeleteRequest) (*emptypb.Empty, error) + Delete(context.Context, *SandboxDeleteRequest) (*SandboxDeleteResponse, error) Pause(context.Context, *SandboxPauseRequest) (*SandboxPauseResponse, error) Checkpoint(context.Context, *SandboxCheckpointRequest) (*SandboxCheckpointResponse, error) mustEmbedUnimplementedSandboxServiceServer() @@ -137,7 +137,7 @@ func (UnimplementedSandboxServiceServer) Update(context.Context, *SandboxUpdateR func (UnimplementedSandboxServiceServer) List(context.Context, *emptypb.Empty) (*SandboxListResponse, error) { return nil, status.Error(codes.Unimplemented, "method List not implemented") } -func (UnimplementedSandboxServiceServer) Delete(context.Context, *SandboxDeleteRequest) (*emptypb.Empty, error) { +func (UnimplementedSandboxServiceServer) Delete(context.Context, *SandboxDeleteRequest) (*SandboxDeleteResponse, error) { return nil, status.Error(codes.Unimplemented, "method Delete not implemented") } func (UnimplementedSandboxServiceServer) Pause(context.Context, *SandboxPauseRequest) (*SandboxPauseResponse, error) { From 33bd7844386e07c693ee7851b957d011960214f8 Mon Sep 17 00:00:00 2001 From: Fred E <7602667+wallscaler@users.noreply.github.com> Date: Wed, 16 Sep 2026 20:28:12 -0400 Subject: [PATCH 6/9] fix(orchestrator): preserve legacy lifecycle compatibility --- .../orchestrator/pkg/dummyserver/sandbox.go | 8 +- .../pkg/dummyserver/sandbox_test.go | 120 ++++++++++++++++++ packages/orchestrator/pkg/server/sandboxes.go | 8 +- 3 files changed, 128 insertions(+), 8 deletions(-) create mode 100644 packages/orchestrator/pkg/dummyserver/sandbox_test.go diff --git a/packages/orchestrator/pkg/dummyserver/sandbox.go b/packages/orchestrator/pkg/dummyserver/sandbox.go index dfd76c74b5..fee5b1128b 100644 --- a/packages/orchestrator/pkg/dummyserver/sandbox.go +++ b/packages/orchestrator/pkg/dummyserver/sandbox.go @@ -126,7 +126,7 @@ func (s *SandboxServer) Delete(_ context.Context, req *orchestrator.SandboxDelet if req.GetSandboxId() == "" { return nil, status.Error(codes.InvalidArgument, "sandbox_id is required") } - if req.GetExecutionId() == "" { + if req.GetWaitForStop() && req.GetExecutionId() == "" { return nil, status.Error(codes.InvalidArgument, "execution_id is required") } @@ -136,7 +136,7 @@ func (s *SandboxServer) Delete(_ context.Context, req *orchestrator.SandboxDelet if !ok { return nil, status.Errorf(codes.NotFound, "sandbox %q not found", req.GetSandboxId()) } - if sbx.GetExecutionId() != req.GetExecutionId() { + if req.GetExecutionId() != "" && sbx.GetExecutionId() != req.GetExecutionId() { return nil, status.Errorf(codes.FailedPrecondition, "sandbox %q execution changed", req.GetSandboxId()) } delete(s.sandboxes, req.GetSandboxId()) @@ -148,7 +148,7 @@ func (s *SandboxServer) Pause(_ context.Context, req *orchestrator.SandboxPauseR if req.GetSandboxId() == "" { return nil, status.Error(codes.InvalidArgument, "sandbox_id is required") } - if req.GetExecutionId() == "" { + if req.GetWaitForStorage() && req.GetExecutionId() == "" { return nil, status.Error(codes.InvalidArgument, "execution_id is required") } @@ -159,7 +159,7 @@ func (s *SandboxServer) Pause(_ context.Context, req *orchestrator.SandboxPauseR if !ok { return nil, status.Errorf(codes.NotFound, "sandbox %q not found", req.GetSandboxId()) } - if sbx.GetExecutionId() != req.GetExecutionId() { + if req.GetExecutionId() != "" && sbx.GetExecutionId() != req.GetExecutionId() { return nil, status.Errorf(codes.FailedPrecondition, "sandbox %q execution changed", req.GetSandboxId()) } delete(s.sandboxes, req.GetSandboxId()) diff --git a/packages/orchestrator/pkg/dummyserver/sandbox_test.go b/packages/orchestrator/pkg/dummyserver/sandbox_test.go new file mode 100644 index 0000000000..8ac80ee80a --- /dev/null +++ b/packages/orchestrator/pkg/dummyserver/sandbox_test.go @@ -0,0 +1,120 @@ +package dummyserver + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + "github.com/e2b-dev/infra/packages/shared/pkg/grpc/orchestrator" +) + +const ( + testSandboxID = "sandbox-1" + testExecutionID = "execution-1" +) + +func createTestSandbox(t *testing.T, server *SandboxServer) { + t.Helper() + + _, err := server.Create(context.Background(), &orchestrator.SandboxCreateRequest{ + Sandbox: &orchestrator.SandboxConfig{ + SandboxId: testSandboxID, + ExecutionId: testExecutionID, + }, + }) + require.NoError(t, err) +} + +func TestDeleteLegacyRequestWithoutExecutionID(t *testing.T) { + server := NewSandbox() + createTestSandbox(t, server) + + response, err := server.Delete(context.Background(), &orchestrator.SandboxDeleteRequest{ + SandboxId: testSandboxID, + }) + + require.NoError(t, err) + require.False(t, response.GetStopCompleted()) +} + +func TestDeleteEvidenceRequestRequiresExactExecutionID(t *testing.T) { + t.Run("missing", func(t *testing.T) { + server := NewSandbox() + createTestSandbox(t, server) + + _, err := server.Delete(context.Background(), &orchestrator.SandboxDeleteRequest{ + SandboxId: testSandboxID, + WaitForStop: true, + }) + + require.Equal(t, codes.InvalidArgument, status.Code(err)) + }) + + t.Run("stale", func(t *testing.T) { + server := NewSandbox() + createTestSandbox(t, server) + + _, err := server.Delete(context.Background(), &orchestrator.SandboxDeleteRequest{ + SandboxId: testSandboxID, + ExecutionId: "stale-execution", + WaitForStop: true, + }) + + require.Equal(t, codes.FailedPrecondition, status.Code(err)) + }) + + t.Run("exact", func(t *testing.T) { + server := NewSandbox() + createTestSandbox(t, server) + + response, err := server.Delete(context.Background(), &orchestrator.SandboxDeleteRequest{ + SandboxId: testSandboxID, + ExecutionId: testExecutionID, + WaitForStop: true, + }) + + require.NoError(t, err) + require.True(t, response.GetStopCompleted()) + }) +} + +func TestPauseLegacyRequestWithoutExecutionID(t *testing.T) { + server := NewSandbox() + createTestSandbox(t, server) + + _, err := server.Pause(context.Background(), &orchestrator.SandboxPauseRequest{ + SandboxId: testSandboxID, + }) + + require.NoError(t, err) +} + +func TestPauseEvidenceRequestRequiresExactExecutionID(t *testing.T) { + t.Run("missing", func(t *testing.T) { + server := NewSandbox() + createTestSandbox(t, server) + + _, err := server.Pause(context.Background(), &orchestrator.SandboxPauseRequest{ + SandboxId: testSandboxID, + WaitForStorage: true, + }) + + require.Equal(t, codes.InvalidArgument, status.Code(err)) + }) + + t.Run("stale", func(t *testing.T) { + server := NewSandbox() + createTestSandbox(t, server) + + _, err := server.Pause(context.Background(), &orchestrator.SandboxPauseRequest{ + SandboxId: testSandboxID, + ExecutionId: "stale-execution", + WaitForStorage: true, + }) + + require.Equal(t, codes.FailedPrecondition, status.Code(err)) + }) +} diff --git a/packages/orchestrator/pkg/server/sandboxes.go b/packages/orchestrator/pkg/server/sandboxes.go index 804edb3449..778c50d73c 100644 --- a/packages/orchestrator/pkg/server/sandboxes.go +++ b/packages/orchestrator/pkg/server/sandboxes.go @@ -686,7 +686,7 @@ func (s *Server) Delete(ctxConn context.Context, in *orchestrator.SandboxDeleteR childSpan.SetAttributes( telemetry.WithSandboxID(in.GetSandboxId()), ) - if in.GetExecutionId() == "" { + if in.GetWaitForStop() && in.GetExecutionId() == "" { return nil, status.Error(codes.InvalidArgument, "execution_id is required") } @@ -696,7 +696,7 @@ func (s *Server) Delete(ctxConn context.Context, in *orchestrator.SandboxDeleteR return nil, status.Errorf(codes.NotFound, "sandbox '%s' not found", in.GetSandboxId()) } - if sbx.Runtime.ExecutionID != in.GetExecutionId() { + if in.GetExecutionId() != "" && sbx.Runtime.ExecutionID != in.GetExecutionId() { return nil, status.Errorf(codes.FailedPrecondition, "sandbox '%s' execution changed", in.GetSandboxId()) } @@ -874,7 +874,7 @@ func (s *Server) Pause(ctx context.Context, in *orchestrator.SandboxPauseRequest telemetry.WithTemplateID(in.GetTemplateId()), telemetry.WithBuildID(in.GetBuildId()), ) - if in.GetExecutionId() == "" { + if in.GetWaitForStorage() && in.GetExecutionId() == "" { return nil, status.Error(codes.InvalidArgument, "execution_id is required") } @@ -884,7 +884,7 @@ func (s *Server) Pause(ctx context.Context, in *orchestrator.SandboxPauseRequest return nil, status.Error(codes.NotFound, "sandbox not found") } - if sbx.Runtime.ExecutionID != in.GetExecutionId() { + if in.GetExecutionId() != "" && sbx.Runtime.ExecutionID != in.GetExecutionId() { return nil, status.Errorf(codes.FailedPrecondition, "sandbox '%s' execution changed", in.GetSandboxId()) } From 40989b74cd65836f571ba218f1af7dd2b31275eb Mon Sep 17 00:00:00 2001 From: Fred E <7602667+wallscaler@users.noreply.github.com> Date: Wed, 16 Sep 2026 20:35:12 -0400 Subject: [PATCH 7/9] fix(runtime): preserve legacy pause lifetime semantics --- .../internal/orchestrator/delete_instance.go | 2 +- .../internal/orchestrator/pause_instance.go | 24 +++++++++++-------- .../orchestrator/pause_instance_test.go | 3 +++ 3 files changed, 18 insertions(+), 11 deletions(-) diff --git a/packages/api/internal/orchestrator/delete_instance.go b/packages/api/internal/orchestrator/delete_instance.go index ad5a5195c9..e5b50938a5 100644 --- a/packages/api/internal/orchestrator/delete_instance.go +++ b/packages/api/internal/orchestrator/delete_instance.go @@ -402,7 +402,7 @@ func (o *Orchestrator) removeSandboxFromNodeWithEvidence( switch stateAction { case sandbox.StateActionPause: - buildID, err := o.pauseSandboxWithEvidence(ctx, node, sbx, filesystemOnly, restoreOnRefusal, remainingLifetime, waitForCompletion) + buildID, err := o.pauseSandboxWithEvidence(ctx, node, sbx, filesystemOnly, restoreOnRefusal, &remainingLifetime, waitForCompletion) if err != nil { if dberrors.IsForeignKeyViolation(err) { killErr := o.killSandboxOnNode(ctx, node, sbx.ToNodeSandbox(), sandbox.KillReasonBaseTemplateMissing, false) diff --git a/packages/api/internal/orchestrator/pause_instance.go b/packages/api/internal/orchestrator/pause_instance.go index ee23171538..6d0eb71120 100644 --- a/packages/api/internal/orchestrator/pause_instance.go +++ b/packages/api/internal/orchestrator/pause_instance.go @@ -29,16 +29,20 @@ import ( type PauseQueueExhaustedError = sandbox.PauseQueueExhaustedError func (o *Orchestrator) pauseSandbox(ctx context.Context, node *nodemanager.Node, sbx sandbox.Sandbox, filesystemOnly bool, restoreOnRefusal bool) error { - _, err := o.pauseSandboxWithEvidence(ctx, node, sbx, filesystemOnly, restoreOnRefusal, 0, false) + _, err := o.pauseSandboxWithEvidence(ctx, node, sbx, filesystemOnly, restoreOnRefusal, nil, false) return err } -func (o *Orchestrator) pauseSandboxWithEvidence(ctx context.Context, node *nodemanager.Node, sbx sandbox.Sandbox, filesystemOnly bool, restoreOnRefusal bool, remainingLifetime time.Duration, waitForStorage bool) (string, error) { +func (o *Orchestrator) pauseSandboxWithEvidence(ctx context.Context, node *nodemanager.Node, sbx sandbox.Sandbox, filesystemOnly bool, restoreOnRefusal bool, remainingLifetime *time.Duration, waitForStorage bool) (string, error) { ctx, span := tracer.Start(ctx, "pause-sandbox") defer span.End() - result, err := o.throttledUpsertSnapshot(ctx, buildUpsertSnapshotParams(sbx, node, filesystemOnly, remainingLifetime)) + params := buildUpsertSnapshotParams(sbx, node, filesystemOnly) + if remainingLifetime != nil { + params = buildUpsertSnapshotParams(sbx, node, filesystemOnly, *remainingLifetime) + } + result, err := o.throttledUpsertSnapshot(ctx, params) if err != nil { telemetry.ReportCriticalError(ctx, "error inserting snapshot for env", err) @@ -149,15 +153,15 @@ func buildUpsertSnapshotParams(sbx sandbox.Sandbox, node *nodemanager.Node, file clusterID = &sbx.ClusterID } - remainingLifetime := time.Duration(0) + var remainingLifetimeSeconds *uint64 if len(remaining) > 0 { - remainingLifetime = remaining[0] - } - remainingLifetimeSeconds := uint64(0) - if remainingLifetime > 0 { + value := uint64(0) // Round up so a valid sub-second remainder cannot serialize as the // legacy zero/unset value and accidentally regain the default lifetime. - remainingLifetimeSeconds = uint64(math.Ceil(remainingLifetime.Seconds())) + if remaining[0] > 0 { + value = uint64(math.Ceil(remaining[0].Seconds())) + } + remainingLifetimeSeconds = &value } return queries.UpsertSnapshotParams{ @@ -188,7 +192,7 @@ func buildUpsertSnapshotParams(sbx sandbox.Sandbox, node *nodemanager.Node, file FilesystemOnly: filesystemOnly, AutoPauseFilesystemOnly: sbx.AutoPauseFilesystemOnly, Iam: sbx.Iam, - RemainingLifetimeSeconds: &remainingLifetimeSeconds, + RemainingLifetimeSeconds: remainingLifetimeSeconds, }, OriginNodeID: node.ID, Status: types.BuildStatusSnapshotting, diff --git a/packages/api/internal/orchestrator/pause_instance_test.go b/packages/api/internal/orchestrator/pause_instance_test.go index 174caf1fec..665111149a 100644 --- a/packages/api/internal/orchestrator/pause_instance_test.go +++ b/packages/api/internal/orchestrator/pause_instance_test.go @@ -44,6 +44,9 @@ func TestBuildUpsertSnapshotParams_PreservesRemainingLifetime(t *testing.T) { SandboxID: "sbx-1", BaseTemplateID: "tmpl", BuildID: uuid.New(), } node := &nodemanager.Node{ID: "node-1"} + legacy := buildUpsertSnapshotParams(sbx, node, false) + assert.Nil(t, legacy.Config.RemainingLifetimeSeconds) + params := buildUpsertSnapshotParams(sbx, node, false, 37*time.Minute) require.NotNil(t, params.Config.RemainingLifetimeSeconds) From 6092be444160791f244f1fb0b449d53fe384180a Mon Sep 17 00:00:00 2001 From: Fred E <7602667+wallscaler@users.noreply.github.com> Date: Wed, 16 Sep 2026 20:42:21 -0400 Subject: [PATCH 8/9] fix(runtime): await pause teardown before evidence --- docs/cathedral-lifecycle-operations.md | 4 ++-- .../orchestrator/delete_instance_test.go | 23 ++++++++++++++++++- .../internal/orchestrator/pause_instance.go | 3 +++ packages/orchestrator/orchestrator.proto | 4 ++++ .../orchestrator/pkg/dummyserver/sandbox.go | 5 +++- .../pkg/dummyserver/sandbox_test.go | 15 ++++++++++++ packages/orchestrator/pkg/server/sandboxes.go | 17 ++++++++++++-- .../pkg/grpc/orchestrator/orchestrator.pb.go | 20 ++++++++++++---- 8 files changed, 81 insertions(+), 10 deletions(-) diff --git a/docs/cathedral-lifecycle-operations.md b/docs/cathedral-lifecycle-operations.md index a92cc7c75d..53a932f09a 100644 --- a/docs/cathedral-lifecycle-operations.md +++ b/docs/cathedral-lifecycle-operations.md @@ -39,8 +39,8 @@ a completed delete remains readable after the live sandbox identity is gone. `completed` is written only after an execution-bound node RPC confirms that the execution stopped. For pause, the snapshot build must also have reached a durable successful state and its build ID is recorded. The Cathedral pause RPC -waits for remote snapshot storage to complete before returning that evidence; -the ordinary runtime pause path remains asynchronous. `404` from ordinary +waits for both remote snapshot storage and Firecracker teardown before returning +that evidence; the ordinary runtime pause path remains asynchronous. `404` from ordinary sandbox GET/list, a registry row disappearing, a legacy delete acknowledgement, or joining an in-flight removal is never terminal evidence. diff --git a/packages/api/internal/orchestrator/delete_instance_test.go b/packages/api/internal/orchestrator/delete_instance_test.go index daa99e459d..65b8c3d59a 100644 --- a/packages/api/internal/orchestrator/delete_instance_test.go +++ b/packages/api/internal/orchestrator/delete_instance_test.go @@ -112,8 +112,12 @@ func (c *pauseStubClient) Pause(_ context.Context, request *orchestrator.Sandbox if c.storageDurable != nil { durable = *c.storageDurable } + stopped := request.GetWaitForStorage() + if c.stopCompleted != nil { + stopped = *c.stopCompleted + } - return &orchestrator.SandboxPauseResponse{StorageDurable: durable}, nil + return &orchestrator.SandboxPauseResponse{StorageDurable: durable, StopCompleted: stopped}, nil } // recordingCollector counts InstanceStopped emissions — the stopped-analytics @@ -669,6 +673,23 @@ func TestRemoveSandboxWithEvidence_PauseWithoutStorageConfirmationStaysUnconfirm assert.Empty(t, evidence.SnapshotBuildID) } +func TestRemoveSandboxWithEvidence_PauseWithoutStopConfirmationStaysUnconfirmed(t *testing.T) { + t.Parallel() + + f := newRefusalFixture(t, true, consts.LocalClusterID, nil) + node, ok := f.o.nodes.Get(f.o.scopedNodeID(consts.LocalClusterID, "node-1")) + require.True(t, ok) + completed := false + node.SetSandboxClient(&pauseStubClient{stopCompleted: &completed}) + + evidence, err := f.o.RemoveSandboxWithEvidence(t.Context(), f.sbx.TeamID, f.sbx.SandboxID, sandbox.RemoveOpts{ + Action: sandbox.StateActionPause, ExpectExecutionID: f.sbx.ExecutionID, + }) + require.ErrorIs(t, err, ErrSandboxOperationFailed) + assert.False(t, evidence.Confirmed) + assert.Empty(t, evidence.SnapshotBuildID) +} + // A fatal (non-retryable) pause failure removes and emits exactly as before. func TestRemoveSandbox_FatalFailureRemovesAndEmits(t *testing.T) { t.Parallel() diff --git a/packages/api/internal/orchestrator/pause_instance.go b/packages/api/internal/orchestrator/pause_instance.go index 6d0eb71120..904d09ac7b 100644 --- a/packages/api/internal/orchestrator/pause_instance.go +++ b/packages/api/internal/orchestrator/pause_instance.go @@ -111,6 +111,9 @@ func snapshotInstance(ctx context.Context, node *nodemanager.Node, sbx sandbox.S if waitForStorage && (response == nil || !response.GetStorageDurable()) { return errors.New("pause completed without durable storage confirmation") } + if waitForStorage && !response.GetStopCompleted() { + return errors.New("pause completed without Firecracker stop confirmation") + } telemetry.ReportEvent(ctx, "Paused sandbox") return nil diff --git a/packages/orchestrator/orchestrator.proto b/packages/orchestrator/orchestrator.proto index 78979ead1d..3acb743666 100644 --- a/packages/orchestrator/orchestrator.proto +++ b/packages/orchestrator/orchestrator.proto @@ -239,6 +239,10 @@ message SchedulingMetadata { message SandboxPauseResponse { SchedulingMetadata scheduling_metadata = 1; bool storage_durable = 2; + // True only when a wait_for_storage request also waited for the exact + // execution's Firecracker stop. Older nodes decode to false so Cathedral + // cannot mistake storage upload alone for a completed pause during rollout. + bool stop_completed = 3; } message SandboxCheckpointRequest { diff --git a/packages/orchestrator/pkg/dummyserver/sandbox.go b/packages/orchestrator/pkg/dummyserver/sandbox.go index fee5b1128b..72cf23ef86 100644 --- a/packages/orchestrator/pkg/dummyserver/sandbox.go +++ b/packages/orchestrator/pkg/dummyserver/sandbox.go @@ -164,7 +164,10 @@ func (s *SandboxServer) Pause(_ context.Context, req *orchestrator.SandboxPauseR } delete(s.sandboxes, req.GetSandboxId()) - return &orchestrator.SandboxPauseResponse{}, nil + return &orchestrator.SandboxPauseResponse{ + StorageDurable: req.GetWaitForStorage(), + StopCompleted: req.GetWaitForStorage(), + }, nil } func (s *SandboxServer) Checkpoint(_ context.Context, _ *orchestrator.SandboxCheckpointRequest) (*orchestrator.SandboxCheckpointResponse, error) { diff --git a/packages/orchestrator/pkg/dummyserver/sandbox_test.go b/packages/orchestrator/pkg/dummyserver/sandbox_test.go index 8ac80ee80a..16484c5f5d 100644 --- a/packages/orchestrator/pkg/dummyserver/sandbox_test.go +++ b/packages/orchestrator/pkg/dummyserver/sandbox_test.go @@ -117,4 +117,19 @@ func TestPauseEvidenceRequestRequiresExactExecutionID(t *testing.T) { require.Equal(t, codes.FailedPrecondition, status.Code(err)) }) + + t.Run("exact", func(t *testing.T) { + server := NewSandbox() + createTestSandbox(t, server) + + response, err := server.Pause(context.Background(), &orchestrator.SandboxPauseRequest{ + SandboxId: testSandboxID, + ExecutionId: testExecutionID, + WaitForStorage: true, + }) + + require.NoError(t, err) + require.True(t, response.GetStorageDurable()) + require.True(t, response.GetStopCompleted()) + }) } diff --git a/packages/orchestrator/pkg/server/sandboxes.go b/packages/orchestrator/pkg/server/sandboxes.go index 778c50d73c..94bf65a884 100644 --- a/packages/orchestrator/pkg/server/sandboxes.go +++ b/packages/orchestrator/pkg/server/sandboxes.go @@ -972,8 +972,14 @@ func (s *Server) Pause(ctx context.Context, in *orchestrator.SandboxPauseRequest // guest and can close the sandbox, which would read as a crash. sbx.SetStopReason(sandbox.StopReasonPaused) - // Stop the old sandbox in background after we're done - defer s.stopSandboxAsync(context.WithoutCancel(ctx), sbx) + // Legacy pauses keep their asynchronous teardown. Evidence pauses attempt + // the stop synchronously below so the response cannot race continued VM use. + stopAttempted := false + defer func() { + if !stopAttempted { + s.stopSandboxAsync(context.WithoutCancel(ctx), sbx) + } + }() // Defer the rootfs reflink off the pause critical path when enabled: pause is a // suspend, so nothing reads the diff until a later resume (which waits on the @@ -1011,6 +1017,12 @@ func (s *Server) Pause(ctx context.Context, in *orchestrator.SandboxPauseRequest return nil, status.Errorf(codes.Internal, "error durably uploading paused sandbox '%s': %s", in.GetSandboxId(), uploadErr) } storageDurable = true + stopAttempted = true + if stopErr := sbx.Stop(ctx); stopErr != nil { + telemetry.ReportCriticalError(ctx, "error stopping durably paused sandbox", stopErr, telemetry.WithSandboxID(in.GetSandboxId())) + + return nil, status.Errorf(codes.Internal, "snapshot for sandbox '%s' is durable but its execution did not stop: %s", in.GetSandboxId(), stopErr) + } } else { s.uploadSnapshotAsync(ctx, sbx, res) } @@ -1057,6 +1069,7 @@ func (s *Server) Pause(ctx context.Context, in *orchestrator.SandboxPauseRequest return &orchestrator.SandboxPauseResponse{ SchedulingMetadata: res.schedulingMetadata, StorageDurable: storageDurable, + StopCompleted: in.GetWaitForStorage(), }, nil } diff --git a/packages/shared/pkg/grpc/orchestrator/orchestrator.pb.go b/packages/shared/pkg/grpc/orchestrator/orchestrator.pb.go index 2bf8c9e30a..15d984e0fe 100644 --- a/packages/shared/pkg/grpc/orchestrator/orchestrator.pb.go +++ b/packages/shared/pkg/grpc/orchestrator/orchestrator.pb.go @@ -1398,8 +1398,12 @@ type SandboxPauseResponse struct { state protoimpl.MessageState `protogen:"open.v1"` SchedulingMetadata *SchedulingMetadata `protobuf:"bytes,1,opt,name=scheduling_metadata,json=schedulingMetadata,proto3" json:"scheduling_metadata,omitempty"` StorageDurable bool `protobuf:"varint,2,opt,name=storage_durable,json=storageDurable,proto3" json:"storage_durable,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // True only when a wait_for_storage request also waited for the exact + // execution's Firecracker stop. Older nodes decode to false so Cathedral + // cannot mistake storage upload alone for a completed pause during rollout. + StopCompleted bool `protobuf:"varint,3,opt,name=stop_completed,json=stopCompleted,proto3" json:"stop_completed,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *SandboxPauseResponse) Reset() { @@ -1446,6 +1450,13 @@ func (x *SandboxPauseResponse) GetStorageDurable() bool { return false } +func (x *SandboxPauseResponse) GetStopCompleted() bool { + if x != nil { + return x.StopCompleted + } + return false +} + type SandboxCheckpointRequest struct { state protoimpl.MessageState `protogen:"open.v1"` SandboxId string `protobuf:"bytes,1,opt,name=sandbox_id,json=sandboxId,proto3" json:"sandbox_id,omitempty"` @@ -1871,10 +1882,11 @@ const file_orchestrator_proto_rawDesc = "" + "\x15rootfs_dropped_builds\x18\x06 \x01(\rR\x13rootfsDroppedBuilds\x12.\n" + "\x13memfile_build_bytes\x18\a \x03(\x04R\x11memfileBuildBytes\x12,\n" + "\x12rootfs_build_bytes\x18\b \x03(\x04R\x10rootfsBuildBytes\x12/\n" + - "\x14rootfs_base_build_id\x18\t \x01(\tR\x11rootfsBaseBuildId\"\x85\x01\n" + + "\x14rootfs_base_build_id\x18\t \x01(\tR\x11rootfsBaseBuildId\"\xac\x01\n" + "\x14SandboxPauseResponse\x12D\n" + "\x13scheduling_metadata\x18\x01 \x01(\v2\x13.SchedulingMetadataR\x12schedulingMetadata\x12'\n" + - "\x0fstorage_durable\x18\x02 \x01(\bR\x0estorageDurable\"\xd6\x01\n" + + "\x0fstorage_durable\x18\x02 \x01(\bR\x0estorageDurable\x12%\n" + + "\x0estop_completed\x18\x03 \x01(\bR\rstopCompleted\"\xd6\x01\n" + "\x18SandboxCheckpointRequest\x12\x1d\n" + "\n" + "sandbox_id\x18\x01 \x01(\tR\tsandboxId\x12\x19\n" + From f11618e72af6996d4df3290078a6d54fd3099649 Mon Sep 17 00:00:00 2001 From: Fred E <7602667+wallscaler@users.noreply.github.com> Date: Wed, 16 Sep 2026 21:49:25 -0400 Subject: [PATCH 9/9] fix(api): preserve v2 create after upstream rebase --- packages/api/internal/handlers/sandbox_create.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/api/internal/handlers/sandbox_create.go b/packages/api/internal/handlers/sandbox_create.go index 1ba9facb94..abf2ace602 100644 --- a/packages/api/internal/handlers/sandbox_create.go +++ b/packages/api/internal/handlers/sandbox_create.go @@ -76,7 +76,7 @@ func (a *APIStore) PostSandboxes(c *gin.Context, params api.PostSandboxesParams) return } - a.createSandbox(c, body, sandbox.SandboxTimeoutDefault) + a.createSandbox(c, body, sandbox.SandboxTimeoutDefault, params.IdempotencyKey) } // PostV2Sandboxes creates a sandbox with secured envd access; the request has no secure field to opt out. @@ -92,7 +92,7 @@ func (a *APIStore) PostV2Sandboxes(c *gin.Context) { return } - a.createSandbox(c, newSandboxFromV2(body), sandbox.SandboxTimeoutDefaultV2) + a.createSandbox(c, newSandboxFromV2(body), sandbox.SandboxTimeoutDefaultV2, nil) } func newSandboxFromV2(body api.NewSandboxV2) api.NewSandbox { @@ -116,7 +116,7 @@ func newSandboxFromV2(body api.NewSandboxV2) api.NewSandbox { } // createSandbox runs the shared create flow; defaultTimeout applies when the body omits timeout. -func (a *APIStore) createSandbox(c *gin.Context, body api.NewSandbox, defaultTimeout time.Duration) { +func (a *APIStore) createSandbox(c *gin.Context, body api.NewSandbox, defaultTimeout time.Duration, cathedralOperationKey *string) { ctx := c.Request.Context() // Get team from context, use TeamContextKey @@ -133,7 +133,7 @@ func (a *APIStore) createSandbox(c *gin.Context, body api.NewSandbox, defaultTim cathedralClaim, proceed := a.inspectCathedralCreate( c, teamInfo.Team.ID, - params.IdempotencyKey, + cathedralOperationKey, body, ) if !proceed {