diff --git a/.circleci/config.yml b/.circleci/config.yml index b1d8ecf..9faeca6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,59 +1,24 @@ version: 2.1 executors: - node-postgres: + build: docker: - - image: cimg/python:3.13.2 - - image: postgres:17-alpine - environment: - POSTGRES_USER: forms - POSTGRES_PASSWORD: forms_ci - POSTGRES_DB: forms_test + - image: cimg/python:3.13.2-browsers resource_class: medium commands: - install: + install-dependencies: steps: - run: - name: Install pinned Node and pnpm + name: Install build and deployment dependencies command: | sudo apt-get update -qq - sudo apt-get install -y -qq jq - export NVM_DIR=/home/circleci/.nvm - git clone --quiet --depth 1 --branch v0.40.3 https://github.com/nvm-sh/nvm.git "$NVM_DIR" - test "$(git -C "$NVM_DIR" rev-parse HEAD)" = 977563e97ddc66facf3a8e31c6cff01d236f09bd - . "$NVM_DIR/nvm.sh" --no-use - nvm install - nvm use - npm install --global pnpm@11.21.0 - pnpm install --frozen-lockfile + sudo apt-get install -y -qq jq python3-pip pip install boto3 awscli - echo 'export NVM_DIR=/home/circleci/.nvm' >> "$BASH_ENV" - echo '. "$NVM_DIR/nvm.sh" --no-use' >> "$BASH_ENV" - echo 'nvm use' >> "$BASH_ENV" jobs: - verify: - executor: node-postgres - environment: - DATABASE_URL: postgresql://forms:forms_ci@localhost:5432/forms_test?schema=forms - TEST_DATABASE_URL: postgresql://forms:forms_ci@localhost:5432/forms_test?schema=forms - steps: - - checkout - - install - - run: - name: Lint, build, and verify with PostgreSQL - command: | - pnpm lint - pnpm build - pnpm typecheck - pnpm test - pnpm migrate:deploy - pnpm test:integration - python3 -m py_compile deploy/bootstrap-dev.py deploy/copy-dev-data.py deploy/release.py deploy/seed-dev.py - deploy: - executor: node-postgres + executor: build parameters: deploy_env: type: enum @@ -68,40 +33,50 @@ jobs: APPNAME: forms-api-v6 steps: - checkout - - install - setup_remote_docker + - install-dependencies + - run: + name: Verify first-release activation and rollback handling + command: python3 -m unittest discover -s deploy/tests - run: name: Build runtime and one-shot migration images command: | - docker build --target migrate -t forms-api-v6:migrate-candidate . - docker build --target runtime -t forms-api-v6:candidate . + docker buildx build --load --target migrate -t forms-api-v6:migrate-candidate . + docker buildx build --load --target runtime -t forms-api-v6:latest . - run: - name: Authenticate, migrate, and deploy through CloudFormation + name: Migrate and deploy using the Topcoder deployment suite no_output_timeout: 35m command: | git clone --quiet --depth 1 --branch v1.4.20 https://github.com/topcoder-platform/tc-deploy-scripts ../buildscript test "$(git -C ../buildscript rev-parse HEAD)" = 5f3745c35463ce0e2475c12269aa7a2b446456e5 - cp ../buildscript/awsconfiguration.sh . + cp ../buildscript/{master_deploy,buildenv,awsconfiguration,psvar-processor}.sh . ./awsconfiguration.sh << parameters.deploy_env >> set +x source awsenvconf - rm -f awsenvconf awsconfiguration.sh + ./psvar-processor.sh -t appenv -p "/config/${APPNAME}/deployvar" + source deployvar_env python3 -u deploy/release.py << parameters.environment >> "<< parameters.environment >>-${CIRCLE_SHA1}-${CIRCLE_BUILD_NUM}" + ./master_deploy.sh \ + -d ECS \ + -e << parameters.deploy_env >> \ + -t latest \ + -j "/config/${APPNAME}/appvar,/config/common/global-appvar" \ + -i "$APPNAME" \ + -p FARGATE + python3 -u deploy/activate.py - store_artifacts: path: deploy/release-<< parameters.environment >>.json destination: release workflows: - verify-and-deploy: + build-and-deploy: jobs: - - verify - deploy: name: deploy-dev context: org-global deploy_env: DEV environment: dev serial-group: << pipeline.project.slug >>/deploy-dev - requires: [verify] filters: branches: only: develop @@ -111,7 +86,6 @@ workflows: deploy_env: PROD environment: production serial-group: << pipeline.project.slug >>/deploy-production - requires: [verify] filters: branches: only: master diff --git a/.env.example b/.env.example index bbe435d..d270309 100644 --- a/.env.example +++ b/.env.example @@ -3,13 +3,10 @@ DATABASE_URL=postgresql://forms:forms_local@127.0.0.1:5546/forms?schema=forms PORT=3006 CORS_ORIGINS=http://localhost:3000,http://localhost:3006 -# Local demo token verification. Deployment: AUTH_MODE=jwks and a trusted HTTPS JWKS URL. -AUTH_MODE=hs256 +# Shared Topcoder JWT authentication: HS256 secret and RS256 issuer-derived JWKS. AUTH_SECRET=local-development-only-change-this-secret-000000 -VALID_ISSUERS=https://api.topcoder-dev.com -AUTH_AUDIENCE=forms-api +VALID_ISSUERS=https://api.topcoder-dev.com,https://auth.topcoder-dev.com/,https://topcoder-dev.auth0.com/ AUTH_CLAIM_NAMESPACE=https://topcoder.com/ -# JWKS_URL=https://topcoder-dev.auth0.com/.well-known/jwks.json # Requests per client IP per minute, per API replica. Shared enforcement belongs at the gateway. THROTTLE_LIMIT=30 @@ -19,3 +16,14 @@ TRUST_PROXY_CIDRS= # Local Docker Compose host ports. FORMS_HTTP_PORT=3006 FORMS_DATABASE_PORT=5546 + +# Optional outbound Kafka publication through Bus API (required for kafka=true submissions). +# The wrapper adds /bus/events; use the API base ending in /v6. +# BUSAPI_URL=https://api.topcoder-dev.com/v6 +# AUTH0_URL=https://topcoder-dev.auth0.com/oauth/token +# AUTH0_AUDIENCE=https://www.topcoder-dev.com +# AUTH0_CLIENT_ID= +# AUTH0_CLIENT_SECRET= +# TOKEN_CACHE_TIME=86400000 +# AUTH0_PROXY_SERVER_URL= +# KAFKA_ERROR_TOPIC=common.error.reporting diff --git a/README.md b/README.md index e24fbae..a498b3f 100644 --- a/README.md +++ b/README.md @@ -66,7 +66,8 @@ Stop the Compose API first if it already occupies port 3006. `pnpm start:dev` us 3. Publish it. The API creates `forms.event_interest_v1` transactionally. 4. Add a Payload form block referencing `event_interest` to a content page. 5. The website fetches the current API schema and submits the exact displayed revision with a UUID `Idempotency-Key`. -6. Read private paginated JSON/CSV reports or query the SQL view with a reporting database role. +6. Optionally include `"kafka": true` alongside the submitted answers to publish a `form.submitted` event through Bus API; see [the event contract and retry behavior](docs/api.md#optional-kafka-publication). +7. Read private paginated JSON/CSV reports or query the SQL view with a reporting database role. Published definitions are immutable. Changes use a new sequential revision; publication retires the previous one. An already-open older page receives a 409 and must reload. Historical reports remain available. Exact retries of accepted submissions return the original receipt, including after retirement. @@ -101,3 +102,7 @@ TEST_DATABASE_URL=postgresql://forms:forms_local@127.0.0.1:5546/forms?schema=for ``` Coverage includes actual PostgreSQL writes, all field types, SQL views, CSV output, concurrent idempotency, immutable versions, JWT access, required-answer/ownership constraints, Payload synchronization over HTTP, and browser rendering/retry behavior. CI executes the same checks with PostgreSQL 17. + +Forms portal endpoints, inclusive UTC date filtering, and full CSV exports are +specified in [docs/api.md](docs/api.md). The development-only `/lets-talk` migration, +Kafka contract, seed command, and rollout order are in [docs/lets-talk.md](docs/lets-talk.md). diff --git a/deploy/README.md b/deploy/README.md index f352bc6..c778e25 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -11,6 +11,8 @@ require database availability. OpenAPI UI is `/v6/forms/api-docs`, with JSON at `/v6/forms/api-docs-json`. The root forms list requires an administrator token; published schemas and anonymous submissions are public. +Inbound JWT handling uses the same `tc-core-library-js` middleware as the other v6 APIs. Both legacy HS256 and Auth0 RS256 tokens are supported in one deployment. The Forms `VALID_ISSUERS` parameter must contain `https://auth.topcoder-dev.com/` for dev browser sign-in; the library resolves its JWKS and normalizes the dev namespaced roles/user ID. Existing `AUTH_MODE` and `AUTH_AUDIENCE` task settings are ignored by the new runtime. Updating the application image is sufficient when the issuer is already allowed; otherwise update the issuer parameter and roll the tasks as well. See [authentication configuration](../docs/operations.md#environment). + ## Initial environment setup Dev uses schema `forms` in database `topcoder-services` on the existing services RDS instance. The database administrator provisions the `forms` login and schema; this service does not create a separate database or rotate that login. All tables, enums, functions, reporting views, and Prisma migration history remain in `forms`. @@ -32,39 +34,76 @@ The copy refuses a nonempty target. The source is retained and fenced against fu ## Releases -`.circleci/config.yml` runs lint, build, type checks, unit tests, migrations, and -real-PostgreSQL integration tests. After verification, `develop` deploys dev and -`master` deploys production. Both use Topcoder's `org-global` context and pinned -`tc-deploy-scripts` credential helper, matching the other services. Deployments are -serialized separately per environment. The CircleCI project must be connected in -the Topcoder organization to use that shared context. +`.circleci/config.yml` uses the same pinned `tc-deploy-scripts` v1.4.20 flow as +`bus-api-v6`: `awsconfiguration.sh` loads credentials, `psvar-processor.sh` loads +`/config/forms-api-v6/deployvar`, and `master_deploy.sh` publishes the runtime image +and updates ECS with service/global SSM secret references. `develop` deploys dev; +`master` deploys production. Both use the Topcoder `org-global` context and are +serialized separately per environment. + +Docker builds the runtime as `forms-api-v6:latest` and a separate migration image. +The existing `release.py` now only pushes and runs the migration image in the +service's private subnets, using `MIGRATION_DATABASE_URL` as its sole secret. +It clones the service's current task definition and records the migration image +digest and task ARN in `deploy/release-.json`. A nonzero migration +exit stops the job before `master_deploy.sh` can deploy the runtime. On timeout, +inspect the reported migration task before retrying. + +The runtime deployment command is the standard shared invocation: + +```sh +./master_deploy.sh -d ECS -e DEV -t latest \ + -j "/config/${APPNAME}/appvar,/config/common/global-appvar" \ + -i "$APPNAME" -p FARGATE +``` -The same release command can run from an authorized workstation: +The shared script tags the runtime image with `CIRCLE_BUILD_NUM`, registers the +ECS task definition, updates the existing service, and checks rollout status. +Its Fargate template uses the shared `ecsTaskExecutionRole` and writes logs to +`/aws/ecs/` with the deployment environment as stream prefix. Ensure +that role can read both SSM prefixes (and decrypt any custom KMS key). +The service's ALB readiness checks and deployment circuit breaker remain in place. +Database migrations must remain compatible with the previous runtime; service +rollback does not undo schema or data migrations. + +Provision `/config/forms-api-v6/deployvar` before the first release in each account: + +| Parameter | Value | +| --- | --- | +| `AWS_REPOSITORY`, `AWS_ECS_SERVICE`, `AWS_ECS_TASK_FAMILY`, `AWS_ECS_CONTAINER_NAME` | `forms-api-v6` | +| `AWS_ECS_CLUSTER` | Existing cluster, e.g. `topcoder-infrastructure` | +| `AWS_ECS_PORTS` | `3000:3000:tcp` | +| `AWS_ECS_FARGATE_CPU`, `AWS_ECS_FARGATE_MEMORY` | `512`, `1024` for the current Forms task size | +| `AWS_ECS_CONTAINER_CPU`, `AWS_ECS_CONTAINER_MEMORY_RESERVATION` | `0`, `512` | +| `AWS_ECS_READONLY_ROOTFILESYSTEM` | `true` | +| `AWS_ECS_TASK_ROLE_ARN` | Existing Forms task role **name**, without its ARN prefix | +| `AWS_ECS_CONTAINER_HEALTH_CMD` | Readiness command, with double quotes escaped for `psvar-processor.sh`'s shell export format | + +Move runtime environment settings into service appvars: `NODE_ENV=production`, +`PORT=3000`, `AWS_REGION=us-east-1`, and the environment's existing `CORS_ORIGINS` +and `TRUST_PROXY_CIDRS`. The shared template obtains runtime settings from SSM, +not from the previous CloudFormation task definition. Production requires its own +origins, credentials, task role, and deployvars. + +Local verification: ```sh nvm use pnpm lint && pnpm build && pnpm test -docker build --target migrate -t forms-api-v6:migrate-candidate . -docker build --target runtime -t forms-api-v6:candidate . -python3 -u deploy/release.py dev dev-UNIQUE_RELEASE_TAG ``` -Use `--network=host` for local Docker builds if the workstation bridge cannot -resolve the registry. `release.py` requires boto3 and Docker and checks account and -stack environment before pushing. Runtime and migration images receive immutable -release tags. The migration task runs in the same private subnets as the service, -uses the configured schema-owner login, and applies only the schema-scoped migrations. -Only exit code zero permits CloudFormation promotion. ECS keeps the previous task -healthy during rollout and uses its deployment circuit breaker to roll back failed -runtime starts. Database migrations must remain compatible with the previous -runtime; the service rollback does not undo data/schema migrations. - -The script writes `release-dev.json` or `release-production.json` containing image -digests and AWS task/stack identifiers. On an observation timeout, inspect the -reported task/stack operation before retrying; do not start a second migration -while the first remains active. Infrastructure edits are applied separately using -CloudFormation with the current ImageTag and environment parameters preserved; -normal app releases retain the existing stack template and parameters. +CloudFormation remains responsible for infrastructure. Application releases now +update the ECS service directly, as in the other v6 services; the stack's ImageTag +and TaskDefinition output no longer track the active application release. When +changing infrastructure, preserve the live service task definition to avoid +restoring the stack's older task definition. A newly bootstrapped service with +DesiredCount=0 is started by `activate.py` after its migrations and first runtime +deployment. This final CircleCI check preserves existing nonzero replica counts, +verifies the requested image, and fails if ECS rolls back or never becomes healthy. + +Production infrastructure and the configuration handoff are recorded in +[production.md](production.md). `production.parameters.json` describes initial +creation at zero replicas; do not reuse its bootstrap image/count for later updates. ## Dev sample @@ -73,3 +112,20 @@ normal app releases retain the existing stack template and parameters. It refuses production and conflicting published definitions. The corresponding CMS seed lives in `payload-cms/scripts/seed-forms-test.ts`. The website resolves the CMS page at `/forms-test` and fetches its schema from the public Forms API at runtime. + +## Runtime appvar injection + +Forms invokes the shared `master_deploy.sh` directly with +`-j /config/forms-api-v6/appvar,/config/common/global-appvar`. It injects SSM ARN +references into the runtime task's `secrets` list; service-specific names take +precedence over matching globals. ECS resolves the values at task startup. +There is no Forms-specific appvar mapping script or extra Python/YAML dependency. + +New releases pick up parameter additions and removals. After changing only an +existing parameter's value, force a new ECS deployment to refresh running tasks. + +For Kafka delivery, create `BUSAPI_URL=https://api.topcoder-dev.com/v6` under the +Forms dev appvar path. The Forms `AUTH0_CLIENT_ID` / `AUTH0_CLIENT_SECRET` combine +with shared `AUTH0_URL`, `AUTH0_AUDIENCE`, and optional `AUTH0_PROXY_SERVER_URL` / +`TOKEN_CACHE_TIME`. Production needs its corresponding URL and authorized credentials. +Do not decrypt or copy shared values into the service path to perform injection. diff --git a/deploy/activate.py b/deploy/activate.py new file mode 100644 index 0000000..4f53e29 --- /dev/null +++ b/deploy/activate.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +"""Start a bootstrapped service and verify the image deployed by master_deploy.sh. + +Run after successful migrations and shared runtime deployment. Reads the shared +AWS_ECS_* deployvars and CIRCLE_BUILD_NUM; keeps existing nonzero replica counts. +""" +import os + +import boto3 + +from release import wait_until + + +def activate_service(ecs, cluster, service_name, expected_image): + """Activate and observe the requested runtime using an ECS client and identifiers. + + Returns the stable service. Raises on missing services, an unexpected image, + rollback, failed deployment, AWS errors, or observation timeout. Only a service + already referencing expected_image can be scaled from zero to one. + """ + def read_service(): + """Return the named active ECS service or raise if it is unavailable.""" + result = ecs.describe_services(cluster=cluster, services=[service_name]) + if result.get('failures') or len(result.get('services', [])) != 1: + raise RuntimeError('The deployment service could not be found.') + service = result['services'][0] + if service['status'] != 'ACTIVE': + raise RuntimeError('The deployment service is not active.') + return service + + service = read_service() + definition_arn = service['taskDefinition'] + definition = ecs.describe_task_definition(taskDefinition=definition_arn)['taskDefinition'] + container = next(c for c in definition['containerDefinitions'] if c['name'] == 'forms-api-v6') + if container['image'] != expected_image: + raise RuntimeError('Shared deployment did not promote the requested runtime image.') + if service['desiredCount'] == 0: + ecs.update_service(cluster=cluster, service=service_name, desiredCount=1) + print('Started the bootstrapped Forms service with one replica.', flush=True) + + def is_stable(): + """Return the healthy service when converged; reject rollback or failure.""" + current = read_service() + if current['taskDefinition'] != definition_arn: + raise RuntimeError('ECS rolled back or replaced the requested runtime.') + deployments = current['deployments'] + if any(d.get('rolloutState') == 'FAILED' for d in deployments): + raise RuntimeError('ECS runtime deployment failed.') + if (len(deployments) == 1 and deployments[0].get('rolloutState') == 'COMPLETED' + and current['runningCount'] == current['desiredCount'] > 0 + and current['pendingCount'] == 0): + arns = ecs.list_tasks(cluster=cluster, serviceName=service_name, + desiredStatus='RUNNING')['taskArns'] + if arns: + tasks = ecs.describe_tasks(cluster=cluster, tasks=arns)['tasks'] + healthy = [task for task in tasks + if task['taskDefinitionArn'] == definition_arn + and task.get('healthStatus') == 'HEALTHY' + and task['lastStatus'] == 'RUNNING'] + if len(healthy) >= current['desiredCount']: + return current + return None + + return wait_until('requested Forms runtime to become stable', is_stable) + + +def main(): + """Read CircleCI/deployvar settings and activate the expected image; errors propagate.""" + session = boto3.Session(region_name=os.environ.get('AWS_REGION', 'us-east-1')) + account = session.client('sts').get_caller_identity()['Account'] + image = (f'{account}.dkr.ecr.{session.region_name}.amazonaws.com/' + f'{os.environ["AWS_REPOSITORY"]}:{os.environ["CIRCLE_BUILD_NUM"]}') + service = activate_service(session.client('ecs'), os.environ['AWS_ECS_CLUSTER'], + os.environ['AWS_ECS_SERVICE'], image) + print('Verified running release: ' + service['taskDefinition'], flush=True) + + +if __name__ == '__main__': + main() diff --git a/deploy/bootstrap-dev.py b/deploy/bootstrap-dev.py index 8de28e2..4174867 100644 --- a/deploy/bootstrap-dev.py +++ b/deploy/bootstrap-dev.py @@ -105,7 +105,6 @@ def main(): Overwrite=True, Description='Forms schema in topcoder-services; user-provided forms login') issuers = json.loads(parameter(ssm, '/config/common/global-appvar/VALID_ISSUERS')) ensure_parameter(ssm, PREFIX + '/VALID_ISSUERS', ','.join(issuers)) - ensure_parameter(ssm, PREFIX + '/AUTH_AUDIENCE', parameter(ssm, '/config/common/global-appvar/AUTH0_AUDIENCE')) print('Encrypted database URLs now select topcoder-services, schema forms, user forms.') diff --git a/deploy/production-parameters-manifest.json b/deploy/production-parameters-manifest.json new file mode 100644 index 0000000..f745f1d --- /dev/null +++ b/deploy/production-parameters-manifest.json @@ -0,0 +1,137 @@ +[ + { + "name": "/config/forms-api-v6/appvar/DATABASE_URL", + "type": "SecureString", + "requiresReplacement": true + }, + { + "name": "/config/forms-api-v6/appvar/MIGRATION_DATABASE_URL", + "type": "SecureString", + "requiresReplacement": true + }, + { + "name": "/config/forms-api-v6/appvar/AUTH0_CLIENT_ID", + "type": "SecureString", + "requiresReplacement": true + }, + { + "name": "/config/forms-api-v6/appvar/AUTH0_CLIENT_SECRET", + "type": "SecureString", + "requiresReplacement": true + }, + { + "name": "/config/forms-api-v6/appvar/BUSAPI_URL", + "type": "SecureString", + "requiresReplacement": false + }, + { + "name": "/config/forms-api-v6/appvar/VALID_ISSUERS", + "type": "SecureString", + "requiresReplacement": false + }, + { + "name": "/config/forms-api-v6/appvar/NODE_ENV", + "type": "SecureString", + "requiresReplacement": false + }, + { + "name": "/config/forms-api-v6/appvar/PORT", + "type": "SecureString", + "requiresReplacement": false + }, + { + "name": "/config/forms-api-v6/appvar/AWS_REGION", + "type": "SecureString", + "requiresReplacement": false + }, + { + "name": "/config/forms-api-v6/appvar/CORS_ORIGINS", + "type": "SecureString", + "requiresReplacement": false + }, + { + "name": "/config/forms-api-v6/appvar/TRUST_PROXY_CIDRS", + "type": "SecureString", + "requiresReplacement": false + }, + { + "name": "/config/forms-api-v6/appvar/AUTH_CLAIM_NAMESPACE", + "type": "SecureString", + "requiresReplacement": false + }, + { + "name": "/config/forms-api-v6/appvar/THROTTLE_LIMIT", + "type": "SecureString", + "requiresReplacement": false + }, + { + "name": "/config/forms-api-v6/appvar/KAFKA_ERROR_TOPIC", + "type": "SecureString", + "requiresReplacement": false + }, + { + "name": "/config/forms-api-v6/deployvar/AWS_REPOSITORY", + "type": "SecureString", + "requiresReplacement": false + }, + { + "name": "/config/forms-api-v6/deployvar/AWS_ECS_SERVICE", + "type": "SecureString", + "requiresReplacement": false + }, + { + "name": "/config/forms-api-v6/deployvar/AWS_ECS_TASK_FAMILY", + "type": "SecureString", + "requiresReplacement": false + }, + { + "name": "/config/forms-api-v6/deployvar/AWS_ECS_CONTAINER_NAME", + "type": "SecureString", + "requiresReplacement": false + }, + { + "name": "/config/forms-api-v6/deployvar/AWS_ECS_CLUSTER", + "type": "SecureString", + "requiresReplacement": false + }, + { + "name": "/config/forms-api-v6/deployvar/AWS_ECS_PORTS", + "type": "SecureString", + "requiresReplacement": false + }, + { + "name": "/config/forms-api-v6/deployvar/AWS_ECS_FARGATE_CPU", + "type": "SecureString", + "requiresReplacement": false + }, + { + "name": "/config/forms-api-v6/deployvar/AWS_ECS_FARGATE_MEMORY", + "type": "SecureString", + "requiresReplacement": false + }, + { + "name": "/config/forms-api-v6/deployvar/AWS_ECS_CONTAINER_CPU", + "type": "SecureString", + "requiresReplacement": false + }, + { + "name": "/config/forms-api-v6/deployvar/AWS_ECS_CONTAINER_MEMORY_RESERVATION", + "type": "SecureString", + "requiresReplacement": false + }, + { + "name": "/config/forms-api-v6/deployvar/AWS_ECS_READONLY_ROOTFILESYSTEM", + "type": "SecureString", + "requiresReplacement": false + }, + { + "name": "/config/forms-api-v6/deployvar/AWS_ECS_TASK_ROLE_ARN", + "type": "SecureString", + "requiresReplacement": false + }, + { + "name": "/config/forms-api-v6/deployvar/AWS_ECS_CONTAINER_HEALTH_CMD", + "type": "SecureString", + "requiresReplacement": false + } +] diff --git a/deploy/production-readiness.json b/deploy/production-readiness.json new file mode 100644 index 0000000..e5b0a4e --- /dev/null +++ b/deploy/production-readiness.json @@ -0,0 +1,56 @@ +{ + "date": "2026-10-06", + "account": "409275337247", + "region": "us-east-1", + "stack": "arn:aws:cloudformation:us-east-1:409275337247:stack/forms-api-v6-production/7c574b20-c10c-11f1-89bd-0affde949709", + "stackStatus": "CREATE_COMPLETE", + "cluster": "topcoder-infrastructure", + "service": "arn:aws:ecs:us-east-1:409275337247:service/topcoder-infrastructure/forms-api-v6", + "taskDefinition": "arn:aws:ecs:us-east-1:409275337247:task-definition/forms-api-v6:1", + "desiredCount": 0, + "secureStringCount": 27, + "outputs": { + "TargetGroupArn": "arn:aws:elasticloadbalancing:us-east-1:409275337247:targetgroup/forms-api-v6-production/693b280b99d3fc76", + "ApiUrl": "https://api.topcoder.com/v6/forms", + "ServiceName": "forms-api-v6", + "TaskSecurityGroupId": "sg-0bf5e59924ba4a023", + "RepositoryUri": "409275337247.dkr.ecr.us-east-1.amazonaws.com/forms-api-v6", + "TaskDefinitionArn": "arn:aws:ecs:us-east-1:409275337247:task-definition/forms-api-v6:1" + }, + "preflightImage": "409275337247.dkr.ecr.us-east-1.amazonaws.com/forms-api-v6:infrastructure-preflight-20261006", + "preflight": [ + { + "subnet": "subnet-0d91bddbea456aa6b", + "taskArn": "arn:aws:ecs:us-east-1:409275337247:task/topcoder-infrastructure/92ebe25821de4175b0f92a023bc96d30", + "exitCode": 0, + "checks": [ + "runtime configuration parses", + "auth.topcoder.com HTTPS reachable: 200", + "topcoder.auth0.com HTTPS reachable: 200", + "auth0proxy.topcoder.com HTTPS reachable: 500", + "api.topcoder.com HTTPS reachable: 404", + "PostgreSQL reachable; RDS certificate verified", + "INFRASTRUCTURE_PREFLIGHT_PASSED" + ] + }, + { + "subnet": "subnet-7f657057", + "taskArn": "arn:aws:ecs:us-east-1:409275337247:task/topcoder-infrastructure/0bff2d9353eb4c37845cc78cfa3105c6", + "exitCode": 0, + "checks": [ + "runtime configuration parses", + "auth.topcoder.com HTTPS reachable: 200", + "topcoder.auth0.com HTTPS reachable: 200", + "auth0proxy.topcoder.com HTTPS reachable: 500", + "api.topcoder.com HTTPS reachable: 404", + "PostgreSQL reachable; RDS certificate verified", + "INFRASTRUCTURE_PREFLIGHT_PASSED" + ] + } + ], + "limitations": [ + "Placeholders must be replaced before release.", + "Database login/schema and real credentials have not been validated.", + "No production application deployment or migration has run." + ] +} diff --git a/deploy/production.md b/deploy/production.md new file mode 100644 index 0000000..4652493 --- /dev/null +++ b/deploy/production.md @@ -0,0 +1,115 @@ +# Production handoff — 2026-10-06 + +Production infrastructure is provisioned in AWS account `409275337247`, region +`us-east-1`. CloudFormation stack `forms-api-v6-production` owns service +`forms-api-v6` on cluster `topcoder-infrastructure`. The service is intentionally +at **zero replicas** until the first configured release. No production database +migration or application startup has been performed. + +## Before merging develop into master + +Update these four **SecureString** parameters under +`/config/forms-api-v6/appvar` in the production account: + +| Parameter | Required value | +| --- | --- | +| `DATABASE_URL` | Production Forms schema-owner connection URL. The placeholder targets the shared production `topcoder-services` database with user `forms`, `schema=forms`, and `sslmode=verify-full`. Replace `REPLACE_ME` with the URL-encoded password and confirm the database/user. | +| `MIGRATION_DATABASE_URL` | Connection URL for the owner of the same `forms` schema. The same Forms login can be used for both URLs. Replace its placeholder independently. | +| `AUTH0_CLIENT_ID` | Production M2M client authorized to publish through Bus API. | +| `AUTH0_CLIENT_SECRET` | Matching production M2M client secret. | + +The `forms` login/schema must already be provisioned in that database. The login +must own `forms`; runtime publication creates reporting views there. CircleCI runs +`prisma migrate deploy` to create the application objects before deploying the +runtime. This setup does not create a database login/schema or copy development +records into production. Existing Forms data would require a separate migration +plan; do not point the baseline at an unrelated existing installation. + +Review these populated SecureStrings under the same appvar path: + +| Parameter | Initial value | +| --- | --- | +| `BUSAPI_URL` | `https://api.topcoder.com/v6` | +| `VALID_ISSUERS` | JSON list: `https://api.topcoder.com`, `https://topcoder.auth0.com/`, `https://auth.topcoder.com/` | +| `CORS_ORIGINS` | `https://www.topcoder.com,https://reports.topcoder.com,https://platform-ui.topcoder.com` | +| `TRUST_PROXY_CIDRS` | `10.25.0.0/16` | +| `AUTH_CLAIM_NAMESPACE` | `https://topcoder.com/` | +| `THROTTLE_LIMIT` | `30` | +| `KAFKA_ERROR_TOPIC` | `common.error.reporting` | +| `NODE_ENV`, `PORT`, `AWS_REGION` | `production`, `3000`, `us-east-1` | + +The existing production SecureStrings in `/config/common/global-appvar` provide +`AUTH_SECRET`, `AUTH0_URL`, `AUTH0_AUDIENCE`, `AUTH0_PROXY_SERVER_URL`, and +`TOKEN_CACHE_TIME`. They were preserved and are referenced by the shared deployment +suite. No shared secrets were copied from dev or replaced with placeholders. +Service appvars override matching globals, including `VALID_ISSUERS`. + +All 13 `/config/forms-api-v6/deployvar` settings are populated as SecureStrings, +including the generated task role name and the correctly escaped readiness +command. They require no placeholder replacement. The complete list of 27 new +parameters is in [production-parameters-manifest.json](production-parameters-manifest.json). + +## First release + +After updating the configuration, merge the prepared `develop` branch into +`master`. CircleCI uses `org-global`, requests production credentials, builds and +pushes the migration image, and runs migrations in the service's private network. +Only a successful migration proceeds to the shared `master_deploy.sh` runtime +release. `activate.py` then starts a zero-count service at one replica and waits +for the requested image's tasks to be healthy. Later releases retain the existing +nonzero replica count. Rollback, wrong-image, and readiness failures fail the job. + +The expected public endpoints are: + +- `https://api.topcoder.com/v6/forms/health/ready` +- `https://api.topcoder.com/v6/forms/api-docs` + +Until the first release, these routes return 503 because there are no runtime +targets. Parameter value changes require new tasks to take effect. + +## Provisioned infrastructure + +- Task definition `forms-api-v6:1`, Fargate, 512 CPU units / 1024 MiB, port 3000. +- ECR repository `forms-api-v6`, immutable release tags and image scanning. +- VPC `vpc-7cc06b19`; private subnets `subnet-0d91bddbea456aa6b` and + `subnet-7f657057` in the ALB's enabled availability zones; public IPs disabled. +- Task security group `sg-0bf5e59924ba4a023`: inbound 3000 only from the services + ALB; outbound HTTPS and PostgreSQL to the database security group. +- PostgreSQL ingress from this task group to `sg-930253f6` on port 5432. +- Target group `forms-api-v6-production`, readiness health check, ALB HTTPS rule + priority 680 for `/v6/forms` and `/v6/forms/*`. +- API Gateway `kthgpat40e`, both Forms routes, existing TLS/VPC integration + `6tcvbtt` and CloudFront origin authorizer `ptbwwm`. +- Initial task/migration log group `/aws/ecs/forms-api-v6-production`; shared + deployment suite runtime logs use `/aws/ecs/topcoder-infrastructure`. +- Stack termination protection enabled. Existing shared execution and CI roles + already grant the required access; no shared IAM policy changes were needed. + +[production.parameters.json](production.parameters.json) records the initial +CloudFormation parameters. It contains `ImageTag=bootstrap` and `DesiredCount=0`; +**do not apply it unchanged to an active service**. Preserve the live task +reference and replica count when performing later infrastructure updates, because +shared-script application deployments do not update CloudFormation's task output. + +## Verification and limits + +Both production subnets passed temporary ECS preflight tasks using the packaged +runtime image and shared execution role. Checks covered ECR pulls, encrypted SSM +injection, runtime configuration parsing, CloudWatch logging, HTTPS access to the +JWKS/Auth0/API endpoints, and PostgreSQL connectivity with RDS certificate +verification. These checks did not authenticate to PostgreSQL, request an M2M +token, publish a Bus event, or run application migrations. Auth0 proxy/API probes +only established connectivity; their unauthenticated responses do not establish +that service credentials work. + +CloudFormation reached `CREATE_COMPLETE`; all 27 new parameters are SecureStrings. +The CI role's ECR, SSM, CloudFormation read, ECS, and role-passing permissions were +checked with IAM policy simulation. The shared deployvar export script correctly +round-tripped the task role and escaped health command. Local lint, build, runtime +Docker build, and six deployment regression tests passed. A production CircleCI +run remains intentionally deferred until the real configuration is supplied and +`master` is merged. + +[production-readiness.json](production-readiness.json) records the AWS identifiers +and both preflight task results. The temporary preflight definition was deregistered; +the stopped tasks/logs and immutable preflight image remain as verification evidence. diff --git a/deploy/production.parameters.json b/deploy/production.parameters.json new file mode 100644 index 0000000..6f0b16e --- /dev/null +++ b/deploy/production.parameters.json @@ -0,0 +1,70 @@ +[ + { + "ParameterKey": "BootstrapOnly", + "ParameterValue": "false" + }, + { + "ParameterKey": "Environment", + "ParameterValue": "production" + }, + { + "ParameterKey": "ClusterName", + "ParameterValue": "topcoder-infrastructure" + }, + { + "ParameterKey": "VpcId", + "ParameterValue": "vpc-7cc06b19" + }, + { + "ParameterKey": "PrivateSubnetIds", + "ParameterValue": "subnet-0d91bddbea456aa6b,subnet-7f657057" + }, + { + "ParameterKey": "AlbSecurityGroupId", + "ParameterValue": "sg-9225f1db" + }, + { + "ParameterKey": "DatabaseSecurityGroupId", + "ParameterValue": "sg-930253f6" + }, + { + "ParameterKey": "HttpsListenerArn", + "ParameterValue": "arn:aws:elasticloadbalancing:us-east-1:409275337247:listener/app/tc-services-alb/eb37e4c9eb22e365/ea2b4e51ee9a1dea" + }, + { + "ParameterKey": "ListenerRulePriority", + "ParameterValue": "680" + }, + { + "ParameterKey": "ApiId", + "ParameterValue": "kthgpat40e" + }, + { + "ParameterKey": "ApiIntegrationId", + "ParameterValue": "6tcvbtt" + }, + { + "ParameterKey": "ApiOriginAuthorizerId", + "ParameterValue": "ptbwwm" + }, + { + "ParameterKey": "ImageTag", + "ParameterValue": "bootstrap" + }, + { + "ParameterKey": "DesiredCount", + "ParameterValue": "0" + }, + { + "ParameterKey": "CorsOrigins", + "ParameterValue": "https://www.topcoder.com,https://reports.topcoder.com,https://platform-ui.topcoder.com" + }, + { + "ParameterKey": "ParameterPrefix", + "ParameterValue": "/config/forms-api-v6/appvar" + }, + { + "ParameterKey": "SecretsKmsKeyArn", + "ParameterValue": "" + } +] diff --git a/deploy/release.py b/deploy/release.py index 0eb90aa..389575d 100644 --- a/deploy/release.py +++ b/deploy/release.py @@ -1,10 +1,10 @@ #!/usr/bin/env python3 -"""Deploy prebuilt runtime and migration images using the existing Forms stack. +"""Run the migration gate before the shared Topcoder ECS deployment script. -CLI: release.py dev|production IMAGE_TAG [RUNTIME_IMAGE] [MIGRATION_IMAGE]. -Requires inherited AWS credentials, boto3, and Docker. Pushes immutable ECR tags, -executes migrations as a private one-shot ECS task, then updates CloudFormation. -Raises on failures; runtime promotion never occurs after failed migrations. +CLI: release.py dev|production IMAGE_TAG [MIGRATION_IMAGE]. +Requires inherited AWS credentials, boto3, and Docker. Pushes the prebuilt migration +image to ECR and runs it as a private one-shot ECS task. Raises on failures so +CircleCI stops before master_deploy.sh promotes the runtime image. """ import base64 import copy @@ -36,17 +36,17 @@ def wait_until(description, check, seconds=1800): def main(): - """Validate release arguments and environment, migrate, promote, and save evidence. + """Validate CLI arguments and environment, run migrations, and save evidence. - Returns None on success; AWS, Docker, failed migration, and failed deployment + Returns None on success; AWS, Docker, and failed migration errors propagate. Credentials are passed through stdin/environment, never argv. """ - if len(sys.argv) not in (3, 5) or sys.argv[1] not in ('dev', 'production'): + if len(sys.argv) not in (3, 4) or sys.argv[1] not in ('dev', 'production'): raise RuntimeError(__doc__) environment, tag = sys.argv[1:3] if not re.fullmatch(r'[A-Za-z0-9_][A-Za-z0-9_.-]{0,110}', tag): raise RuntimeError('Invalid immutable release tag.') - local_runtime, local_migration = sys.argv[3:] or ['forms-api-v6:candidate', 'forms-api-v6:migrate-candidate'] + local_migration = sys.argv[3] if len(sys.argv) == 4 else 'forms-api-v6:migrate-candidate' session = boto3.Session(region_name=os.environ.get('AWS_REGION', 'us-east-1')) account = session.client('sts').get_caller_identity()['Account'] if (account == '811668436784') != (environment == 'dev'): @@ -65,27 +65,25 @@ def main(): user, password = base64.b64decode(authorization['authorizationToken']).decode().split(':', 1) subprocess.run(['docker', 'login', '--username', user, '--password-stdin', authorization['proxyEndpoint']], input=password, text=True, check=True, stdout=subprocess.DEVNULL) - images = [] - for local, suffix in [(local_runtime, ''), (local_migration, '-migrate')]: - destination = f'{repository}:{tag}{suffix}' - subprocess.run(['docker', 'tag', local, destination], check=True) - subprocess.run(['docker', 'push', destination], check=True) - digest = ecr.describe_images(repositoryName='forms-api-v6', imageIds=[{'imageTag': tag + suffix}])['imageDetails'][0]['imageDigest'] - images.append(f'{repository}@{digest}') - current = ecs.describe_task_definition(taskDefinition=output['TaskDefinitionArn'])['taskDefinition'] + destination = f'{repository}:{tag}-migrate' + subprocess.run(['docker', 'tag', local_migration, destination], check=True) + subprocess.run(['docker', 'push', destination], check=True) + digest = ecr.describe_images(repositoryName='forms-api-v6', imageIds=[{'imageTag': tag + '-migrate'}])['imageDetails'][0]['imageDigest'] + migration_image = f'{repository}@{digest}' + service = ecs.describe_services(cluster=settings['ClusterName'], services=[output['ServiceName']])['services'][0] + current = ecs.describe_task_definition(taskDefinition=service['taskDefinition'])['taskDefinition'] allowed = {'family', 'taskRoleArn', 'executionRoleArn', 'networkMode', 'containerDefinitions', 'volumes', 'placementConstraints', 'requiresCompatibilities', 'cpu', 'memory', 'runtimePlatform', 'ephemeralStorage'} migration = {k: copy.deepcopy(v) for k, v in current.items() if k in allowed} migration['family'] = 'forms-api-v6-migrate' container = migration['containerDefinitions'][0] - container['image'] = images[1] + container['image'] = migration_image container['readonlyRootFilesystem'] = False container.pop('healthCheck', None) container['portMappings'] = [] container['secrets'] = [{'name': 'DATABASE_URL', 'valueFrom': settings['ParameterPrefix'] + '/MIGRATION_DATABASE_URL'}] container['command'] = ['/bin/sh', '-c', 'pnpm migrate:deploy'] migration_definition = ecs.register_task_definition(**migration)['taskDefinition']['taskDefinitionArn'] - service = ecs.describe_services(cluster=settings['ClusterName'], services=[output['ServiceName']])['services'][0] result = ecs.run_task(cluster=settings['ClusterName'], taskDefinition=migration_definition, launchType='FARGATE', networkConfiguration=service['networkConfiguration'], startedBy='forms-release') @@ -103,30 +101,9 @@ def migration_finished(): completed = wait_until('migration task ' + migration_arn, migration_finished) if any(c.get('exitCode') != 0 for c in completed['containers']): - raise RuntimeError('Migration failed; inspect /aws/ecs/forms-api-v6-' + environment + '. Runtime was not promoted.') - parameters = [({'ParameterKey': x['ParameterKey'], 'ParameterValue': tag} if x['ParameterKey'] == 'ImageTag' - else {'ParameterKey': x['ParameterKey'], 'ParameterValue': str(max(1, int(settings['DesiredCount'])))} if x['ParameterKey'] == 'DesiredCount' - else {'ParameterKey': x['ParameterKey'], 'UsePreviousValue': True}) for x in stack['Parameters']] - result = cfn.update_stack(StackName=stack_name, UsePreviousTemplate=True, - Parameters=parameters, Capabilities=['CAPABILITY_IAM']) - print('Updating stack: ' + result['StackId'], flush=True) - - def stack_finished(): - """Read the same stack operation; return completed stack or raise on rollback.""" - state = cfn.describe_stacks(StackName=stack_name)['Stacks'][0] - if state['StackStatus'] == 'UPDATE_COMPLETE': - return state - if state['StackStatus'] not in ('UPDATE_IN_PROGRESS', 'UPDATE_COMPLETE_CLEANUP_IN_PROGRESS'): - raise RuntimeError('Stack promotion failed or rolled back: ' + state['StackStatus']) - return None - - wait_until('CloudFormation promotion', stack_finished) - live = ecs.describe_services(cluster=settings['ClusterName'], services=[output['ServiceName']])['services'][0] - definition = ecs.describe_task_definition(taskDefinition=live['taskDefinition'])['taskDefinition'] - if definition['containerDefinitions'][0]['image'] != f'{repository}:{tag}' or live['runningCount'] < 1: - raise RuntimeError('ECS did not retain the requested release.') - evidence = {'environment': environment, 'tag': tag, 'runtimeImage': images[0], 'migrationImage': images[1], - 'migrationTask': migration_arn, 'taskDefinition': live['taskDefinition'], 'stack': stack_name} + raise RuntimeError('Migration failed; inspect the migration task logs. Runtime was not promoted.') + evidence = {'environment': environment, 'tag': tag, 'migrationImage': migration_image, + 'migrationTask': migration_arn, 'stack': stack_name} Path('deploy/release-' + environment + '.json').write_text(json.dumps(evidence, indent=2) + '\n') print(json.dumps(evidence, indent=2)) diff --git a/deploy/seed-dev.py b/deploy/seed-dev.py index c376dc2..a7505e7 100644 --- a/deploy/seed-dev.py +++ b/deploy/seed-dev.py @@ -22,7 +22,7 @@ def encoded(value): def main(): - """Create or verify event_interest using dev configuration; raise on drift or HTTP errors.""" + """Publish the dev sample with a shared-auth-compatible M2M token; raise on drift or HTTP errors.""" session = boto3.Session(region_name='us-east-1') if session.client('sts').get_caller_identity()['Account'] != '811668436784': raise RuntimeError('The sample form is restricted to development.') @@ -31,10 +31,11 @@ def setting(name): """Read one exact decrypted SSM setting; propagate AWS errors without logging values.""" return ssm.get_parameter(Name=name, WithDecryption=True)['Parameter']['Value'] now = int(time.time()) + raw_issuers = setting('/config/forms-api-v6/appvar/VALID_ISSUERS').strip() + issuers = json.loads(raw_issuers) if raw_issuers.startswith('[') else raw_issuers.split(',') head = encoded({'alg': 'HS256', 'typ': 'JWT'}) body = encoded({'sub': 'forms-dev-bootstrap@clients', 'iat': now, 'exp': now + 300, - 'iss': setting('/config/forms-api-v6/appvar/VALID_ISSUERS').split(',')[0], - 'aud': setting('/config/forms-api-v6/appvar/AUTH_AUDIENCE'), + 'iss': issuers[0].strip(), 'azp': 'forms-dev-bootstrap', 'gty': 'client-credentials', 'scope': 'manage:forms'}) message = head + '.' + body signature = hmac.new(setting('/config/common/global-appvar/AUTH_SECRET').encode(), message.encode(), hashlib.sha256).digest() diff --git a/deploy/service.yaml b/deploy/service.yaml index 96b9337..d13513e 100644 --- a/deploy/service.yaml +++ b/deploy/service.yaml @@ -54,7 +54,7 @@ Parameters: Description: Bootstrap at zero; deploy after database migration succeeds. CorsOrigins: Type: String - Default: https://www.topcoder-dev.com + Default: https://www.topcoder-dev.com,https://reports.topcoder-dev.com,https://platform-ui.topcoder-dev.com ParameterPrefix: Type: String Default: /config/forms-api-v6/appvar @@ -226,14 +226,12 @@ Resources: - {Name: NODE_ENV, Value: production} - {Name: PORT, Value: '3000'} - {Name: AWS_REGION, Value: !Ref 'AWS::Region'} - - {Name: AUTH_MODE, Value: hs256} - {Name: CORS_ORIGINS, Value: !Ref CorsOrigins} - {Name: TRUST_PROXY_CIDRS, Value: '10.0.0.0/8'} Secrets: - {Name: DATABASE_URL, ValueFrom: !Sub '${ParameterPrefix}/DATABASE_URL'} - {Name: AUTH_SECRET, ValueFrom: /config/common/global-appvar/AUTH_SECRET} - {Name: VALID_ISSUERS, ValueFrom: !Sub '${ParameterPrefix}/VALID_ISSUERS'} - - {Name: AUTH_AUDIENCE, ValueFrom: !Sub '${ParameterPrefix}/AUTH_AUDIENCE'} HealthCheck: Command: [CMD-SHELL, 'node -e "fetch(\"http://127.0.0.1:3000/v6/forms/health/ready\").then(r=>process.exit(r.ok?0:1)).catch(()=>process.exit(1))"'] Interval: 30 diff --git a/deploy/status.md b/deploy/status.md index 01056b7..ba53fed 100644 --- a/deploy/status.md +++ b/deploy/status.md @@ -1,5 +1,8 @@ # Deployment status — 2026-09-09 +This is a historical dev checkpoint. See [production.md](production.md) for the +2026-10-06 production provisioning and current release prerequisites. + ## Completed and verified in dev - CloudFormation stack `forms-api-v6-dev` owns service `forms-api-v6` in cluster diff --git a/deploy/tests/test_activate.py b/deploy/tests/test_activate.py new file mode 100644 index 0000000..3801725 --- /dev/null +++ b/deploy/tests/test_activate.py @@ -0,0 +1,81 @@ +"""Deployment regressions: bootstrap activation, readiness, and rollback detection.""" +import sys +import unittest +from copy import deepcopy +from pathlib import Path +from unittest.mock import Mock, patch + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from activate import activate_service + + +class ActivationTests(unittest.TestCase): + """Verify that CircleCI cannot succeed with zero tasks or a rolled-back image.""" + + def setUp(self): + """Build an isolated ECS client with one healthy requested task per test.""" + self.service = {'status': 'ACTIVE', 'taskDefinition': 'requested', + 'desiredCount': 1, 'runningCount': 1, 'pendingCount': 0, + 'deployments': [{'rolloutState': 'COMPLETED'}]} + self.ecs = Mock() + self.ecs.describe_services.return_value = {'services': [self.service]} + self.ecs.describe_task_definition.return_value = {'taskDefinition': { + 'containerDefinitions': [{'name': 'forms-api-v6', 'image': 'repo:42'}]}} + self.ecs.list_tasks.return_value = {'taskArns': ['task']} + self.ecs.describe_tasks.return_value = {'tasks': [{ + 'taskDefinitionArn': 'requested', 'lastStatus': 'RUNNING', 'healthStatus': 'HEALTHY'}]} + self.poll = patch('activate.wait_until', side_effect=lambda label, check: check()) + self.poll.start() + self.addCleanup(self.poll.stop) + + def run_activation(self): + """Invoke activation against the fake ECS client and expected release image.""" + return activate_service(self.ecs, 'cluster', 'service', 'repo:42') + + def test_bootstrap_starts_one_replica(self): + """A successfully promoted zero-count service starts without another manual step.""" + bootstrap = dict(self.service, desiredCount=0, runningCount=0) + self.ecs.describe_services.side_effect = [ + {'services': [bootstrap]}, {'services': [self.service]}] + self.assertEqual(self.run_activation(), self.service) + self.ecs.update_service.assert_called_once_with( + cluster='cluster', service='service', desiredCount=1) + + def test_existing_capacity_is_preserved(self): + """Normal releases retain the operator's nonzero replica count.""" + self.service.update(desiredCount=2, runningCount=2) + task = self.ecs.describe_tasks.return_value['tasks'][0] + self.ecs.describe_tasks.return_value['tasks'].append(deepcopy(task)) + self.assertEqual(self.run_activation(), self.service) + self.ecs.update_service.assert_not_called() + + def test_wrong_image_never_starts(self): + """A stale shared-script success must not activate the bootstrap image.""" + self.ecs.describe_task_definition.return_value['taskDefinition'][ + 'containerDefinitions'][0]['image'] = 'repo:old' + with self.assertRaisesRegex(RuntimeError, 'requested runtime image'): + self.run_activation() + self.ecs.update_service.assert_not_called() + + def test_rollback_is_failure(self): + """A healthy previous release is still a failure for this CircleCI deployment.""" + self.ecs.describe_services.side_effect = [ + {'services': [self.service]}, + {'services': [dict(self.service, taskDefinition='previous')]}] + with self.assertRaisesRegex(RuntimeError, 'rolled back'): + self.run_activation() + + def test_running_but_unhealthy_is_not_success(self): + """Running counts alone do not prove that readiness checks passed.""" + self.ecs.describe_tasks.return_value['tasks'][0]['healthStatus'] = 'UNKNOWN' + self.assertIsNone(self.run_activation()) + + def test_failed_rollout_is_failure(self): + """An ECS circuit breaker failure must fail the release observation.""" + self.service['deployments'][0]['rolloutState'] = 'FAILED' + with self.assertRaisesRegex(RuntimeError, 'deployment failed'): + self.run_activation() + + +if __name__ == '__main__': + unittest.main() diff --git a/docs/api.md b/docs/api.md index 873e365..aa53c99 100644 --- a/docs/api.md +++ b/docs/api.md @@ -23,9 +23,11 @@ Keys are lowercase snake_case, begin with a letter, and contain at most 40 chara ## Authentication -Authorization is deny-by-default. Public routes may omit a token; supplying an invalid token still returns 401. Configure either HTTPS JWKS/RS256 or the legacy HS256 shared-secret mode. The two verification paths have separate algorithm allowlists. Signature, issuer, audience, expiry, issued-at presence, and subject are required. +Authorization is deny-by-default. Public routes may omit a token; supplying an invalid token still returns 401. Forms uses `tc-core-library-js`'s `middleware.jwtAuthenticator`, the same entry point used by Reports, Engagements, Support, and Bus API. It verifies HS256 tokens using `AUTH_SECRET` and RS256 tokens using the trusted issuer's `/.well-known/jwks.json`, with cached signing keys. The issuer must be an exact member of `VALID_ISSUERS`. Forms rejects other algorithms before invoking the shared verifier, requires `exp` and `iat`, and preserves the generic 401 response for authentication failures. -Roles are matched case-insensitively. Roles and userId are read from direct claims or the exact `AUTH_CLAIM_NAMESPACE` prefix, default `https://topcoder.com/`. Subject is retained for editor audit; member forms require an actual `userId` claim. A missing userId is not silently replaced with an Auth0 subject. +Audience handling follows the shared Topcoder library: there is no Forms-specific audience check. This permits both legacy member JWTs without `aud` and Auth0 browser tokens whose `aud` is the Topcoder application client ID. `AUTH_MODE`, `AUTH_AUDIENCE`, and `JWKS_URL` no longer select or restrict inbound authentication. Trust is established by signature, issuer and token lifetime; Forms still enforces the human-role/M2M-scope rules below. + +The shared middleware normalizes Topcoder namespaced roles and user IDs, including `https://topcoder-dev.com/roles` and `https://topcoder-dev.com/userId`, before Forms builds its actor. Roles are matched case-insensitively. The actor adapter also supports direct claims and an `AUTH_CLAIM_NAMESPACE` fallback, default `https://topcoder.com/`. Subject is retained for editor audit; legacy human tokens without `sub` use their verified numeric `userId`. Member forms require an actual `userId`; an Auth0 subject is never silently substituted for member identity. Standard M2M tokens include `azp`, `sub`, `gty=client-credentials`, and space-separated `scope` claims. | Caller | Manage access | Report access | Submit access | | --------- | --------------------------------------------- | ---------------------------------------- | ------------------------------------------------------- | @@ -69,9 +71,42 @@ Idempotency-Key: 6b3d585c-f134-4e06-a6d5-6ea42fc1c7ce Receipts contain no answers or member details. All answers must belong to the pinned stored revision; client-supplied field definitions and extra answer keys are rejected. Server validation does not coerce number or boolean strings. Decimal values are exact strings; multi-select answers are arrays of option keys. Every answer and its selections commit atomically with the envelope. +## Optional Kafka publication + +Include `"kafka": true` in the submission body alongside `version`, `answers`, and `sourcePage` to publish the accepted submission to **`form.submitted`** through the authenticated Topcoder Bus API. The exact string `"true"` is also accepted; omitted, `false`, or `"false"` does not publish. Other non-null values are rejected. This is envelope metadata, not an answer field or query parameter. Answer values retain their strict type validation. + +```json +{ + "version": 1, + "answers": { "email": "member@example.com" }, + "sourcePage": "/events", + "kafka": true +} +``` + +The Bus API envelope uses `topic: "form.submitted"`, `originator: "forms-api-v6"`, `mime-type: "application/json"`, the submission timestamp, and the submission UUID as `key`. Its `payload` is: + +```json +{ + "submissionId": "c1ab5d27-722a-431f-a811-46737109de1f", + "formKey": "event_interest", + "version": 1, + "submittedAt": "2026-09-28T01:00:00.000Z", + "memberId": null, + "sourcePage": "/events", + "answers": { "email": "member@example.com" } +} +``` + +Answers use their validated, canonical values (including exact decimal strings, booleans, numbers, date strings, and option keys); omitted optional answers stay omitted. Member identity comes from verified claims. The honeypot and Kafka flag are excluded from the event. The HTTP receipt remains unchanged. + +Publication happens only after the submission and all answers commit. A successful delivery is recorded separately and identical retries, including concurrent retries, do not publish again. Changing the Kafka opt-in for an existing retry key returns 409; omission and false are equivalent and preserve compatibility with older submissions. + +If Bus API is unavailable or unconfigured, the submission remains saved and the request returns **503**. Retry the identical body and `Idempotency-Key` to resume delivery, even after retirement. There is no background retry worker. A crash or lost acknowledgement after Bus API accepts an event but before the delivery receipt commits can cause redelivery; consumers must deduplicate by `submissionId`. Invalid or rejected submissions never publish. + ## Retry and error handling -Generate one cryptographically random UUID for a logical submission attempt. Reuse it with the same revision, answers, member identity, and source page after a timeout or lost response. The service canonicalizes decimals and multi-select order before hashing. An identical retry returns the original receipt; changed content with that key returns 409. This key remains reserved while the submission is retained. +Generate one cryptographically random UUID for a logical submission attempt. Reuse it with the same revision, answers, member identity, source page, and Kafka opt-in after a timeout or lost response. The service canonicalizes decimals and multi-select order before hashing. An identical retry returns the original receipt; changed content with that key returns 409. This key remains reserved while the submission is retained. A previously accepted attempt can be retried after a version is retired. A new attempt targeting a draft/retired version returns 409. A member-form retry still requires the member token. The API does not replay a submission under a different member identity. @@ -98,3 +133,32 @@ Envelope/DTO errors use Nest's standard `message` array. Clients should handle b | 500/503 | Service/database failure; preserve input and retry appropriately. | Reports are ordered by creation timestamp and UUID. Pagination does not create a database snapshot; use SQL/reporting jobs when an exact export snapshot is required. CSV is quoted and formula-prefixed text is neutralized for spreadsheets; arrays are JSON-encoded only in the exported CSV cell. SQL data remains fully relational. + +## Forms portal reporting + +All routes below require **Report** access and return `Cache-Control: no-store`. +They do not grant form management privileges. + +| Route (under `/v6`) | Result | +| --- | --- | +| `GET /forms/reports/directory?after=key` | `{ data: [{ key, title }], nextCursor }`; 100 forms per page, most recent published/retired title, no drafts. | +| `GET /forms/:key/submissions?limit=25&after=uuid&startDate=2026-09-01&endDate=2026-09-30` | `{ form, columns, labels, data, total, nextCursor }` across all published/retired revisions. | +| `GET /forms/:key/submissions/export?startDate=2026-09-01&endDate=2026-09-30` | Complete streaming CSV with one header, independently of the table page. Omit both dates for all data. | + +Dates are optional **inclusive UTC calendar dates** in `YYYY-MM-DD`; an end date +includes timestamps through 23:59:59.999. Invalid calendar dates and reversed +ranges return 400. Existing version-specific JSON/CSV endpoints accept these same +filters. A cursor must belong to the selected form/revision and date range. + +The all-revision report unions field keys in first-seen order. `labels` contains +the latest published label for each key; absent answers are null. Existing metadata +columns (`submission_id`, `submitted_at`, `member_id`, `source_page`, `form_version`) +remain present. Rows sort by timestamp then UUID; `total` counts matching rows +before pagination. Decimal values remain strings and multi-select values remain +arrays. A field reused with a different type retains each revision's original value. + +The complete export accepts only date filters, not `limit` or `after`. It fetches +1,000 rows per batch, applies backpressure, stops when the client disconnects, and +uses the established CSV escaping/formula protection. Columns and an upper +submission timestamp are captured at export start. This is not a repeatable-read +database snapshot: transactions already in flight can commit during export. diff --git a/docs/data-model.md b/docs/data-model.md index 4d59f73..2047a33 100644 --- a/docs/data-model.md +++ b/docs/data-model.md @@ -17,6 +17,7 @@ erDiagram FormVersion ||--o{ FormField : defines FormField ||--o{ FieldOption : choices FormVersion ||--o{ Submission : receives + Submission ||--o| SubmissionEvent : delivery Submission ||--o{ Answer : contains FormField ||--o{ Answer : constrains Answer ||--o{ AnswerSelection : selects @@ -32,6 +33,7 @@ erDiagram | `FormField` | Named key unique within a version; explicit type, position, required flag, text length and numeric bounds. | | `FieldOption` | Named option key and label, owned by a field/version. | | `Submission` | Exact version, server timestamp, UUID idempotency key, canonical request hash, verified member ID, and optional source page path. | +| `SubmissionEvent` | Optional one-to-one delivery receipt with submission UUID and nullable publication timestamp; cascades on submission deletion. | | `Answer` | One scalar value in a type-specific column, or a multi-select answer parent; unique per submission/field. | | `AnswerSelection` | One row per selected option, with composite ownership foreign keys and duplicate prevention. | @@ -48,6 +50,8 @@ The API additionally validates email syntax, exact decimal input precision, real API form registration and revisions are serialized on the stable form row. Publication, retirement, and submission acceptance use the same lock. This favors simple consistency for occasional website forms; it serializes submissions to the same form. Revisit this locking strategy if measured submission volume requires greater throughput. +Kafka opt-in creates a `SubmissionEvent` row atomically with the submission. Its publication timestamp is updated after Bus API acceptance in a separate transaction; submission envelopes and answers remain immutable. Delivery retries lock this row to prevent concurrent duplicate sends. A failed or unacknowledged delivery remains pending until the caller retries. No JSON payload is stored; the validated request and immutable revision reconstruct the event. + ## Field semantics - `INTEGER` is PostgreSQL `integer` (signed 32-bit), sent as a JSON number. @@ -103,3 +107,31 @@ SQL views are computed rather than materialized. They are indexed through the su The service supports `DRAFT -> PUBLISHED -> RETIRED`. Published versions cannot return to draft; retired versions cannot be reopened. Create the next sequential revision to reopen or change a form. Draft content becomes immutable through the API when first saved; local Payload drafts can be edited freely until synchronized. Versions and historical reporting views are retained. The service does not impose an arbitrary retention period or add deletion endpoints. An approved retention policy can delete whole submission envelopes in batches; cascading foreign keys remove child data. Definition records remain for interpretation of retained historical data. Reports and database grants expose personal submission information only to their authorized readers. + +## Submission processing + +`ProcessorFlow`, `ProcessorEvent`, and `ProcessorDelivery` support forms-processor-v6. +The API owns their migrations; the processor uses these schema-qualified tables +without running DDL at startup. A flow has a stable ID, form key, enabled flag, +AND-combined answer predicates (`rules.all`), action name, and JSON action settings. +The seeded `lets-talk-sales-email` flow matches all `lets-talk` submissions and is +disabled with TBD recipients, sender, and template. Manage it externally in SQL; +set `updatedAt` when changing settings. Disabling delivery preserves queued work. + +The inbox stores the submitted event before Kafka commit. Routing records a unique +receipt per submission/flow, including disabled matching flows. Retries read current +action settings. No matching flow means the event is retained but has no actions. +Flow additions/rule edits apply to events not yet routed; replay requires explicitly +clearing `routedAt` and never deletes existing delivery receipts. Bus API email-event acceptance +and the receipt cannot be atomic; a crash in between can cause a duplicate email. + +Processor events contain answers and must be included in personal-data retention +and erasure procedures separately from `Submission`. Deleting a ProcessorEvent +cascades its delivery records and removes deduplication protection, so retain it +through the Kafka retention/replay window. No automatic deletion policy is enabled. +See forms-processor-v6/README.md for rules, settings, replay, and operational SQL. + +The `sendgrid-email` action name is retained for compatibility, but forms-processor-v6 +now publishes the v3 email contract to `external.action.email` through Bus API. +`deliveredAt` records Bus API acceptance; email-service-v6 handles provider delivery +and retries. Optional `fromEmail` overrides that service's default sender. diff --git a/docs/lets-talk.md b/docs/lets-talk.md new file mode 100644 index 0000000..226253f --- /dev/null +++ b/docs/lets-talk.md @@ -0,0 +1,57 @@ +# Development /lets-talk migration + +`examples/lets-talk.json` mirrors the form schema returned by +`https://www.topcoder.com/__api/forms/XpPgFzW8lmX8frlStzzg0` on 2026-09-28. +It preserves the six visible fields, their order and required flags, interest +labels, hidden attribution fields, and the HubSpot success message. +The stable Forms API key is `lets_talk` and the title is “Let’s talk” for reporting discovery. No historical HubSpot submissions are imported. + +Option values are normalized to API-safe keys: + +| HubSpot value | Forms API key | +| --- | --- | +| `Crowdsourcing` | `crowdsourcing` | +| `App Design & Development` | `app_design_development` | +| `Freelancers` | `freelancers` | +| `Landing Page` (hidden lead source) | `landing_page` | + +The website supplies the existing hidden campaign defaults, including +`source__c=/lets-talk`; empty UTM fields remain empty. These are validated answers, +not trusted identity. `sourcePage` is the current pathname. The site submits +`kafka: true` so each accepted attempt publishes `form.submitted` through the +existing Bus API integration. Consumers must deduplicate by `submissionId`. + +## Rollout after review + +1. Deploy this API revision and the existing `SubmissionEvent` migration. Configure + outbound `BUSAPI_URL`, `AUTH0_URL`, `AUTH0_AUDIENCE`, `AUTH0_CLIENT_ID`, + `AUTH0_CLIENT_SECRET`, and `KAFKA_ERROR_TOPIC` as described in operations.md. +2. Ensure `CORS_ORIGINS` / CloudFormation `CorsOrigins` includes the exact website + and reports origins (`https://www.topcoder-dev.com`, + `https://reports.topcoder-dev.com`, `https://platform-ui.topcoder-dev.com` for the + combined portal). Existing stacks retain parameter values: update the parameter + explicitly; changing the template default alone does not update a deployed stack. +3. Publish the reviewed definition from this repository root, supplying an existing + administrator bearer token or M2M token with `manage:forms` through the environment: + + ```sh + nvm use + pnpm exec tsx scripts/publish-lets-talk.ts + ``` + + `FORMS_ADMIN_TOKEN` must be set without putting it in source control. The script + hardcodes the development API, creates no submissions, and fails on differing + immutable content. Re-running an identical version is safe; subsequent schema + changes require a reviewed next version. +4. Deploy the companion `topcoder-website` develop change and the `platform-ui` dev + change. No CMS content mutation is required: the website selects the known form + component only on the development `/lets-talk` route. Production and other + routes continue using their existing forms. +5. Verify the public schema, then submit an explicitly marked development test with + Kafka configured. Confirm the receipt, event, table row, date filtering, and CSV. + A Bus API outage returns 503 after saving; retry unchanged inputs to finish delivery. + +The React integration accepts `kafka` (false by default) and `hiddenAnswers` +(empty by default). Hidden answers are merged into the API payload and omitted +from visible controls. They must match stored field types and validation rules. +The website maintains its analytics adapter around the same rendering contract. diff --git a/docs/operations.md b/docs/operations.md index a8a5f91..7381311 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -6,20 +6,48 @@ | ---------------------- | --------------------------------------------------------------------------------------------------------------------------- | | `DATABASE_URL` | Required PostgreSQL URL with `schema=forms`; dev uses `topcoder-services`. Both migrations and runtime use it. | | `PORT` | HTTP listen port, default 3000; `.env.example` uses 3006 on the host. | -| `AUTH_MODE` | `jwks` (default) or `hs256`. | -| `JWKS_URL` | Required HTTPS URL in JWKS mode; only RS256 is accepted. | -| `AUTH_SECRET` | At least 32 characters in HS256 mode; only HS256 is accepted. | -| `VALID_ISSUERS` | Required comma-separated exact JWT issuers. Unlike some older APIs, this setting is not a JSON array. | -| `AUTH_AUDIENCE` | Required expected JWT audience. | -| `AUTH_CLAIM_NAMESPACE` | Exact roles/userId custom-claim prefix; default `https://topcoder.com/`. | +| `AUTH_SECRET` | Required shared Topcoder secret, at least 32 characters; used for HS256 tokens. | +| `VALID_ISSUERS` | Required exact trusted JWT issuers, as a JSON string array or comma-separated list. RS256 JWKS URLs derive from these issuers. | +| `AUTH_CLAIM_NAMESPACE` | Actor fallback prefix, default `https://topcoder.com/`; the shared middleware normalizes Topcoder namespaces first. | | `CORS_ORIGINS` | Comma-separated exact HTTP(S) origins, no trailing slash/wildcard. Empty means no browser origins are allowed. | | `THROTTLE_LIMIT` | Requests per minute per client IP and handler, per replica; default 30. Health probes are exempt. | | `TRUST_PROXY_CIDRS` | Explicit trusted ingress proxy addresses/CIDRs. Empty by default. | +Inbound authentication uses the pinned `tc-core-library-js` authenticator for both HS256 and RS256. For the dev reports application, `VALID_ISSUERS` must include `https://auth.topcoder-dev.com/` exactly (including the trailing slash); its public signing keys are fetched from `https://auth.topcoder-dev.com/.well-known/jwks.json`. Keep the legacy API issuer and other intended trusted issuers in the same list. The runtime needs outbound HTTPS access to those JWKS endpoints. + +`AUTH_MODE`, `AUTH_AUDIENCE`, and `JWKS_URL` are obsolete and ignored, so an existing ECS task definition with `AUTH_MODE=hs256` does not block RS256 after deploying the new image. The updated template removes these obsolete mappings. The library does not enforce a Forms-specific audience; the Topcoder browser client ID is accepted through the same trusted-issuer policy as other v6 services. No browser token, client secret, or private signing key needs to be copied into Forms configuration. + All Forms tables, enums, functions, migration history, and reporting views live in the `forms` schema. The database can be shared with other services. Models and raw SQL explicitly qualify this schema; migrations do not change other schemas. PostgreSQL TLS options belong in the connection URL/driver configuration. The service does not disable certificate verification. The Prisma client uses the PostgreSQL driver adapter and generated TypeScript code. Connection URLs are configured in `prisma.config.ts`, consistent with the [Prisma 7 migration guide](https://www.prisma.io/docs/orm/more/upgrade-guides/upgrading-versions/upgrading-to-prisma-7). Client generation and compilation require no live database. Migrations and runtime require `DATABASE_URL`. +## Outbound Bus API + +ECS releases inject all service appvars from `/config/forms-api-v6/appvar` and +shared appvars from `/config/common/global-appvar` as SSM secret references, with +service values taking precedence. See [deployment injection](../deploy/README.md#runtime-appvar-injection) +for configuration-only rolls and required execution-role permissions. + +Ordinary submissions need no Bus API configuration. To accept `kafka=true` submissions successfully, configure: + +| Variable | Meaning | +| --- | --- | +| `BUSAPI_URL` | HTTP(S) API base ending in `/v6`, e.g. `https://api.topcoder-dev.com/v6`. The shared wrapper appends `/bus/events`. | +| `AUTH0_URL` | Auth0 token endpoint used by the shared Topcoder M2M client. | +| `AUTH0_AUDIENCE` | Outbound M2M audience used to acquire a Bus API token. | +| `AUTH0_CLIENT_ID`, `AUTH0_CLIENT_SECRET` | Service credentials authorized to publish Bus API events. Required when `BUSAPI_URL` is set. | +| `TOKEN_CACHE_TIME` | Optional M2M token cache duration in milliseconds, 0–86400000; otherwise uses the wrapper default. | +| `AUTH0_PROXY_SERVER_URL` | Optional Auth0 proxy supported by the shared wrapper. | +| `KAFKA_ERROR_TOPIC` | Wrapper error-topic setting, default `common.error.reporting`; submission topic is always `form.submitted`. | + +For ECS, expose the five required Bus API/Auth0 settings to the runtime container through its task-definition secrets (the existing template only maps inbound authentication and database settings). Store them under the service parameter prefix so its existing SSM read permissions apply; never put credentials in the template. + +Apply migration `20260928010000_submission_events` before deploying this version. It adds the `forms.SubmissionEvent` delivery table; it does not change existing submissions or reporting views. The service role needs SELECT/INSERT/UPDATE on this table. Ensure `form.submitted` is available through Bus API and downstream consumers deduplicate on the payload's `submissionId`. + +Delivery failures return 503 after saving the submission; callers must retry the same body and key. Pending attempts have `publishedAt IS NULL` in `forms.SubmissionEvent`. No background delivery job is included. Provider error bodies, credentials, and answers are not logged by the publisher. Delivery uses a separate transaction with a row lock and a 15-second transaction timeout; failure to record an accepted event may result in redelivery. + +The shared wrapper and its Topcoder core dependency are pinned to Git commits. `pnpm-workspace.yaml` allows Git subdependencies for this integration and explicitly skips optional native DTrace builds. + ## Database access The checked-in migrations create tables, enums, foreign keys, CHECKs, indexes, and lifecycle/answer triggers inside `forms`. Use `pnpm migrate:deploy`; **do not use `prisma db push`**, which does not reproduce the custom integrity triggers and checks. diff --git a/examples/lets-talk.json b/examples/lets-talk.json new file mode 100644 index 0000000..aa046bf --- /dev/null +++ b/examples/lets-talk.json @@ -0,0 +1,124 @@ +{ + "title": "Let’s talk", + "description": "", + "successMessage": "Thanks for submitting the form. Someone will reach out to you soon.", + "access": "ANONYMOUS", + "fields": [ + { + "key": "interested_in", + "label": "What are you interested in?", + "type": "SINGLE_SELECT", + "required": true, + "options": [ + { + "key": "crowdsourcing", + "label": "Crowdsourced Innovation" + }, + { + "key": "app_design_development", + "label": "App Design and Development" + }, + { + "key": "freelancers", + "label": "Flexible Talent Access" + } + ] + }, + { + "key": "firstname", + "label": "First name", + "type": "TEXT", + "required": true, + "maxLength": 500 + }, + { + "key": "lastname", + "label": "Last name", + "type": "TEXT", + "required": true, + "maxLength": 500 + }, + { + "key": "email", + "label": "Work Email", + "type": "EMAIL", + "required": true, + "maxLength": 254 + }, + { + "key": "company", + "label": "Company name", + "type": "TEXT", + "required": true, + "maxLength": 500 + }, + { + "key": "briefly_describe_your_inquiry", + "label": "What can we help you with?", + "type": "TEXTAREA", + "required": false, + "maxLength": 10000 + }, + { + "key": "leadsource", + "label": "Leadsource", + "type": "SINGLE_SELECT", + "required": false, + "options": [ + { + "key": "landing_page", + "label": "Landing Page" + } + ] + }, + { + "key": "source__c", + "label": "Landing Page Link", + "type": "TEXT", + "required": false, + "maxLength": 500 + }, + { + "key": "lead_source_for_campaign__c", + "label": "Lead Source For Campaign C", + "type": "TEXT", + "required": false, + "maxLength": 500 + }, + { + "key": "primary_campaign__c", + "label": "Primary Campaign C", + "type": "TEXT", + "required": false, + "maxLength": 500 + }, + { + "key": "insta_page_name__c", + "label": "Insta Page Name C", + "type": "TEXT", + "required": false, + "maxLength": 500 + }, + { + "key": "utm_medium__c", + "label": "Utm Medium C", + "type": "TEXT", + "required": false, + "maxLength": 500 + }, + { + "key": "utm_campaign__c", + "label": "Utm Campaign C", + "type": "TEXT", + "required": false, + "maxLength": 500 + }, + { + "key": "utm_source__c", + "label": "Utm Source C", + "type": "TEXT", + "required": false, + "maxLength": 500 + } + ] +} diff --git a/integrations/react/TopcoderForm.tsx b/integrations/react/TopcoderForm.tsx index c7a2cf6..89c485b 100644 --- a/integrations/react/TopcoderForm.tsx +++ b/integrations/react/TopcoderForm.tsx @@ -13,13 +13,15 @@ import { isPublicForm, type FormField, type PublicForm } from '../contracts'; export interface TopcoderFormProps { formKey: string; apiBaseUrl: string; + kafka?: boolean; + hiddenAnswers?: Readonly>; getAccessToken?: () => Promise; } /** * Renders an API-owned schema referenced by a Payload block, preserving inputs and retry keys on failure. * Cancels pending work when the block reference changes so an old receipt cannot complete a new form. - * @param props Stable form key, public API base ending in /v6, and optional signed-in member token provider. + * @param props Stable form key, public API base ending in /v6, optional token provider, Kafka opt-in, and API-validated hidden answer defaults. * @returns Accessible loading, form, error, or receipt UI; submission failures are displayed without throwing. * @throws No errors during ordinary rendering; fetch/token failures are caught and displayed. */ @@ -27,6 +29,8 @@ export function TopcoderForm({ formKey, apiBaseUrl, getAccessToken, + kafka = false, + hiddenAnswers = {}, }: TopcoderFormProps): ReactNode { const [schema, setSchema] = useState(null); const [state, setState] = useState< @@ -84,7 +88,8 @@ export function TopcoderForm({ const data = new FormData(event.currentTarget); const body = JSON.stringify({ version: schema.version, - answers: collectAnswers(schema.fields, data), + answers: { ...collectAnswers(schema.fields, data), ...hiddenAnswers }, + ...(kafka ? { kafka: true } : {}), sourcePage: window.location.pathname, website: data.get('_website') ?? '', }); @@ -130,9 +135,11 @@ export function TopcoderForm({ ? 'This form changed or this attempt conflicts with a previous submission. Reload the page before submitting again.' : response.status === 401 ? 'Please sign in again to submit this form.' - : response.status === 429 - ? 'Too many requests. Please wait a minute and try again.' - : 'Your submission could not be saved. Check the fields and try again.', + : response.status === 503 + ? 'Your submission may be saved, but delivery is not confirmed. Please retry without changing your answers.' + : response.status === 429 + ? 'Too many requests. Please wait a minute and try again.' + : 'Your submission could not be saved. Check the fields and try again.', ); setState('ready'); return; @@ -187,14 +194,16 @@ export function TopcoderForm({ > {schema.title} - {schema.fields.map((field) => ( - - ))} + {schema.fields + .filter((field) => !Object.hasOwn(hiddenAnswers, field.key)) + .map((field) => ( + + ))}