markmnl · markmnl · Sep 10, 2026 · Sep 10, 2026 · Sep 10, 2026 · Sep 10, 2026
@@ -34,13 +34,39 @@ jobs:
         with:
           go-version: "stable"
 
+      - name: Select component refs
+        id: refs
+        env:
+          GH_TOKEN: ${{ github.token }}
+          PR_BRANCH: ${{ github.head_ref }}
+          INPUT_FMSGD: ${{ inputs.fmsgd_ref }}
+          INPUT_FMSGID: ${{ inputs.fmsgid_ref }}
+          INPUT_WEBAPI: ${{ inputs.fmsg_webapi_ref }}
+          INPUT_CLI: ${{ inputs.fmsg_cli_ref }}
+        run: |
+          pick_ref() {
+            local repo="$1" explicit="$2"
+            if [ -n "$explicit" ]; then
+              printf '%s\n' "$explicit"
+            elif [ -n "$PR_BRANCH" ] && gh api "repos/markmnl/$repo/git/ref/heads/$PR_BRANCH" >/dev/null 2>&1; then
+              printf '%s\n' "$PR_BRANCH"
+            else
+              echo main
+            fi
+          }
+          echo "fmsgd=$(pick_ref fmsgd "$INPUT_FMSGD")" >> "$GITHUB_OUTPUT"
+          echo "fmsgid=$(pick_ref fmsgid "$INPUT_FMSGID")" >> "$GITHUB_OUTPUT"
+          echo "webapi=$(pick_ref fmsg-webapi "$INPUT_WEBAPI")" >> "$GITHUB_OUTPUT"
+          echo "cli=$(pick_ref fmsg-cli "$INPUT_CLI")" >> "$GITHUB_OUTPUT"
+
       - name: Run integration tests
         run: bash test/run-tests.sh
         env:
-          FMSGD_REF: ${{ github.event.inputs.fmsgd_ref }}
-          FMSGID_REF: ${{ github.event.inputs.fmsgid_ref }}
-          FMSG_WEBAPI_REF: ${{ github.event.inputs.fmsg_webapi_ref }}
-          FMSG_CLI_REF: ${{ github.event.inputs.fmsg_cli_ref }}
+          FMSGD_REF: ${{ steps.refs.outputs.fmsgd }}
+          FMSGID_REF: ${{ steps.refs.outputs.fmsgid }}
+          FMSG_WEBAPI_REF: ${{ steps.refs.outputs.webapi }}
+          FMSG_CLI_REF: ${{ steps.refs.outputs.cli }}
+          FMSG_CHALLENGE_MODE: ALWAYS
 
       - name: Upload integration logs
         if: failure()

@@ -219,3 +219,30 @@ On first startup (empty data volume), PostgreSQL runs the scripts in `docker/pos
 > ```
 
 
+
+### Testing message finalization
+
+The message schema requires a SHA-256 and durable wire representation for sent
+messages. The initialization SQL is for new databases only. To upgrade an existing
+stack, stop the daemon and API, back up the database and shared data volume, then run
+[the standalone `fmsg-backfill` binary](https://github.com/markmnl/fmsgd#immutable-message-finalization-and-upgrades)
+with access to the database and stored payload paths. It embeds the schema upgrade;
+run without `-apply` for a full dry run and with `-apply` to commit. Start the matching
+daemon/API only after migration succeeds. The migration binary is not bundled in the
+daemon image. Retain the shared message volume, including `.fmsg-wire-*` directories.
+The daemon owns the schema; the initialization file here is its bootstrap copy.
+
+Test `015-message-sha256.sh` verifies local-only hashing, later federation with
+compression, batch-hash replies, and notification-only add-to. Test `009` now creates
+batch replies through the API. To exercise challenge responses on every exchange:
+
+```sh
+FMSG_CHALLENGE_MODE=ALWAYS ./test/run-tests-podman.sh
+```
+
+For coordinated feature branches, set `FMSGD_REF` and `FMSG_WEBAPI_REF` to those
+branches. `FMSG_TEST_NETWORK` optionally changes the shared test network name
+(default `fmsg-test`).
+
+Pull-request CI selects the matching component branch when it exists, otherwise
+`main`. Manual workflow inputs take precedence. CI forces challenge responses.
@@ -22,6 +22,7 @@ services:
 
   fmsgd:
     environment:
+      FMSG_CHALLENGE_MODE: ${FMSG_CHALLENGE_MODE:-HAS_NOT_PARTICIPATED}
       FMSG_TLS_CERT: /opt/fmsg/tls/fmsg.${FMSG_DOMAIN}.crt
       FMSG_TLS_KEY: /opt/fmsg/tls/fmsg.${FMSG_DOMAIN}.key
       FMSG_TLS_INSECURE_SKIP_VERIFY: "true"
@@ -58,3 +59,4 @@ networks:
   default:
   fmsg-test:
     external: true
+    name: ${FMSG_TEST_NETWORK:-fmsg-test}
@@ -43,7 +43,7 @@ cleanup() {
   COMPOSE_PROJECT_NAME=example FMSG_DOMAIN=example.com FMSG_WEBAPI_HOST_PORT=8182 \
     docker compose -f docker-compose.yml -f ../test/docker-compose.test.yml down -v 2>/dev/null || true
 
-  docker network rm fmsg-test 2>/dev/null || true
+  docker network rm "${FMSG_TEST_NETWORK:-fmsg-test}" 2>/dev/null || true
   rm -rf "$REPO_ROOT/test/.tls"
   rm -rf "$TEST_LOG_DIR"
   echo "==> Cleanup complete."
@@ -260,7 +260,7 @@ if [ "$SKIP_START" != "true" ]; then
 
   # ── Create shared Docker network ──────────────────────────
   echo "==> Creating fmsg-test network..."
-  docker network create fmsg-test
+  docker network create "${FMSG_TEST_NETWORK:-fmsg-test}"
 
   # ── Generate self-signed TLS certificates ─────────────────
   echo "==> Generating self-signed TLS certificates..."

@@ -103,3 +103,11 @@ api_json_get() {
 
   curl -s -H "Authorization: Bearer $token" "$api_url$path"
 }
+
+# Send JSON through the authenticated API for fields the CLI does not expose.
+api_json_write() {
+  local api_url="$1" api_key="$2" method="$3" path="$4" body="$5" token
+  token=$(curl -fsS -X POST -H "Authorization: Bearer $api_key" "$api_url/fmsg/token" | jq -er '.access_token')
+  curl -fsS -X "$method" -H "Authorization: Bearer $token" \
+    -H 'Content-Type: application/json' --data "$body" "$api_url$path"
+}
@@ -37,15 +37,15 @@ to more than one recipient.
   resolveLocalDelivery. Also asserts receiving hosts retain the complete _to_
   list (SPEC §11), which folds in most of what 014 was for.
 
-- [ ] **014 — same-domain / local delivery** (bob -> carol, both `@example.com`)
+- [x] **014 — same-domain / local delivery** (bob -> carol, both `@example.com`)
   *Reduced by 013, which already covers a local recipient alongside a remote
   one. What remains untested is a message with ONLY local recipients, which
   never reaches fmsgd's outbound sender at all.*
   Exercises `resolveLocalDelivery`
   (`fmsg-webapi/internal/handlers/messages.go:97`), which bypasses the fmsgd
   wire and is covered by nothing today.
 
-- [ ] **015 — notification-only add-to, code 11** (SPEC §10.4 step 1, §12, §11)
+- [x] **015 — notification-only add-to, code 11** (SPEC §10.4 step 1, §12, §11)
   Tests 004/009/010 all take the **65** path. Setup: alice(hairpin) ->
   bob(example); then **bob** adds `@carol@example.com`, so hairpin.local hosts
   only `from` and must respond 11 and record the batch. Key assertion: alice can
@@ -115,3 +115,7 @@ expose, so they follow test 009's precedent of injecting a pending outbound row
 directly into the sender's database. Those tests will break whenever
 `fmsgd/dd.sql` changes shape — a deliberate maintenance cost, currently paid
 once.
+
+Tests `009` and `015-message-sha256.sh` cover API batch-hash replies, local-only
+identities and subsequent federation, and compressed notification-only add-to.
+Run with `FMSG_CHALLENGE_MODE=ALWAYS` to assert challenge-response coverage.
@@ -10,14 +10,8 @@
 # hash from the add-to header plus its stored copy of the original data, and
 # the originating host persists the hash of batches it sends.
 #
-# Regression coverage for fmsgd#35 (batch identity is the batch message
-# hash, stored on msg_add_to_batch) and fmsgd#39 (replies resolving batch
-# hashes; ensureBatchHash on the originator). fmsg-webapi cannot yet compose
-# a reply referencing a batch, so this test injects the pending outbound
-# reply directly into the sender host's database in the same shape
-# fmsg-webapi writes — psha256 carries the parent hash; the relational pid
-# stays null so the populate-psha256 trigger passes it through — and lets
-# fmsgd deliver it cross-instance.
+# Compose the reply through the API using the batch hash. No direct database
+# injection is needed: the API preserves the exact protocol parent reference.
 set -euo pipefail
 
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
@@ -74,21 +68,13 @@ if [ "$HAIRPIN_BATCH_HASH" != "$BATCH_HASH" ]; then
 fi
 echo "    Originator and receiver agree on the batch hash"
 
-echo "    Injecting carol's pending reply to the batch (pid = batch hash) at example.com"
-PAYLOAD_PATH="/opt/fmsg/data/example.com/carol/out/test-009-$TEST_TOKEN"
-printf %s "$REPLY_TEXT" | docker exec -i example-fmsgd-1 sh -c "mkdir -p /opt/fmsg/data/example.com/carol/out && cat > $PAYLOAD_PATH"
-REPLY_SIZE=$(printf %s "$REPLY_TEXT" | wc -c)
-
-# Wrapped in a CTE because psql prints the "INSERT 0 1" command tag even
-# with -tA; selecting from the CTE yields just the id.
-REPLY_ROW_ID=$(psql_example "with ins as (
-  insert into msg (version, no_reply, is_important, is_deflate, from_addr, topic, type, size, filepath, time_sent, psha256)
-  values (1, false, false, false, '$CAROL_ADDR', '', 'text/plain;charset=UTF-8', $REPLY_SIZE, '$PAYLOAD_PATH', extract(epoch from now()), decode('$BATCH_HASH', 'hex'))
-  returning id
-) select id from ins")
-[ -n "$REPLY_ROW_ID" ] || fail_test "could not insert carol's reply row at example.com"
-psql_example "insert into msg_to (msg_id, addr) values ($REPLY_ROW_ID, '$ALICE_ADDR')" > /dev/null
-echo "    Injected pending reply row ID: $REPLY_ROW_ID"
+echo "    Creating carol's reply through the API using the batch hash"
+REPLY_INPUT=$(jq -n --arg from "$CAROL_ADDR" --arg to "$ALICE_ADDR" \
+  --arg parent "$BATCH_HASH" --arg body "$REPLY_TEXT" \
+  '{version:1, from:$from, to:[$to], pid:$parent, topic:"", type:"text/plain;charset=UTF-8", data:$body}')
+REPLY_ROW_ID=$(api_json_write "$EXAMPLE_API_URL" "$CAROL_API_KEY" POST /fmsg "$REPLY_INPUT" | jq -er '.id')
+REPLY_HASH=$(api_json_write "$EXAMPLE_API_URL" "$CAROL_API_KEY" POST "/fmsg/$REPLY_ROW_ID/send" '{}' | jq -er '.sha256')
+[[ "$REPLY_HASH" =~ ^[0-9a-f]{64}$ ]] || fail_test "reply was sent without a hash"
 
 echo "    Waiting for cross-instance delivery of the batch reply to $ALICE_ADDR..."
 ALICE_REPLY_ID=$(wait_for_message_id_by_data "$HAIRPIN_API_URL" "$ALICE_API_KEY" "$REPLY_TEXT" 30)

@@ -0,0 +1,67 @@
+#!/usr/bin/env bash
+# Local-only identities remain stable when later federated through add-to.
+set -euo pipefail
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+source "$SCRIPT_DIR/../test-lib.sh"
+
+TEST_TOKEN="$(date +%s)-$$"
+BODY="$(printf 'A compressible message for immutable local delivery. %.0s' {1..40}) [$TEST_TOKEN]"
+INPUT=$(jq -n --arg from "$BOB_ADDR" --arg to "$CAROL_ADDR" --arg body "$BODY" \
+  '{version:1,from:$from,to:[$to],type:"text/plain;charset=UTF-8",data:$body}')
+ROOT_ID=$(api_json_write "$EXAMPLE_API_URL" "$BOB_API_KEY" POST /fmsg "$INPUT" | jq -er '.id')
+api_json_get "$EXAMPLE_API_URL" "$BOB_API_KEY" "/fmsg/$ROOT_ID" | jq -e '.time == null and .sha256 == null' >/dev/null
+ROOT_HASH=$(api_json_write "$EXAMPLE_API_URL" "$BOB_API_KEY" POST "/fmsg/$ROOT_ID/send" '{}' | jq -er '.sha256')
+[[ "$ROOT_HASH" =~ ^[0-9a-f]{64}$ ]] || fail_test "local-only send did not return a hash"
+api_json_get "$EXAMPLE_API_URL" "$CAROL_API_KEY" "/fmsg/$ROOT_HASH" | jq -e --arg hash "$ROOT_HASH" '.sha256 == $hash and .time != null' >/dev/null
+
+echo "    Local-only delivery has an immediately usable identity"
+REPLY_INPUT=$(jq -n --arg from "$CAROL_ADDR" --arg to "$BOB_ADDR" --arg pid "$ROOT_HASH" \
+  '{version:1,from:$from,to:[$to],pid:$pid,type:"text/plain;charset=UTF-8",data:"local reply"}')
+REPLY_ID=$(api_json_write "$EXAMPLE_API_URL" "$CAROL_API_KEY" POST /fmsg "$REPLY_INPUT" | jq -er '.id')
+api_json_write "$EXAMPLE_API_URL" "$CAROL_API_KEY" POST "/fmsg/$REPLY_ID/send" '{}' | jq -e '.sha256 | test("^[0-9a-f]{64}$")' >/dev/null
+api_json_get "$EXAMPLE_API_URL" "$CAROL_API_KEY" "/fmsg/$REPLY_ID" | jq -e --arg hash "$ROOT_HASH" '.psha256 == $hash' >/dev/null
+api_json_write "$EXAMPLE_API_URL" "$CAROL_API_KEY" POST "/fmsg/$ROOT_HASH/react" '{"emoji":"👍"}' | jq -e '.sha256 | test("^[0-9a-f]{64}$")' >/dev/null
+
+echo "    Adding a remote participant after local-only delivery"
+ADD_INPUT=$(jq -n --arg addr "$ALICE_ADDR" '{add_to:[$addr]}')
+BATCH_HASH=$(api_json_write "$EXAMPLE_API_URL" "$BOB_API_KEY" POST "/fmsg/$ROOT_HASH/add-to" "$ADD_INPUT" | jq -er '.sha256')
+ALICE_ROOT=$(wait_for_message_id_by_data "$HAIRPIN_API_URL" "$ALICE_API_KEY" "$BODY" 30)
+api_json_get "$HAIRPIN_API_URL" "$ALICE_API_KEY" "/fmsg/$ROOT_HASH" | jq -e --arg root "$ROOT_HASH" --arg batch "$BATCH_HASH" \
+  '.sha256 == $root and any(.add_to[]; .sha256 == $batch)' >/dev/null
+api_json_get "$EXAMPLE_API_URL" "$BOB_API_KEY" "/fmsg/$ROOT_ID" | jq -e --arg hash "$ROOT_HASH" '.sha256 == $hash' >/dev/null
+
+REPLY_TEXT="remote batch reply [$TEST_TOKEN]"
+REPLY_INPUT=$(jq -n --arg from "$ALICE_ADDR" --arg to "$BOB_ADDR" --arg pid "$BATCH_HASH" --arg body "$REPLY_TEXT" \
+  '{version:1,from:$from,to:[$to],pid:$pid,type:"text/plain;charset=UTF-8",data:$body}')
+REPLY_ID=$(api_json_write "$HAIRPIN_API_URL" "$ALICE_API_KEY" POST /fmsg "$REPLY_INPUT" | jq -er '.id')
+REPLY_HASH=$(api_json_write "$HAIRPIN_API_URL" "$ALICE_API_KEY" POST "/fmsg/$REPLY_ID/send" '{}' | jq -er '.sha256')
+BOB_REPLY=$(wait_for_message_id_by_data "$EXAMPLE_API_URL" "$BOB_API_KEY" "$REPLY_TEXT" 30)
+api_json_get "$EXAMPLE_API_URL" "$BOB_API_KEY" "/fmsg/$BOB_REPLY" | jq -e --arg parent "$BATCH_HASH" --arg hash "$REPLY_HASH" \
+  '.psha256 == $parent and .sha256 == $hash' >/dev/null
+
+echo "    Verifying a locally hashed batch on the notification-only (11) path"
+NOTIFY_BODY="$(printf 'Compression also survives a batch notification. %.0s' {1..30}) [$TEST_TOKEN]"
+INPUT=$(jq -n --arg from "$ALICE_ADDR" --arg to "$BOB_ADDR" --arg body "$NOTIFY_BODY" \
+  '{version:1,from:$from,to:[$to],type:"text/plain;charset=UTF-8",data:$body}')
+ID=$(api_json_write "$HAIRPIN_API_URL" "$ALICE_API_KEY" POST /fmsg "$INPUT" | jq -er '.id')
+HASH=$(api_json_write "$HAIRPIN_API_URL" "$ALICE_API_KEY" POST "/fmsg/$ID/send" '{}' | jq -er '.sha256')
+wait_for_message_id_by_data "$EXAMPLE_API_URL" "$BOB_API_KEY" "$NOTIFY_BODY" 30 >/dev/null
+ADD_INPUT=$(jq -n --arg addr "$CAROL_ADDR" '{add_to:[$addr]}')
+NOTIFY_HASH=$(api_json_write "$EXAMPLE_API_URL" "$BOB_API_KEY" POST "/fmsg/$HASH/add-to" "$ADD_INPUT" | jq -er '.sha256')
+FOUND=false
+for attempt in $(seq 1 30); do
+  if api_json_get "$HAIRPIN_API_URL" "$ALICE_API_KEY" "/fmsg/$HASH" | jq -e --arg hash "$NOTIFY_HASH" 'any(.add_to[]?; .sha256 == $hash)' >/dev/null; then
+    FOUND=true
+    break
+  fi
+  sleep 1
+done
+[ "$FOUND" = true ] || fail_test "notification-only host did not retain the batch identity"
+CODE=$(docker exec example-postgres-1 psql -U postgres -d fmsgd -tAc \
+  "select n.response_code from msg_add_to_notify n join msg_add_to_batch b on b.id=n.batch_id where b.sha256=decode('$NOTIFY_HASH','hex')")
+[ "$CODE" = 11 ] || fail_test "expected notification code 11, got $CODE"
+
+if [ "${FMSG_CHALLENGE_MODE:-}" = ALWAYS ]; then
+  docker logs example-fmsgd-1 2>&1 | grep 'CHALLENGE RESP' >/dev/null || fail_test "no challenge response recorded"
+fi
+echo "    OK: hashes survive local delivery, reactions, compression, federation, batch replies and notifications"