From c425ed896f112a1d44f6a8f1f49a3328098290ec Mon Sep 17 00:00:00 2001 From: Mark Mennell Date: Thu, 10 Sep 2026 15:43:24 +0800 Subject: [PATCH 1/4] Test finalized message hashes across local and federated delivery --- README.md | 21 +++++ docker/fmsgd/Dockerfile | 4 +- docker/postgres/init/002-fmsgd-dd.sql | 107 +++++++++++++++++++++++- test/docker-compose.test.yml | 2 + test/run-tests.sh | 4 +- test/test-lib.sh | 8 ++ test/tests-to-add.md | 8 +- test/tests/009-reply-to-add-to-batch.sh | 32 ++----- test/tests/015-message-sha256.sh | 67 +++++++++++++++ 9 files changed, 224 insertions(+), 29 deletions(-) create mode 100755 test/tests/015-message-sha256.sh diff --git a/README.md b/README.md index 3a1187d..ecc552e 100644 --- a/README.md +++ b/README.md @@ -219,3 +219,24 @@ On first startup (empty data volume), PostgreSQL runs the scripts in `docker/pos > ``` + +### Testing message finalization + +The message schema now requires a SHA-256 whenever a message becomes sent. Deploy +compatible daemon/API versions with this schema; pause writers and federation during +an existing-stack upgrade and run `/opt/fmsgd/fmsg-backfill -domain example.com -apply` +from the daemon container before resuming. Run the command without `-apply` to list +pending records. Retain the shared message data volume, including `.fmsg-wire-*` +directories. The daemon owns this schema; the initialization file here is its copy. + +Test `015-message-sha256.sh` verifies local-only hashing, later federation with +compression, batch-hash replies, and notification-only add-to. Test `009` now creates +batch replies through the API. To exercise challenge responses on every exchange: + +```sh +FMSG_CHALLENGE_MODE=ALWAYS ./test/run-tests-podman.sh +``` + +For coordinated feature branches, set `FMSGD_REF` and `FMSG_WEBAPI_REF` to those +branches. `FMSG_TEST_NETWORK` optionally changes the shared test network name +(default `fmsg-test`). diff --git a/docker/fmsgd/Dockerfile b/docker/fmsgd/Dockerfile index 1d7edbb..6983072 100644 --- a/docker/fmsgd/Dockerfile +++ b/docker/fmsgd/Dockerfile @@ -6,7 +6,8 @@ ARG CACHEBUST WORKDIR /build RUN git clone --branch "$FMSGD_REF" --depth 1 https://github.com/markmnl/fmsgd.git . && \ - go build -o fmsgd ./cmd/fmsgd + go build -o fmsgd ./cmd/fmsgd && \ + go build -o fmsg-backfill ./cmd/fmsg-backfill FROM debian:bookworm-slim @@ -18,6 +19,7 @@ RUN useradd -r -s /bin/false fmsg WORKDIR /opt/fmsgd COPY --from=builder /build/fmsgd /opt/fmsgd/fmsgd +COPY --from=builder /build/fmsg-backfill /opt/fmsgd/fmsg-backfill RUN mkdir -p /opt/fmsg/data && \ chown -R fmsg:fmsg /opt/fmsgd /opt/fmsg/data diff --git a/docker/postgres/init/002-fmsgd-dd.sql b/docker/postgres/init/002-fmsgd-dd.sql index 0f2ebb4..a5c124c 100644 --- a/docker/postgres/init/002-fmsgd-dd.sql +++ b/docker/postgres/init/002-fmsgd-dd.sql @@ -192,7 +192,7 @@ begin raise exception 'cannot clear sha256 for message %: it has replies', NEW.id; end if; - if OLD.sha256 is distinct from NEW.sha256 then + if OLD.sha256 is not null and OLD.sha256 is distinct from NEW.sha256 then raise exception 'cannot change sha256 for message %: it has replies', NEW.id; end if; end if; @@ -389,3 +389,108 @@ create constraint trigger trg_recipients_added after insert on msg_add_to_batch deferrable initially deferred for each row execute function notify_recipients_added(); + +-- Durable protocol representations, shared by the API finalizer and daemon. +-- NULL on legacy rows; received add-to variants belong to their batch only. +alter table msg add column if not exists wire_message jsonb; +alter table msg_add_to_batch add column if not exists wire_message jsonb; +create index if not exists msg_add_to_batch_sha256_idx on msg_add_to_batch (sha256) where sha256 is not null; +create index if not exists msg_pid_idx on msg (pid) where pid is not null; + +-- Preserve protocol identity; local relational pid links and delivery/read +-- metadata are bookkeeping and may still change. Legacy NULL hashes may be +-- filled once without changing the timestamp, including parents with replies. +create or replace function protect_msg_identity() returns trigger as $$ +begin + if OLD.time_sent is not null then + if NEW.time_sent is distinct from OLD.time_sent then + raise exception 'sent message timestamp is immutable'; + end if; + if OLD.sha256 is not null and + (NEW.sha256 is distinct from OLD.sha256 or + row(NEW.version,NEW.psha256,NEW.no_reply,NEW.is_important,NEW.is_terminal, + NEW.is_deflate,NEW.from_addr,NEW.topic,NEW.type,NEW.size,NEW.filepath) + is distinct from + row(OLD.version,OLD.psha256,OLD.no_reply,OLD.is_important,OLD.is_terminal, + OLD.is_deflate,OLD.from_addr,OLD.topic,OLD.type,OLD.size,OLD.filepath) or + (OLD.wire_header is not null and NEW.wire_header is distinct from OLD.wire_header) or + (OLD.wire_message is not null and NEW.wire_message is distinct from OLD.wire_message)) then + raise exception 'sent message content and hash are immutable'; + end if; + end if; + return NEW; +end; +$$ language plpgsql; +drop trigger if exists trg_msg_identity on msg; +create trigger trg_msg_identity before update on msg for each row execute function protect_msg_identity(); + +-- Validate at commit so receiving hosts can assemble rows and recipients in +-- one transaction. Existing unhashed rows are backfilled by fmsg-backfill. +create or replace function require_sent_msg_hash() returns trigger as $$ +begin + if TG_OP='UPDATE' then + if OLD.time_sent is not distinct from NEW.time_sent and OLD.sha256 is not distinct from NEW.sha256 then return null; end if; + end if; + if exists (select 1 from msg where id=NEW.id and time_sent is not null + and (sha256 is null or octet_length(sha256) <> 32)) then + raise exception 'sent message % requires a 32-byte sha256', NEW.id; + end if; + return null; +end; +$$ language plpgsql; +drop trigger if exists trg_msg_require_hash on msg; +create constraint trigger trg_msg_require_hash after insert or update on msg + deferrable initially deferred for each row execute function require_sent_msg_hash(); + +create or replace function protect_msg_parts() returns trigger as $$ +declare + message_id bigint; + frozen boolean; +begin + -- Delivery and read receipts do not change a protocol recipient. + if TG_OP='UPDATE' then + if TG_TABLE_NAME='msg_to' then + if row(NEW.id,NEW.msg_id,NEW.addr) is not distinct from row(OLD.id,OLD.msg_id,OLD.addr) then return NEW; end if; + end if; + if TG_TABLE_NAME='msg_add_to' then + if row(NEW.id,NEW.msg_id,NEW.batch_id,NEW.addr) is not distinct from row(OLD.id,OLD.msg_id,OLD.batch_id,OLD.addr) then return NEW; end if; + end if; + end if; + if TG_OP='DELETE' then message_id=OLD.msg_id; else message_id=NEW.msg_id; end if; + if TG_OP='UPDATE' and NEW.msg_id <> OLD.msg_id then raise exception 'cannot move message parts'; end if; + if TG_TABLE_NAME='msg_add_to' then + if TG_OP='DELETE' then + select sha256 is not null into frozen from msg_add_to_batch where id=OLD.batch_id for update; + else + if TG_OP='UPDATE' and NEW.batch_id <> OLD.batch_id then raise exception 'cannot move batch recipients'; end if; + select sha256 is not null into frozen from msg_add_to_batch where id=NEW.batch_id and msg_id=message_id for update; + if not found then raise exception 'batch does not belong to message'; end if; + end if; + else + select time_sent is not null and sha256 is not null into frozen from msg where id=message_id for update; + end if; + if frozen then raise exception 'finalized message parts are immutable'; end if; + if TG_OP='DELETE' then return OLD; end if; + return NEW; +end; +$$ language plpgsql; +-- AFTER INSERT allows an ON CONFLICT DO NOTHING receipt to remain a no-op. +drop trigger if exists trg_msg_to_content on msg_to; +create trigger trg_msg_to_content after insert or update or delete on msg_to for each row execute function protect_msg_parts(); +drop trigger if exists trg_msg_attachment_content on msg_attachment; +create trigger trg_msg_attachment_content after insert or update or delete on msg_attachment for each row execute function protect_msg_parts(); +drop trigger if exists trg_msg_add_to_content on msg_add_to; +create trigger trg_msg_add_to_content after insert or update or delete on msg_add_to for each row execute function protect_msg_parts(); + +create or replace function protect_batch_identity() returns trigger as $$ +begin + if OLD.sha256 is not null and + row(NEW.msg_id,NEW.add_to_from,NEW.time_added,NEW.sha256,NEW.wire_message) + is distinct from row(OLD.msg_id,OLD.add_to_from,OLD.time_added,OLD.sha256,OLD.wire_message) then + raise exception 'finalized add-to batch is immutable'; + end if; + return NEW; +end; +$$ language plpgsql; +drop trigger if exists trg_batch_identity on msg_add_to_batch; +create trigger trg_batch_identity before update on msg_add_to_batch for each row execute function protect_batch_identity(); diff --git a/test/docker-compose.test.yml b/test/docker-compose.test.yml index 8158b76..f7df767 100644 --- a/test/docker-compose.test.yml +++ b/test/docker-compose.test.yml @@ -22,6 +22,7 @@ services: fmsgd: environment: + FMSG_CHALLENGE_MODE: ${FMSG_CHALLENGE_MODE:-HAS_NOT_PARTICIPATED} FMSG_TLS_CERT: /opt/fmsg/tls/fmsg.${FMSG_DOMAIN}.crt FMSG_TLS_KEY: /opt/fmsg/tls/fmsg.${FMSG_DOMAIN}.key FMSG_TLS_INSECURE_SKIP_VERIFY: "true" @@ -58,3 +59,4 @@ networks: default: fmsg-test: external: true + name: ${FMSG_TEST_NETWORK:-fmsg-test} diff --git a/test/run-tests.sh b/test/run-tests.sh index 8971b58..c906f2e 100755 --- a/test/run-tests.sh +++ b/test/run-tests.sh @@ -43,7 +43,7 @@ cleanup() { COMPOSE_PROJECT_NAME=example FMSG_DOMAIN=example.com FMSG_WEBAPI_HOST_PORT=8182 \ docker compose -f docker-compose.yml -f ../test/docker-compose.test.yml down -v 2>/dev/null || true - docker network rm fmsg-test 2>/dev/null || true + docker network rm "${FMSG_TEST_NETWORK:-fmsg-test}" 2>/dev/null || true rm -rf "$REPO_ROOT/test/.tls" rm -rf "$TEST_LOG_DIR" echo "==> Cleanup complete." @@ -260,7 +260,7 @@ if [ "$SKIP_START" != "true" ]; then # ── Create shared Docker network ────────────────────────── echo "==> Creating fmsg-test network..." - docker network create fmsg-test + docker network create "${FMSG_TEST_NETWORK:-fmsg-test}" # ── Generate self-signed TLS certificates ───────────────── echo "==> Generating self-signed TLS certificates..." diff --git a/test/test-lib.sh b/test/test-lib.sh index d8b102f..a4e00b8 100644 --- a/test/test-lib.sh +++ b/test/test-lib.sh @@ -103,3 +103,11 @@ api_json_get() { curl -s -H "Authorization: Bearer $token" "$api_url$path" } + +# Send JSON through the authenticated API for fields the CLI does not expose. +api_json_write() { + local api_url="$1" api_key="$2" method="$3" path="$4" body="$5" token + token=$(curl -fsS -X POST -H "Authorization: Bearer $api_key" "$api_url/fmsg/token" | jq -er '.access_token') + curl -fsS -X "$method" -H "Authorization: Bearer $token" \ + -H 'Content-Type: application/json' --data "$body" "$api_url$path" +} diff --git a/test/tests-to-add.md b/test/tests-to-add.md index 62f3d7e..f876a6a 100644 --- a/test/tests-to-add.md +++ b/test/tests-to-add.md @@ -37,7 +37,7 @@ to more than one recipient. resolveLocalDelivery. Also asserts receiving hosts retain the complete _to_ list (SPEC §11), which folds in most of what 014 was for. -- [ ] **014 — same-domain / local delivery** (bob -> carol, both `@example.com`) +- [x] **014 — same-domain / local delivery** (bob -> carol, both `@example.com`) *Reduced by 013, which already covers a local recipient alongside a remote one. What remains untested is a message with ONLY local recipients, which never reaches fmsgd's outbound sender at all.* @@ -45,7 +45,7 @@ to more than one recipient. (`fmsg-webapi/internal/handlers/messages.go:97`), which bypasses the fmsgd wire and is covered by nothing today. -- [ ] **015 — notification-only add-to, code 11** (SPEC §10.4 step 1, §12, §11) +- [x] **015 — notification-only add-to, code 11** (SPEC §10.4 step 1, §12, §11) Tests 004/009/010 all take the **65** path. Setup: alice(hairpin) -> bob(example); then **bob** adds `@carol@example.com`, so hairpin.local hosts only `from` and must respond 11 and record the batch. Key assertion: alice can @@ -115,3 +115,7 @@ expose, so they follow test 009's precedent of injecting a pending outbound row directly into the sender's database. Those tests will break whenever `fmsgd/dd.sql` changes shape — a deliberate maintenance cost, currently paid once. + +Tests `009` and `015-message-sha256.sh` cover API batch-hash replies, local-only +identities and subsequent federation, and compressed notification-only add-to. +Run with `FMSG_CHALLENGE_MODE=ALWAYS` to assert challenge-response coverage. diff --git a/test/tests/009-reply-to-add-to-batch.sh b/test/tests/009-reply-to-add-to-batch.sh index 63b4a06..e4e6645 100755 --- a/test/tests/009-reply-to-add-to-batch.sh +++ b/test/tests/009-reply-to-add-to-batch.sh @@ -10,14 +10,8 @@ # hash from the add-to header plus its stored copy of the original data, and # the originating host persists the hash of batches it sends. # -# Regression coverage for fmsgd#35 (batch identity is the batch message -# hash, stored on msg_add_to_batch) and fmsgd#39 (replies resolving batch -# hashes; ensureBatchHash on the originator). fmsg-webapi cannot yet compose -# a reply referencing a batch, so this test injects the pending outbound -# reply directly into the sender host's database in the same shape -# fmsg-webapi writes — psha256 carries the parent hash; the relational pid -# stays null so the populate-psha256 trigger passes it through — and lets -# fmsgd deliver it cross-instance. +# Compose the reply through the API using the batch hash. No direct database +# injection is needed: the API preserves the exact protocol parent reference. set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -74,21 +68,13 @@ if [ "$HAIRPIN_BATCH_HASH" != "$BATCH_HASH" ]; then fi echo " Originator and receiver agree on the batch hash" -echo " Injecting carol's pending reply to the batch (pid = batch hash) at example.com" -PAYLOAD_PATH="/opt/fmsg/data/example.com/carol/out/test-009-$TEST_TOKEN" -printf %s "$REPLY_TEXT" | docker exec -i example-fmsgd-1 sh -c "mkdir -p /opt/fmsg/data/example.com/carol/out && cat > $PAYLOAD_PATH" -REPLY_SIZE=$(printf %s "$REPLY_TEXT" | wc -c) - -# Wrapped in a CTE because psql prints the "INSERT 0 1" command tag even -# with -tA; selecting from the CTE yields just the id. -REPLY_ROW_ID=$(psql_example "with ins as ( - insert into msg (version, no_reply, is_important, is_deflate, from_addr, topic, type, size, filepath, time_sent, psha256) - values (1, false, false, false, '$CAROL_ADDR', '', 'text/plain;charset=UTF-8', $REPLY_SIZE, '$PAYLOAD_PATH', extract(epoch from now()), decode('$BATCH_HASH', 'hex')) - returning id -) select id from ins") -[ -n "$REPLY_ROW_ID" ] || fail_test "could not insert carol's reply row at example.com" -psql_example "insert into msg_to (msg_id, addr) values ($REPLY_ROW_ID, '$ALICE_ADDR')" > /dev/null -echo " Injected pending reply row ID: $REPLY_ROW_ID" +echo " Creating carol's reply through the API using the batch hash" +REPLY_INPUT=$(jq -n --arg from "$CAROL_ADDR" --arg to "$ALICE_ADDR" \ + --arg parent "$BATCH_HASH" --arg body "$REPLY_TEXT" \ + '{version:1, from:$from, to:[$to], pid:$parent, topic:"", type:"text/plain;charset=UTF-8", data:$body}') +REPLY_ROW_ID=$(api_json_write "$EXAMPLE_API_URL" "$CAROL_API_KEY" POST /fmsg "$REPLY_INPUT" | jq -er '.id') +REPLY_HASH=$(api_json_write "$EXAMPLE_API_URL" "$CAROL_API_KEY" POST "/fmsg/$REPLY_ROW_ID/send" '{}' | jq -er '.sha256') +[[ "$REPLY_HASH" =~ ^[0-9a-f]{64}$ ]] || fail_test "reply was sent without a hash" echo " Waiting for cross-instance delivery of the batch reply to $ALICE_ADDR..." ALICE_REPLY_ID=$(wait_for_message_id_by_data "$HAIRPIN_API_URL" "$ALICE_API_KEY" "$REPLY_TEXT" 30) diff --git a/test/tests/015-message-sha256.sh b/test/tests/015-message-sha256.sh new file mode 100755 index 0000000..1b74732 --- /dev/null +++ b/test/tests/015-message-sha256.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +# Local-only identities remain stable when later federated through add-to. +set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/../test-lib.sh" + +TEST_TOKEN="$(date +%s)-$$" +BODY="$(printf 'A compressible message for immutable local delivery. %.0s' {1..40}) [$TEST_TOKEN]" +INPUT=$(jq -n --arg from "$BOB_ADDR" --arg to "$CAROL_ADDR" --arg body "$BODY" \ + '{version:1,from:$from,to:[$to],type:"text/plain;charset=UTF-8",data:$body}') +ROOT_ID=$(api_json_write "$EXAMPLE_API_URL" "$BOB_API_KEY" POST /fmsg "$INPUT" | jq -er '.id') +api_json_get "$EXAMPLE_API_URL" "$BOB_API_KEY" "/fmsg/$ROOT_ID" | jq -e '.time == null and .sha256 == null' >/dev/null +ROOT_HASH=$(api_json_write "$EXAMPLE_API_URL" "$BOB_API_KEY" POST "/fmsg/$ROOT_ID/send" '{}' | jq -er '.sha256') +[[ "$ROOT_HASH" =~ ^[0-9a-f]{64}$ ]] || fail_test "local-only send did not return a hash" +api_json_get "$EXAMPLE_API_URL" "$CAROL_API_KEY" "/fmsg/$ROOT_HASH" | jq -e --arg hash "$ROOT_HASH" '.sha256 == $hash and .time != null' >/dev/null + +echo " Local-only delivery has an immediately usable identity" +REPLY_INPUT=$(jq -n --arg from "$CAROL_ADDR" --arg to "$BOB_ADDR" --arg pid "$ROOT_HASH" \ + '{version:1,from:$from,to:[$to],pid:$pid,type:"text/plain;charset=UTF-8",data:"local reply"}') +REPLY_ID=$(api_json_write "$EXAMPLE_API_URL" "$CAROL_API_KEY" POST /fmsg "$REPLY_INPUT" | jq -er '.id') +api_json_write "$EXAMPLE_API_URL" "$CAROL_API_KEY" POST "/fmsg/$REPLY_ID/send" '{}' | jq -e '.sha256 | test("^[0-9a-f]{64}$")' >/dev/null +api_json_get "$EXAMPLE_API_URL" "$CAROL_API_KEY" "/fmsg/$REPLY_ID" | jq -e --arg hash "$ROOT_HASH" '.psha256 == $hash' >/dev/null +api_json_write "$EXAMPLE_API_URL" "$CAROL_API_KEY" POST "/fmsg/$ROOT_HASH/react" '{"emoji":"👍"}' | jq -e '.sha256 | test("^[0-9a-f]{64}$")' >/dev/null + +echo " Adding a remote participant after local-only delivery" +ADD_INPUT=$(jq -n --arg addr "$ALICE_ADDR" '{add_to:[$addr]}') +BATCH_HASH=$(api_json_write "$EXAMPLE_API_URL" "$BOB_API_KEY" POST "/fmsg/$ROOT_HASH/add-to" "$ADD_INPUT" | jq -er '.sha256') +ALICE_ROOT=$(wait_for_message_id_by_data "$HAIRPIN_API_URL" "$ALICE_API_KEY" "$BODY" 30) +api_json_get "$HAIRPIN_API_URL" "$ALICE_API_KEY" "/fmsg/$ROOT_HASH" | jq -e --arg root "$ROOT_HASH" --arg batch "$BATCH_HASH" \ + '.sha256 == $root and any(.add_to[]; .sha256 == $batch)' >/dev/null +api_json_get "$EXAMPLE_API_URL" "$BOB_API_KEY" "/fmsg/$ROOT_ID" | jq -e --arg hash "$ROOT_HASH" '.sha256 == $hash' >/dev/null + +REPLY_TEXT="remote batch reply [$TEST_TOKEN]" +REPLY_INPUT=$(jq -n --arg from "$ALICE_ADDR" --arg to "$BOB_ADDR" --arg pid "$BATCH_HASH" --arg body "$REPLY_TEXT" \ + '{version:1,from:$from,to:[$to],pid:$pid,type:"text/plain;charset=UTF-8",data:$body}') +REPLY_ID=$(api_json_write "$HAIRPIN_API_URL" "$ALICE_API_KEY" POST /fmsg "$REPLY_INPUT" | jq -er '.id') +REPLY_HASH=$(api_json_write "$HAIRPIN_API_URL" "$ALICE_API_KEY" POST "/fmsg/$REPLY_ID/send" '{}' | jq -er '.sha256') +BOB_REPLY=$(wait_for_message_id_by_data "$EXAMPLE_API_URL" "$BOB_API_KEY" "$REPLY_TEXT" 30) +api_json_get "$EXAMPLE_API_URL" "$BOB_API_KEY" "/fmsg/$BOB_REPLY" | jq -e --arg parent "$BATCH_HASH" --arg hash "$REPLY_HASH" \ + '.psha256 == $parent and .sha256 == $hash' >/dev/null + +echo " Verifying a locally hashed batch on the notification-only (11) path" +NOTIFY_BODY="$(printf 'Compression also survives a batch notification. %.0s' {1..30}) [$TEST_TOKEN]" +INPUT=$(jq -n --arg from "$ALICE_ADDR" --arg to "$BOB_ADDR" --arg body "$NOTIFY_BODY" \ + '{version:1,from:$from,to:[$to],type:"text/plain;charset=UTF-8",data:$body}') +ID=$(api_json_write "$HAIRPIN_API_URL" "$ALICE_API_KEY" POST /fmsg "$INPUT" | jq -er '.id') +HASH=$(api_json_write "$HAIRPIN_API_URL" "$ALICE_API_KEY" POST "/fmsg/$ID/send" '{}' | jq -er '.sha256') +wait_for_message_id_by_data "$EXAMPLE_API_URL" "$BOB_API_KEY" "$NOTIFY_BODY" 30 >/dev/null +ADD_INPUT=$(jq -n --arg addr "$CAROL_ADDR" '{add_to:[$addr]}') +NOTIFY_HASH=$(api_json_write "$EXAMPLE_API_URL" "$BOB_API_KEY" POST "/fmsg/$HASH/add-to" "$ADD_INPUT" | jq -er '.sha256') +FOUND=false +for attempt in $(seq 1 30); do + if api_json_get "$HAIRPIN_API_URL" "$ALICE_API_KEY" "/fmsg/$HASH" | jq -e --arg hash "$NOTIFY_HASH" 'any(.add_to[]; .sha256 == $hash)' >/dev/null; then + FOUND=true + break + fi + sleep 1 +done +[ "$FOUND" = true ] || fail_test "notification-only host did not retain the batch identity" +CODE=$(docker exec example-postgres-1 psql -U postgres -d fmsgd -tAc \ + "select n.response_code from msg_add_to_notify n join msg_add_to_batch b on b.id=n.batch_id where b.sha256=decode('$NOTIFY_HASH','hex')") +[ "$CODE" = 11 ] || fail_test "expected notification code 11, got $CODE" + +if [ "${FMSG_CHALLENGE_MODE:-}" = ALWAYS ]; then + docker logs example-fmsgd-1 2>&1 | grep 'CHALLENGE RESP' >/dev/null || fail_test "no challenge response recorded" +fi +echo " OK: hashes survive local delivery, reactions, compression, federation, batch replies and notifications" From 611cfd0da50e4bc05b580257c892473c905109c5 Mon Sep 17 00:00:00 2001 From: Mark Mennell Date: Thu, 10 Sep 2026 15:49:43 +0800 Subject: [PATCH 2/4] Test coordinated component branches in pull request CI --- .github/workflows/integration-test.yml | 34 +++++++++++++++++++++++--- README.md | 3 +++ 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 7664de6..e20ddf9 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -34,13 +34,39 @@ jobs: with: go-version: "stable" + - name: Select component refs + id: refs + env: + GH_TOKEN: ${{ github.token }} + PR_BRANCH: ${{ github.head_ref }} + INPUT_FMSGD: ${{ inputs.fmsgd_ref }} + INPUT_FMSGID: ${{ inputs.fmsgid_ref }} + INPUT_WEBAPI: ${{ inputs.fmsg_webapi_ref }} + INPUT_CLI: ${{ inputs.fmsg_cli_ref }} + run: | + pick_ref() { + local repo="$1" explicit="$2" + if [ -n "$explicit" ]; then + printf '%s\n' "$explicit" + elif [ -n "$PR_BRANCH" ] && gh api "repos/markmnl/$repo/git/ref/heads/$PR_BRANCH" >/dev/null 2>&1; then + printf '%s\n' "$PR_BRANCH" + else + echo main + fi + } + echo "fmsgd=$(pick_ref fmsgd "$INPUT_FMSGD")" >> "$GITHUB_OUTPUT" + echo "fmsgid=$(pick_ref fmsgid "$INPUT_FMSGID")" >> "$GITHUB_OUTPUT" + echo "webapi=$(pick_ref fmsg-webapi "$INPUT_WEBAPI")" >> "$GITHUB_OUTPUT" + echo "cli=$(pick_ref fmsg-cli "$INPUT_CLI")" >> "$GITHUB_OUTPUT" + - name: Run integration tests run: bash test/run-tests.sh env: - FMSGD_REF: ${{ github.event.inputs.fmsgd_ref }} - FMSGID_REF: ${{ github.event.inputs.fmsgid_ref }} - FMSG_WEBAPI_REF: ${{ github.event.inputs.fmsg_webapi_ref }} - FMSG_CLI_REF: ${{ github.event.inputs.fmsg_cli_ref }} + FMSGD_REF: ${{ steps.refs.outputs.fmsgd }} + FMSGID_REF: ${{ steps.refs.outputs.fmsgid }} + FMSG_WEBAPI_REF: ${{ steps.refs.outputs.webapi }} + FMSG_CLI_REF: ${{ steps.refs.outputs.cli }} + FMSG_CHALLENGE_MODE: ALWAYS - name: Upload integration logs if: failure() diff --git a/README.md b/README.md index ecc552e..cf7706d 100644 --- a/README.md +++ b/README.md @@ -240,3 +240,6 @@ FMSG_CHALLENGE_MODE=ALWAYS ./test/run-tests-podman.sh For coordinated feature branches, set `FMSGD_REF` and `FMSG_WEBAPI_REF` to those branches. `FMSG_TEST_NETWORK` optionally changes the shared test network name (default `fmsg-test`). + +Pull-request CI selects the matching component branch when it exists, otherwise +`main`. Manual workflow inputs take precedence. CI forces challenge responses. From f8b2cd2ed2d3e7751bc5f34855249b72495f1373 Mon Sep 17 00:00:00 2001 From: Mark Mennell Date: Thu, 10 Sep 2026 16:11:46 +0800 Subject: [PATCH 3/4] Use strict bootstrap schema and standalone offline migration --- README.md | 15 +- docker/fmsgd/Dockerfile | 4 +- docker/postgres/init/002-fmsgd-dd.sql | 249 ++++++++++---------------- 3 files changed, 106 insertions(+), 162 deletions(-) diff --git a/README.md b/README.md index cf7706d..01a6360 100644 --- a/README.md +++ b/README.md @@ -222,12 +222,15 @@ On first startup (empty data volume), PostgreSQL runs the scripts in `docker/pos ### Testing message finalization -The message schema now requires a SHA-256 whenever a message becomes sent. Deploy -compatible daemon/API versions with this schema; pause writers and federation during -an existing-stack upgrade and run `/opt/fmsgd/fmsg-backfill -domain example.com -apply` -from the daemon container before resuming. Run the command without `-apply` to list -pending records. Retain the shared message data volume, including `.fmsg-wire-*` -directories. The daemon owns this schema; the initialization file here is its copy. +The message schema requires a SHA-256 and durable wire representation for sent +messages. The initialization SQL is for new databases only. To upgrade an existing +stack, stop the daemon and API, back up the database and shared data volume, then run +[the standalone `fmsg-backfill` binary](https://github.com/markmnl/fmsgd#immutable-message-finalization-and-upgrades) +with access to the database and stored payload paths. It embeds the schema upgrade; +run without `-apply` for a full dry run and with `-apply` to commit. Start the matching +daemon/API only after migration succeeds. The migration binary is not bundled in the +daemon image. Retain the shared message volume, including `.fmsg-wire-*` directories. +The daemon owns the schema; the initialization file here is its bootstrap copy. Test `015-message-sha256.sh` verifies local-only hashing, later federation with compression, batch-hash replies, and notification-only add-to. Test `009` now creates diff --git a/docker/fmsgd/Dockerfile b/docker/fmsgd/Dockerfile index 6983072..1d7edbb 100644 --- a/docker/fmsgd/Dockerfile +++ b/docker/fmsgd/Dockerfile @@ -6,8 +6,7 @@ ARG CACHEBUST WORKDIR /build RUN git clone --branch "$FMSGD_REF" --depth 1 https://github.com/markmnl/fmsgd.git . && \ - go build -o fmsgd ./cmd/fmsgd && \ - go build -o fmsg-backfill ./cmd/fmsg-backfill + go build -o fmsgd ./cmd/fmsgd FROM debian:bookworm-slim @@ -19,7 +18,6 @@ RUN useradd -r -s /bin/false fmsg WORKDIR /opt/fmsgd COPY --from=builder /build/fmsgd /opt/fmsgd/fmsgd -COPY --from=builder /build/fmsg-backfill /opt/fmsgd/fmsg-backfill RUN mkdir -p /opt/fmsg/data && \ chown -R fmsg:fmsg /opt/fmsgd /opt/fmsg/data diff --git a/docker/postgres/init/002-fmsgd-dd.sql b/docker/postgres/init/002-fmsgd-dd.sql index a5c124c..d4ac2b8 100644 --- a/docker/postgres/init/002-fmsgd-dd.sql +++ b/docker/postgres/init/002-fmsgd-dd.sql @@ -1,27 +1,10 @@ \connect fmsgd -/**************************************************************** - * - * PostgreSQL database objects data definition for fmsgd - * - * This script is IDEMPOTENT: every statement is safe to re-run - * (create table/index if not exists, alter table add column if - * not exists, create or replace function, drop trigger if exists - * before create trigger). Migrating an existing database is - * therefore just re-running the whole script, e.g.: - * - * psql -d fmsgd -v ON_ERROR_STOP=1 -f dd.sql - * - * Keep it that way: add new objects and columns only with - * idempotent statements, and name indexes explicitly to match - * PostgreSQL's default generated names so indexes that already - * exist unnamed on live databases are recognised, not duplicated. - * - ****************************************************************/ - --- database with encoding UTF8 should already be created and connected - -create table if not exists msg ( +-- PostgreSQL bootstrap schema for a new, empty fmsg message database. +-- Existing installations use the standalone fmsg-backfill binary before +-- starting this version. This file is not an upgrade script. + +create table msg ( id bigserial primary key, version int not null, pid bigint references msg (id), @@ -37,13 +20,12 @@ create table if not exists msg ( psha256 bytea, size int not null, -- spec allows uint32 but we don't enforced by FMSG_MAX_MSG_SIZE filepath text not null, - wire_header bytea -- received messages: the exact wire header bytes (fields 1-13), so any hash can always be faithfully recomputed (SPEC §11); null for locally-authored messages + wire_header bytea, -- exact protocol header (fields 1-13) + wire_message jsonb -- durable original wire representation; null for drafts or originals received only through add-to ); -create index if not exists msg_lower_idx on msg ((lower(from_addr))); -alter table msg add column if not exists wire_header bytea; -- upgrade path for databases created before this column -alter table msg add column if not exists is_terminal boolean not null default false; -- upgrade path (SPEC v0.6.0) +create index msg_lower_idx on msg ((lower(from_addr))); -create table if not exists msg_to ( +create table msg_to ( id bigserial primary key, msg_id bigint not null references msg (id), addr varchar(255) not null, @@ -54,7 +36,7 @@ create table if not exists msg_to ( attempt_count int not null default 0, -- number of failed delivery attempts; used for exponential back-off unique (msg_id, addr) ); -create index if not exists msg_to_lower_idx on msg_to ((lower(addr))); +create index msg_to_lower_idx on msg_to ((lower(addr))); -- Each add-to delivery for a shared message is one batch: a single sender -- (add_to_from) added a set of recipients at a point in time. Storing batches @@ -62,19 +44,18 @@ create index if not exists msg_to_lower_idx on msg_to ((lower(addr))); -- which a single flat recipient list cannot preserve (SPEC §12). A batch's -- identity is its message hash (sha256), which covers the batch's time: the -- same addresses re-issued at a new time are a distinct batch, not a --- duplicate (SPEC §11/§12). sha256 is null for rows recorded before this --- column existed and for locally originated batches not yet hashed. -create table if not exists msg_add_to_batch ( +-- duplicate (SPEC §11/§12). Batches of a draft finalize when it is sent. +create table msg_add_to_batch ( id bigserial primary key, msg_id bigint not null references msg (id), add_to_from varchar(255) not null, -- sender that added this batch's recipients time_added double precision not null, -- the batch message's wire time field (for locally originated batches, when the batch was created) - sha256 bytea -- batch message hash: the batch's identity (SPEC §11) + sha256 bytea, -- finalized batch identity (SPEC §11) + wire_message jsonb -- durable batch wire representation ); -alter table msg_add_to_batch add column if not exists sha256 bytea; -create index if not exists msg_add_to_batch_msg_id_idx on msg_add_to_batch (msg_id); +create index msg_add_to_batch_msg_id_idx on msg_add_to_batch (msg_id); -create table if not exists msg_add_to ( +create table msg_add_to ( id bigserial primary key, msg_id bigint not null references msg (id), batch_id bigint not null references msg_add_to_batch (id), -- batch this recipient was added in @@ -86,15 +67,10 @@ create table if not exists msg_add_to ( attempt_count int not null default 0, -- number of failed delivery attempts; used for exponential back-off unique (batch_id, addr) ); --- An address is unique within a batch, not across batches: distinct batches --- may re-add the same address (each batch is its own sibling branch, SPEC --- §12). Migrate existing databases off the old per-message constraint. -alter table msg_add_to drop constraint if exists msg_add_to_msg_id_addr_key; -create unique index if not exists msg_add_to_batch_id_addr_key on msg_add_to (batch_id, addr); -create index if not exists msg_add_to_lower_idx on msg_add_to ((lower(addr))); -create index if not exists msg_add_to_batch_id_idx on msg_add_to (batch_id); - -create table if not exists msg_attachment ( +create index msg_add_to_lower_idx on msg_add_to ((lower(addr))); +create index msg_add_to_batch_id_idx on msg_add_to (batch_id); + +create table msg_attachment ( msg_id bigint references msg (id), position smallint not null default 0, flags smallint not null default 0, @@ -105,12 +81,38 @@ create table if not exists msg_attachment ( primary key (msg_id, filename) ); +-- Sender-side state for add-to participant notification (SPEC §10.2): an +-- add-to message is sent to every participant domain of the message being +-- added to -- the domains of from and every to address as well as the new +-- recipients' -- so all participants learn recipients were added, not only +-- the domains hosting the new recipients. Domains hosting a recipient of the +-- batch itself learn through normal recipient delivery; every other +-- participant domain gets one row here per batch and receives the add-to as +-- a notification-only exchange completing at code 11. Rows are created by +-- the Web API when recipients are added through it (the local domain itself +-- needs no row -- this database is its record). +create table msg_add_to_notify ( + id bigserial primary key, + batch_id bigint not null references msg_add_to_batch (id), + domain varchar(255) not null, + time_notified double precision, -- time remote host acknowledged the batch; null means pending + time_last_attempt double precision, -- time of last failed attempt; drives exponential back-off + response_code smallint, -- response code of last attempt + attempt_count int not null default 0, + unique (batch_id, domain) +); + +create index msg_add_to_batch_sha256_idx on msg_add_to_batch (sha256) where sha256 is not null; +create index msg_pid_idx on msg (pid) where pid is not null; + +-- Functions and triggers. + -- keep protocol parent hash populated for locally-created replies that set -- the relational parent id. A reply cannot reference a draft parent or a -- terminal parent (SPEC v0.6.0 §3: a Sending Host must not transmit a reply -- to a terminal message, so refuse to create one), and any explicit psha256 -- must match the referenced parent's sha256. -create or replace function populate_msg_psha256_from_pid() returns trigger as $$ +create function populate_msg_psha256_from_pid() returns trigger as $$ declare parent_time_sent double precision; parent_sha256 bytea; @@ -137,9 +139,8 @@ begin raise exception 'cannot set pid %: parent message is terminal', NEW.pid; end if; - if parent_sha256 is null or octet_length(parent_sha256) = 0 then - -- parent was delivered locally only and has no sha256 yet; psha256 cannot be populated - return NEW; + if parent_sha256 is null or octet_length(parent_sha256) <> 32 then + raise exception 'parent message % has no finalized identity', NEW.pid; end if; if NEW.psha256 is null or octet_length(NEW.psha256) = 0 then @@ -159,14 +160,13 @@ begin end; $$ language plpgsql; -drop trigger if exists trg_msg_populate_psha256 on msg; create trigger trg_msg_populate_psha256 before insert or update of pid, psha256 on msg for each row execute function populate_msg_psha256_from_pid(); -- recipients cannot be added to a terminal message (SPEC §12): refuse to -- create a batch for one, so the sender never has such a unit to transmit. -create or replace function prevent_add_to_terminal_msg() returns trigger as $$ +create function prevent_add_to_terminal_msg() returns trigger as $$ begin if exists (select 1 from msg where id = NEW.msg_id and is_terminal) then raise exception 'cannot add recipients to message %: it is terminal', NEW.msg_id; @@ -175,36 +175,10 @@ begin end; $$ language plpgsql; -drop trigger if exists trg_msg_add_to_batch_terminal on msg_add_to_batch; create trigger trg_msg_add_to_batch_terminal before insert on msg_add_to_batch for each row execute function prevent_add_to_terminal_msg(); --- once a message has replies, it must remain referenceable by protocol hash. -create or replace function prevent_referenced_msg_from_becoming_unreferenceable() returns trigger as $$ -begin - if exists (select 1 from msg child where child.pid = NEW.id) then - if NEW.time_sent is null then - raise exception 'cannot make message % a draft: it has replies', NEW.id; - end if; - - if OLD.sha256 is not null and (NEW.sha256 is null or octet_length(NEW.sha256) = 0) then - raise exception 'cannot clear sha256 for message %: it has replies', NEW.id; - end if; - - if OLD.sha256 is not null and OLD.sha256 is distinct from NEW.sha256 then - raise exception 'cannot change sha256 for message %: it has replies', NEW.id; - end if; - end if; - return NEW; -end; -$$ language plpgsql; - -drop trigger if exists trg_msg_prevent_unreferenceable_parent on msg; -create trigger trg_msg_prevent_unreferenceable_parent - before update of time_sent, sha256 on msg - for each row execute function prevent_referenced_msg_from_becoming_unreferenceable(); - -- Notify the sender's outgoing worker (channel new_msg_to) whenever new -- delivery work appears. One function serves all three triggers, dispatching -- on the table it fired for: @@ -216,7 +190,7 @@ create trigger trg_msg_prevent_unreferenceable_parent -- message whose recipient rows follow in the same -- transaction); notify that recipient. -- The payload is advisory only: the worker re-polls fully on any wake-up. -create or replace function notify_msg_sent() returns trigger as $$ +create function notify_msg_sent() returns trigger as $$ begin if TG_TABLE_NAME = 'msg' then if OLD.time_sent is null and NEW.time_sent is not null then @@ -234,17 +208,14 @@ begin end; $$ language plpgsql; -drop trigger if exists trg_msg_to_insert on msg_to; create trigger trg_msg_to_insert after insert on msg_to for each row execute function notify_msg_sent(); -drop trigger if exists trg_msg_add_to_insert on msg_add_to; create trigger trg_msg_add_to_insert after insert on msg_add_to for each row execute function notify_msg_sent(); -drop trigger if exists trg_msg_sent on msg; create trigger trg_msg_sent after update on msg for each row execute function notify_msg_sent(); @@ -261,7 +232,7 @@ create trigger trg_msg_sent -- msg row is written before its msg_to/msg_add_to rows (FK ordering), so a -- plain row trigger would see no recipients. At commit every recipient row in -- the transaction is visible. -create or replace function notify_new_msg() returns trigger as $$ +create function notify_new_msg() returns trigger as $$ begin if (TG_OP = 'INSERT' and NEW.time_sent is not null) or (TG_OP = 'UPDATE' and OLD.time_sent is null and NEW.time_sent is not null) then @@ -275,7 +246,6 @@ begin end; $$ language plpgsql; -drop trigger if exists trg_new_msg on msg; create constraint trigger trg_new_msg after insert or update on msg deferrable initially deferred @@ -292,7 +262,7 @@ create constraint trigger trg_new_msg -- it's the sender whose UI needs to react. Unlike trg_new_msg this does not -- need to be deferred: the msg row referenced by msg_id already exists (FK) -- by the time msg_to/msg_add_to is updated. -create or replace function notify_delivered() returns trigger as $$ +create function notify_delivered() returns trigger as $$ begin perform pg_notify('delivered', NEW.msg_id::text || ',' || m.from_addr) from msg m where m.id = NEW.msg_id; @@ -300,45 +270,22 @@ begin end; $$ language plpgsql; -drop trigger if exists trg_msg_to_delivered on msg_to; create trigger trg_msg_to_delivered after update of time_delivered on msg_to for each row when (OLD.time_delivered is null and NEW.time_delivered is not null) execute function notify_delivered(); -drop trigger if exists trg_msg_add_to_delivered on msg_add_to; create trigger trg_msg_add_to_delivered after update of time_delivered on msg_add_to for each row when (OLD.time_delivered is null and NEW.time_delivered is not null) execute function notify_delivered(); --- Sender-side state for add-to participant notification (SPEC §10.2): an --- add-to message is sent to every participant domain of the message being --- added to -- the domains of from and every to address as well as the new --- recipients' -- so all participants learn recipients were added, not only --- the domains hosting the new recipients. Domains hosting a recipient of the --- batch itself learn through normal recipient delivery; every other --- participant domain gets one row here per batch and receives the add-to as --- a notification-only exchange completing at code 11. Rows are created by --- the Web API when recipients are added through it (the local domain itself --- needs no row -- this database is its record). -create table if not exists msg_add_to_notify ( - id bigserial primary key, - batch_id bigint not null references msg_add_to_batch (id), - domain varchar(255) not null, - time_notified double precision, -- time remote host acknowledged the batch; null means pending - time_last_attempt double precision, -- time of last failed attempt; drives exponential back-off - response_code smallint, -- response code of last attempt - attempt_count int not null default 0, - unique (batch_id, domain) -); - -- Wake the sender's outgoing worker (channel new_msg_to) for a pending -- participant notification, mirroring notify_msg_sent for recipient rows. -- The payload is advisory only: the worker re-polls fully on any wake-up. -create or replace function notify_add_to_notify_pending() returns trigger as $$ +create function notify_add_to_notify_pending() returns trigger as $$ begin perform pg_notify('new_msg_to', b.msg_id::text || ',' || NEW.domain) from msg_add_to_batch b @@ -348,7 +295,6 @@ begin end; $$ language plpgsql; -drop trigger if exists trg_msg_add_to_notify_insert on msg_add_to_notify; create trigger trg_msg_add_to_notify_insert after insert on msg_add_to_notify for each row execute function notify_add_to_notify_pending(); @@ -365,7 +311,7 @@ create trigger trg_msg_add_to_notify_insert -- new_msg. Like trg_new_msg this is a deferred constraint trigger: the -- batch's own msg_add_to rows are inserted after the batch row, so only at -- commit is the full recipient set visible. -create or replace function notify_recipients_added() returns trigger as $$ +create function notify_recipients_added() returns trigger as $$ begin if not exists (select 1 from msg where id = NEW.msg_id and time_sent is not null) then return NEW; @@ -384,65 +330,66 @@ begin end; $$ language plpgsql; -drop trigger if exists trg_recipients_added on msg_add_to_batch; create constraint trigger trg_recipients_added after insert on msg_add_to_batch deferrable initially deferred for each row execute function notify_recipients_added(); --- Durable protocol representations, shared by the API finalizer and daemon. --- NULL on legacy rows; received add-to variants belong to their batch only. -alter table msg add column if not exists wire_message jsonb; -alter table msg_add_to_batch add column if not exists wire_message jsonb; -create index if not exists msg_add_to_batch_sha256_idx on msg_add_to_batch (sha256) where sha256 is not null; -create index if not exists msg_pid_idx on msg (pid) where pid is not null; - --- Preserve protocol identity; local relational pid links and delivery/read --- metadata are bookkeeping and may still change. Legacy NULL hashes may be --- filled once without changing the timestamp, including parents with replies. -create or replace function protect_msg_identity() returns trigger as $$ +-- Sent protocol fields are immutable. Relational pid links and delivery/read +-- metadata remain bookkeeping and may change. +create function protect_msg_identity() returns trigger as $$ begin - if OLD.time_sent is not null then - if NEW.time_sent is distinct from OLD.time_sent then - raise exception 'sent message timestamp is immutable'; - end if; - if OLD.sha256 is not null and - (NEW.sha256 is distinct from OLD.sha256 or - row(NEW.version,NEW.psha256,NEW.no_reply,NEW.is_important,NEW.is_terminal, - NEW.is_deflate,NEW.from_addr,NEW.topic,NEW.type,NEW.size,NEW.filepath) - is distinct from - row(OLD.version,OLD.psha256,OLD.no_reply,OLD.is_important,OLD.is_terminal, - OLD.is_deflate,OLD.from_addr,OLD.topic,OLD.type,OLD.size,OLD.filepath) or - (OLD.wire_header is not null and NEW.wire_header is distinct from OLD.wire_header) or - (OLD.wire_message is not null and NEW.wire_message is distinct from OLD.wire_message)) then - raise exception 'sent message content and hash are immutable'; - end if; + if OLD.time_sent is not null and + row(NEW.time_sent,NEW.sha256,NEW.version,NEW.psha256,NEW.no_reply, + NEW.is_important,NEW.is_terminal,NEW.is_deflate,NEW.from_addr, + NEW.topic,NEW.type,NEW.size,NEW.filepath,NEW.wire_header,NEW.wire_message) + is distinct from + row(OLD.time_sent,OLD.sha256,OLD.version,OLD.psha256,OLD.no_reply, + OLD.is_important,OLD.is_terminal,OLD.is_deflate,OLD.from_addr, + OLD.topic,OLD.type,OLD.size,OLD.filepath,OLD.wire_header,OLD.wire_message) then + raise exception 'sent message content, timestamp and hash are immutable'; end if; return NEW; end; $$ language plpgsql; -drop trigger if exists trg_msg_identity on msg; create trigger trg_msg_identity before update on msg for each row execute function protect_msg_identity(); --- Validate at commit so receiving hosts can assemble rows and recipients in --- one transaction. Existing unhashed rows are backfilled by fmsg-backfill. -create or replace function require_sent_msg_hash() returns trigger as $$ +-- Validate after all rows in the transaction have been assembled. An original +-- first received via add-to has its payload representation on the received batch. +create function require_sent_msg_hash() returns trigger as $$ begin - if TG_OP='UPDATE' then - if OLD.time_sent is not distinct from NEW.time_sent and OLD.sha256 is not distinct from NEW.sha256 then return null; end if; + if exists (select 1 from msg m where m.id=NEW.id and m.time_sent is not null + and (m.sha256 is null or octet_length(m.sha256) <> 32 or + (m.wire_message is null and not exists ( + select 1 from msg_add_to_batch b where b.msg_id=m.id + and b.sha256 is not null and b.wire_message is not null)))) then + raise exception 'sent message % requires a 32-byte sha256 and a wire representation', NEW.id; end if; - if exists (select 1 from msg where id=NEW.id and time_sent is not null - and (sha256 is null or octet_length(sha256) <> 32)) then - raise exception 'sent message % requires a 32-byte sha256', NEW.id; + if exists (select 1 from msg_add_to_batch b join msg m on m.id=b.msg_id + where m.id=NEW.id and m.time_sent is not null + and (b.sha256 is null or octet_length(b.sha256) <> 32 or b.wire_message is null)) then + raise exception 'sent message % has an unfinalized add-to batch', NEW.id; end if; return null; end; $$ language plpgsql; -drop trigger if exists trg_msg_require_hash on msg; create constraint trigger trg_msg_require_hash after insert or update on msg deferrable initially deferred for each row execute function require_sent_msg_hash(); -create or replace function protect_msg_parts() returns trigger as $$ +create function require_sent_batch_hash() returns trigger as $$ +begin + if exists (select 1 from msg_add_to_batch b join msg m on m.id=b.msg_id + where b.id=NEW.id and m.time_sent is not null + and (b.sha256 is null or octet_length(b.sha256) <> 32 or b.wire_message is null)) then + raise exception 'sent batch % requires a 32-byte sha256 and a wire representation', NEW.id; + end if; + return null; +end; +$$ language plpgsql; +create constraint trigger trg_batch_require_hash after insert or update on msg_add_to_batch + deferrable initially deferred for each row execute function require_sent_batch_hash(); + +create function protect_msg_parts() returns trigger as $$ declare message_id bigint; frozen boolean; @@ -467,7 +414,7 @@ begin if not found then raise exception 'batch does not belong to message'; end if; end if; else - select time_sent is not null and sha256 is not null into frozen from msg where id=message_id for update; + select time_sent is not null into frozen from msg where id=message_id for update; end if; if frozen then raise exception 'finalized message parts are immutable'; end if; if TG_OP='DELETE' then return OLD; end if; @@ -475,14 +422,11 @@ begin end; $$ language plpgsql; -- AFTER INSERT allows an ON CONFLICT DO NOTHING receipt to remain a no-op. -drop trigger if exists trg_msg_to_content on msg_to; create trigger trg_msg_to_content after insert or update or delete on msg_to for each row execute function protect_msg_parts(); -drop trigger if exists trg_msg_attachment_content on msg_attachment; create trigger trg_msg_attachment_content after insert or update or delete on msg_attachment for each row execute function protect_msg_parts(); -drop trigger if exists trg_msg_add_to_content on msg_add_to; create trigger trg_msg_add_to_content after insert or update or delete on msg_add_to for each row execute function protect_msg_parts(); -create or replace function protect_batch_identity() returns trigger as $$ +create function protect_batch_identity() returns trigger as $$ begin if OLD.sha256 is not null and row(NEW.msg_id,NEW.add_to_from,NEW.time_added,NEW.sha256,NEW.wire_message) @@ -492,5 +436,4 @@ begin return NEW; end; $$ language plpgsql; -drop trigger if exists trg_batch_identity on msg_add_to_batch; create trigger trg_batch_identity before update on msg_add_to_batch for each row execute function protect_batch_identity(); From fd91b8198a82dfaa9a1578612afcd4bae8b7bbaa Mon Sep 17 00:00:00 2001 From: Mark Mennell Date: Thu, 10 Sep 2026 16:14:41 +0800 Subject: [PATCH 4/4] Handle pending notification batches during hash test polling --- test/tests/015-message-sha256.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/tests/015-message-sha256.sh b/test/tests/015-message-sha256.sh index 1b74732..3ec08cc 100755 --- a/test/tests/015-message-sha256.sh +++ b/test/tests/015-message-sha256.sh @@ -50,7 +50,7 @@ ADD_INPUT=$(jq -n --arg addr "$CAROL_ADDR" '{add_to:[$addr]}') NOTIFY_HASH=$(api_json_write "$EXAMPLE_API_URL" "$BOB_API_KEY" POST "/fmsg/$HASH/add-to" "$ADD_INPUT" | jq -er '.sha256') FOUND=false for attempt in $(seq 1 30); do - if api_json_get "$HAIRPIN_API_URL" "$ALICE_API_KEY" "/fmsg/$HASH" | jq -e --arg hash "$NOTIFY_HASH" 'any(.add_to[]; .sha256 == $hash)' >/dev/null; then + if api_json_get "$HAIRPIN_API_URL" "$ALICE_API_KEY" "/fmsg/$HASH" | jq -e --arg hash "$NOTIFY_HASH" 'any(.add_to[]?; .sha256 == $hash)' >/dev/null; then FOUND=true break fi