Finalize local message hashes and accept SHA-256 API references

.github/workflows/build-test.yml

@@ -8,6 +8,20 @@ on:
 jobs:
   build-and-test:
     runs-on: ubuntu-latest
+    services:
+      postgres:
+        image: postgres:17-alpine
+        env:
+          POSTGRES_HOST_AUTH_METHOD: trust
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd "pg_isready -U postgres"
+          --health-interval 5s
+          --health-timeout 5s
+          --health-retries 10
+    env:
+      FMSG_TEST_DATABASE_URL: postgres://postgres@localhost:5432/postgres?sslmode=disable
     steps:
       - uses: actions/checkout@v4
 
@@ -19,4 +33,5 @@ jobs:
         run: go build ./...
 
       - name: Test
-        run: go test ./...
+        run: |
+          FMSG_TEST_DD="$(go list -m -f '{{.Dir}}' github.com/markmnl/fmsgd)/dd.sql" go test -race ./...

README.md

@@ -633,9 +633,33 @@ Deletes a draft message and all its attachments from the database and disk. Only
 | `403`  | Not the owner, or message already sent |
 | `404`  | Message not found |
 
+### Message hashes and references
+
+A sent or received message includes `sha256`, a lowercase 64-character hexadecimal
+protocol hash; a draft returns `null`. `psha256` is the exact parent hash or `null`.
+Both fields appear in message detail, inbox/sent lists and WebSocket message
+payloads. Structured thread entries include them too, retaining `message_sha256`
+as an alias for existing clients. Each `add_to` batch exposes its own `sha256`.
+These hashes describe the protocol message, not the API JSON or mutable read and
+delivery state.
+
+Every message `:id` route accepts either its positive numeric ID or its 64-character
+SHA-256 (hexadecimal input is case-insensitive). This includes detail, body and
+attachment downloads, thread routes, read, react, and add-to. Hashes do not grant
+access: the same identity checks apply. Invalid references return `400`; unknown
+hashes return `404`. Drafts have no hash, and draft-only operations still reject
+sent messages.
+
+Create/update accepts `pid` as a numeric parent ID or a hexadecimal hash string.
+Response `pid` remains numeric. The hash may identify an original message or an
+add-to batch; a batch reference is retained exactly in `psha256`. A participant
+added only through a batch must use that batch's hash when replying. The parent
+must be sent and non-terminal, and the caller must participate in that specific
+original or batch. Client-provided `sha256`/`psha256` values are not trusted.
+
 ### POST `/fmsg/:id/send`
 
-Marks a draft message as sent by setting `time_sent` to the current timestamp. Only the owner may send.
+Atomically stamps `time_sent`, computes the protocol `sha256`, and retains the finalized wire representation. This happens for local-only delivery too. Only the owner may send. Compression and common media type encoding are chosen before hashing; future federation reuses that representation. A failure leaves the message a draft. Concurrent edits, attachment changes, deletion, and send are serialized; sent content is immutable.
 
 **Send is the validation gate.** Drafts are a workspace: create/update accept
 incomplete messages (no recipients, no type) by design, but a draft may only
@@ -657,7 +681,7 @@ third-party delivery's outcome, so the reply is attempted and the wire's
 originating domain always passes (it retains its outgoing messages). Local
 recipients are unaffected.
 
-**Response:** `200 OK` with `{"id": <int>, "time": <float64>}`.
+**Response:** `200 OK` with `{"id": <int>, "time": <float64>, "sha256": "<64 hex characters>"}`.
 
 **Errors:**
 
@@ -697,7 +721,7 @@ This endpoint records the add-to as a new `msg_add_to_batch` row (capturing the
 
 New addresses must be distinct among themselves (case-insensitive).
 
-**Response:** `200 OK` with `{"id": <int>, "added": <int>}`.
+**Response:** `200 OK` with `{"id": <int>, "added": <int>, "batch_id": <int>, "sha256": "<64 hex characters>"}`. On a draft, the batch hash is `null` until send finalizes it.
 
 **Errors:**
 
@@ -731,7 +755,7 @@ Setting the same reaction the caller already has is idempotent: nothing is
 sent and the existing reaction message is returned with `200 OK`. Clearing when
 the caller has no reaction returns `200 OK` with `null` values.
 
-**Response:** `201 Created` with the reaction message's `{"id": <int>, "time": <number>}`.
+**Response:** `201 Created` with the reaction message's `{"id": <int>, "time": <number>, "sha256": "<64 hex characters>"}`.
 
 **Errors:**
 
@@ -815,8 +839,8 @@ truncated.
       "to": ["@agent@example.net"],
       "type": "text/plain",
       "size": 5,
-      "message_sha256": "0123456789abcdef",
-      "body": {"type": "text/plain", "size": 5, "text": "hello", "cache_key": "sha256:0123456789abcdef:body", "cacheable": true},
+      "message_sha256": "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
+      "body": {"type": "text/plain", "size": 5, "text": "hello", "cache_key": "sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef:body", "cacheable": true},
       "attachments": []
     }
   ]
@@ -967,3 +991,23 @@ CREATE TABLE push_subscription (
     PRIMARY KEY (addr, endpoint)
 );
 ```
+
+### Finalization schema upgrade and PostgreSQL tests
+
+This version requires the matching `fmsgd/dd.sql` schema and finalization-capable
+daemon. For existing installations, stop both services and run the standalone
+`fmsg-backfill` binary before starting the new versions. The binary embeds the schema
+upgrade, prepares existing sent messages and batches, and backfills local-only hashes
+in one transaction. Run without `-apply` for a full dry run, then with `-apply` to commit.
+Do not rerun the daemon's bootstrap `dd.sql` on an existing database. See
+[the daemon upgrade instructions](https://github.com/markmnl/fmsgd#immutable-message-finalization-and-upgrades).
+All sent messages have hashes; only drafts expose a `null` message hash.
+
+`go test ./...` runs the unit tests. PostgreSQL integration tests create and remove
+isolated schemas; set `FMSG_TEST_DATABASE_URL` to a disposable test database and
+`FMSG_TEST_DD` to the matching daemon schema (defaults to the sibling checkout):
+
+```sh
+FMSG_TEST_DATABASE_URL='postgres://postgres@localhost/fmsg_test?sslmode=disable' \
+FMSG_TEST_DD=/path/to/fmsgd/dd.sql go test ./...
+```

cmd/fmsg-webapi/main.go

@@ -185,21 +185,21 @@ func main() {
 
 		fmsg.GET("", msgHandler.List)
 		fmsg.GET("/sent", msgHandler.Sent)
-		fmsg.POST("", msgHandler.Create)
+		fmsg.POST("", msgHandler.Atomic((*handlers.MessageHandler).Create))
 		fmsg.GET("/:id", msgHandler.Get)
-		fmsg.PUT("/:id", msgHandler.Update)
-		fmsg.DELETE("/:id", msgHandler.Delete)
-		fmsg.POST("/:id/send", msgHandler.Send)
+		fmsg.PUT("/:id", msgHandler.Atomic((*handlers.MessageHandler).Update))
+		fmsg.DELETE("/:id", msgHandler.Atomic((*handlers.MessageHandler).Delete))
+		fmsg.POST("/:id/send", msgHandler.Atomic((*handlers.MessageHandler).Send))
 		fmsg.POST("/:id/read", msgHandler.MarkRead)
-		fmsg.POST("/:id/add-to", msgHandler.AddRecipients)
-		fmsg.POST("/:id/react", msgHandler.React)
+		fmsg.POST("/:id/add-to", msgHandler.Atomic((*handlers.MessageHandler).AddRecipients))
+		fmsg.POST("/:id/react", msgHandler.Atomic((*handlers.MessageHandler).React))
 		fmsg.GET("/:id/data", msgHandler.DownloadData)
 		fmsg.GET("/:id/thread", msgHandler.ThreadText)
 		fmsg.GET("/:id/thread/messages", msgHandler.ThreadMessages)
 
-		fmsg.POST("/:id/attach", attHandler.Upload)
+		fmsg.POST("/:id/attach", attHandler.Atomic((*handlers.AttachmentHandler).Upload))
 		fmsg.GET("/:id/attach/:filename", attHandler.Download)
-		fmsg.DELETE("/:id/attach/:filename", attHandler.DeleteAttachment)
+		fmsg.DELETE("/:id/attach/:filename", attHandler.Atomic((*handlers.AttachmentHandler).DeleteAttachment))
 
 		if pushHandler != nil {
 			fmsg.POST("/push/subscribe", pushHandler.Subscribe)

go.mod

@@ -10,6 +10,7 @@ require (
 	github.com/gorilla/websocket v1.5.3
 	github.com/jackc/pgx/v5 v5.8.0
 	github.com/joho/godotenv v1.5.1
+	github.com/markmnl/fmsgd v0.5.1-0.20260910081105-17cb088bca99
 	golang.org/x/sync v0.20.0
 )
 

go.sum

@@ -58,6 +58,10 @@ github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzh
 github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
 github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ=
 github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI=
+github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
+github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
+github.com/markmnl/fmsgd v0.5.1-0.20260910081105-17cb088bca99 h1:LTzLNjH5hKj5X9YfVrJotqn3dMH3UJ0QQYIFrLzmIaQ=
+github.com/markmnl/fmsgd v0.5.1-0.20260910081105-17cb088bca99/go.mod h1:DNykk/1Z25IM+yzSxjtJs51nEYYaA1H03dAMG37JERk=
 github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
 github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=

internal/handlers/attachments.go

@@ -19,6 +19,7 @@ import (
 
 // AttachmentHandler holds dependencies for attachment routes.
 type AttachmentHandler struct {
+	query         queries
 	DB            *db.DB
 	DataDir       string
 	MaxAttachSize int64
@@ -33,7 +34,7 @@ func NewAttachmentHandler(database *db.DB, dataDir string, maxAttachSize, maxMsg
 // Upload handles POST /fmsg/:id/attachments.
 func (h *AttachmentHandler) Upload(c *gin.Context) {
 	identity := middleware.GetIdentity(c)
-	msgID, ok := parseID(c)
+	msgID, ok := resolveID(c, h.q())
 	if !ok {
 		return
 	}
@@ -43,7 +44,7 @@ func (h *AttachmentHandler) Upload(c *gin.Context) {
 	// Load the message to check ownership and draft status.
 	var fromAddr string
 	var timeSent *float64
-	err := h.DB.Pool.QueryRow(ctx,
+	err := h.q().QueryRow(ctx,
 		"SELECT from_addr, time_sent FROM msg WHERE id = $1", msgID,
 	).Scan(&fromAddr, &timeSent)
 	if err != nil {
@@ -113,9 +114,10 @@ func (h *AttachmentHandler) Upload(c *gin.Context) {
 		return
 	}
 
+	createdFile(c, finalPath)
 	// Check total message size and persist attachment in a transaction to
 	// prevent concurrent uploads from exceeding MaxMsgSize.
-	tx, err := h.DB.Pool.Begin(ctx)
+	tx, err := h.q().Begin(ctx)
 	if err != nil {
 		_ = os.Remove(finalPath)
 		log.Printf("upload attachment: begin tx: %v", err)
@@ -169,7 +171,7 @@ func (h *AttachmentHandler) Upload(c *gin.Context) {
 // Download handles GET /fmsg/:id/attachments/:filename.
 func (h *AttachmentHandler) Download(c *gin.Context) {
 	identity := middleware.GetIdentity(c)
-	msgID, ok := parseID(c)
+	msgID, ok := resolveID(c, h.q())
 	if !ok {
 		return
 	}
@@ -185,7 +187,7 @@ func (h *AttachmentHandler) Download(c *gin.Context) {
 
 	// Check ownership or recipient access.
 	var fromAddr string
-	err := h.DB.Pool.QueryRow(ctx, "SELECT from_addr FROM msg WHERE id = $1", msgID).Scan(&fromAddr)
+	err := h.q().QueryRow(ctx, "SELECT from_addr FROM msg WHERE id = $1", msgID).Scan(&fromAddr)
 	if err != nil {
 		if errors.Is(err, pgx.ErrNoRows) {
 			c.JSON(http.StatusNotFound, gin.H{"error": "message not found"})
@@ -199,7 +201,7 @@ func (h *AttachmentHandler) Download(c *gin.Context) {
 	// Check recipients (to or add_to) if not owner.
 	if fromAddr != identity {
 		var recipientCount int
-		if err = h.DB.Pool.QueryRow(ctx,
+		if err = h.q().QueryRow(ctx,
 			`SELECT COUNT(*) FROM (
 				SELECT 1 FROM msg_to WHERE msg_id = $1 AND lower(addr) = lower($2)
 				UNION ALL
@@ -213,7 +215,7 @@ func (h *AttachmentHandler) Download(c *gin.Context) {
 
 	// Look up the attachment filepath.
 	var storedPath string
-	err = h.DB.Pool.QueryRow(ctx,
+	err = h.q().QueryRow(ctx,
 		"SELECT filepath FROM msg_attachment WHERE msg_id = $1 AND filename = $2",
 		msgID, filename,
 	).Scan(&storedPath)
@@ -242,7 +244,7 @@ func (h *AttachmentHandler) Download(c *gin.Context) {
 // DeleteAttachment handles DELETE /fmsg/:id/attachments/:filename.
 func (h *AttachmentHandler) DeleteAttachment(c *gin.Context) {
 	identity := middleware.GetIdentity(c)
-	msgID, ok := parseID(c)
+	msgID, ok := resolveID(c, h.q())
 	if !ok {
 		return
 	}
@@ -257,7 +259,7 @@ func (h *AttachmentHandler) DeleteAttachment(c *gin.Context) {
 
 	var fromAddr string
 	var timeSent *float64
-	err := h.DB.Pool.QueryRow(ctx,
+	err := h.q().QueryRow(ctx,
 		"SELECT from_addr, time_sent FROM msg WHERE id = $1", msgID,
 	).Scan(&fromAddr, &timeSent)
 	if err != nil {
@@ -281,7 +283,7 @@ func (h *AttachmentHandler) DeleteAttachment(c *gin.Context) {
 
 	// Get filepath before deleting.
 	var storedPath string
-	err = h.DB.Pool.QueryRow(ctx,
+	err = h.q().QueryRow(ctx,
 		"SELECT filepath FROM msg_attachment WHERE msg_id = $1 AND filename = $2",
 		msgID, filename,
 	).Scan(&storedPath)
@@ -295,7 +297,7 @@ func (h *AttachmentHandler) DeleteAttachment(c *gin.Context) {
 		return
 	}
 
-	if _, err = h.DB.Pool.Exec(ctx,
+	if _, err = h.q().Exec(ctx,
 		"DELETE FROM msg_attachment WHERE msg_id = $1 AND filename = $2", msgID, filename,
 	); err != nil {
 		log.Printf("delete attachment: db: %v", err)
@@ -307,7 +309,7 @@ func (h *AttachmentHandler) DeleteAttachment(c *gin.Context) {
 	cleanPath := filepath.Clean(storedPath)
 	cleanDataDir := filepath.Clean(h.DataDir)
 	if strings.HasPrefix(cleanPath, cleanDataDir+string(filepath.Separator)) {
-		_ = os.Remove(cleanPath)
+		afterCommit(c, func() { _ = os.Remove(cleanPath) })
 	}
 
 	c.Status(http.StatusNoContent)