From 3a4f6e6906340132b906c7fb97fbafbf9b2fc7b7 Mon Sep 17 00:00:00 2001 From: Lokesh Date: Sun, 4 Oct 2026 11:16:50 +0400 Subject: [PATCH 1/5] fix(witness): complete a step a crash left on only some replicas The gateway writes each step to all three replicas and the endpoint needs all three receipts. A control plane that stopped part way left some replicas one step ahead, and lineage recovery then refused that lineage forever, so its pairing never worked again until a re-pair. Before recovery, the gateway now brings the replicas left behind forward by the same endpoint-signed step. A replica takes it only for a lineage it has not recovered, only as a direct successor of its own head, and only when fresh heads both other replicas signed for one endpoint-signed read sit at its head or that successor. A missed registration completes as an ordinary registration. Heads for the comparison come from currentHead, which does not keep the answer as the read's response, so the same read sees the new head afterwards. No replica moves backward and every head is one the endpoint signed: an endpoint restored from before its pending step is still caught as a fork, and replicas holding two different successors stay unrecovered. Signed-off-by: Lokesh --- .../src/witness-deployment-test.ts | 63 ++++++ .../test/witness-deployment-test.test.ts | 148 +++++++++++++- services/control-plane/src/witness.ts | 192 +++++++++++++++++- 3 files changed, 395 insertions(+), 8 deletions(-) diff --git a/services/aws-control-plane/src/witness-deployment-test.ts b/services/aws-control-plane/src/witness-deployment-test.ts index 8cdbde69..cba3aa7f 100644 --- a/services/aws-control-plane/src/witness-deployment-test.ts +++ b/services/aws-control-plane/src/witness-deployment-test.ts @@ -35,8 +35,10 @@ import { witnessLineageHash, } from "@axl/control-plane"; import { + parseWitnessReplicaReceipt, parseWitnessRequest, WITNESS_RECEIPT_SIGNATURE_DOMAIN, + type WitnessReplicaReceipt, type WitnessReplicaTrust, type WitnessRequest, witnessBytesEqual, @@ -356,6 +358,16 @@ export async function createDeploymentTestWitness( ); } +function sameHead(left: WitnessReplicaReceipt, right: WitnessReplicaReceipt): boolean { + return ( + left.result === right.result && + left.counter === right.counter && + witnessBytesEqual(left.commitment, right.commitment) && + witnessBytesEqual(left.predecessorCommitment, right.predecessorCommitment) && + left.revocationGeneration === right.revocationGeneration + ); +} + class RecoveringWitnessGateway extends WitnessGateway { readonly #replicas: readonly WitnessReplica[]; readonly #admit: ( @@ -437,6 +449,7 @@ class RecoveringWitnessGateway extends WitnessGateway { } if (ready()) return; const attempt = (async () => { + await this.#repairSplit(admission, lineage, exactRequest); const receipts = await Promise.all( this.#replicas.map((replica) => replica.recoveryHead(admission, exactRequest)), ); @@ -463,6 +476,56 @@ class RecoveringWitnessGateway extends WitnessGateway { } } + /** + * Each step goes to all three replicas, so a crash part way through one leaves some replicas a + * step ahead of the rest, with no certificate issued and the step still pending on the endpoint. + * Recovery needs all three to agree, so first bring the others forward by that same + * endpoint-signed step: a missed registration through an ordinary submission, since those + * replicas never held the lineage, and a missed advance through `catchUpLineage`, which checks + * the step against both other replicas' fresh heads. Anything else is left for recovery to refuse. + */ + async #repairSplit(admission: WitnessAdmission, lineage: Uint8Array, exactRead: Uint8Array) { + const raw = await Promise.all( + this.#replicas.map((replica) => + replica.currentHead(admission, exactRead).catch(() => undefined), + ), + ); + const heads = raw.map((bytes) => + bytes === undefined ? undefined : parseWitnessReplicaReceipt(bytes), + ); + const held = heads.filter((head): head is WitnessReplicaReceipt => head !== undefined); + const top = held.reduce( + (best, head) => (best === undefined || head.counter > best.counter ? head : best), + undefined, + ); + if (top === undefined || heads.every((head) => head !== undefined && sameHead(head, top))) { + return; + } + const step = await this.#replicas[heads.indexOf(top)]?.headStep(lineage); + if (step === undefined) return; + const kind = parseWitnessRequest(step).kind; + await Promise.all( + this.#replicas.map(async (replica, own) => { + const head = heads[own]; + if (head !== undefined && sameHead(head, top)) return; + if (head === undefined) { + if (kind === "register") await replica.submit(admission, step); + return; + } + if (kind !== "advance" || replica.lineageReady(lineage)) return; + await replica.catchUpLineage( + admission, + step, + raw.flatMap((receiptBytes, index) => + index === own || receiptBytes === undefined + ? [] + : [{ requestBytes: exactRead, receiptBytes }], + ), + ); + }), + ); + } + /** Move every replica from bootstrap to ready with the first registered lineage's signed read. */ async #completeBootstrap(principal: HostedWitnessPrincipal, exactRequest: Uint8Array) { let request: WitnessRequest; diff --git a/services/aws-control-plane/test/witness-deployment-test.test.ts b/services/aws-control-plane/test/witness-deployment-test.test.ts index c8f678ec..1cb4622a 100644 --- a/services/aws-control-plane/test/witness-deployment-test.test.ts +++ b/services/aws-control-plane/test/witness-deployment-test.test.ts @@ -10,9 +10,10 @@ import { sign, } from "node:crypto"; import test, { type TestContext } from "node:test"; +import { setTimeout as delay } from "node:timers/promises"; import { DynamoDBClient } from "@aws-sdk/client-dynamodb"; -import { WitnessServiceError } from "@axl/control-plane"; +import { type WitnessReplicaStorage, WitnessServiceError } from "@axl/control-plane"; import { parseWitnessQuorumCertificate, WITNESS_REQUEST_SIGNATURE_DOMAIN, @@ -112,6 +113,8 @@ function endpoint(session: number) { read: () => request(2), /** Advance from counter `expected`, whose commitment is filled with `expected`. */ advance: (expected: bigint) => request(3, { expected, commitment: Number(expected) + 1 }), + /** A different successor of `expected`, as an endpoint restored from a backup would propose. */ + diverge: (expected: bigint) => request(3, { expected, commitment: 0x7f }), }; } @@ -127,7 +130,28 @@ function keys(): DeploymentTestWitnessKey[] { }); } -async function durable(context: TestContext) { +/** + * Replica storage whose writes fail while its replica is in `crashed`, as if the process died + * before reaching it. The failure waits for the other replicas' writes to land first. + */ +function crashable(storage: WitnessReplicaStorage, replica: number, crashed: Set) { + return { + transact: async ( + lineageHash: Uint8Array, + transaction: Parameters[1], + ): Promise => { + if (crashed.has(replica)) { + await delay(300); + throw new Error("The control plane stopped before this replica's write"); + } + return storage.transact(lineageHash, transaction) as Promise; + }, + listLineageHashes: () => storage.listLineageHashes(), + read: (lineageHash: Uint8Array) => storage.read(lineageHash), + } satisfies WitnessReplicaStorage; +} + +async function durable(context: TestContext, crashed = new Set()) { const db = await startFakeDynamoDb(); db.pageSize = 5; const client = new DynamoDBClient({ @@ -145,11 +169,15 @@ async function durable(context: TestContext) { keys: witnessKeys, accountId: ACCOUNT_ID, stores: (key) => ({ - storage: new DynamoWitnessReplicaStorage({ - tableName: key === witnessKeys[0] ? tableName : "witness", - replicaId: key.replicaId, - client, - }), + storage: crashable( + new DynamoWitnessReplicaStorage({ + tableName: key === witnessKeys[0] ? tableName : "witness", + replicaId: key.replicaId, + client, + }), + witnessKeys.indexOf(key), + crashed, + ), journal: new DynamoWitnessHighWaterJournal({ tableName: "journal", replicaId: key.replicaId, @@ -217,6 +245,112 @@ test("paired endpoints keep their witness heads across a control-plane restart", }); }); +const unavailable = (error: unknown) => + error instanceof WitnessServiceError && error.code === "witness_unavailable"; + +for (const crashedReplicas of [[2], [1, 2]]) { + test(`a step that reached ${3 - crashedReplicas.length} of 3 replicas before a crash completes after the restart`, async (context) => { + const crashed = new Set(); + const start = await durable(context, crashed); + const phone = endpoint(9); + let witness = await start(); + await witness.submit(principal, phone.register()); + await witness.submit(principal, phone.read()); + await witness.submit(principal, phone.advance(1n)); + + const pending = phone.advance(2n); + for (const replica of crashedReplicas) crashed.add(replica); + await assert.rejects(witness.submit(principal, pending)); + crashed.clear(); + + witness = await start(); + // The endpoint reads first, as it does before every mutation; the replicas left behind take + // the step, so all three agree on it. + assert.deepEqual(result(await witness.submit(principal, phone.read())), { + result: "head", + counter: 3n, + }); + // The endpoint resends its pending step and gets that step's certificate. + assert.deepEqual(result(await witness.submit(principal, pending)), { + result: "advanced", + counter: 3n, + }); + assert.deepEqual(result(await witness.submit(principal, phone.advance(3n))), { + result: "advanced", + counter: 4n, + }); + }); +} + +test("a registration that reached some replicas before a crash completes after the restart", async (context) => { + const crashed = new Set(); + const start = await durable(context, crashed); + let witness = await start(); + // Another pairing keeps every replica non-empty, as on a running stack, and its first read ends + // bootstrap. + const other = endpoint(10); + await witness.submit(principal, other.register()); + await witness.submit(principal, other.read()); + const phone = endpoint(11); + const registration = phone.register(); + crashed.add(0); + await assert.rejects(witness.submit(principal, registration)); + crashed.clear(); + + witness = await start(); + assert.deepEqual(result(await witness.submit(principal, phone.read())), { + result: "head", + counter: 1n, + }); + assert.deepEqual(result(await witness.submit(principal, registration)), { + result: "registered", + counter: 1n, + }); + assert.deepEqual(result(await witness.submit(principal, phone.advance(1n))), { + result: "advanced", + counter: 2n, + }); +}); + +test("after a split is repaired, an endpoint restored from before its pending step is caught", async (context) => { + const crashed = new Set(); + const start = await durable(context, crashed); + const phone = endpoint(12); + let witness = await start(); + await witness.submit(principal, phone.register()); + await witness.submit(principal, phone.read()); + crashed.add(1); + await assert.rejects(witness.submit(principal, phone.advance(1n))); + crashed.clear(); + + witness = await start(); + await witness.submit(principal, phone.read()); + // A copy of the endpoint from before the pending step proposes another successor: a fork. + assert.equal( + result(await witness.submit(principal, phone.diverge(1n))).result, + "conflicting_successor", + ); +}); + +test("a replica behind replicas that disagree with each other stays unrecovered", async (context) => { + const crashed = new Set(); + const start = await durable(context, crashed); + const phone = endpoint(13); + let witness = await start(); + await witness.submit(principal, phone.register()); + await witness.submit(principal, phone.read()); + // Replica 0 takes one successor and replica 1 another; replica 2 takes neither. + crashed.add(1).add(2); + await assert.rejects(witness.submit(principal, phone.advance(1n))); + crashed.clear(); + crashed.add(0).add(2); + await assert.rejects(witness.submit(principal, phone.diverge(1n))); + crashed.clear(); + + witness = await start(); + await assert.rejects(witness.submit(principal, phone.read()), unavailable); +}); + test("replicas that disagree about holding state refuse to start", async (context) => { const start = await durable(context); const witness = await start(); diff --git a/services/control-plane/src/witness.ts b/services/control-plane/src/witness.ts index aa82ac7d..d0f064a6 100644 --- a/services/control-plane/src/witness.ts +++ b/services/control-plane/src/witness.ts @@ -82,7 +82,8 @@ export interface WitnessSecurityAuditEvent { | "receipt_invalid" | "replica_disagreement" | "journal_inconsistent" - | "replica_behind"; + | "replica_behind" + | "replica_caught_up"; readonly lineageHash?: string; readonly replicaId?: string; } @@ -953,6 +954,174 @@ export class WitnessReplica implements WitnessReplicaClient { }); } + /** + * The endpoint-signed request behind this replica's head of `lineage`, so that a replica a crash + * left one step behind can take the same step. Undefined unless the head is an accepted step. + */ + async headStep(lineage: Uint8Array): Promise { + const stored = await this.#options.storage.read(lineage); + if (stored === undefined) return undefined; + const state = rebuild(stored); + const last = stored.ledger.at(-1); + if ( + last === undefined || + (last.kind !== "registered" && last.kind !== "advanced") || + last.requestBytes === undefined || + state.forkResult !== undefined || + state.revocation !== undefined || + last.counter !== state.head.counter || + !witnessBytesEqual(last.commitment, state.head.commitment) + ) { + return undefined; + } + return last.requestBytes.slice(); + } + + /** + * Take the one step a crash between replica writes left this replica without. The gateway writes + * each step to all three replicas and the endpoint needs all three receipts, so a crash part way + * leaves some replicas one step ahead with no certificate issued, and the endpoint still holds + * that step pending. `exactStep` is the endpoint-signed request the ahead replicas accepted. + * + * It applies only to an unrecovered lineage of a resumed replica, only as a direct successor of + * this replica's own head, and only when fresh heads that both other replicas signed for one + * endpoint-signed read each sit at this replica's head or at that successor, at least one at the + * successor. The step then goes through every check of an ordinary submission. No replica moves + * backward, and every head a replica holds is one the endpoint signed, so rollback detection is + * unchanged. The lineage still recovers through `recoverLineage` afterwards. + */ + async catchUpLineage( + admission: WitnessAdmission, + exactStep: Uint8Array, + peerEvidence: readonly WitnessPeerHeadEvidence[], + ): Promise { + if (this.#state !== "lineage_recovery") { + throw new WitnessServiceError( + "witness_unavailable", + "Lineage catch-up applies only to a resumed replica", + 503, + ); + } + const step = parseWitnessRequest(exactStep); + if (step.kind !== "advance") { + throw new WitnessServiceError("bad_request", "Lineage catch-up takes an advance", 400); + } + const lineage = lineageHash(step); + const key = witnessBytesHex(lineage); + await this.#withLineage(key, async () => { + if (this.#recoveredLineages.has(key)) return; + const stored = await this.#options.storage.read(lineage); + if (stored === undefined) { + throw new WitnessServiceError("witness_unavailable", "Recovery lineage is missing", 503); + } + const { record, state } = await this.#validateLocalState(lineage, stored); + const successor = operation(step); + if ( + state.forkResult !== undefined || + state.revocation !== undefined || + step.expectedCounter !== state.head.counter || + !witnessBytesEqual(step.expectedCommitment ?? zero(48), state.head.commitment) || + successor.counter !== state.head.counter + 1n + ) { + throw new WitnessServiceError( + "witness_unavailable", + "The catch-up step does not follow this replica's head", + 503, + ); + } + const evidence = peerEvidence.map((value) => ({ + request: parseWitnessRequest(value.requestBytes), + receipt: parseWitnessReplicaReceipt(value.receiptBytes), + })); + const [first, second] = evidence; + if ( + evidence.length !== 2 || + first === undefined || + second === undefined || + !witnessBytesEqual(first.request.exactBytes, second.request.exactBytes) + ) { + throw new WitnessServiceError( + "witness_unavailable", + "Catch-up requires fresh heads from both other replicas for one read", + 503, + ); + } + const read = first.request; + if ( + read.kind !== "read" || + !witnessBytesEqual(lineageHash(read), lineage) || + !witnessBytesEqual(read.credentialFingerprint, record.binding.credentialFingerprint) || + !endpointSignatureValid(read, record.binding.verificationKey) + ) { + throw new WitnessServiceError( + "witness_auth_failed", + "Catch-up read request endpoint proof is invalid", + 401, + ); + } + const expectedPeerIds = new Set(this.#otherReplicaIds().map(witnessBytesHex)); + const actualPeerIds = new Set( + evidence.map(({ receipt }) => witnessBytesHex(receipt.replicaId)), + ); + if ( + actualPeerIds.size !== 2 || + [...actualPeerIds].some((replicaId) => !expectedPeerIds.has(replicaId)) + ) { + throw new WitnessServiceError( + "witness_receipt_invalid", + "Catch-up evidence must come from both other distinct replicas", + 400, + ); + } + const now = this.#options.clock.nowMs(); + let ahead = 0; + for (const { receipt } of evidence) { + verifyReceiptForRequest(read, receipt, this.#recoveryTrust); + if ( + receipt.result !== "head" || + receipt.issuedAtMs > now || + now - receipt.issuedAtMs > WITNESS_RECOVERY_EVIDENCE_MAX_AGE_MS || + receipt.revocationGeneration !== state.revocationGeneration + ) { + throw new WitnessServiceError( + "witness_receipt_invalid", + "Catch-up peer evidence is stale or has the wrong terminal state", + 400, + ); + } + const peer = { + counter: receipt.counter, + commitment: receipt.commitment, + predecessorCommitment: receipt.predecessorCommitment, + }; + if ( + sameHead(peer, successor) && + witnessBytesEqual(peer.predecessorCommitment, state.head.commitment) + ) { + ahead += 1; + } else if (!sameHead(peer, state.head)) { + throw new WitnessServiceError("witness_unavailable", "Recovery peer heads conflict", 503); + } + } + if (ahead === 0) { + throw new WitnessServiceError( + "witness_unavailable", + "No other replica holds the catch-up step", + 503, + ); + } + const receipt = parseWitnessReplicaReceipt(await this.#submit(admission, exactStep)); + if (receipt.result !== "advanced" || !sameHead(receipt, successor)) { + throw new WitnessServiceError( + "service_unavailable", + "The catch-up step was not accepted", + 503, + ); + } + await this.#audit("replica_caught_up", lineage); + }); + } + /** Run `work` after every earlier operation on the lineage `key` in this process. */ async #withLineage(key: string, work: () => Promise): Promise { const prior = this.#lineageQueues.get(key) ?? Promise.resolve(); @@ -1518,6 +1687,26 @@ export class WitnessReplica implements WitnessReplicaClient { async recoveryHead( admission: WitnessAdmission, exactReadRequest: Uint8Array, + ): Promise { + return this.#recoveryHead(admission, exactReadRequest, true); + } + + /** + * `recoveryHead` without keeping the answer as this read's response. A split repair compares + * heads with it before it brings replicas forward, after which the same read must see the new + * head. The receipt serves only as catch-up evidence inside the gateway. + */ + async currentHead( + admission: WitnessAdmission, + exactReadRequest: Uint8Array, + ): Promise { + return this.#recoveryHead(admission, exactReadRequest, false); + } + + async #recoveryHead( + admission: WitnessAdmission, + exactReadRequest: Uint8Array, + retain: boolean, ): Promise { const request = parseWitnessRequest(exactReadRequest); if (request.kind !== "read") { @@ -1567,6 +1756,7 @@ export class WitnessReplica implements WitnessReplicaClient { receiptFields(request, hash, result, state, this.#options.clock), this.#options.signer, ); + if (!retain) return receipt; return this.#retainResponse(lineage, exactReadRequest, hash, receipt, record); } From e513bb58f94c76b2b3bc5d25ce6ea0dc813b1bd1 Mon Sep 17 00:00:00 2001 From: Lokesh Date: Sun, 4 Oct 2026 11:16:50 +0400 Subject: [PATCH 2/5] fix(remote): one daemon serves remote access, and the newest tab wins Two daemons for one user (axl and axl --unsafe keep separate state) share the installation's relay identity. With /remote in both, each replaced the other's relay route forever: both phones hung without an error and each daemon received the other phone's requests. The daemon serving remote access now holds a claim file under ~/.axl/remote, and another daemon's /remote is refused with its process and state directory before its own pairing is touched; its restore waits. On the phone, the tab opened last always wins. A tab listens for takeovers from the start, a new tab repeats its request while it waits and then takes the endpoint lock from a tab that never answers (a suspended background tab) through the binding's new takeOverDeviceEndpoint, and the owner is marked in localStorage so a tab that wakes later stops instead of reconnecting over the new one. A tab that let go keeps saying why. A request that overtakes the pairing activation is dropped quietly instead of being recorded as a failure; the phone resends both. The phone E2E gains three scenarios: tabs opened back to back, a frozen tab with no tab channel, and a second daemon's refused /remote. Signed-off-by: Lokesh --- packages/e2ee/bindings/browser/index.d.ts | 7 + .../e2ee/bindings/browser/loader/index.js | 20 ++ .../bindings/browser/scripts/check-abi.mjs | 1 + packages/runtime/src/remote-claim.ts | 129 ++++++++++++ packages/runtime/src/remote-host.ts | 42 +++- packages/runtime/src/remote-production.ts | 7 + packages/runtime/test/remote-claim.test.ts | 73 +++++++ packages/runtime/test/remote-host.test.ts | 34 ++++ packages/web/e2e/remote/remote.spec.ts | 73 ++++++- packages/web/e2e/remote/stack.ts | 39 +++- packages/web/src/remote/main.ts | 192 +++++++++++++++--- 11 files changed, 576 insertions(+), 41 deletions(-) create mode 100644 packages/runtime/src/remote-claim.ts create mode 100644 packages/runtime/test/remote-claim.test.ts diff --git a/packages/e2ee/bindings/browser/index.d.ts b/packages/e2ee/bindings/browser/index.d.ts index 45c2a15f..64f8ec3b 100644 --- a/packages/e2ee/bindings/browser/index.d.ts +++ b/packages/e2ee/bindings/browser/index.d.ts @@ -164,5 +164,12 @@ export declare function createDeviceEndpoint( export declare function openDeviceEndpoint(session: { readonly cryptoSessionId: Uint8Array; }): Promise; +/** + * Take a crypto session's endpoint from another tab of this profile that does not let go; that + * tab's endpoint fails closed as for any lost lock. + */ +export declare function takeOverDeviceEndpoint(session: { + readonly cryptoSessionId: Uint8Array; +}): Promise; /** Permanently closes this module instance. Repeated close calls are harmless. */ export declare function closeBrowserBinding(): void; diff --git a/packages/e2ee/bindings/browser/loader/index.js b/packages/e2ee/bindings/browser/loader/index.js index 941d730d..f33bdbbd 100644 --- a/packages/e2ee/bindings/browser/loader/index.js +++ b/packages/e2ee/bindings/browser/loader/index.js @@ -434,6 +434,26 @@ export const openDeviceEndpoint = ({ cryptoSessionId } = {}) => nullResult(value); return deviceEndpointHandle(); }); +/** + * Take a crypto session's endpoint from another tab or window of this browser profile that holds + * it and does not let go, such as a background tab the browser suspended. The worker there loses + * the session's lifetime lock, so its endpoint fails closed and later calls are refused, exactly as + * for any lost lock; committed state is not touched. Resolves once the lock is free, after which + * `openDeviceEndpoint` can take it. + */ +export const takeOverDeviceEndpoint = async ({ cryptoSessionId } = {}) => { + if (!(cryptoSessionId instanceof Uint8Array) || cryptoSessionId.byteLength !== 16) { + throw new AxlE2eeError("invalid_argument"); + } + if (!globalThis.navigator?.locks) throw new AxlE2eeError("storage_unavailable"); + // The name the worker's store holds for the endpoint's lifetime (worker/storage.js). + const sessionHex = [...cryptoSessionId].map((byte) => byte.toString(16).padStart(2, "0")).join(""); + await globalThis.navigator.locks.request( + `axl-e2ee-v1:${sessionHex}`, + { mode: "exclusive", steal: true }, + () => undefined, + ); +}; export const closeBrowserBinding = () => { if (state === "closed") return; state = "closed"; diff --git a/packages/e2ee/bindings/browser/scripts/check-abi.mjs b/packages/e2ee/bindings/browser/scripts/check-abi.mjs index 34b32cea..fd637ba3 100644 --- a/packages/e2ee/bindings/browser/scripts/check-abi.mjs +++ b/packages/e2ee/bindings/browser/scripts/check-abi.mjs @@ -227,6 +227,7 @@ const valueExports = [ "inspectPairingInvitation", "openDaemonEndpoint", "openDeviceEndpoint", + "takeOverDeviceEndpoint", ].sort(); const declared = [...declarations.matchAll(/^export declare (?:class|const|function) ([A-Za-z0-9_]+)/gmu)] .map((match) => match[1]) diff --git a/packages/runtime/src/remote-claim.ts b/packages/runtime/src/remote-claim.ts new file mode 100644 index 00000000..18eca275 --- /dev/null +++ b/packages/runtime/src/remote-claim.ts @@ -0,0 +1,129 @@ +// SPDX-FileCopyrightText: 2026 Lokesh +// SPDX-License-Identifier: Apache-2.0 + +/** + * One daemon at a time serves remote access for an installation. + * + * Every daemon a user runs (`axl`, `axl --unsafe`, another sandbox) keeps its own state directory, + * so each can hold its own pairing, but they share the installation and so its relay identity. Two + * of them connected at once would replace each other's relay route over and over, and neither + * phone would get through. The daemon serving remote access holds a claim file under the shared + * Axl home; another daemon's `/remote` is refused with a reason, and its restore waits until the + * claim is released or its owner has exited. + */ + +import { randomUUID } from "node:crypto"; +import { link, mkdir, open, readFile, rm, unlink } from "node:fs/promises"; +import { dirname } from "node:path"; + +interface ClaimRecord { + readonly version: 1; + readonly token: string; + readonly pid: number; + readonly stateDirectory: string; +} + +export interface RemoteAccessClaim { + release(): Promise; +} + +/** Another daemon serves remote access for this installation. */ +export class RemoteAccessBusyError extends Error { + readonly code = "remote_busy"; + readonly pid: number; + readonly stateDirectory: string; + + constructor(pid: number, stateDirectory: string) { + super( + `Remote access is in use by another Axl daemon (process ${pid}, state in ${stateDirectory}). Use /remote in that daemon's sessions, or stop it first.`, + ); + this.name = "RemoteAccessBusyError"; + this.pid = pid; + this.stateDirectory = stateDirectory; + } +} + +function alive(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch (cause) { + return (cause as NodeJS.ErrnoException).code === "EPERM"; + } +} + +async function readRecord(path: string): Promise { + let text: string; + try { + text = await readFile(path, "utf8"); + } catch (cause) { + if ((cause as NodeJS.ErrnoException).code === "ENOENT") return undefined; + throw cause; + } + try { + const value = JSON.parse(text) as Partial; + if ( + value.version === 1 && + typeof value.token === "string" && + Number.isSafeInteger(value.pid) && + typeof value.stateDirectory === "string" + ) { + return value as ClaimRecord; + } + } catch { + // A torn or foreign file is treated like a claim whose owner is gone. + } + return { version: 1, token: "", pid: 0, stateDirectory: "" }; +} + +/** + * Claim remote access at `path` for the daemon whose state is in `stateDirectory`. A claim left by + * a process that has exited is taken over; a live one rejects with `RemoteAccessBusyError`. + */ +export async function claimRemoteAccess( + path: string, + stateDirectory: string, +): Promise { + await mkdir(dirname(path), { recursive: true, mode: 0o700 }); + for (let attempt = 0; attempt < 3; attempt += 1) { + const record: ClaimRecord = { + version: 1, + token: randomUUID(), + pid: process.pid, + stateDirectory, + }; + const candidate = `${path}.${record.token}.tmp`; + try { + const handle = await open(candidate, "wx", 0o600); + try { + await handle.writeFile(`${JSON.stringify(record)}\n`, "utf8"); + await handle.sync(); + } finally { + await handle.close(); + } + // link() fails if the claim exists, so exactly one daemon creates it. + await link(candidate, path); + return { + async release() { + const current = await readRecord(path).catch(() => undefined); + if (current?.token === record.token) await unlink(path).catch(() => undefined); + }, + }; + } catch (cause) { + if ((cause as NodeJS.ErrnoException).code !== "EEXIST") throw cause; + const existing = await readRecord(path); + if (existing === undefined) continue; + if (existing.pid !== process.pid && existing.pid > 1 && alive(existing.pid)) { + throw new RemoteAccessBusyError(existing.pid, existing.stateDirectory); + } + // Its owner exited, or it is this process's own leftover: remove it unless it changed. + const current = await readRecord(path); + if (current !== undefined && current.token === existing.token) { + await unlink(path).catch(() => undefined); + } + } finally { + await rm(candidate, { force: true }); + } + } + throw new Error("Could not claim remote access"); +} diff --git a/packages/runtime/src/remote-host.ts b/packages/runtime/src/remote-host.ts index 5a287e65..a241bb4e 100644 --- a/packages/runtime/src/remote-host.ts +++ b/packages/runtime/src/remote-host.ts @@ -83,6 +83,8 @@ import { uuidToBytes, } from "@axl/sdk"; +import type { RemoteAccessClaim } from "./remote-claim.ts"; + /** Version 2 records the device ID minted for each pairing. */ const STATE_VERSION = 2; const DEVICE_SCOPES: readonly RemoteDeviceScope[] = ["observe", "steer"]; @@ -148,7 +150,9 @@ function startFailure(cause: unknown, origin: string, logPath: string): RemotePa ? "Remote access is not enabled for this account." : code === "helper_unavailable" ? `The key helper is not available. Details are in ${logPath}.` - : `Remote pairing could not start. Details are in ${logPath}.`; + : code === "remote_busy" && cause instanceof Error + ? cause.message + : `Remote pairing could not start. Details are in ${logPath}.`; return new RemotePairingStartError(reason, { cause }); } @@ -212,6 +216,11 @@ export interface HostedRemoteSettings { readonly linkAccessToken?: string; /** Run before a pairing starts or a paired session is restored; it may run again. */ prepare?(): Promise; + /** + * Claim remote access among the user's daemons before serving a session, held until the session + * closes. It rejects with code `remote_busy` while another daemon serves this installation. + */ + claim?(): Promise; /** The daemon endpoint for one crypto session, with its storage under `root`. */ endpoint( root: string, @@ -323,6 +332,8 @@ export class HostedRemoteHost implements RemotePairingService { #restoreAfterStart = false; /** The device of the completed pairing, whether or not its session is being served yet. */ #paired: DeviceId | undefined; + /** This daemon's claim on remote access, held while it has a session open. */ + #claim: RemoteAccessClaim | undefined; private constructor( config: HostedRemoteSettings, @@ -512,6 +523,8 @@ export class HostedRemoteHost implements RemotePairingService { // One pairing at a time: the device of the pairing this one replaces loses access for good. try { await this.#config.prepare?.(); + // Before the current pairing is replaced: another daemon serving remote access keeps it. + await this.#acquireClaim(); } catch (cause) { this.#fail(`remote: pairing could not start: ${describe(cause)}`); // Nothing about the current pairing has changed yet, so go back to serving it. @@ -588,7 +601,18 @@ export class HostedRemoteHost implements RemotePairingService { this.#log(`remote: device ${deviceId} revoked`); } + async #acquireClaim(): Promise { + if (this.#claim === undefined) this.#claim = await this.#config.claim?.(); + } + + async #releaseClaim(): Promise { + const claim = this.#claim; + this.#claim = undefined; + await claim?.release().catch(() => undefined); + } + async #openSession(cryptoSessionId: CryptoSessionId, deviceId: DeviceId): Promise { + await this.#acquireClaim(); const endpoint = await this.#config.endpoint( join(this.#root, "sessions", cryptoSessionId), uuidToBytes(this.#config.accountId), @@ -705,6 +729,12 @@ export class HostedRemoteHost implements RemotePairingService { return; } const envelope = parseRemoteE2eeEnvelope(delivery.opaquePayload); + if (envelope.messageClass === "application_request" && session.claimHash !== undefined) { + // The device sends its first request right behind its activation and resends both until a + // request is answered. One that arrives first, because the relay lost the activation, is + // dropped here and comes again after the activation. + return; + } if (envelope.messageClass !== "pair_activation" || session.claimHash === undefined) { throw new Error(`Unexpected ${envelope.messageClass} before pairing completed`); } @@ -822,10 +852,12 @@ export class HostedRemoteHost implements RemotePairingService { async #closeSession(): Promise { const session = this.#session; this.#session = undefined; - if (session === undefined) return; - session.relay.close(); - if (session.bridge === undefined) session.endpoint.close(); - else await session.bridge.shutdown(); + if (session !== undefined) { + session.relay.close(); + if (session.bridge === undefined) session.endpoint.close(); + else await session.bridge.shutdown(); + } + await this.#releaseClaim(); } /** Forget every session directory except `keep`. */ diff --git a/packages/runtime/src/remote-production.ts b/packages/runtime/src/remote-production.ts index edc37d50..d9ebede6 100755 --- a/packages/runtime/src/remote-production.ts +++ b/packages/runtime/src/remote-production.ts @@ -19,6 +19,7 @@ import { type RemoteAccountFile, RemoteAccountSession, } from "./remote-account.ts"; +import { claimRemoteAccess } from "./remote-claim.ts"; import { type HostedDaemonEndpoint, HostedRemoteHost } from "./remote-host.ts"; interface HostedBinding { @@ -71,6 +72,12 @@ export function openProductionRemoteHost( accessToken: async () => (await credentials()).accessToken(), possession: async (ticket) => (await credentials()).proof(ticket), prepare: async () => (await credentials()).register(), + // Shared by every daemon this user runs, whatever its state directory. + claim: () => + claimRemoteAccess( + join(axlHome, "remote", `${account.installationId}.lock`), + stateDirectory, + ), async endpoint(root, accountId, installationId, cryptoSessionId) { binding ??= import(pathToFileURL(account.binding).href) as Promise; const loaded = await binding; diff --git a/packages/runtime/test/remote-claim.test.ts b/packages/runtime/test/remote-claim.test.ts new file mode 100644 index 00000000..f33111a3 --- /dev/null +++ b/packages/runtime/test/remote-claim.test.ts @@ -0,0 +1,73 @@ +// SPDX-FileCopyrightText: 2026 Lokesh +// SPDX-License-Identifier: Apache-2.0 + +import assert from "node:assert/strict"; +import { spawn } from "node:child_process"; +import { once } from "node:events"; +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; + +import { claimRemoteAccess, RemoteAccessBusyError } from "../src/remote-claim.ts"; + +async function scratch(context: test.TestContext): Promise { + const directory = await mkdtemp(join(tmpdir(), "axl-remote-claim-")); + context.after(() => rm(directory, { recursive: true, force: true })); + return directory; +} + +/** A claim file as another daemon process would leave it. */ +async function foreign(path: string, pid: number, stateDirectory: string): Promise { + await writeFile( + path, + `${JSON.stringify({ version: 1, token: "other", pid, stateDirectory })}\n`, + { mode: 0o600 }, + ); +} + +test("a daemon claims remote access, and releasing it lets the next one in", async (context) => { + const path = join(await scratch(context), "remote", "installation.lock"); + const claim = await claimRemoteAccess(path, "/home/user/.axl"); + const record = JSON.parse(await readFile(path, "utf8")); + assert.equal(record.pid, process.pid); + assert.equal(record.stateDirectory, "/home/user/.axl"); + await claim.release(); + await assert.rejects(readFile(path), { code: "ENOENT" }); + const next = await claimRemoteAccess(path, "/home/user/.axl/unsafe"); + await next.release(); +}); + +test("a live daemon's claim refuses another, naming it", async (context) => { + const path = join(await scratch(context), "installation.lock"); + const owner = spawn(process.execPath, ["-e", "setTimeout(() => {}, 60_000)"]); + context.after(() => owner.kill()); + await once(owner, "spawn"); + await foreign(path, owner.pid as number, "/home/user/.axl/unsafe"); + await assert.rejects( + claimRemoteAccess(path, "/home/user/.axl"), + (error) => + error instanceof RemoteAccessBusyError && + error.code === "remote_busy" && + error.pid === owner.pid && + error.message.includes("/home/user/.axl/unsafe"), + ); +}); + +test("a claim left by a daemon that exited is taken over", async (context) => { + const path = join(await scratch(context), "installation.lock"); + const owner = spawn(process.execPath, ["-e", ""]); + await once(owner, "exit"); + await foreign(path, owner.pid as number, "/home/user/.axl/unsafe"); + const claim = await claimRemoteAccess(path, "/home/user/.axl"); + assert.equal(JSON.parse(await readFile(path, "utf8")).pid, process.pid); + await claim.release(); +}); + +test("releasing never removes a claim another daemon took since", async (context) => { + const path = join(await scratch(context), "installation.lock"); + const claim = await claimRemoteAccess(path, "/home/user/.axl"); + await foreign(path, 999_999, "/home/user/.axl/unsafe"); + await claim.release(); + assert.equal(JSON.parse(await readFile(path, "utf8")).token, "other"); +}); diff --git a/packages/runtime/test/remote-host.test.ts b/packages/runtime/test/remote-host.test.ts index 45552a96..4b8a1628 100644 --- a/packages/runtime/test/remote-host.test.ts +++ b/packages/runtime/test/remote-host.test.ts @@ -10,6 +10,7 @@ import test, { type TestContext } from "node:test"; import { type AxlDaemon, RemoteDeviceAuthorityStore, RemotePairingStartError } from "@axl/daemon"; import { parseDeviceId, parseInstallationId, parseSessionId } from "@axl/protocol"; +import { RemoteAccessBusyError } from "../src/remote-claim.ts"; import { HostedRemoteHost, type HostedRemoteSettings } from "../src/remote-host.ts"; const PAIRED = JSON.stringify({ @@ -103,6 +104,39 @@ test("a /remote refused for the account names the reason", async (context) => { }); }); +test("a /remote while another daemon serves remote access names that daemon and keeps the pairing", async (context) => { + const root = await mkdtemp(join(tmpdir(), "axl-remote-host-")); + let host: HostedRemoteHost | undefined; + context.after(async () => { + await host?.close(); + await rm(root, { recursive: true, force: true, maxRetries: 3 }); + }); + await writeFile(join(root, "host.json"), PAIRED); + const busy = new RemoteAccessBusyError(4242, "/home/user/.axl/unsafe"); + let opened = 0; + host = await HostedRemoteHost.open( + { + ...settings(async () => undefined), + claim: async () => { + throw busy; + }, + endpoint: async () => { + opened += 1; + throw new Error("no endpoint in this test"); + }, + }, + root, + ); + await host.attach({} as AxlDaemon); + await assert.rejects(host.start(), { + name: "RemotePairingStartError", + message: busy.message, + }); + assert.equal(opened, 0, "no session was opened while the other daemon held remote access"); + assert.equal(await readFile(join(root, "host.json"), "utf8"), PAIRED, "nothing was replaced"); + assert.match(host.status().lastError?.message ?? "", /another Axl daemon \(process 4242/u); +}); + test("unpairing forgets the paired device and ends its shares", async (context) => { const offline = () => Object.assign(new Error("connect ECONNREFUSED"), { code: "ECONNREFUSED" }); const { root, host, output } = await pairedHost(context, offline); diff --git a/packages/web/e2e/remote/remote.spec.ts b/packages/web/e2e/remote/remote.spec.ts index 5707e510..161cb560 100644 --- a/packages/web/e2e/remote/remote.spec.ts +++ b/packages/web/e2e/remote/remote.spec.ts @@ -9,7 +9,7 @@ import assert from "node:assert/strict"; import { appendFileSync, writeFileSync } from "node:fs"; import { join } from "node:path"; import { setTimeout as delay } from "node:timers/promises"; - +import type { SessionId } from "@axl/sdk"; import { type Browser, type BrowserContext, @@ -19,8 +19,6 @@ import { test, } from "@playwright/test"; -import type { SessionId } from "@axl/sdk"; - import { type RemoteStack, startStack } from "./stack.ts"; const directory = process.env.AXL_REMOTE_E2E_DIRECTORY; @@ -233,6 +231,75 @@ test("a second tab takes over and sees every reply exactly once", async () => { await second.close(); }); +test("tabs opened back to back: the newest one works and the others say why", async () => { + test.setTimeout(180_000); + const tabs: Page[] = []; + for (let index = 0; index < 3; index += 1) { + const tab = await context.newPage(); + await tab.goto(`${stack.origin}/remote/?debug`); + tabs.push(tab); + await delay(300); + } + const newest = tabs[2] as Page; + await expect(newest.locator("#thread")).toBeVisible({ timeout: 60_000 }); + await through("newest of three tabs", newest, "p9b in the newest tab", 60_000); + for (const older of [page, ...tabs.slice(0, 2)]) { + await expect(older.locator("#status")).toHaveText(/moved to another tab/u, { timeout: 30_000 }); + await expect(older.locator("#retry")).toHaveText("Use this tab"); + } + for (const tab of tabs) await tab.close(); +}); + +test("a tab that never lets go loses the pairing to a new one, and stays out when it wakes", async () => { + test.setTimeout(240_000); + // A suspended tab neither hears other tabs nor releases the endpoint: no tab channel, frozen. + const stuck = await context.newPage(); + await stuck.addInitScript(() => { + Reflect.deleteProperty(globalThis, "BroadcastChannel"); + }); + await stuck.goto(`${stack.origin}/remote/?debug`); + await expect(stuck.locator("#thread")).toBeVisible({ timeout: 60_000 }); + const lifecycle = await context.newCDPSession(stuck); + await lifecycle.send("Page.setWebLifecycleState", { state: "frozen" }); + + const fresh = await context.newPage(); + const started = Date.now(); + await fresh.goto(`${stack.origin}/remote/?debug`); + await expect(fresh.locator("#thread")).toBeVisible({ timeout: 60_000 }); + timings["took over a stuck tab"] = Date.now() - started; + await through("beside a stuck tab", fresh, "p9c beside a stuck tab", 60_000); + + // Woken, the old tab sees the new owner and stops instead of reconnecting over it. + await lifecycle.send("Page.setWebLifecycleState", { state: "active" }); + await lifecycle.detach(); + await stuck.evaluate(() => document.dispatchEvent(new Event("visibilitychange"))); + await expect(stuck.locator("#status")).toHaveText(/moved to another tab/u, { timeout: 30_000 }); + await delay(10_000); + await through("after the stuck tab woke", fresh, "p9d after the stuck tab woke", 60_000); + await stuck.close(); + await fresh.close(); +}); + +test("a second daemon's /remote is refused while the first serves the phone", async () => { + test.setTimeout(180_000); + const current = await context.newPage(); + await current.goto(`${stack.origin}/remote/?debug`); + await expect(current.locator("#thread")).toBeVisible({ timeout: 60_000 }); + // `axl --unsafe` runs its own daemon, with its own state, for the same installation. + const unsafe = await stack.startUnsafeDaemon(); + try { + const other = await unsafe.createSession(); + await assert.rejects(unsafe.pair(other), /another Axl daemon \(process \d+/u); + assert.equal((await unsafe.remoteStatus()).phase, "unpaired"); + // The first daemon kept its relay route: the phone gets through without a reconnect. + await through("second daemon refused", current, "p9e beside a second daemon", 30_000); + assert.equal((await stack.remoteStatus()).relay, "connected"); + } finally { + await unsafe.stop(); + } + await current.close(); +}); + test("pairing again moves the daemon to the new phone and locks the old one out", async ({ browser, }) => { diff --git a/packages/web/e2e/remote/stack.ts b/packages/web/e2e/remote/stack.ts index b6ee97dd..8d3a6778 100644 --- a/packages/web/e2e/remote/stack.ts +++ b/packages/web/e2e/remote/stack.ts @@ -729,9 +729,10 @@ export async function startStack(directory) { }); } const socketPath = join(home, ".axl/axl.sock"); - const startDaemon = async () => { + const startDaemon = async (unsafe = false) => { + const target = unsafe ? join(home, ".axl/unsafe/axl.sock") : socketPath; const daemon = new Service( - "daemon", + unsafe ? "daemon-unsafe" : "daemon", process.execPath, [ join(repositoryRoot, "packages/cli/dist/main.js"), @@ -742,6 +743,7 @@ export async function startStack(directory) { "echo", "--thinking", "off", + ...(unsafe ? ["--unsafe"] : []), ], { env: { @@ -764,7 +766,7 @@ export async function startStack(directory) { await until( "the daemon socket", async () => { - const client = await connectUnixClient(socketPath); + const client = await connectUnixClient(target); client.close(); return true; }, @@ -774,8 +776,8 @@ export async function startStack(directory) { }; let daemon = await startDaemon(); - const client = async (work) => { - const connected = await connectUnixClient(socketPath); + const client = async (work, path = socketPath) => { + const connected = await connectUnixClient(path); try { return await work(connected); } finally { @@ -814,6 +816,33 @@ export async function startStack(directory) { ).sessionId, dropConnections: (side = "all") => originServer.drop(side), stallConnections: (side = "all") => originServer.stall(side), + /** A second daemon for the same user, as `axl --unsafe` starts one, with its own state. */ + async startUnsafeDaemon() { + const extra = await startDaemon(true); + const path = join(home, ".axl/unsafe/axl.sock"); + return { + createSession: async (): Promise => + ( + await client( + (connected) => + connected.request( + "session.create", + { cwd: workspace }, + { idempotencyKey: randomUUID() }, + ), + path, + ) + ).sessionId, + pair: (sessionId: SessionId) => + client( + async (connected) => + (await connected.startRemotePairing({ shareSessionId: sessionId })).link, + path, + ), + remoteStatus: () => client((connected) => connected.remoteStatus(), path), + stop: () => extra.stop(), + }; + }, async restartDaemon() { await daemon.stop(); daemon = await startDaemon(); diff --git a/packages/web/src/remote/main.ts b/packages/web/src/remote/main.ts index 2008ff52..4d0f5fa6 100644 --- a/packages/web/src/remote/main.ts +++ b/packages/web/src/remote/main.ts @@ -26,8 +26,10 @@ * Phones suspend pages and drop sockets, so the page keeps its own connection honest: it probes * the relay on a heartbeat and whenever it becomes visible again, the session resends requests * the daemon has not answered, and an open conversation resumes from its last acknowledged cursor - * after any reconnect. One tab owns the pairing at a time; a newer tab asks the older one to let - * go of the endpoint instead of failing on its lock. + * after any reconnect. One tab owns the pairing at a time, and the tab opened last wins: it asks + * older tabs to let go of the endpoint, takes it from one that never answers (a tab the phone + * suspended in the background), and marks itself the owner, so an older tab that wakes up later + * stops instead of reconnecting over it. "Use this tab" reloads, which makes that tab the newest. */ import { @@ -82,6 +84,7 @@ interface DeviceBinding { openDeviceEndpoint(session: { readonly cryptoSessionId: Uint8Array; }): Promise; + takeOverDeviceEndpoint(session: { readonly cryptoSessionId: Uint8Array }): Promise; } /** What `session.subscribe` answers: the subscription and, for a fresh view, its snapshot. */ @@ -112,11 +115,14 @@ const SEND_TIMEOUT_MS = 30 * 60_000; const SEND_REOPEN_ATTEMPTS = 2; /** Probe the relay this often; a suspended page's socket usually dies without a close event. */ const HEARTBEAT_MS = 25_000; -/** How long a new tab waits for an older one to release the endpoint. */ -const HANDOFF_WAIT_MS = 5_000; +/** How long a new tab waits for an older one to release the endpoint before taking it. */ +const HANDOFF_WAIT_MS = 4_000; /** How long the pairing waits for the daemon to answer before resending its activation. */ const CONFIRM_TIMEOUT_MS = 5_000; const TAB_CHANNEL = "axl-remote-tabs"; +/** The tab that owns the pairing, as `/`; other tabs see it change. */ +const OWNER_KEY = "axl.remote.owner"; +const MOVED = "This pairing moved to another tab"; /** * Acknowledge a view's cursor this long after new events arrive. Every acknowledgement is a full * encrypted round trip through the phone's one witness queue, so it waits for any request the page @@ -127,6 +133,8 @@ const ACK_DELAY_MS = 10_000; /** A sent prompt the transcript never showed stops being drawn as pending after this long. */ const PENDING_SETTLE_MS = 15_000; const TAB_ID = crypto.randomUUID(); +/** When this tab loaded; between tabs of one pairing, the later one wins. */ +const TAB_STARTED = Date.now(); const STEP_LABELS: Readonly> = { claim: "Create this device's pairing claim", notice: "Reach the daemon through the relay", @@ -186,7 +194,12 @@ function connection(state: "online" | "connecting" | "offline"): void { view.connection.dataset.state = state; } -function status(text: string, tone: "normal" | "error" = "normal"): void { +/** Set once this tab has let go of the pairing; whatever was in flight then fails quietly. */ +let letGo = false; + +function status(text: string, tone: "normal" | "error" = "normal", final = false): void { + // Requests cut off when the tab let go must not replace the reason it shows. + if (letGo && !final) return; view.status.textContent = text; view.status.classList.toggle("error", tone === "error"); } @@ -307,28 +320,110 @@ function tabChannel(): BroadcastChannel | undefined { return typeof BroadcastChannel === "function" ? new BroadcastChannel(TAB_CHANNEL) : undefined; } +/** Whether the tab that loaded at `started` came after this one. */ +function newerTab(started: unknown, tab: unknown): boolean { + return ( + typeof started === "number" && + typeof tab === "string" && + (started > TAB_STARTED || (started === TAB_STARTED && tab > TAB_ID)) + ); +} + +function markOwner(link: RemotePairingLink): void { + try { + localStorage.setItem( + OWNER_KEY, + JSON.stringify({ session: link.cryptoSessionId, tab: TAB_ID, started: TAB_STARTED }), + ); + } catch { + // Without storage, the tab channel and the endpoint lock still keep one owner. + } +} + +/** The tab marked as owning this pairing, if one is. */ +function owner( + link: RemotePairingLink, +): { readonly tab?: unknown; readonly started?: unknown } | undefined { + try { + const value = JSON.parse(localStorage.getItem(OWNER_KEY) ?? "null") as { + readonly session?: unknown; + readonly tab?: unknown; + readonly started?: unknown; + } | null; + return value?.session === link.cryptoSessionId ? value : undefined; + } catch { + return undefined; + } +} + +/** Whether another tab has taken this pairing since this one did. */ +function ownedElsewhere(link: RemotePairingLink): boolean { + const current = owner(link); + return current !== undefined && current.tab !== TAB_ID; +} + +/** Whether a tab opened after this one already owns the pairing. */ +function ownedByNewer(link: RemotePairingLink): boolean { + const current = owner(link); + return current !== undefined && newerTab(current.started, current.tab); +} + /** * Open the endpoint, first asking any older tab holding this pairing to let go. The binding keeps - * one exclusive lock per pairing, so without the handoff a second tab fails with lifecycle_busy. + * one exclusive lock per pairing. The request repeats while this tab waits, for a tab that was + * still starting when it was first sent; a tab that never answers, because the phone suspended it, + * loses the lock to this one, and its endpoint fails closed. Resolves undefined, having taken + * nothing, once a newer tab wants the pairing. */ async function openExclusive( binding: DeviceBinding, link: RemotePairingLink, channel: BroadcastChannel | undefined, -): Promise { - channel?.postMessage({ type: "takeover", session: link.cryptoSessionId, tab: TAB_ID }); + superseded: () => boolean, +): Promise { const deadline = Date.now() + HANDOFF_WAIT_MS; - for (;;) { + let takenOver = false; + for (let round = 0; ; round += 1) { + if (superseded() || ownedByNewer(link)) return undefined; + if (round % 4 === 0) { + channel?.postMessage({ + type: "takeover", + session: link.cryptoSessionId, + tab: TAB_ID, + started: TAB_STARTED, + }); + } try { return await openEndpoint(binding, link); } catch (cause) { if ((cause as { readonly code?: unknown }).code !== "lifecycle_busy") throw cause; - if (Date.now() > deadline) throw cause; + if (Date.now() > deadline) { + if (takenOver) throw cause; + trace("another tab did not let go of the pairing; taking it over"); + status("Taking over from another tab"); + await binding.takeOverDeviceEndpoint({ + cryptoSessionId: uuidToBytes(link.cryptoSessionId), + }); + takenOver = true; + continue; + } await sleep(250); } } } +/** Show that another tab has this pairing now, with a button to take it back. */ +function showMoved(): void { + letGo = true; + connection("offline"); + show("pairing"); + view.steps.replaceChildren(); + status(MOVED, "normal", true); + view.hint.textContent = "Tap the button to use this pairing here instead."; + view.retry.textContent = "Use this tab"; + view.retry.hidden = false; +} + /** This device's key for `deviceId`, created on first use and kept in IndexedDB. */ async function deviceKey(deviceId: string): Promise { const database = await new Promise((resolve, reject) => { @@ -1105,6 +1200,31 @@ async function main(): Promise { view.hint.textContent = "Run /remote in the Axl terminal again and open the new link."; return; } + // A newer tab for this pairing may ask for it at any point, even while this one is starting. + const channel = tabChannel(); + let superseded = false; + let onTakeover = () => { + superseded = true; + }; + if (channel !== undefined) { + channel.onmessage = (event: MessageEvent) => { + const message = event.data as { + type?: unknown; + session?: unknown; + tab?: unknown; + started?: unknown; + }; + // Only a newer tab takes the pairing; an older one that is still starting waits for this + // tab's owner mark and gives up. + if ( + message.type === "takeover" && + message.session === link.cryptoSessionId && + newerTab(message.started, message.tab) + ) { + onTakeover(); + } + }; + } // Enrollment comes first: a link already used on another device must be refused before this // browser touches any E2EE or witness state for its device ID. const token = tokenFor(signIn, link); @@ -1133,46 +1253,58 @@ async function main(): Promise { if (document.visibilityState === "visible") keepFresh(); }); } - const channel = tabChannel(); status("Opening this device's keys"); - const endpoint = traced(await openExclusive(binding, link, channel)); + const opened = await openExclusive(binding, link, channel, () => superseded); + if (opened === undefined || superseded) { + // A newer tab asked for the pairing while this one was starting. + relay.close(); + await opened?.close().catch(() => undefined); + showMoved(); + return; + } + markOwner(link); + const endpoint = traced(opened); const session = new RemoteBrowserSession({ endpoint, relay, ...link, trace }); // A newer tab for the same pairing takes over; this one lets go of the endpoint and its lock. let released = false; - const release = async (reason: string) => { + const release = async (moved: boolean) => { if (released) return; released = true; + letGo = true; session.close(); relay.close(); await endpoint.close().catch(() => undefined); + if (moved) { + showMoved(); + return; + } show("pairing"); view.steps.replaceChildren(); - status(reason); + status("Paused", "normal", true); view.hint.textContent = "Tap the button to use this pairing here instead."; view.retry.textContent = "Use this tab"; view.retry.hidden = false; }; - if (channel !== undefined) { - channel.onmessage = (event: MessageEvent) => { - const message = event.data as { type?: unknown; session?: unknown; tab?: unknown }; - if ( - message.type === "takeover" && - message.session === link.cryptoSessionId && - message.tab !== TAB_ID - ) { - void release("This pairing moved to another tab"); - } - }; - } + onTakeover = () => void release(true); + // A tab that missed the request, because the phone had suspended it, sees the new owner here. + addEventListener("storage", (event) => { + if (event.key === OWNER_KEY && ownedElsewhere(link)) void release(true); + }); // A page restored from the back-forward cache let go of everything when it was hidden. - addEventListener("pagehide", () => void release("Paused")); + addEventListener("pagehide", () => void release(false)); addEventListener("pageshow", (event) => { if (event.persisted) location.reload(); }); relay.onState((state) => { trace(`relay ${state}`); + if (released) return; + // Another tab connecting replaces this one's relay route; reconnecting would replace it back. + if (state !== "connected" && ownedElsewhere(link)) { + void release(true); + return; + } connection(state === "connected" ? "online" : "connecting"); if (state === "reconnecting") status("Reconnecting to the relay"); else if (state === "connected" && view.status.textContent === "Reconnecting to the relay") { @@ -1191,7 +1323,9 @@ async function main(): Promise { // A phone that wakes the page or regains its network checks the socket at once, rather than // waiting for the next heartbeat to notice it died while suspended. const wake = () => { - if (document.visibilityState === "visible" && !released) void relay.checkAlive(); + if (document.visibilityState !== "visible" || released) return; + if (ownedElsewhere(link)) void release(true); + else void relay.checkAlive(); }; document.addEventListener("visibilitychange", wake); addEventListener("online", wake); @@ -1226,6 +1360,8 @@ addEventListener("hashchange", () => location.reload()); view.retry.addEventListener("click", () => location.reload()); main().catch((cause: unknown) => { + // A tab that handed the pairing over already says so; what it was doing then was cut off. + if (letGo) return; connection("offline"); const code = (cause as { readonly code?: unknown }).code; const known = typeof code === "string" && REFUSALS[code] !== undefined; From d4be9f44687f70b38af2d24fd5c3ec1923d43714 Mon Sep 17 00:00:00 2001 From: Lokesh Date: Sun, 4 Oct 2026 11:17:03 +0400 Subject: [PATCH 3/5] fix(e2ee): tolerate two minutes of clock skew between phone and daemon A daemon refused the KeyPackage of a phone whose clock was even a second ahead of its own (OpenMLS checks that the lifetime has begun), and a phone refused an invitation issued after its own clock read. WSL clocks drift after sleep, so real pairings could fail with an opaque error. The same check was the intermittent join failure in the Chrome and Safari barrier jobs: the test peer's clock is captured before the device creates its KeyPackage. The device now starts its KeyPackage lifetime two minutes early, as OpenMLS's default lifetimes do, and still ends it ten minutes from now; the daemon accepts that span and the exact ten-minute span of older devices. The device accepts an invitation issued up to two minutes ahead of its clock. Each side's checks against its own clock stay exact. The barrier scenario runs its peer 1.5 s behind on purpose. Signed-off-by: Lokesh --- .../bindings/browser/test/barrier-scenario.js | 4 +- packages/e2ee/src/lib.rs | 45 ++++++++++++++++++- packages/e2ee/src/pairing.rs | 21 ++++++++- 3 files changed, 65 insertions(+), 5 deletions(-) diff --git a/packages/e2ee/bindings/browser/test/barrier-scenario.js b/packages/e2ee/bindings/browser/test/barrier-scenario.js index bd8be7ea..312b5697 100644 --- a/packages/e2ee/bindings/browser/test/barrier-scenario.js +++ b/packages/e2ee/bindings/browser/test/barrier-scenario.js @@ -175,7 +175,9 @@ function createOptions(peer, witness, operationId) { export async function runDeviceBarrierScenario() { const witness = new TestWitness(); - const peer = new TestPeerDaemonFixture(0x91, Date.now()); + // The peer daemon's clock runs behind this device's, as a laptop's can after sleep. A KeyPackage + // must still be accepted (this was the intermittent join failure on slow CI browsers). + const peer = new TestPeerDaemonFixture(0x91, Date.now() - 1_500); const session = peer.crypto_session_id(); const databaseName = `axl-e2ee-production-v1:${hex(session)}`; const lockName = `axl-e2ee-v1:${hex(session)}`; diff --git a/packages/e2ee/src/lib.rs b/packages/e2ee/src/lib.rs index f7b79940..d37e5ce7 100644 --- a/packages/e2ee/src/lib.rs +++ b/packages/e2ee/src/lib.rs @@ -43,6 +43,9 @@ pub const SUITE_VALUE: u16 = 0x004e; pub const HANDSHAKE_MAX_BYTES: usize = 16 * 1024; /// KeyPackage lifetime fixed by the pairing profile. pub const KEY_PACKAGE_LIFETIME_SECONDS: u64 = 10 * 60; +/// How far a daemon's and a device's clocks may disagree during pairing. Each side allows it +/// only for the other side's timestamps; its checks against its own clock stay exact. +pub const PAIRING_CLOCK_SKEW_SECONDS: u64 = 2 * 60; /// Maximum authenticated-data length. pub const AAD_MAX_BYTES: usize = 512; /// Maximum application plaintext length. @@ -1350,12 +1353,16 @@ impl Phone { CoreProvider::new().map_err(|_| Error::Crypto("provider initialization failed"))?; ensure_suite(&provider)?; let (credential, signer) = make_credential(&provider, &identity)?; + // Starting the lifetime a little in the past lets a daemon whose clock is behind this + // device's accept the KeyPackage, as OpenMLS's own default lifetimes do. It still ends + // the full lifetime from now. let now_seconds = now_ms / 1_000; + let not_before = now_seconds.saturating_sub(PAIRING_CLOCK_SKEW_SECONDS); let not_after = now_seconds .checked_add(KEY_PACKAGE_LIFETIME_SECONDS) .ok_or(Error::ClockRollback)?; let bundle = KeyPackage::builder() - .key_package_lifetime(Lifetime::init(now_seconds, not_after)) + .key_package_lifetime(Lifetime::init(not_before, not_after)) .build(SUITE, &provider, &signer, credential) .map_err(|_| Error::Crypto("KeyPackage creation failed"))?; let bytes = bundle @@ -1787,7 +1794,10 @@ fn validate_phone_key_package( } let lifetime = package.life_time(); let now_seconds = now_ms / 1_000; - if lifetime.not_after().checked_sub(lifetime.not_before()) != Some(KEY_PACKAGE_LIFETIME_SECONDS) + // The lifetime starts the clock skew early; devices built before that started it now. + let span = lifetime.not_after().checked_sub(lifetime.not_before()); + if (span != Some(KEY_PACKAGE_LIFETIME_SECONDS + PAIRING_CLOCK_SKEW_SECONDS) + && span != Some(KEY_PACKAGE_LIFETIME_SECONDS)) || now_seconds < lifetime.not_before() || now_seconds >= lifetime.not_after() { @@ -1942,6 +1952,37 @@ impl<'a> Cursor<'a> { mod tests { use super::*; + /// A daemon consuming the KeyPackage of a phone whose clock is `ahead_ms` ahead of its own. + fn consume_from_phone_ahead_by(ahead_ms: u64) -> Result { + let context = PairContext { + crypto_session_id: [21; 16], + group_id: [22; 32], + account_id: [23; 16], + installation_id: [24; 16], + device_id: [25; 16], + }; + let phone_identity = Identity::device( + context.account_id, + context.installation_id, + context.device_id, + )?; + let now_ms = SystemClock.now_ms()?; + let (_, package) = Phone::create_at(phone_identity, now_ms + ahead_ms)?; + let mut daemon = Daemon::create( + Identity::daemon(context.account_id, context.installation_id), + context, + )?; + daemon.consume_key_package(package) + } + + #[test] + fn a_daemon_accepts_a_phone_clock_ahead_by_up_to_the_pairing_skew() { + consume_from_phone_ahead_by(0).unwrap(); + consume_from_phone_ahead_by(1_500).unwrap(); + consume_from_phone_ahead_by((PAIRING_CLOCK_SKEW_SECONDS - 5) * 1_000).unwrap(); + assert!(consume_from_phone_ahead_by((PAIRING_CLOCK_SKEW_SECONDS + 5) * 1_000).is_err()); + } + #[test] fn rejects_invalid_key_package_lifetime_and_capabilities() { let context = PairContext { diff --git a/packages/e2ee/src/pairing.rs b/packages/e2ee/src/pairing.rs index 104f7887..2ee8b412 100644 --- a/packages/e2ee/src/pairing.rs +++ b/packages/e2ee/src/pairing.rs @@ -313,8 +313,16 @@ impl PairingInvitation { self.verify_at(now_ms) } + /// A device's check of the daemon's invitation against the device's own clock, which may be + /// up to `PAIRING_CLOCK_SKEW_SECONDS` behind the daemon's. The daemon keeps its exact checks. pub(crate) fn verify_at(&self, now_ms: u64) -> Result<(), PairingError> { - self.validate_time(now_ms)?; + self.validate_time_shape()?; + if now_ms.saturating_add(crate::PAIRING_CLOCK_SKEW_SECONDS * 1_000) < self.issued_at_ms { + return Err(PairingError::ClockRollback); + } + if now_ms >= self.expires_at_ms { + return Err(PairingError::Expired); + } self.verify_signature() } @@ -1540,7 +1548,16 @@ mod tests { extended.validate_time_shape(), Err(PairingError::InvalidTime) ); - fixture.clock.set(fixture.invitation.issued_at_ms - 1); + // A device clock may be behind the daemon's by up to the pairing skew, and no further. + let skew_ms = crate::PAIRING_CLOCK_SKEW_SECONDS * 1_000; + fixture.clock.set(fixture.invitation.issued_at_ms - skew_ms); + fixture + .invitation + .verify_with_clock(fixture.clock.as_ref()) + .unwrap(); + fixture + .clock + .set(fixture.invitation.issued_at_ms - skew_ms - 1); assert_eq!( fixture.invitation.verify_with_clock(fixture.clock.as_ref()), Err(PairingError::ClockRollback) From 7d9c87b549efaf40df484edcb6a555b25b805b86 Mon Sep 17 00:00:00 2001 From: Lokesh Date: Sun, 4 Oct 2026 11:17:03 +0400 Subject: [PATCH 4/5] docs(remote): record the witness catch-up, one daemon at a time, and pairing skew The hosted-path checkpoint now records how each of its gates was settled instead of listing them as open. Signed-off-by: Lokesh --- docs/architecture/remote-e2ee-openmls.md | 2 ++ docs/architecture/remote-hosted-path.md | 18 ++++++++++-------- docs/architecture/remote-production-wsl.md | 4 ++++ 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/docs/architecture/remote-e2ee-openmls.md b/docs/architecture/remote-e2ee-openmls.md index c9b0f2df..74c66bf1 100644 --- a/docs/architecture/remote-e2ee-openmls.md +++ b/docs/architecture/remote-e2ee-openmls.md @@ -169,6 +169,8 @@ Limits and lifecycle: | KeyPackage | 16 KiB | 10 minutes, reserved atomically for 60 seconds, consumed once by the daemon | | Welcome | 16 KiB | 10 minutes, byte-identical retry until device activation acknowledgement | +The daemon's and the device's clocks may disagree by up to two minutes. The device starts its KeyPackage lifetime two minutes in the past, as OpenMLS's own default lifetimes do, and still ends it ten minutes from now, so a daemon whose clock is behind accepts it. The daemon accepts that twelve-minute span and the exact ten-minute span of devices built before it. The device accepts an invitation issued up to two minutes after its own clock reads. Each side's checks against its own clock stay exact: the daemon's invitation expiry and rollback detection, and every durable clock-rollback check. + A KeyPackage reservation binds account, installation, device, crypto session, profile ID, profile revision, credential fingerprint, and KeyPackage hash. One concurrent reservation wins. Failed group creation releases the reservation only while it remains live and no Welcome exists. Successful group creation consumes it permanently. A consumed, expired, malformed, wrong-profile, or wrong-credential KeyPackage is deleted and cannot be retried. The daemon creates a fresh group ID, adds the reserved KeyPackage, persists the group transition and exact Welcome bytes atomically, and only then publishes the Welcome. The device validates the complete pairing transcript, profile, group ID, daemon credential, member count of two, and its own leaf before persisting the joined state. It returns an MLS-protected activation acknowledgement. The control plane deletes the Welcome after that acknowledgement or expiry. Expiry or ambiguous state requires a new invitation and KeyPackage. KeyPackages and Welcomes never authorize daemon scopes. diff --git a/docs/architecture/remote-hosted-path.md b/docs/architecture/remote-hosted-path.md index bf3e4087..50039f2a 100644 --- a/docs/architecture/remote-hosted-path.md +++ b/docs/architecture/remote-hosted-path.md @@ -3,7 +3,7 @@ # Remote hosted-path checkpoint -Status: real E2EE integration adapters implemented with local test stores; production startup disabled +Status: checkpoint record from the Session 60 integration. The production path it led to is described in [Production remote access from a WSL daemon](remote-production-wsl.md), [from macOS and Linux desktop daemons](remote-production-desktop.md), and [Pair once, share per session](remote-session-sharing.md). The gates below record how each one was settled. ## Scope @@ -88,12 +88,14 @@ Every sealed envelope costs a native endpoint transaction, a witness round, and - **Superseded grants.** An application envelope sealed under an older hosted grant generation is never opened or executed. The bridge answers it once per operation with `daemon_rejected` (`stale_grant_generation`), sealed under the current generation and sent to the frame's source route without moving the reply route. The SDK removes the matching record from the outbox and reports the request as failed, instead of retrying ciphertext that can never be accepted. The bridge remembers the last 256 rejected operations; each rejection is still bounded by the sender's relay frame budget. An envelope from a generation ahead of the daemon's is refused without a reply. - **Route following.** The daemon follows the relay's route view for the paired device. A device that reconnects receives deliveries on its new route before it sends anything. When the device route disappears, the bridge keeps the last one and sends fail at the relay until the device returns. -## Remaining gates +## Gates -Before production remote control: +This checkpoint listed five gates before production remote control. Where they stand: -- Person 1 must provide the reviewed OpenMLS prepared-envelope transaction and browser/WASM persistence strategy. -- MLS application, Update, commit, and epoch-ready delivery classes need an ordered priority contract. -- Production identity, datastore, workload authentication, quotas, deployment, and TLS termination must be selected. -- Permission lifecycle, action-digest binding, policy generations, and race resolution must be implemented and reviewed. -- Ordinary-session remote exposure must receive an explicit enablement review. +- **Browser persistence:** settled. The worker barrier and the IndexedDB strategy in [Endpoint E2EE platform bindings](e2ee-platform-bindings.md) are implemented, including one tab per pairing: a newer tab takes the endpoint's lifetime lock from an older one, and the older endpoint fails closed. +- **Delivery-class priority:** still open as a written contract. The daemon delivery shape above orders replies after the deliveries that preceded them. +- **Production identity, datastore, and deployment:** settled. Accounts sign in through Cognito, the daemon admits its relay connections with its installation key, tickets, the witness, and its journal live in DynamoDB, and the stack serves TLS on its own domain. +- **Permission lifecycle:** not needed for the first release. The phone answers only the agent's questions; it never approves tool actions. +- **Ordinary-session exposure:** settled by sharing per session. The phone reaches only the sessions where `/remote` ran, a share ends after 24 hours idle, and remote access stays behind the per-account opt-in. + +The gates that remain are the evidence runs and the independent security review in the WSL and desktop documents. diff --git a/docs/architecture/remote-production-wsl.md b/docs/architecture/remote-production-wsl.md index ad9cde55..42f77d42 100755 --- a/docs/architecture/remote-production-wsl.md +++ b/docs/architecture/remote-production-wsl.md @@ -90,6 +90,8 @@ Remote access is opt-in per person: the control plane accepts a token only when The control plane never accepts a static account token in production. +One daemon at a time serves remote access for an installation. A user can run several daemons, for example `axl` and `axl --unsafe`, each with its own state directory and so its own pairing, but they share the installation and its relay identity; two connected at once would replace each other's relay route without end and neither phone would get through. The daemon serving remote access holds a claim file in `~/.axl/remote/`, which names its process and state directory. Another daemon's `/remote` is refused with that name, and its restore waits until the claim is released or its owner has exited. + ## Production control plane A production runtime replaces the deployment-test runtime's single configured account: @@ -107,6 +109,8 @@ It replaces the deployment-test runtime on the existing stack (`infra/aws/hosted The witness keeps its current shape: three replicas inside the control plane, each with its own Ed25519 signing key and its own DynamoDB records and high-water journal, recovering lineage by lineage after a restart. The gateway requires all three receipts. +A restart can find a lineage split: the process stopped after some replicas stored a step and before the rest did. No certificate was issued for that step, and the endpoint still holds it pending. Recovery needs all three replicas to agree, so before it runs, the gateway brings the replicas left behind forward by that same endpoint-signed step. A replica takes it only for a lineage it has not recovered, only as a direct successor of its own head, and only when fresh heads that both other replicas signed for one endpoint-signed read each sit at its head or at that successor. A registration that some replicas missed is completed as an ordinary registration, since they never held the lineage. Anything else, such as two replicas holding different successors, is left unrecovered and the endpoint fails closed. No replica moves backward and every head is one the endpoint signed, so rollback detection is unchanged: an endpoint restored from before its pending step proposes a conflicting successor and is caught. + This does not meet the RFC's "independently administered" requirement. One account's credentials can reach all three replicas, so they protect against a lost or rolled-back table, not against a compromised operator account. Spreading the replicas over regions of the same account was considered and rejected for now: it would add two cross-region round trips to every message and make any one region an outage for all of remote access, while buying no independence. Three replicas in separately administered accounts, each its own service, are a prerequisite for widening the opt-in beyond the owner. Replica trust (the three public verification keys) is pinned into the production daemon and browser builds, never taken from a link or a server response. From be4b1331cab7a84b9fce6fdb14d5554b05f523a9 Mon Sep 17 00:00:00 2001 From: Lokesh Date: Sun, 4 Oct 2026 11:52:12 +0400 Subject: [PATCH 5/5] fix(remote): keep trying for a while after taking a stuck tab's pairing A tab that took the endpoint lock from a tab that never let go tried to open the endpoint once more and gave up if the lock still looked held. The lock manager can see the taken lock's release after the worker's next request, and on a slow runner the new tab then said the pairing was open elsewhere. The tab now keeps trying for the handoff window after the takeover, taking the lock again every few attempts. Signed-off-by: Lokesh --- packages/web/src/remote/main.ts | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/packages/web/src/remote/main.ts b/packages/web/src/remote/main.ts index 4d0f5fa6..92c6276e 100644 --- a/packages/web/src/remote/main.ts +++ b/packages/web/src/remote/main.ts @@ -382,7 +382,9 @@ async function openExclusive( superseded: () => boolean, ): Promise { const deadline = Date.now() + HANDOFF_WAIT_MS; - let takenOver = false; + // Set once this tab takes the endpoint: the lock manager may see the taken lock's release + // after the worker's next request, so the tab keeps trying (and taking) until this passes. + let takeoverDeadline: number | undefined; for (let round = 0; ; round += 1) { if (superseded() || ownedByNewer(link)) return undefined; if (round % 4 === 0) { @@ -397,15 +399,17 @@ async function openExclusive( return await openEndpoint(binding, link); } catch (cause) { if ((cause as { readonly code?: unknown }).code !== "lifecycle_busy") throw cause; - if (Date.now() > deadline) { - if (takenOver) throw cause; - trace("another tab did not let go of the pairing; taking it over"); - status("Taking over from another tab"); + const now = Date.now(); + if (takeoverDeadline !== undefined && now > takeoverDeadline) throw cause; + if (now > deadline && (takeoverDeadline === undefined || round % 4 === 0)) { + if (takeoverDeadline === undefined) { + trace("another tab did not let go of the pairing; taking it over"); + status("Taking over from another tab"); + takeoverDeadline = now + HANDOFF_WAIT_MS; + } await binding.takeOverDeviceEndpoint({ cryptoSessionId: uuidToBytes(link.cryptoSessionId), }); - takenOver = true; - continue; } await sleep(250); }