From 87d506f774003fa81e4fd43b6e5f708c4b83cc47 Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Fri, 4 Sep 2026 18:25:24 -0600 Subject: [PATCH 01/38] feat(servercomm): add secure HTTP transport --- SimpleAPI/pom.xml | 18 +- .../bencodez/simpleapi/file/DurableFiles.java | 65 ++ .../http/HttpBackendTransportConnector.java | 492 ++++++++++++ .../http/HttpClientCredentialStore.java | 376 +++++++++ .../servercomm/http/HttpConnectionCode.java | 112 +++ .../http/HttpEnrollmentAuthority.java | 212 +++++ .../http/HttpInboundDeliveryStore.java | 173 ++++ .../servercomm/http/HttpPinnedTls.java | 103 +++ .../http/HttpProxyTransportServer.java | 499 ++++++++++++ .../servercomm/http/HttpTlsIdentity.java | 467 +++++++++++ .../http/HttpTransportProtocol.java | 231 ++++++ .../servercomm/http/HttpTransportSecrets.java | 64 ++ .../http/HttpTransportRuntimeTest.java | 759 ++++++++++++++++++ .../http/HttpTransportSecurityTest.java | 401 +++++++++ 14 files changed, 3971 insertions(+), 1 deletion(-) create mode 100644 SimpleAPI/src/main/java/com/bencodez/simpleapi/file/DurableFiles.java create mode 100644 SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpBackendTransportConnector.java create mode 100644 SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpClientCredentialStore.java create mode 100644 SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpConnectionCode.java create mode 100644 SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentAuthority.java create mode 100644 SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpInboundDeliveryStore.java create mode 100644 SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpPinnedTls.java create mode 100644 SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java create mode 100644 SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpTlsIdentity.java create mode 100644 SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpTransportProtocol.java create mode 100644 SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpTransportSecrets.java create mode 100644 SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportRuntimeTest.java create mode 100644 SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportSecurityTest.java diff --git a/SimpleAPI/pom.xml b/SimpleAPI/pom.xml index c02f0ba..f9559f7 100644 --- a/SimpleAPI/pom.xml +++ b/SimpleAPI/pom.xml @@ -16,6 +16,7 @@ 21 21 21 + 1.85 src/main/java @@ -71,6 +72,11 @@ ${project.name} + + org.apache.maven.plugins + maven-surefire-plugin + 3.5.4 + org.apache.maven.plugins maven-shade-plugin @@ -155,6 +161,16 @@ + + org.bouncycastle + bcpkix-jdk18on + ${bouncycastle.version} + + + org.bouncycastle + bcprov-jdk18on + ${bouncycastle.version} + org.spigotmc spigot-api @@ -407,4 +423,4 @@ - \ No newline at end of file + diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/file/DurableFiles.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/file/DurableFiles.java new file mode 100644 index 0000000..3969f64 --- /dev/null +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/file/DurableFiles.java @@ -0,0 +1,65 @@ +package com.bencodez.simpleapi.file; + +import java.io.IOException; +import java.nio.channels.FileChannel; +import java.nio.file.AccessDeniedException; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.Locale; + +/** Cross-platform helpers for forcing file contents and published directory entries. */ +public final class DurableFiles { + private DurableFiles() { } + + public static void forceFile(Path file) throws IOException { + try (FileChannel channel = FileChannel.open(file, StandardOpenOption.WRITE, LinkOption.NOFOLLOW_LINKS)) { + channel.force(true); + } + } + + public static void forceDirectory(Path directory) throws IOException { + if (directory == null) return; + try { + try (FileChannel channel = FileChannel.open(directory, StandardOpenOption.READ)) { + channel.force(true); + } + } catch (AccessDeniedException unsupportedDirectoryHandle) { + // The Windows NIO provider cannot open directory handles. File contents are + // still forced before atomic publication; do not make persistence unusable. + if (!isWindowsName(System.getProperty("os.name", ""))) throw unsupportedDirectoryHandle; + } catch (UnsupportedOperationException unsupportedDirectoryForce) { + // Some providers support atomic moves but expose no directory-force operation. + } + } + + public static boolean deleteIfExists(Path target) throws IOException { + boolean deleted = Files.deleteIfExists(target); + if (deleted) forceDirectory(target.toAbsolutePath().normalize().getParent()); + return deleted; + } + + public static void forceMoveDirectories(Path source, Path target) throws IOException { + try { + Path sourceParent = source.toAbsolutePath().normalize().getParent(); + Path targetParent = target.toAbsolutePath().normalize().getParent(); + forceDirectory(targetParent); + if (sourceParent != null && !sourceParent.equals(targetParent)) forceDirectory(sourceParent); + } catch (IOException failure) { + throw new PublishedException(failure); + } + } + + public static boolean isWindowsName(String name) { + return name != null && name.trim().toLowerCase(Locale.ROOT).startsWith("windows"); + } + + /** Indicates that an atomic rename completed before metadata writeback failed. */ + @SuppressWarnings("serial") + public static final class PublishedException extends IOException { + public PublishedException(IOException cause) { + super("File was published but its directory metadata could not be forced", cause); + } + } +} diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpBackendTransportConnector.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpBackendTransportConnector.java new file mode 100644 index 0000000..aa9997d --- /dev/null +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpBackendTransportConnector.java @@ -0,0 +1,492 @@ +package com.bencodez.simpleapi.servercomm.http; + +import com.bencodez.simpleapi.servercomm.codec.JsonEnvelope; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.net.http.HttpTimeoutException; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.security.KeyStore; +import java.security.cert.X509Certificate; +import java.time.Clock; +import java.time.Duration; +import java.util.ArrayDeque; +import java.util.Collection; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Flow; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Consumer; +import javax.net.ssl.KeyManagerFactory; +import javax.net.ssl.SSLContext; + +/** Backend-side, persistent HTTP/1.1 long-poll connector. */ +public final class HttpBackendTransportConnector implements AutoCloseable { + public static final Duration CLIENT_TIMEOUT = Duration.ofSeconds(35); + static final int CALLBACK_QUEUE_CAPACITY = 128; + private volatile HttpClientCredentialStore.HttpClientProfile profile; + private final String serverId; + private final Consumer onEnvelope; + private volatile HttpClient client; + private volatile HttpClientCredentialStore.ClientCredential credential; + private final Path credentialDirectory; + private final HttpInboundDeliveryStore inboundDeliveries; + private final URI transportEndpoint; + private final ThreadPoolExecutor callbackExecutor; + private final AtomicBoolean running = new AtomicBoolean(); + private final CountDownLatch firstResponse = new CountDownLatch(1); + private final Object state = new Object(); + private final Object renewal = new Object(); + private final LinkedHashMap outgoing = new LinkedHashMap<>(); + private final Set received = new LinkedHashSet<>(), processing = new LinkedHashSet<>(); + private final ArrayDeque acknowledgements = new ArrayDeque<>(); + private final String session = UUID.randomUUID().toString(); + private volatile Thread poller; + private long sequence; + private volatile long nextRenewalCheckNanos; + + /** In-memory test constructor; production transport must use a directory-backed constructor. */ + HttpBackendTransportConnector(HttpConnectionCode code, String serverId, + HttpClientCredentialStore.ClientCredential credential, Consumer onEnvelope) throws Exception { + this(profile(code, serverId), credential, onEnvelope); + } + + /** In-memory test constructor; production transport must use a directory-backed constructor. */ + HttpBackendTransportConnector(HttpClientCredentialStore.EnrolledClient enrolled, Consumer onEnvelope) throws Exception { + this(enrolled, onEnvelope, null); + } + + /** In-memory test constructor; production transport must use a directory-backed constructor. */ + HttpBackendTransportConnector(HttpClientCredentialStore.HttpClientProfile profile, + HttpClientCredentialStore.ClientCredential credential, Consumer onEnvelope) throws Exception { + this(profile, credential, onEnvelope, null); + } + + private HttpBackendTransportConnector(HttpClientCredentialStore.EnrolledClient enrolled, Consumer onEnvelope, + Path credentialDirectory) throws Exception { + this(enrolled == null ? null : enrolled.profile(), enrolled == null ? null : enrolled.credential(), onEnvelope, credentialDirectory); + } + + private HttpBackendTransportConnector(HttpClientCredentialStore.HttpClientProfile profile, + HttpClientCredentialStore.ClientCredential credential, Consumer onEnvelope, Path credentialDirectory) throws Exception { + if (profile == null || credential == null || onEnvelope == null) throw new IllegalArgumentException("HTTP backend transport configuration is invalid"); + if (!matchesCredential(profile, credential)) throw new IllegalArgumentException("HTTP client certificate does not match transport profile"); + this.profile = profile; this.serverId = profile.serverId(); this.onEnvelope = onEnvelope; + this.credential = credential; + this.credentialDirectory = credentialDirectory; + inboundDeliveries = credentialDirectory == null ? null : new HttpInboundDeliveryStore(credentialDirectory); + if (inboundDeliveries != null) for (var entry : inboundDeliveries.snapshot().entrySet()) { + if (entry.getValue() == HttpInboundDeliveryStore.State.COMPLETED) { + received.add(entry.getKey()); + queueAck(entry.getKey()); + } + } + client = client(profile, credential); + transportEndpoint = profile.endpoint().resolve("v1/transport"); + // GlobalMessageHandler routes mutate backend vote state and must observe the + // wire order. One bounded lane preserves batch ordering without running work on + // the long-poll thread; bounded admission below backpressures this poller. + callbackExecutor = executor("SimpleAPI-HTTP-callback", 1, CALLBACK_QUEUE_CAPACITY); + } + + /** Convenience constructor for the owner-only credential directory produced by {@link #enroll}. */ + public HttpBackendTransportConnector(HttpConnectionCode code, String serverId, Path credentials, + Consumer onEnvelope) throws Exception { + this(HttpClientCredentialStore.loadEnrolled(credentials), onEnvelope, credentials); + if (code == null || !profile(code, serverId).equals(this.profile)) throw new IllegalArgumentException("HTTP transport profile does not match connection code"); + } + + /** Starts normal transport using only the persisted certificate and non-secret profile. */ + public HttpBackendTransportConnector(Path credentials, Consumer onEnvelope) throws Exception { + this(HttpClientCredentialStore.loadEnrolled(credentials), onEnvelope, credentials); + } + + /** Performs enrollment network I/O; call this from a connector/setup worker, never a platform main thread. */ + public static HttpClientCredentialStore.ClientCredential enroll(HttpConnectionCode code, String serverId, Path credentials) throws Exception { + if (code == null || credentials == null || serverId == null || !serverId.matches("[A-Za-z0-9][A-Za-z0-9._-]{0,63}")) throw new IllegalArgumentException("Enrollment configuration is invalid"); + if (!code.serverId().equals(HttpTlsIdentity.canonicalServerId(serverId))) + throw new IllegalArgumentException("HTTP connection code belongs to a different backend"); + code.requireActive(Clock.systemUTC()); + byte[] payload = ("{\"server\":\"" + serverId + "\",\"token\":\"" + code.enrollmentToken() + "\"}").getBytes(StandardCharsets.UTF_8); + HttpClient client = HttpClient.newBuilder().version(HttpClient.Version.HTTP_1_1).followRedirects(HttpClient.Redirect.NEVER) + .connectTimeout(Duration.ofSeconds(5)).sslContext(HttpPinnedTls.clientContext(code)).build(); + HttpRequest request = HttpRequest.newBuilder(code.endpoint().resolve("v1/enroll")).timeout(CLIENT_TIMEOUT) + .header("Content-Type", "application/json").header("Cache-Control", "no-store").POST(HttpRequest.BodyPublishers.ofByteArray(payload)).build(); + LimitedResponse response = sendLimited(client, request); + if (response.statusCode() != 201) throw new IllegalArgumentException("Enrollment was rejected"); + HttpTlsIdentity.IssuedClientCertificate issued = HttpTransportProtocol.parseEnrollmentResponse(serverId, response.body()); + HttpClientCredentialStore.saveEnrolled(credentials, code, issued); return HttpClientCredentialStore.load(credentials); + } + + public void start() { + if (!running.compareAndSet(false, true)) return; + poller = new Thread(this::pollLoop, "SimpleAPI-HTTP-poll"); poller.setDaemon(true); poller.start(); + } + /** Waits for one authenticated, protocol-valid transport response. */ + public boolean awaitFirstResponse(long deadlineNanos) throws InterruptedException { + long remaining = deadlineNanos - System.nanoTime(); + return remaining > 0L && firstResponse.await(remaining, TimeUnit.NANOSECONDS) && running.get(); + } + /** + * Inserts an in-memory at-least-once delivery. It survives retry/lost responses while this process remains alive; + * callers needing restart durability must retain the application operation independently. + */ + public boolean send(JsonEnvelope envelope) { + if (envelope == null || !running.get()) return false; + try { HttpTransportProtocol.validateEnvelope(envelope); } + catch (IllegalArgumentException invalid) { return false; } + synchronized (state) { + if (outgoing.size() >= HttpTransportProtocol.MAX_QUEUE) return false; + String id = UUID.randomUUID().toString(); outgoing.put(id, new HttpTransportProtocol.Delivery(id, envelope)); return true; + } + } + /** A synchronous single poll, useful for lifecycle-controlled integrations and tests. */ + public synchronized boolean pollOnce() { + return pollOnce(CLIENT_TIMEOUT, true, true); + } + private boolean pollOnce(Duration timeout, boolean requireRunning, boolean acceptIncoming) { + if (requireRunning && !running.get()) return false; + List acks = List.of(); boolean acknowledgementsConfirmed = false; + try { + if (requireRunning) maybeRenewCredential(); + List messages; long requestSequence; + synchronized (state) { + acks = first(acknowledgements); requestSequence = sequence++; + messages = HttpTransportProtocol.fittingMessages(serverId, session, requestSequence, acks, outgoing.values()); + for (int index = 0; index < acks.size(); index++) acknowledgements.removeFirst(); + } + HttpRequest request = HttpRequest.newBuilder(transportEndpoint).timeout(timeout).header("Content-Type", "application/json") + .header("Cache-Control", "no-store").POST(HttpRequest.BodyPublishers.ofByteArray(HttpTransportProtocol.request(serverId, session, requestSequence, acks, messages))).build(); + LimitedResponse response = sendLimited(client, request); + if (response.statusCode() != 200) return false; + HttpTransportProtocol.Packet packet = HttpTransportProtocol.parsePacket(response.body()); + if (!serverId.equals(packet.server()) || !session.equals(packet.session()) || packet.sequence() != requestSequence) return false; + confirmAcknowledgements(acks); + acknowledgementsConfirmed = true; + synchronized (state) { for (String ack : packet.acks()) outgoing.remove(ack); } + if (acceptIncoming) for (HttpTransportProtocol.Delivery delivery : accept(packet.messages())) dispatch(delivery); + firstResponse.countDown(); + return true; + } catch (Exception failure) { return false; + } finally { if (!acknowledgementsConfirmed) requeueAcknowledgements(acks); } + } + /** Stops normal polling and gives already-queued outbound messages a bounded final delivery attempt. */ + public boolean flushOutgoing(long deadlineNanos) { + running.set(false); + firstResponse.countDown(); + Thread current = poller; + if (current != null) current.interrupt(); + if (!joinPoller(current, deadlineNanos)) return false; + while (queuedOutgoing() != 0) { + long remaining = deadlineNanos - System.nanoTime(); + if (remaining <= 0L) return false; + Duration timeout = Duration.ofNanos(Math.min(CLIENT_TIMEOUT.toNanos(), remaining)); + synchronized (this) { + if (pollOnce(timeout, false, false)) continue; + } + // The proxy may retain its one-active-poll guard briefly after the old + // client request is interrupted. Retry that transient 409 without busy + // spinning, but never extend the caller's shutdown deadline. + remaining = deadlineNanos - System.nanoTime(); + if (remaining <= 0L) return false; + try { TimeUnit.NANOSECONDS.sleep(Math.min(TimeUnit.MILLISECONDS.toNanos(50), remaining)); } + catch (InterruptedException interrupted) { Thread.currentThread().interrupt(); return false; } + } + return true; + } + @Override public void close() { + running.getAndSet(false); + firstResponse.countDown(); + // Revoke this connector's journal writer before a replacement snapshots it. + // In-flight transitions serialize with seal(): either COMPLETED is already + // durable, or the delivery remains durably RUNNING and fail-closed. + if (inboundDeliveries != null) inboundDeliveries.seal(); + Thread current = poller; if (current != null) current.interrupt(); + callbackExecutor.shutdown(); try { if (!callbackExecutor.awaitTermination(5, TimeUnit.SECONDS)) callbackExecutor.shutdownNow(); } + catch (InterruptedException interrupted) { Thread.currentThread().interrupt(); callbackExecutor.shutdownNow(); } + // The owning transport may release the credential-directory semaphore as soon + // as close returns. Wait for the interrupted poller so an in-flight renewal + // cannot activate an old credential generation after that ownership handoff. + joinPoller(current); + } + boolean pollerAlive() { Thread current = poller; return current != null && current.isAlive(); } + private static boolean joinPoller(Thread poller, long deadlineNanos) { + if (poller == null || poller == Thread.currentThread()) return true; + boolean interrupted = false; + while (poller.isAlive()) { + long remaining = deadlineNanos - System.nanoTime(); + if (remaining <= 0L) { + if (interrupted) Thread.currentThread().interrupt(); + return false; + } + try { TimeUnit.NANOSECONDS.timedJoin(poller, remaining); } + catch (InterruptedException stopRequested) { interrupted = true; poller.interrupt(); } + } + if (interrupted) Thread.currentThread().interrupt(); + return true; + } + private static void joinPoller(Thread poller) { + if (poller == null || poller == Thread.currentThread()) return; + boolean interrupted = false; + while (poller.isAlive()) try { poller.join(); } + catch (InterruptedException stopRequested) { interrupted = true; poller.interrupt(); } + if (interrupted) Thread.currentThread().interrupt(); + } + + private void pollLoop() { + long retry = 1000L; + while (running.get()) { if (pollOnce()) { retry = 1000L; continue; } try { Thread.sleep(retry); } catch (InterruptedException interrupted) { Thread.currentThread().interrupt(); break; } retry = Math.min(30_000L, retry * 2); } + } + List accept(List deliveries) { + synchronized (state) { + List accepted = new java.util.ArrayList<>(); + for (HttpTransportProtocol.Delivery delivery : deliveries) { + HttpInboundDeliveryStore.State persisted = inboundDeliveries == null ? null : inboundDeliveries.state(delivery.id()); + if (received.contains(delivery.id()) || persisted == HttpInboundDeliveryStore.State.COMPLETED) { + received.add(delivery.id()); queueAck(delivery.id()); continue; + } + // A callback that was running when the process stopped may already have + // produced external side effects. Keep the proxy copy without replaying or + // acknowledging it; arbitrary plugin callbacks cannot share this journal. + if (persisted == HttpInboundDeliveryStore.State.RUNNING) continue; + if (!processing.contains(delivery.id())) { + processing.add(delivery.id()); accepted.add(delivery); + } + } + return accepted; + } + } + void dispatch(HttpTransportProtocol.Delivery delivery) { + Runnable callback = () -> { + boolean success = false; + try { + if (inboundDeliveries != null) { + if (inboundDeliveries.state(delivery.id()) == null) inboundDeliveries.reserve(delivery.id()); + inboundDeliveries.markRunning(delivery.id()); + } + onEnvelope.accept(delivery.envelope()); + if (inboundDeliveries != null) inboundDeliveries.markCompleted(delivery.id()); + success = true; + } catch (IOException persistenceFailure) { + // Never run before RUNNING is durable and never acknowledge until + // COMPLETED is durable. An uncertain transition stays fail-closed. + } catch (RuntimeException callbackFailure) { + // The callback may have failed after partial external effects. Leave RUNNING + // unacknowledged so a restart cannot silently lose or duplicate the delivery. + } + completeIncoming(delivery.id(), success); + }; + if (!executeOrdered(callbackExecutor, callback)) completeIncoming(delivery.id(), false); + } + void completeIncoming(String id, boolean success) { synchronized (state) { + processing.remove(id); + if (success) { received.add(id); while (received.size() > HttpTransportProtocol.MAX_QUEUE) received.remove(received.iterator().next()); queueAck(id); } + } } + private void queueAck(String id) { if (acknowledgements.size() < HttpTransportProtocol.MAX_QUEUE && !acknowledgements.contains(id)) acknowledgements.add(id); } + private void requeueAcknowledgements(List ids) { synchronized (state) { + for (int index = ids.size() - 1; index >= 0; index--) { + String id = ids.get(index); + if (!acknowledgements.contains(id)) { + while (acknowledgements.size() >= HttpTransportProtocol.MAX_QUEUE) acknowledgements.removeLast(); + acknowledgements.addFirst(id); + } + } + } } + private void confirmAcknowledgements(Collection ids) { + for (String id : ids) { + if (inboundDeliveries != null) try { inboundDeliveries.remove(id); } + catch (IOException cleanupFailure) { continue; } + synchronized (state) { received.remove(id); } + } + } + int queuedOutgoing() { synchronized (state) { return outgoing.size(); } } + List drainAcknowledgements() { synchronized (state) { return drain(acknowledgements); } } + private static List first(Collection values) { List output = new java.util.ArrayList<>(); for (T value : values) { output.add(value); if (output.size() == HttpTransportProtocol.MAX_BATCH) break; } return output; } + private static List drain(ArrayDeque values) { List output = new java.util.ArrayList<>(); while (!values.isEmpty() && output.size() < HttpTransportProtocol.MAX_BATCH) output.add(values.remove()); return output; } + private static ThreadPoolExecutor executor(String name, int threads, int queue) { ThreadFactory factory = task -> { Thread thread = new Thread(task, name); thread.setDaemon(true); return thread; }; return new ThreadPoolExecutor(threads, threads, 0L, TimeUnit.MILLISECONDS, new ArrayBlockingQueue<>(queue), factory, new ThreadPoolExecutor.AbortPolicy()); } + static boolean executeOrdered(ThreadPoolExecutor executor, Runnable task) { + try { executor.execute(task); return true; } + catch (RejectedExecutionException fullOrClosed) { + if (executor.isShutdown()) return false; + try { + while (!executor.isShutdown()) { + if (!executor.getQueue().offer(task, 100L, TimeUnit.MILLISECONDS)) continue; + if (executor.isShutdown() && executor.remove(task)) return false; + return true; + } + } catch (InterruptedException interrupted) { Thread.currentThread().interrupt(); } + return false; + } + } + private static HttpClientCredentialStore.HttpClientProfile profile(HttpConnectionCode code, String serverId) { + if (code == null || serverId == null) throw new IllegalArgumentException("HTTP backend transport configuration is invalid"); + if (!code.serverId().equals(HttpTlsIdentity.canonicalServerId(serverId))) throw new IllegalArgumentException("HTTP connection code belongs to a different backend"); + return new HttpClientCredentialStore.HttpClientProfile(serverId, code.endpoint(), code.serverCertificatePin(), code.caCertificatePin()); + } + private static boolean matchesCredential(HttpClientCredentialStore.HttpClientProfile profile, + HttpClientCredentialStore.ClientCredential credential) { + try { + credential.certificate().checkValidity(); credential.certificate().verify(credential.caCertificate().getPublicKey()); + String expected = "urn:votingplugin:http-backend:" + profile.serverId(); + var names = credential.certificate().getSubjectAlternativeNames(); if (names == null) return false; + for (java.util.List name : names) if (name.size() == 2 && Integer.valueOf(6).equals(name.get(0)) && expected.equals(name.get(1))) return true; + return false; + } catch (Exception invalid) { return false; } + } + private void maybeRenewCredential() { + synchronized (renewal) { + Path directory = credentialDirectory; + if (directory == null || !HttpTlsIdentity.needsRenewal(credential.certificate(), Clock.systemUTC())) return; + long now = System.nanoTime(); + if (nextRenewalCheckNanos != 0L && now - nextRenewalCheckNanos < 0L) return; + nextRenewalCheckNanos = now + Duration.ofHours(6).toNanos(); + try { + byte[] body = HttpTransportProtocol.renewalRequest(serverId); + HttpRequest request = HttpRequest.newBuilder(profile.endpoint().resolve("v1/renew")).timeout(CLIENT_TIMEOUT) + .header("Content-Type", "application/json").header("Cache-Control", "no-store") + .POST(HttpRequest.BodyPublishers.ofByteArray(body)).build(); + LimitedResponse response = sendLimited(client, request); + if (response.statusCode() != 201) return; + HttpTlsIdentity.IssuedClientCertificate issued = HttpTransportProtocol.parseEnrollmentResponse(serverId, response.body()); + HttpClientCredentialStore.StagedCredential staged = HttpClientCredentialStore.stageReplacement(directory, issued); + HttpClientCredentialStore.ClientCredential replacement = staged.credential(); + HttpClientCredentialStore.HttpClientProfile replacementProfile = staged.profile(); + if (!matchesCredential(replacementProfile, replacement)) throw new IllegalArgumentException("Renewed HTTP certificate is invalid"); + HttpClient replacementClient = client(replacementProfile, replacement); + HttpClientCredentialStore.activateReplacement(directory, staged); + profile = replacementProfile; + client = replacementClient; + credential = replacement; + } catch (Exception ignored) { /* The active generation is unchanged; retry on the bounded schedule. */ } + } + } + private static HttpClient client(HttpClientCredentialStore.HttpClientProfile profile, + HttpClientCredentialStore.ClientCredential credential) throws Exception { + return HttpClient.newBuilder().version(HttpClient.Version.HTTP_1_1).followRedirects(HttpClient.Redirect.NEVER) + .connectTimeout(Duration.ofSeconds(5)).sslContext(clientContext(profile, credential)).build(); + } + static LimitedResponse sendLimited(HttpClient client, HttpRequest request) throws IOException, InterruptedException { + CompletableFuture> exchange = client.sendAsync(request, + ignored -> new LimitedBodySubscriber(HttpTransportProtocol.MAX_BODY_BYTES)); + HttpResponse response; + try { + Duration timeout = request.timeout().orElse(CLIENT_TIMEOUT); + response = exchange.get(timeout.toMillis(), TimeUnit.MILLISECONDS); + } catch (TimeoutException timeout) { + exchange.cancel(true); + throw new HttpTimeoutException("HTTP transport response timed out"); + } catch (InterruptedException interrupted) { + exchange.cancel(true); + throw interrupted; + } catch (ExecutionException failure) { + Throwable cause = failure.getCause(); + if (cause instanceof IOException ioFailure) throw ioFailure; + throw new IOException("HTTP transport request failed", cause); + } + long declaredLength = response.headers().firstValueAsLong("Content-Length").orElse(-1L); + if (declaredLength > HttpTransportProtocol.MAX_BODY_BYTES) + throw new IOException("HTTP transport response exceeds its limit"); + return new LimitedResponse(response.statusCode(), response.body()); + } + + private static final class LimitedBodySubscriber implements HttpResponse.BodySubscriber { + private final int maximum; + private final ByteArrayOutputStream body = new ByteArrayOutputStream(); + private final CompletableFuture result = new CompletableFuture<>(); + private Flow.Subscription subscription; + + private LimitedBodySubscriber(int maximum) { + this.maximum = maximum; + } + + @Override + public CompletionStage getBody() { + return result; + } + + @Override + public void onSubscribe(Flow.Subscription subscription) { + if (this.subscription != null) { + subscription.cancel(); + return; + } + this.subscription = subscription; + subscription.request(1); + } + + @Override + public void onNext(List buffers) { + try { + for (ByteBuffer buffer : buffers) { + if (buffer.remaining() > maximum - body.size()) { + subscription.cancel(); + result.completeExceptionally(new IOException("HTTP transport response exceeds its limit")); + return; + } + byte[] chunk = new byte[buffer.remaining()]; + buffer.get(chunk); + body.writeBytes(chunk); + } + subscription.request(1); + } catch (RuntimeException failure) { + subscription.cancel(); + result.completeExceptionally(failure); + } + } + + @Override + public void onError(Throwable failure) { + result.completeExceptionally(failure); + } + + @Override + public void onComplete() { + result.complete(body.toByteArray()); + } + } + static byte[] readLimited(InputStream body) throws IOException { + byte[] bytes = body.readNBytes(HttpTransportProtocol.MAX_BODY_BYTES + 1); + if (bytes.length > HttpTransportProtocol.MAX_BODY_BYTES) + throw new IOException("HTTP transport response exceeds its limit"); + return bytes; + } + record LimitedResponse(int statusCode, byte[] body) { } + private static SSLContext clientContext(HttpClientCredentialStore.HttpClientProfile profile, HttpClientCredentialStore.ClientCredential credential) throws Exception { + String caPin = HttpTransportSecrets.certificatePin(credential.caCertificate()); + if (!HttpTransportSecrets.constantTimeEquals(profile.caCertificatePin().getBytes(StandardCharsets.US_ASCII), + caPin.getBytes(StandardCharsets.US_ASCII))) throw new IllegalArgumentException("HTTP authority does not match transport profile"); + char[] password = credential.password(); + try { + KeyStore store = KeyStore.getInstance("PKCS12"); store.load(null, new char[0]); + store.setKeyEntry("client", credential.privateKey(), password, + new java.security.cert.Certificate[] { credential.certificate(), credential.caCertificate() }); + KeyManagerFactory keys = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); keys.init(store, password); + KeyStore trustStore = KeyStore.getInstance(KeyStore.getDefaultType()); trustStore.load(null, new char[0]); + trustStore.setCertificateEntry("http-transport-ca", credential.caCertificate()); + javax.net.ssl.TrustManagerFactory trusts = javax.net.ssl.TrustManagerFactory.getInstance( + javax.net.ssl.TrustManagerFactory.getDefaultAlgorithm()); + trusts.init(trustStore); + SSLContext context = SSLContext.getInstance("TLS"); context.init(keys.getKeyManagers(), trusts.getTrustManagers(), null); return context; + } finally { java.util.Arrays.fill(password, '\0'); } + } +} diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpClientCredentialStore.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpClientCredentialStore.java new file mode 100644 index 0000000..9b42bc4 --- /dev/null +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpClientCredentialStore.java @@ -0,0 +1,376 @@ +package com.bencodez.simpleapi.servercomm.http; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.nio.file.attribute.PosixFilePermission; +import java.net.URI; +import java.security.KeyStore; +import java.security.PrivateKey; +import java.security.cert.X509Certificate; +import java.util.Base64; +import java.util.EnumSet; +import java.util.Properties; +import com.bencodez.simpleapi.file.DurableFiles; + +/** Owner-only persistence for the client certificate bundle returned by enrollment. */ +public final class HttpClientCredentialStore { + private static final String BUNDLE_FILE = "http-transport-client.p12"; + private static final String PASSWORD_FILE = "http-transport-client-password"; + private static final String PROFILE_FILE = "http-transport-profile.properties"; + private static final String CONNECTION_CODE_DIGEST_FILE = "http-transport-connection-code.sha256"; + private static final String GENERATIONS_DIRECTORY = "http-transport-client-generations"; + private static final String CURRENT_FILE = "http-transport-client-current"; + private HttpClientCredentialStore() { } + + public static void save(Path directory, HttpTlsIdentity.IssuedClientCertificate issued) throws IOException { + if (issued == null) throw new IllegalArgumentException("Issued credential is required"); + Files.createDirectories(directory); + byte[] bundle = issued.pkcs12(); + try { writePrivate(safe(directory.resolve(BUNDLE_FILE)), bundle); } + finally { java.util.Arrays.fill(bundle, (byte) 0); } + char[] password = issued.password(); + try { writePrivate(safe(directory.resolve(PASSWORD_FILE)), asciiBytes(password)); } + finally { java.util.Arrays.fill(password, '\0'); } + } + + /** Persists the certificate plus the non-secret normal-transport profile after enrollment. */ + public static void saveEnrolled(Path directory, HttpConnectionCode code, HttpTlsIdentity.IssuedClientCertificate issued) + throws IOException { + if (code == null || issued == null) throw new IllegalArgumentException("Connection code and credential are required"); + HttpClientProfile profile = new HttpClientProfile(HttpTlsIdentity.canonicalServerId(issued.serverId()), code.endpoint(), + code.serverCertificatePin(), code.caCertificatePin()); + try { + StagedCredential staged = stage(directory, issued, profile, connectionCodeDigest(code)); + activateReplacement(directory, staged); + } catch (IOException failure) { throw failure; + } catch (Exception failure) { throw new IOException("Could not persist HTTP client credential", failure); } + } + + private static void writeProfile(Path directory, HttpClientProfile profile) throws IOException { + Properties properties = new Properties(); + properties.setProperty("version", "1"); + properties.setProperty("serverId", profile.serverId()); + properties.setProperty("endpoint", profile.endpoint().toASCIIString()); + properties.setProperty("serverPin", profile.serverCertificatePin()); + properties.setProperty("caPin", profile.caCertificatePin()); + java.io.ByteArrayOutputStream bytes = new java.io.ByteArrayOutputStream(); + properties.store(bytes, "VotingPlugin HTTP transport profile"); + writePrivate(safe(directory.resolve(PROFILE_FILE)), bytes.toByteArray()); + } + + public static ClientCredential load(Path directory) throws Exception { + return loadCredential(activeDirectory(directory)); + } + + private static ClientCredential loadCredential(Path directory) throws Exception { + Path bundle = safe(directory.resolve(BUNDLE_FILE)); + Path passwordFile = safe(directory.resolve(PASSWORD_FILE)); + if (!Files.isRegularFile(bundle, LinkOption.NOFOLLOW_LINKS) || !Files.isRegularFile(passwordFile, LinkOption.NOFOLLOW_LINKS)) + throw new IOException("HTTP client certificate has not been enrolled"); + byte[] passwordBytes = Files.readAllBytes(passwordFile); + if (passwordBytes.length < 40 || passwordBytes.length > 128) throw new IOException("HTTP client password is invalid"); + char[] password = new String(passwordBytes, StandardCharsets.US_ASCII).toCharArray(); + java.util.Arrays.fill(passwordBytes, (byte) 0); + try { + KeyStore store = KeyStore.getInstance("PKCS12"); + try (var input = Files.newInputStream(bundle, LinkOption.NOFOLLOW_LINKS)) { store.load(input, password); } + PrivateKey privateKey = (PrivateKey) store.getKey("client", password); + java.security.cert.Certificate[] chain = store.getCertificateChain("client"); + if (privateKey == null || chain == null || chain.length != 2 + || !(chain[0] instanceof X509Certificate client) || !(chain[1] instanceof X509Certificate authority)) + throw new IOException("HTTP client certificate bundle is invalid"); + return new ClientCredential(privateKey, client, authority, password); + } finally { java.util.Arrays.fill(password, '\0'); } + } + + /** Writes and validates a replacement generation without touching the active credential. */ + static StagedCredential stageReplacement(Path directory, HttpTlsIdentity.IssuedClientCertificate issued) throws Exception { + Path active = activeDirectory(directory); + return stage(directory, issued, loadProfileFile(active), readConnectionCodeDigest(active)); + } + + private static StagedCredential stage(Path directory, HttpTlsIdentity.IssuedClientCertificate issued, + HttpClientProfile profile, String connectionCodeDigest) throws Exception { + if (directory == null || issued == null) throw new IllegalArgumentException("Credential replacement is required"); + Path credentialDirectory = directory.toAbsolutePath().normalize(); + boolean created = !Files.exists(credentialDirectory, LinkOption.NOFOLLOW_LINKS); + Path generations = credentialDirectory.resolve(GENERATIONS_DIRECTORY); + Files.createDirectories(generations); + // Credential files cannot make the newly created credential-root entry durable. + // Persist its parent before an enrolled transport can activate this root. + if (created) DurableFiles.forceDirectory(credentialDirectory.getParent()); + if (Files.isSymbolicLink(generations) || !Files.isDirectory(generations, LinkOption.NOFOLLOW_LINKS)) + throw new IOException("HTTP credential generation directory is unsafe"); + String name = java.util.UUID.randomUUID().toString(); + Path generation = generations.resolve(name); + Files.createDirectory(generation); + setOwnerOnlyDirectory(generation); + try { + save(generation, issued); + ClientCredential replacement = loadCredential(generation); + profile = new HttpClientProfile(profile.serverId(), profile.endpoint(), profile.serverCertificatePin(), + HttpTransportSecrets.certificatePin(replacement.caCertificate())); + writeProfile(generation, profile); + if (connectionCodeDigest != null) writePrivate(safe(generation.resolve(CONNECTION_CODE_DIGEST_FILE)), + connectionCodeDigest.getBytes(StandardCharsets.US_ASCII)); + EnrolledClient enrolled = loadEnrolled(generation); + // Each file is durable within the generation, but the generation name is + // published by its parent. Persist it before CURRENT can activate it. + DurableFiles.forceDirectory(generations); + return new StagedCredential(name, enrolled.credential(), enrolled.profile()); + } catch (Exception failure) { + try { Files.deleteIfExists(generation.resolve(BUNDLE_FILE)); Files.deleteIfExists(generation.resolve(PASSWORD_FILE)); + Files.deleteIfExists(generation.resolve(PROFILE_FILE)); Files.deleteIfExists(generation.resolve(CONNECTION_CODE_DIGEST_FILE)); + Files.deleteIfExists(generation); } + catch (IOException cleanup) { failure.addSuppressed(cleanup); } + throw failure; + } + } + + /** Atomically makes a fully validated generation durable and active. */ + static void activateReplacement(Path directory, StagedCredential staged) throws IOException { + if (directory == null || staged == null || !staged.name().matches("[0-9a-f-]{36}")) + throw new IllegalArgumentException("Staged credential is invalid"); + Path generations = directory.toAbsolutePath().normalize().resolve(GENERATIONS_DIRECTORY); + Path generation = generations.resolve(staged.name()).normalize(); + if (!generation.getParent().equals(generations) || Files.isSymbolicLink(generation) + || !Files.isRegularFile(generation.resolve(BUNDLE_FILE), LinkOption.NOFOLLOW_LINKS) + || !Files.isRegularFile(generation.resolve(PASSWORD_FILE), LinkOption.NOFOLLOW_LINKS) + || !Files.isRegularFile(generation.resolve(PROFILE_FILE), LinkOption.NOFOLLOW_LINKS)) + throw new IOException("Staged HTTP credential is incomplete"); + writePrivate(safe(directory.resolve(CURRENT_FILE)), staged.name().getBytes(StandardCharsets.US_ASCII)); + } + + static record StagedCredential(String name, ClientCredential credential, HttpClientProfile profile) { } + + public static HttpClientProfile loadProfile(Path directory) throws IOException { + return loadProfileFile(activeDirectory(directory)); + } + + private static HttpClientProfile loadProfileFile(Path directory) throws IOException { + Path profile = safe(directory.resolve(PROFILE_FILE)); + if (!Files.isRegularFile(profile, LinkOption.NOFOLLOW_LINKS) || Files.size(profile) > 8192) + throw new IOException("HTTP transport profile has not been enrolled"); + Properties properties = new Properties(); + try (var input = Files.newInputStream(profile, LinkOption.NOFOLLOW_LINKS)) { properties.load(input); } + if (properties.size() != 5 || !"1".equals(properties.getProperty("version"))) + throw new IOException("HTTP transport profile is invalid"); + try { + return new HttpClientProfile(properties.getProperty("serverId"), URI.create(properties.getProperty("endpoint")), + properties.getProperty("serverPin"), properties.getProperty("caPin")); + } catch (IllegalArgumentException failure) { throw new IOException("HTTP transport profile is invalid", failure); } + } + + public static boolean hasEnrolledProfile(Path directory) { + try { loadEnrolled(directory); return true; } + catch (Exception unavailable) { return false; } + } + + /** Returns whether this exact one-time code created the active credential, without persisting the code itself. */ + public static boolean matchesEnrollmentCode(Path directory, HttpConnectionCode code) throws IOException { + if (code == null) throw new IllegalArgumentException("Connection code is required"); + String stored = readConnectionCodeDigest(activeDirectory(directory)); + if (stored == null) return false; + byte[] storedBytes = stored.getBytes(StandardCharsets.US_ASCII); + return HttpTransportSecrets.constantTimeEquals(storedBytes, + connectionCodeDigest(code.encode()).getBytes(StandardCharsets.US_ASCII)) + || HttpTransportSecrets.constantTimeEquals(storedBytes, + connectionCodeDigest(code.encodeLegacy()).getBytes(StandardCharsets.US_ASCII)); + } + + /** Loads and cross-checks the persisted client key material and bound normal-transport profile. */ + public static EnrolledClient loadEnrolled(Path directory) throws Exception { + Path active = activeDirectory(directory); + return loadEnrolledDirectory(active); + } + + private static EnrolledClient loadEnrolledDirectory(Path active) throws Exception { + ClientCredential credential = loadCredential(active); + HttpClientProfile profile = loadProfileFile(active); + if (!matchesProfile(credential, profile)) throw new IOException("HTTP client certificate does not match its profile"); + return new EnrolledClient(profile, credential); + } + + /** Captures the exact credential generation used by a transport before a staged replacement starts. */ + public static ActiveCredentialGeneration snapshotActiveGeneration(Path directory) throws Exception { + Path root = directory.toAbsolutePath().normalize(); + Path active = activeDirectory(root); + EnrolledClient enrolled = loadEnrolledDirectory(active); + return new ActiveCredentialGeneration(active.equals(root) ? "" : active.getFileName().toString(), + enrolled.profile(), readConnectionCodeDigest(active)); + } + + /** + * Restores a pre-replacement generation unless the replacement already activated a + * newer credential for the same backend endpoint. A successful renewal or same-endpoint + * re-enrollment may revoke the snapshotted certificate at the proxy, so that newer + * generation is the only safe rollback identity. + */ + public static void restoreActiveGenerationAfterReplacement(Path directory, + ActiveCredentialGeneration snapshot) throws Exception { + if (directory == null || snapshot == null) throw new IllegalArgumentException("Credential rollback is required"); + HttpClientProfile previous = snapshot.profile(); + Path active = null; + HttpClientProfile current = null; + if (previous != null) try { + active = activeDirectory(directory); + current = loadEnrolledDirectory(active).profile(); + } catch (Exception unavailable) { active = null; current = null; } + if (active != null && previous.serverId().equals(current.serverId()) + && previous.endpoint().equals(current.endpoint())) { + restoreConnectionCodeDigest(active, snapshot.connectionCodeDigest()); + return; + } + restoreActiveGeneration(directory, snapshot); + } + + /** Atomically restores a previously validated credential generation after replacement rollback. */ + public static void restoreActiveGeneration(Path directory, ActiveCredentialGeneration snapshot) throws Exception { + if (directory == null || snapshot == null) throw new IllegalArgumentException("Credential rollback is required"); + Path root = directory.toAbsolutePath().normalize(); + Path generations = root.resolve(GENERATIONS_DIRECTORY); + Path target = snapshot.name().isEmpty() ? root : generations.resolve(snapshot.name()).normalize(); + if (!snapshot.name().isEmpty() && (!target.getParent().equals(generations) + || Files.isSymbolicLink(target) || !Files.isDirectory(target, LinkOption.NOFOLLOW_LINKS))) + throw new IOException("HTTP client credential generation is invalid"); + loadEnrolledDirectory(target); + Path current = safe(root.resolve(CURRENT_FILE)); + if (snapshot.name().isEmpty()) { + Files.deleteIfExists(current); + DurableFiles.forceDirectory(root); + } else { + writePrivate(current, snapshot.name().getBytes(StandardCharsets.US_ASCII)); + } + } + + public record ClientCredential(PrivateKey privateKey, X509Certificate certificate, X509Certificate caCertificate, char[] password) { + public ClientCredential { password = password.clone(); } + @Override public char[] password() { return password.clone(); } + } + + public record HttpClientProfile(String serverId, URI endpoint, String serverCertificatePin, String caCertificatePin) { + public HttpClientProfile { + serverId = HttpTlsIdentity.canonicalServerId(serverId); + HttpConnectionCode validation = new HttpConnectionCode(serverId, endpoint, serverCertificatePin, caCertificatePin, + java.time.Instant.now().plusSeconds(1), HttpTransportSecrets.randomToken()); + endpoint = validation.endpoint(); + serverCertificatePin = validation.serverCertificatePin(); + caCertificatePin = validation.caCertificatePin(); + } + } + + public record EnrolledClient(HttpClientProfile profile, ClientCredential credential) { } + + public record ActiveCredentialGeneration(String name, HttpClientProfile profile, String connectionCodeDigest) { + public ActiveCredentialGeneration(String name) { this(name, null, null); } + public ActiveCredentialGeneration { + if (name == null || (!name.isEmpty() && !name.matches("[0-9a-f-]{36}"))) + throw new IllegalArgumentException("HTTP client credential generation is invalid"); + if (connectionCodeDigest != null && !connectionCodeDigest.matches("[0-9a-f]{64}")) + throw new IllegalArgumentException("HTTP connection-code marker is invalid"); + } + } + + private static boolean matchesProfile(ClientCredential credential, HttpClientProfile profile) { + try { + String authorityPin = HttpTransportSecrets.certificatePin(credential.caCertificate()); + if (!HttpTransportSecrets.constantTimeEquals(profile.caCertificatePin().getBytes(StandardCharsets.US_ASCII), + authorityPin.getBytes(StandardCharsets.US_ASCII))) return false; + credential.certificate().checkValidity(); + credential.certificate().verify(credential.caCertificate().getPublicKey()); + java.util.List usage = credential.certificate().getExtendedKeyUsage(); + boolean[] keyUsage = credential.certificate().getKeyUsage(); + if (usage == null || !usage.contains(org.bouncycastle.asn1.x509.KeyPurposeId.id_kp_clientAuth.getId()) + || keyUsage == null || !keyUsage[0]) return false; + String expected = "urn:votingplugin:http-backend:" + profile.serverId(); + var names = credential.certificate().getSubjectAlternativeNames(); + if (names == null) return false; + for (java.util.List name : names) if (name.size() == 2 + && Integer.valueOf(6).equals(name.get(0)) && expected.equals(name.get(1))) return true; + return false; + } catch (Exception invalid) { return false; } + } + + private static Path safe(Path file) throws IOException { + if (Files.isSymbolicLink(file)) throw new IOException("Refusing unsafe HTTP credential path"); + return file.toAbsolutePath().normalize(); + } + + private static Path activeDirectory(Path directory) throws IOException { + Path root = directory.toAbsolutePath().normalize(); + Path current = safe(root.resolve(CURRENT_FILE)); + if (!Files.exists(current, LinkOption.NOFOLLOW_LINKS)) return root; + if (!Files.isRegularFile(current, LinkOption.NOFOLLOW_LINKS) || Files.size(current) > 64) + throw new IOException("HTTP client credential pointer is invalid"); + String name = Files.readString(current, StandardCharsets.US_ASCII); + if (!name.matches("[0-9a-f-]{36}")) throw new IOException("HTTP client credential pointer is invalid"); + Path generations = root.resolve(GENERATIONS_DIRECTORY); + Path generation = generations.resolve(name).normalize(); + if (!generation.getParent().equals(generations) || Files.isSymbolicLink(generation) || !Files.isDirectory(generation, LinkOption.NOFOLLOW_LINKS)) + throw new IOException("HTTP client credential generation is invalid"); + return generation; + } + + private static String readConnectionCodeDigest(Path directory) throws IOException { + Path digest = safe(directory.resolve(CONNECTION_CODE_DIGEST_FILE)); + if (!Files.exists(digest, LinkOption.NOFOLLOW_LINKS)) return null; + if (!Files.isRegularFile(digest, LinkOption.NOFOLLOW_LINKS) || Files.size(digest) != 64) + throw new IOException("HTTP connection-code marker is invalid"); + String value = Files.readString(digest, StandardCharsets.US_ASCII); + if (!value.matches("[0-9a-f]{64}")) throw new IOException("HTTP connection-code marker is invalid"); + return value; + } + + private static String connectionCodeDigest(HttpConnectionCode code) { + return connectionCodeDigest(code.encode()); + } + + private static String connectionCodeDigest(String encoded) { + return HttpTransportSecrets.sha256Hex(encoded.getBytes(StandardCharsets.US_ASCII)); + } + + private static void restoreConnectionCodeDigest(Path directory, String digest) throws IOException { + Path marker = safe(directory.resolve(CONNECTION_CODE_DIGEST_FILE)); + if (digest == null) { + Files.deleteIfExists(marker); + DurableFiles.forceDirectory(directory); + } else { + writePrivate(marker, digest.getBytes(StandardCharsets.US_ASCII)); + } + } + + private static void writePrivate(Path file, byte[] contents) throws IOException { + Path temporary = Files.createTempFile(file.getParent(), file.getFileName().toString(), ".tmp"); + try { + setOwnerOnly(temporary); + Files.write(temporary, contents, StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING); + DurableFiles.forceFile(temporary); + try { Files.move(temporary, file, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); } + catch (java.nio.file.AtomicMoveNotSupportedException unsupported) { Files.move(temporary, file, StandardCopyOption.REPLACE_EXISTING); } + setOwnerOnly(file); + DurableFiles.forceDirectory(file.getParent()); + } finally { Files.deleteIfExists(temporary); } + } + + private static void setOwnerOnly(Path path) throws IOException { + try { Files.setPosixFilePermissions(path, EnumSet.of(PosixFilePermission.OWNER_READ, PosixFilePermission.OWNER_WRITE)); } + catch (UnsupportedOperationException ignored) { } + } + + private static void setOwnerOnlyDirectory(Path path) throws IOException { + try { Files.setPosixFilePermissions(path, EnumSet.of(PosixFilePermission.OWNER_READ, + PosixFilePermission.OWNER_WRITE, PosixFilePermission.OWNER_EXECUTE)); } + catch (UnsupportedOperationException ignored) { } + } + + private static byte[] asciiBytes(char[] characters) { + byte[] output = new byte[characters.length]; + for (int index = 0; index < characters.length; index++) output[index] = (byte) characters[index]; + return output; + } +} diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpConnectionCode.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpConnectionCode.java new file mode 100644 index 0000000..872bf4b --- /dev/null +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpConnectionCode.java @@ -0,0 +1,112 @@ +package com.bencodez.simpleapi.servercomm.http; + +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.charset.StandardCharsets; +import java.time.Clock; +import java.time.Instant; +import java.util.Base64; +import java.util.Locale; + +/** + * A copy/paste connection code. It deliberately includes no long-lived credential: + * its only secret is a short-lived, single-use enrollment token. The trailing MAC is a + * corruption check keyed by that included token; it is not a proxy signature and cannot stop + * someone who can replace the whole code from replacing it with another valid code. + */ +public record HttpConnectionCode(String serverId, URI endpoint, String serverCertificatePin, String caCertificatePin, + Instant expiresAt, String enrollmentToken) { + private static final String LEGACY_VERSION = "VPH1"; + private static final String VERSION = "VPH2"; + private static final int MAX_CODE_LENGTH = 4096; + + public HttpConnectionCode { + serverId = HttpTlsIdentity.canonicalServerId(serverId); + endpoint = validateEndpoint(endpoint); + serverCertificatePin = validatePin(serverCertificatePin, "server certificate pin"); + caCertificatePin = validatePin(caCertificatePin, "CA certificate pin"); + if (expiresAt == null) throw new IllegalArgumentException("Expiry is required"); + enrollmentToken = validateToken(enrollmentToken); + } + + public String encode() { + String serverPart = Base64.getUrlEncoder().withoutPadding() + .encodeToString(serverId.getBytes(StandardCharsets.UTF_8)); + return encode(VERSION, serverPart); + } + + String encodeLegacy() { + return encode(LEGACY_VERSION, serverId); + } + + private String encode(String version, String serverPart) { + String endpointPart = Base64.getUrlEncoder().withoutPadding() + .encodeToString(endpoint.toASCIIString().getBytes(StandardCharsets.UTF_8)); + String unsigned = String.join(".", version, serverPart, endpointPart, serverCertificatePin, caCertificatePin, + Long.toString(expiresAt.getEpochSecond()), enrollmentToken); + byte[] token = Base64.getUrlDecoder().decode(enrollmentToken); + return unsigned + "." + HttpTransportSecrets.hmacSha256Url(token, unsigned); + } + + public boolean isActive(Clock clock) { + return expiresAt.isAfter(clock.instant()); + } + + public void requireActive(Clock clock) { + if (!isActive(clock)) throw new IllegalArgumentException("Connection code has expired"); + } + + public static HttpConnectionCode parse(String code) { + if (code == null || code.length() > MAX_CODE_LENGTH || code.indexOf('\n') >= 0 || code.indexOf('\r') >= 0) + throw new IllegalArgumentException("Connection code is invalid"); + String[] parts = code.split("\\.", -1); + if (parts.length != 8 || (!VERSION.equals(parts[0]) && !LEGACY_VERSION.equals(parts[0]))) + throw new IllegalArgumentException("Connection code is invalid"); + try { + String serverId = LEGACY_VERSION.equals(parts[0]) ? parts[1] + : new String(Base64.getUrlDecoder().decode(parts[1]), StandardCharsets.UTF_8); + String endpoint = new String(Base64.getUrlDecoder().decode(parts[2]), StandardCharsets.UTF_8); + String unsigned = String.join(".", parts[0], parts[1], parts[2], parts[3], parts[4], parts[5], parts[6]); + byte[] token = Base64.getUrlDecoder().decode(parts[6]); + String expected = HttpTransportSecrets.hmacSha256Url(token, unsigned); + if (!HttpTransportSecrets.constantTimeEquals(expected.getBytes(StandardCharsets.US_ASCII), + parts[7].getBytes(StandardCharsets.US_ASCII))) throw new IllegalArgumentException("Connection code is invalid"); + return new HttpConnectionCode(serverId, new URI(endpoint), parts[3], parts[4], Instant.ofEpochSecond(Long.parseLong(parts[5])), + parts[6]); + } catch (IllegalArgumentException | URISyntaxException failure) { + throw new IllegalArgumentException("Connection code is invalid", failure); + } + } + + private static URI validateEndpoint(URI value) { + if (value == null || !"https".equalsIgnoreCase(value.getScheme()) || value.getHost() == null + || value.getUserInfo() != null || value.getFragment() != null || value.getRawQuery() != null) + throw new IllegalArgumentException("Endpoint must be an absolute HTTPS URL without credentials or query"); + if (value.getPort() == 0 || value.getPort() > 65535 || value.getPort() < -1) + throw new IllegalArgumentException("Endpoint port is invalid"); + String path = value.getRawPath(); + if (path == null || path.isEmpty()) path = "/"; + if (!path.endsWith("/")) path += "/"; + try { + return new URI("https", null, value.getHost().toLowerCase(Locale.ROOT), value.getPort(), path, null, null); + } catch (URISyntaxException failure) { + throw new IllegalArgumentException("Endpoint is invalid", failure); + } + } + + private static String validatePin(String pin, String name) { + if (pin == null || !pin.matches("[0-9a-fA-F]{64}")) throw new IllegalArgumentException(name + " is invalid"); + return pin.toLowerCase(Locale.ROOT); + } + + private static String validateToken(String token) { + if (token == null || token.length() < 43 || token.length() > 128 || !token.matches("[A-Za-z0-9_-]+")) + throw new IllegalArgumentException("Enrollment token is invalid"); + try { + if (Base64.getUrlDecoder().decode(token).length < 32) throw new IllegalArgumentException("Enrollment token is invalid"); + return token; + } catch (IllegalArgumentException failure) { + throw new IllegalArgumentException("Enrollment token is invalid", failure); + } + } +} diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentAuthority.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentAuthority.java new file mode 100644 index 0000000..b223acd --- /dev/null +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentAuthority.java @@ -0,0 +1,212 @@ +package com.bencodez.simpleapi.servercomm.http; + +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.nio.file.attribute.PosixFilePermission; +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.Properties; +import java.util.Base64; +import java.util.EnumSet; +import com.bencodez.simpleapi.file.DurableFiles; + +/** + * Single-use enrollment tokens and client-certificate binding. Token material is never retained; + * only SHA-256 hashes are kept until expiry. This type is thread-safe. + */ +public final class HttpEnrollmentAuthority { + private static final Duration MAX_ENROLLMENT_LIFETIME = Duration.ofMinutes(15); + private final HttpTlsIdentity identity; + private final Clock clock; + private final Path stateFile; + private final Map enrollments = new HashMap<>(); + private final Map bindings = new HashMap<>(); + private final Set revokedCertificatePins = new HashSet<>(); + private boolean persistenceFailure; + + /** Creates a restart-safe authority. State contains only public certificate pins and revocations. */ + public HttpEnrollmentAuthority(HttpTlsIdentity identity, Path stateDirectory) throws java.io.IOException { + this(identity, Clock.systemUTC(), stateFile(stateDirectory)); + loadState(); + } + + HttpEnrollmentAuthority(HttpTlsIdentity identity, Clock clock) { + this(identity, clock, null); + } + + HttpEnrollmentAuthority(HttpTlsIdentity identity, Clock clock, Path stateFile) { + if (identity == null || clock == null) throw new IllegalArgumentException("Identity and clock are required"); + this.identity = identity; + this.clock = clock; + this.stateFile = stateFile; + } + + public synchronized HttpConnectionCode createConnectionCode(String serverId, URI endpoint, Duration lifetime) { + serverId = HttpTlsIdentity.canonicalServerId(serverId); + if (lifetime == null || lifetime.isNegative() || lifetime.isZero() || lifetime.compareTo(MAX_ENROLLMENT_LIFETIME) > 0) + throw new IllegalArgumentException("Enrollment lifetime must be between one second and fifteen minutes"); + expireEnrollments(); + Instant expiresAt = clock.instant().plus(lifetime); + String token = HttpTransportSecrets.randomToken(); + byte[] tokenHash = HttpTransportSecrets.sha256(token.getBytes(StandardCharsets.US_ASCII)); + String lookup = java.util.Base64.getUrlEncoder().withoutPadding().encodeToString(tokenHash); + enrollments.put(lookup, new Enrollment(tokenHash, expiresAt, serverId)); + return new HttpConnectionCode(serverId, endpoint, identity.serverCertificatePin(), identity.caCertificatePin(), expiresAt, token); + } + + public synchronized HttpTlsIdentity.IssuedClientCertificate enroll(String serverId, String enrollmentToken) throws Exception { + serverId = HttpTlsIdentity.canonicalServerId(serverId); + if (enrollmentToken == null || enrollmentToken.length() > 128) throw new IllegalArgumentException("Enrollment was rejected"); + expireEnrollments(); + byte[] suppliedHash = HttpTransportSecrets.sha256(enrollmentToken.getBytes(StandardCharsets.US_ASCII)); + String lookup = java.util.Base64.getUrlEncoder().withoutPadding().encodeToString(suppliedHash); + Enrollment enrollment = enrollments.get(lookup); + if (enrollment == null || !HttpTransportSecrets.constantTimeEquals(enrollment.tokenHash(), suppliedHash)) + throw new IllegalArgumentException("Enrollment was rejected"); + if (!serverId.equals(enrollment.serverId())) throw new IllegalArgumentException("Enrollment was rejected"); + enrollments.remove(lookup); // consume only after the token and its intended backend both match. + ClientBinding existing = bindings.get(serverId); + if (existing != null && !existing.revoked()) throw new IllegalStateException("Server id is already enrolled"); + HttpTlsIdentity.IssuedClientCertificate issued = identity.issueClientCertificate(serverId); + bindings.put(serverId, new ClientBinding(HttpTransportSecrets.certificatePin(issued.certificate()), null, false)); + try { persistState(); } + catch (java.io.IOException failure) { persistenceFailure = true; throw failure; } + return issued; + } + + public synchronized boolean authenticate(String serverId, java.security.cert.X509Certificate certificate) { + if (persistenceFailure || serverId == null || certificate == null) return false; + try { serverId = HttpTlsIdentity.canonicalServerId(serverId); } + catch (IllegalArgumentException invalid) { return false; } + if (!identity.validClientCertificate(serverId, certificate)) return false; + ClientBinding binding = bindings.get(serverId); + String pin = HttpTransportSecrets.certificatePin(certificate); + if (binding == null || binding.revoked() || revokedCertificatePins.contains(pin)) return false; + if (samePin(binding.certificatePin(), pin)) return true; + if (!samePin(binding.pendingCertificatePin(), pin)) return false; + bindings.put(serverId, new ClientBinding(pin, null, false)); + revokedCertificatePins.add(binding.certificatePin()); + try { persistState(); return true; } + catch (java.io.IOException failure) { persistenceFailure = true; return false; } + } + + /** Issues a replacement while the currently bound certificate is still valid. The old binding remains active + * until the replacement successfully authenticates, making a lost renewal response safe to retry. */ + public synchronized HttpTlsIdentity.IssuedClientCertificate renew(String serverId, + java.security.cert.X509Certificate currentCertificate) throws Exception { + if (!authenticate(serverId, currentCertificate)) throw new IllegalArgumentException("Certificate renewal was rejected"); + serverId = HttpTlsIdentity.canonicalServerId(serverId); + ClientBinding binding = bindings.get(serverId); + HttpTlsIdentity.IssuedClientCertificate issued = identity.issueClientCertificate(serverId); + bindings.put(serverId, new ClientBinding(binding.certificatePin(), + HttpTransportSecrets.certificatePin(issued.certificate()), false)); + try { persistState(); } + catch (java.io.IOException failure) { persistenceFailure = true; throw failure; } + return issued; + } + + public synchronized void revoke(String serverId) { + try { serverId = HttpTlsIdentity.canonicalServerId(serverId); } + catch (IllegalArgumentException invalid) { return; } + final String revokedServer = serverId; + enrollments.entrySet().removeIf(entry -> revokedServer.equals(entry.getValue().serverId())); + ClientBinding binding = bindings.get(serverId); + if (binding != null) { + bindings.put(serverId, new ClientBinding(binding.certificatePin(), binding.pendingCertificatePin(), true)); + revokedCertificatePins.add(binding.certificatePin()); + if (binding.pendingCertificatePin() != null) revokedCertificatePins.add(binding.pendingCertificatePin()); + try { persistState(); } + catch (java.io.IOException failure) { persistenceFailure = true; throw new IllegalStateException("Could not persist HTTP certificate revocation", failure); } + } + } + + private synchronized void loadState() throws java.io.IOException { + if (stateFile == null || !Files.exists(stateFile, LinkOption.NOFOLLOW_LINKS)) return; + if (!Files.isRegularFile(stateFile, LinkOption.NOFOLLOW_LINKS) || Files.size(stateFile) > 65536) + throw new java.io.IOException("HTTP enrollment state is invalid"); + Properties properties = new Properties(); + try (var input = Files.newInputStream(stateFile, LinkOption.NOFOLLOW_LINKS)) { properties.load(input); } + for (String key : properties.stringPropertyNames()) { + if (key.startsWith("binding.")) { + String serverId = new String(Base64.getUrlDecoder().decode(key.substring("binding.".length())), StandardCharsets.UTF_8); + serverId = HttpTlsIdentity.canonicalServerId(serverId); + String[] value = properties.getProperty(key, "").split(":", -1); + if (!((value.length == 2 && "1".equals(properties.getProperty("version"))) + || (value.length == 3 && "2".equals(properties.getProperty("version")))) + || !value[0].matches("[0-9a-f]{64}")) + throw new java.io.IOException("HTTP enrollment state is invalid"); + String pending = value.length == 3 && !"-".equals(value[1]) ? value[1] : null; + String revoked = value[value.length - 1]; + if ((pending != null && !pending.matches("[0-9a-f]{64}")) || !("0".equals(revoked) || "1".equals(revoked))) + throw new java.io.IOException("HTTP enrollment state is invalid"); + bindings.put(serverId, new ClientBinding(value[0], pending, "1".equals(revoked))); + if ("1".equals(revoked)) { revokedCertificatePins.add(value[0]); if (pending != null) revokedCertificatePins.add(pending); } + } else if (!"version".equals(key)) throw new java.io.IOException("HTTP enrollment state is invalid"); + } + if (!("1".equals(properties.getProperty("version")) || "2".equals(properties.getProperty("version")))) + throw new java.io.IOException("HTTP enrollment state is invalid"); + } + + private synchronized void persistState() throws java.io.IOException { + if (stateFile == null) return; + Properties properties = new Properties(); + properties.setProperty("version", "2"); + for (Map.Entry entry : bindings.entrySet()) { + String key = Base64.getUrlEncoder().withoutPadding().encodeToString(entry.getKey().getBytes(StandardCharsets.UTF_8)); + properties.setProperty("binding." + key, entry.getValue().certificatePin() + ":" + + (entry.getValue().pendingCertificatePin() == null ? "-" : entry.getValue().pendingCertificatePin()) + + ":" + (entry.getValue().revoked() ? "1" : "0")); + } + java.io.ByteArrayOutputStream bytes = new java.io.ByteArrayOutputStream(); + properties.store(bytes, "VotingPlugin HTTP certificate bindings"); + Path temporary = Files.createTempFile(stateFile.getParent(), stateFile.getFileName().toString(), ".tmp"); + try { + setOwnerOnly(temporary); + Files.write(temporary, bytes.toByteArray(), StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING); + DurableFiles.forceFile(temporary); + try { Files.move(temporary, stateFile, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); } + catch (java.nio.file.AtomicMoveNotSupportedException unsupported) { Files.move(temporary, stateFile, StandardCopyOption.REPLACE_EXISTING); } + setOwnerOnly(stateFile); + DurableFiles.forceDirectory(stateFile.getParent()); + } finally { Files.deleteIfExists(temporary); } + } + + private static Path stateFile(Path directory) throws java.io.IOException { + if (directory == null) throw new IllegalArgumentException("State directory is required"); + Files.createDirectories(directory); + Path file = directory.toAbsolutePath().normalize().resolve("http-transport-clients.properties"); + if (Files.isSymbolicLink(file)) throw new java.io.IOException("Refusing unsafe HTTP enrollment state path"); + return file; + } + + private static void setOwnerOnly(Path path) throws java.io.IOException { + try { Files.setPosixFilePermissions(path, EnumSet.of(PosixFilePermission.OWNER_READ, PosixFilePermission.OWNER_WRITE)); } + catch (UnsupportedOperationException ignored) { } + } + + private void expireEnrollments() { + Instant now = clock.instant(); + enrollments.entrySet().removeIf(entry -> !entry.getValue().expiresAt().isAfter(now)); + } + + private record Enrollment(byte[] tokenHash, Instant expiresAt, String serverId) { + private Enrollment { tokenHash = tokenHash.clone(); } + @Override public byte[] tokenHash() { return tokenHash.clone(); } + } + private static boolean samePin(String expected, String actual) { + return expected != null && actual != null && HttpTransportSecrets.constantTimeEquals( + expected.getBytes(StandardCharsets.US_ASCII), actual.getBytes(StandardCharsets.US_ASCII)); + } + + private record ClientBinding(String certificatePin, String pendingCertificatePin, boolean revoked) { } +} diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpInboundDeliveryStore.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpInboundDeliveryStore.java new file mode 100644 index 0000000..22022d2 --- /dev/null +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpInboundDeliveryStore.java @@ -0,0 +1,173 @@ +package com.bencodez.simpleapi.servercomm.http; + +import com.bencodez.simpleapi.file.DurableFiles; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.nio.file.attribute.PosixFilePermission; +import java.util.EnumSet; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.UUID; + +/** Crash-durable state for proxy deliveries around a non-transactional application callback. */ +final class HttpInboundDeliveryStore { + private static final String DIRECTORY = "http-transport-inbound-deliveries"; + private static final int MAX_ENTRIES = HttpTransportProtocol.MAX_QUEUE; + private final Path root; + private final Map entries = new LinkedHashMap<>(); + private boolean sealed; + + HttpInboundDeliveryStore(Path credentialDirectory) throws IOException { + Path credentials = credentialDirectory.toAbsolutePath().normalize(); + if (Files.isSymbolicLink(credentials) || !Files.isDirectory(credentials, LinkOption.NOFOLLOW_LINKS)) + throw new IOException("HTTP credential directory is unsafe"); + ownerOnlyDirectory(credentials); + root = credentials.resolve(DIRECTORY).normalize(); + if (!root.getParent().equals(credentials)) throw new IOException("HTTP inbound delivery directory is invalid"); + boolean created = false; + try { Files.createDirectory(root); created = true; } + catch (java.nio.file.FileAlreadyExistsException existing) { } + try { + requireRoot(); + ownerOnlyDirectory(root); + } finally { + if (created) DurableFiles.forceDirectory(credentials); + } + load(); + } + + synchronized State state(String id) { return entries.get(canonical(id)); } + + synchronized void reserve(String id) throws IOException { + requireWritable(); + id = canonical(id); + State existing = entries.get(id); + if (existing == State.RESERVED) return; + if (existing != null) throw new IOException("HTTP inbound delivery fence is already active"); + if (entries.size() >= MAX_ENTRIES) throw new IOException("HTTP inbound delivery fence is full"); + requireRoot(); + Path target = file(id, State.RESERVED); + if (Files.exists(target, LinkOption.NOFOLLOW_LINKS)) + throw new IOException("HTTP inbound delivery fence is inconsistent"); + Path temporary = Files.createTempFile(root, ".pending-", ".tmp"); + try { + ownerOnlyFile(temporary); + Files.writeString(temporary, id, StandardCharsets.US_ASCII, StandardOpenOption.TRUNCATE_EXISTING); + DurableFiles.forceFile(temporary); + move(temporary, target); + ownerOnlyFile(target); + DurableFiles.forceDirectory(root); + entries.put(id, State.RESERVED); + } finally { Files.deleteIfExists(temporary); } + } + + synchronized void markRunning(String id) throws IOException { transition(id, State.RESERVED, State.RUNNING); } + synchronized void markCompleted(String id) throws IOException { transition(id, State.RUNNING, State.COMPLETED); } + synchronized void seal() { sealed = true; } + + synchronized void remove(String id) throws IOException { + requireWritable(); + id = canonical(id); + State state = entries.get(id); + if (state == null) return; + requireRoot(); + DurableFiles.deleteIfExists(file(id, state)); + entries.remove(id); + } + + synchronized Map snapshot() { return Map.copyOf(entries); } + + private void transition(String id, State expected, State replacement) throws IOException { + requireWritable(); + id = canonical(id); + if (entries.get(id) != expected) throw new IOException("HTTP inbound delivery fence state is invalid"); + requireRoot(); + Path source = file(id, expected), target = file(id, replacement); + if (Files.isSymbolicLink(source) || !Files.isRegularFile(source, LinkOption.NOFOLLOW_LINKS) + || Files.exists(target, LinkOption.NOFOLLOW_LINKS)) + throw new IOException("HTTP inbound delivery fence state is unsafe"); + move(source, target); + DurableFiles.forceDirectory(root); + entries.put(id, replacement); + } + + private void load() throws IOException { + try (DirectoryStream files = Files.newDirectoryStream(root)) { + for (Path file : files) { + String name = file.getFileName().toString(); + if (name.startsWith(".pending-") && name.endsWith(".tmp") && !Files.isSymbolicLink(file) + && Files.isRegularFile(file, LinkOption.NOFOLLOW_LINKS)) { + DurableFiles.deleteIfExists(file); + continue; + } + State state = State.fromFileName(name); + if (state == null || Files.isSymbolicLink(file) || !Files.isRegularFile(file, LinkOption.NOFOLLOW_LINKS) + || Files.size(file) > 64L) + throw new IOException("HTTP inbound delivery fence contains an invalid entry"); + String id; + try { id = canonical(name.substring(0, name.length() - state.suffix.length())); } + catch (IllegalArgumentException invalid) { + throw new IOException("HTTP inbound delivery fence entry is invalid", invalid); + } + if (!name.equals(id + state.suffix) || !Files.readString(file, StandardCharsets.US_ASCII).equals(id)) + throw new IOException("HTTP inbound delivery fence entry is invalid"); + State existing = entries.get(id); + if (existing == null) entries.put(id, state); + else { + // A provider without atomic moves may expose both names after an + // interrupted transition. Preserve the furthest fail-closed state: + // RUNNING never replays, and COMPLETED alone may be acknowledged. + State retained = existing.ordinal() >= state.ordinal() ? existing : state; + State obsolete = retained == existing ? state : existing; + DurableFiles.deleteIfExists(file(id, obsolete)); + entries.put(id, retained); + } + if (entries.size() > MAX_ENTRIES) throw new IOException("HTTP inbound delivery fence exceeds its bound"); + } + } + } + + private Path file(String id, State state) { return root.resolve(id + state.suffix); } + private void requireWritable() throws IOException { + if (sealed) throw new IOException("HTTP inbound delivery store ownership has ended"); + } + private static void move(Path source, Path target) throws IOException { + try { Files.move(source, target, StandardCopyOption.ATOMIC_MOVE); } + catch (java.nio.file.AtomicMoveNotSupportedException unsupported) { Files.move(source, target); } + } + private static String canonical(String id) { + if (id == null) throw new IllegalArgumentException("HTTP delivery id is invalid"); + String canonical = UUID.fromString(id).toString(); + if (!canonical.equals(id)) throw new IllegalArgumentException("HTTP delivery id is not canonical"); + return canonical; + } + private void requireRoot() throws IOException { + if (Files.isSymbolicLink(root) || !Files.isDirectory(root, LinkOption.NOFOLLOW_LINKS)) + throw new IOException("HTTP inbound delivery directory is unsafe"); + } + private static void ownerOnlyFile(Path path) throws IOException { + try { Files.setPosixFilePermissions(path, EnumSet.of(PosixFilePermission.OWNER_READ, PosixFilePermission.OWNER_WRITE)); } + catch (UnsupportedOperationException ignored) { } + } + private static void ownerOnlyDirectory(Path path) throws IOException { + try { Files.setPosixFilePermissions(path, EnumSet.of(PosixFilePermission.OWNER_READ, + PosixFilePermission.OWNER_WRITE, PosixFilePermission.OWNER_EXECUTE)); } + catch (UnsupportedOperationException ignored) { } + } + + enum State { + RESERVED(".reserved"), RUNNING(".running"), COMPLETED(".completed"); + private final String suffix; + State(String suffix) { this.suffix = suffix; } + private static State fromFileName(String name) { + for (State state : values()) if (name.endsWith(state.suffix)) return state; + return null; + } + } +} diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpPinnedTls.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpPinnedTls.java new file mode 100644 index 0000000..4240897 --- /dev/null +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpPinnedTls.java @@ -0,0 +1,103 @@ +package com.bencodez.simpleapi.servercomm.http; + +import java.security.cert.X509Certificate; +import java.security.KeyStore; +import javax.net.ssl.KeyManagerFactory; +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLParameters; +import javax.net.ssl.TrustManager; +import javax.net.ssl.X509TrustManager; + +/** Builds the backend TLS context. A public CA store is intentionally not consulted. */ +public final class HttpPinnedTls { + private HttpPinnedTls() { } + + public static SSLContext clientContext(HttpConnectionCode code) throws Exception { + if (code == null) throw new IllegalArgumentException("Connection code is required"); + SSLContext context = SSLContext.getInstance("TLS"); + context.init(null, new TrustManager[] { new PinnedServerTrustManager(code.serverCertificatePin(), code.caCertificatePin()) }, null); + return context; + } + + /** + * Normal transport context: presents the enrolled client certificate and trusts only the + * pinned private authority. Callers must not override the HttpClient default endpoint-identification + * settings; hostname verification remains enabled and the proxy leaf may renew under the same CA. + */ + public static SSLContext mutualTlsContext(HttpConnectionCode code, HttpClientCredentialStore.ClientCredential credential) + throws Exception { + if (code == null || credential == null || credential.privateKey() == null || credential.certificate() == null || credential.caCertificate() == null) + throw new IllegalArgumentException("Enrolled client credential is required"); + char[] password = credential.password(); + try { + String authorityPin = HttpTransportSecrets.certificatePin(credential.caCertificate()); + if (!HttpTransportSecrets.constantTimeEquals(code.caCertificatePin().getBytes(java.nio.charset.StandardCharsets.US_ASCII), + authorityPin.getBytes(java.nio.charset.StandardCharsets.US_ASCII))) + throw new IllegalArgumentException("HTTP authority does not match connection code"); + KeyStore store = KeyStore.getInstance("PKCS12"); + store.load(null, new char[0]); + store.setKeyEntry("client", credential.privateKey(), password, + new java.security.cert.Certificate[] { credential.certificate(), credential.caCertificate() }); + KeyManagerFactory managers = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); + managers.init(store, password); + KeyStore trustStore = KeyStore.getInstance(KeyStore.getDefaultType()); + trustStore.load(null, new char[0]); + trustStore.setCertificateEntry("http-transport-ca", credential.caCertificate()); + javax.net.ssl.TrustManagerFactory trusts = javax.net.ssl.TrustManagerFactory.getInstance( + javax.net.ssl.TrustManagerFactory.getDefaultAlgorithm()); + trusts.init(trustStore); + SSLContext context = SSLContext.getInstance("TLS"); + context.init(managers.getKeyManagers(), trusts.getTrustManagers(), null); + return context; + } finally { java.util.Arrays.fill(password, '\0'); } + } + + public static boolean matchesServerPin(HttpConnectionCode code, X509Certificate certificate) { + if (code == null || certificate == null) return false; + String actual = HttpTransportSecrets.certificatePin(certificate); + return HttpTransportSecrets.constantTimeEquals(code.serverCertificatePin() + .getBytes(java.nio.charset.StandardCharsets.US_ASCII), actual.getBytes(java.nio.charset.StandardCharsets.US_ASCII)); + } + + /** TLS 1.3 is used where the runtime exposes it; hostname verification is deliberately left enabled. */ + public static SSLParameters secureParameters(SSLContext context) { + SSLParameters parameters = context.getDefaultSSLParameters(); + for (String protocol : context.getSupportedSSLParameters().getProtocols()) { + if ("TLSv1.3".equals(protocol)) { + parameters.setProtocols(new String[] { "TLSv1.3" }); + break; + } + } + return parameters; + } + + private static final class PinnedServerTrustManager implements X509TrustManager { + private final String expectedPin; + private final String expectedCaPin; + private PinnedServerTrustManager(String expectedPin, String expectedCaPin) { + this.expectedPin = expectedPin; + this.expectedCaPin = expectedCaPin; + } + @Override public void checkClientTrusted(X509Certificate[] chain, String authType) { throw new UnsupportedOperationException(); } + @Override public void checkServerTrusted(X509Certificate[] chain, String authType) throws java.security.cert.CertificateException { + if (chain == null || chain.length < 2) throw new java.security.cert.CertificateException("Server certificate chain is incomplete"); + chain[0].checkValidity(); + chain[chain.length - 1].checkValidity(); + try { chain[0].verify(chain[chain.length - 1].getPublicKey()); } + catch (java.security.GeneralSecurityException invalid) { + throw new java.security.cert.CertificateException("Server certificate signature is invalid", invalid); + } + if (chain[chain.length - 1].getBasicConstraints() < 0) + throw new java.security.cert.CertificateException("Server certificate authority is invalid"); + String actual = HttpTransportSecrets.certificatePin(chain[0]); + if (!HttpTransportSecrets.constantTimeEquals(expectedPin.getBytes(java.nio.charset.StandardCharsets.US_ASCII), + actual.getBytes(java.nio.charset.StandardCharsets.US_ASCII))) + throw new java.security.cert.CertificateException("Server certificate pin does not match"); + String issuer = HttpTransportSecrets.certificatePin(chain[chain.length - 1]); + if (!HttpTransportSecrets.constantTimeEquals(expectedCaPin.getBytes(java.nio.charset.StandardCharsets.US_ASCII), + issuer.getBytes(java.nio.charset.StandardCharsets.US_ASCII))) + throw new java.security.cert.CertificateException("Server certificate authority pin does not match"); + } + @Override public X509Certificate[] getAcceptedIssuers() { return new X509Certificate[0]; } + } +} diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java new file mode 100644 index 0000000..cc45cc4 --- /dev/null +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java @@ -0,0 +1,499 @@ +package com.bencodez.simpleapi.servercomm.http; + +import com.bencodez.simpleapi.servercomm.codec.JsonEnvelope; +import com.bencodez.simpleapi.file.DurableFiles; +import com.sun.net.httpserver.Headers; +import com.sun.net.httpserver.HttpsConfigurator; +import com.sun.net.httpserver.HttpsExchange; +import com.sun.net.httpserver.HttpsParameters; +import com.sun.net.httpserver.HttpsServer; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.InetSocketAddress; +import java.net.URI; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.security.cert.Certificate; +import java.security.cert.X509Certificate; +import java.time.Duration; +import java.util.ArrayDeque; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Semaphore; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; +import java.util.function.LongSupplier; +import javax.net.ssl.SSLParameters; +import javax.net.ssl.SSLPeerUnverifiedException; + +/** + * One HTTPS listener for enrollment and the backend-to-proxy long-poll transport. + * Every normal request is certificate-authenticated in the handler, rather than relying on TLS WANT auth. + */ +public final class HttpProxyTransportServer implements AutoCloseable { + static { + // JDK HttpServer reads these once when its internal server configuration is initialized. + // Set conservative process-wide bounds before this transport creates its listener. + setDefault("sun.net.httpserver.maxReqTime", "10"); + setDefault("sun.net.httpserver.maxRspTime", "10"); + setDefault("jdk.httpserver.maxConnections", "144"); + setDefault("sun.net.httpserver.maxReqHeaders", "32"); + setDefault("sun.net.httpserver.maxReqHeaderSize", "16384"); + } + // Keep an idle request open long enough to reuse the TLS connection, but bound backend-origin + // latency when a message is queued immediately after the request body has already been sent. + public static final Duration LONG_POLL = Duration.ofSeconds(2); + private final HttpTlsIdentity identity; + private final HttpEnrollmentAuthority authority; + private final HttpsServer server; + private final ThreadPoolExecutor listenerExecutor; + private final ThreadPoolExecutor handlerExecutor; + private final Semaphore admission = new Semaphore(64); + private final Map backends = new HashMap<>(); + private final DurableOutgoingQueue durableOutgoing; + private final Consumer onEnvelope; + private final DeliveryAcknowledgement onAcknowledged; + private volatile boolean closed; + + public HttpProxyTransportServer(InetSocketAddress bind, HttpTlsIdentity identity, HttpEnrollmentAuthority authority, + Consumer onEnvelope) throws Exception { + this(bind, identity, authority, null, onEnvelope, (serverId, deliveryId) -> { }); + } + + public HttpProxyTransportServer(InetSocketAddress bind, HttpTlsIdentity identity, HttpEnrollmentAuthority authority, + Path outgoingDirectory, Consumer onEnvelope) throws Exception { + this(bind, identity, authority, outgoingDirectory, onEnvelope, (serverId, deliveryId) -> { }); + } + + public HttpProxyTransportServer(InetSocketAddress bind, HttpTlsIdentity identity, HttpEnrollmentAuthority authority, + Path outgoingDirectory, Consumer onEnvelope, + DeliveryAcknowledgement onAcknowledged) throws Exception { + if (bind == null || identity == null || authority == null || onEnvelope == null || onAcknowledged == null) + throw new IllegalArgumentException("HTTP transport configuration is required"); + this.identity = identity; this.authority = authority; this.onEnvelope = onEnvelope; + this.onAcknowledged = onAcknowledged; + durableOutgoing = outgoingDirectory == null ? null : new DurableOutgoingQueue(outgoingDirectory); + if (durableOutgoing != null) for (Map.Entry> pending + : durableOutgoing.load().entrySet()) { + BackendState state = new BackendState(pending.getKey(), durableOutgoing, onAcknowledged); + state.restore(pending.getValue()); + backends.put(pending.getKey(), state); + } + server = HttpsServer.create(bind, 32); + server.setHttpsConfigurator(new HttpsConfigurator(identity.serverContext()) { + @Override public void configure(HttpsParameters parameters) { + SSLParameters ssl = HttpPinnedTls.secureParameters(getSSLContext()); + ssl.setWantClientAuth(true); parameters.setSSLParameters(ssl); + } + }); + // Long polls are blocking by design. Capacity is bounded by admission, while enough workers + // remain available for all admitted polls plus setup requests. + listenerExecutor = executor("SimpleAPI-HTTP-listener", 72, 72); + // The proxy router mutates shared presence, vote, and reward state. A separate + // bounded FIFO lane keeps wire order without blocking long-poll workers. + handlerExecutor = executor("SimpleAPI-HTTP-handler", 1, HttpBackendTransportConnector.CALLBACK_QUEUE_CAPACITY); + server.setExecutor(listenerExecutor); + server.createContext("/v1/enroll", exchange -> enroll((HttpsExchange) exchange)); + server.createContext("/v1/renew", exchange -> renew((HttpsExchange) exchange)); + server.createContext("/v1/transport", exchange -> transport((HttpsExchange) exchange)); + } + + private void renew(HttpsExchange exchange) throws IOException { + if (!"/v1/renew".equals(exchange.getRequestURI().getPath()) || exchange.getRequestURI().getRawQuery() != null) { reply(exchange, 404, new byte[0]); return; } + if (!"POST".equals(exchange.getRequestMethod())) { reply(exchange, 405, new byte[0]); return; } + if (!json(exchange)) { reply(exchange, 415, new byte[0]); return; } + if (!boundedFixedBody(exchange, 1024) || !admission.tryAcquire()) { reply(exchange, 429, new byte[0]); return; } + try { + String serverId = HttpTransportProtocol.parseRenewal(read(exchange.getRequestBody(), 1024)); + X509Certificate certificate = peerCertificate(exchange); + if (certificate == null || !authority.authenticate(serverId, certificate)) { reply(exchange, 401, new byte[0]); return; } + HttpTlsIdentity.IssuedClientCertificate issued = authority.renew(serverId, certificate); + reply(exchange, 201, HttpTransportProtocol.enrollmentResponse(issued)); + } catch (IllegalArgumentException rejected) { reply(exchange, 403, new byte[0]); + } catch (Exception failure) { reply(exchange, 503, new byte[0]); + } finally { admission.release(); } + } + + public void start() { if (closed) throw new IllegalStateException("HTTP transport is closed"); server.start(); } + public int port() { return server.getAddress().getPort(); } + public URI endpoint(String host) { return URI.create("https://" + host + ":" + port() + "/"); } + + /** Queues a proxy-origin envelope durably before reporting acceptance. */ + public boolean send(String serverId, JsonEnvelope envelope) { + return send(serverId, UUID.randomUUID().toString(), envelope); + } + + /** Queues a proxy-origin envelope with a stable, caller-persisted delivery ID. */ + public boolean send(String serverId, String deliveryId, JsonEnvelope envelope) { + if (closed || serverId == null || envelope == null) return false; + try { + serverId = HttpTlsIdentity.canonicalServerId(serverId); + HttpTransportProtocol.validId(deliveryId); + HttpTransportProtocol.validateEnvelope(envelope); + } + catch (IllegalArgumentException invalid) { return false; } + BackendState backend; + final String canonicalServerId = serverId; + synchronized (backends) { backend = backends.computeIfAbsent(serverId, + ignored -> new BackendState(canonicalServerId, durableOutgoing, onAcknowledged)); } + return backend.enqueue(new HttpTransportProtocol.Delivery(deliveryId, envelope)); + } + + @Override public void close() { + if (closed) return; closed = true; server.stop(1); + shutdown(handlerExecutor); shutdown(listenerExecutor); + synchronized (backends) { for (BackendState backend : backends.values()) backend.signal(); backends.clear(); } + } + + private void enroll(HttpsExchange exchange) throws IOException { + if (!"/v1/enroll".equals(exchange.getRequestURI().getPath()) || exchange.getRequestURI().getRawQuery() != null) { reply(exchange, 404, new byte[0]); return; } + if (!"POST".equals(exchange.getRequestMethod())) { reply(exchange, 405, new byte[0]); return; } + if (!json(exchange)) { reply(exchange, 415, new byte[0]); return; } + if (!boundedFixedBody(exchange, 8192) || !admission.tryAcquire()) { reply(exchange, 429, new byte[0]); return; } + try { + HttpTransportProtocol.Enrollment request = HttpTransportProtocol.parseEnrollment(read(exchange.getRequestBody(), 8192)); + HttpTlsIdentity.IssuedClientCertificate issued = authority.enroll(request.server(), request.token()); + reply(exchange, 201, HttpTransportProtocol.enrollmentResponse(issued)); + } catch (Exception rejected) { reply(exchange, 403, new byte[0]); } + finally { admission.release(); } + } + + private void transport(HttpsExchange exchange) throws IOException { + if (!"/v1/transport".equals(exchange.getRequestURI().getPath()) || exchange.getRequestURI().getRawQuery() != null) { reply(exchange, 404, new byte[0]); return; } + if (!"POST".equals(exchange.getRequestMethod())) { reply(exchange, 405, new byte[0]); return; } + if (!json(exchange)) { reply(exchange, 415, new byte[0]); return; } + if (!boundedFixedBody(exchange, HttpTransportProtocol.MAX_BODY_BYTES) || !admission.tryAcquire()) { reply(exchange, 429, new byte[0]); return; } + try { + HttpTransportProtocol.Packet packet = HttpTransportProtocol.parsePacket(read(exchange.getRequestBody(), HttpTransportProtocol.MAX_BODY_BYTES)); + X509Certificate certificate = peerCertificate(exchange); + if (certificate == null || !authority.authenticate(packet.server(), certificate)) { reply(exchange, 401, new byte[0]); return; } + BackendState backend; + synchronized (backends) { backend = backends.computeIfAbsent(packet.server(), + ignored -> new BackendState(packet.server(), durableOutgoing, onAcknowledged)); } + if (!backend.beginPoll(packet.session())) { reply(exchange, 409, new byte[0]); return; } + try { + handlePacket(packet, backend); + Response response = backend.await(packet.server(), packet.session(), packet.sequence()); + reply(exchange, 200, HttpTransportProtocol.response(packet.server(), packet.session(), packet.sequence(), response.acks(), response.messages())); + } finally { backend.endPoll(); } + } catch (IllegalArgumentException rejected) { reply(exchange, 400, new byte[0]); + } catch (Exception failure) { reply(exchange, 503, new byte[0]); + } finally { admission.release(); } + } + + private void handlePacket(HttpTransportProtocol.Packet packet, BackendState backend) throws IOException { + List accepted; + synchronized (backend) { + if (!backend.allowRequest()) throw new IllegalArgumentException("transport rate limited"); + if (!backend.acceptSession(packet.session(), packet.sequence())) throw new IllegalArgumentException("stale session request"); + } + backend.acknowledge(packet.acks()); + synchronized (backend) { accepted = backend.acceptIncoming(packet.messages()); } + for (HttpTransportProtocol.Delivery delivery : accepted) dispatch(packet.server(), backend, delivery); + } + private void dispatch(String serverId, BackendState backend, HttpTransportProtocol.Delivery delivery) { + Runnable callback = () -> { + boolean success = false; + try { onEnvelope.accept(new ReceivedEnvelope(serverId, delivery.id(), normalizeBackendIdentity(serverId, delivery.envelope()))); success = true; } + catch (RuntimeException ignored) { } + synchronized (backend) { backend.completeIncoming(delivery.id(), success); } + }; + if (!HttpBackendTransportConnector.executeOrdered(handlerExecutor, callback)) + synchronized (backend) { backend.completeIncoming(delivery.id(), false); } + } + private static JsonEnvelope normalizeBackendIdentity(String serverId, JsonEnvelope envelope) { + // The authenticated TLS identity is authoritative; never forward a forged `server` field. + return envelope.toBuilder().put("server", serverId).build(); + } + private static X509Certificate peerCertificate(HttpsExchange exchange) { + try { Certificate[] peer = exchange.getSSLSession().getPeerCertificates(); + return peer.length > 0 && peer[0] instanceof X509Certificate certificate ? certificate : null; + } catch (SSLPeerUnverifiedException absent) { return null; } + } + private static byte[] read(InputStream input, int maximum) throws IOException { + ByteArrayOutputStream output = new ByteArrayOutputStream(); byte[] buffer = new byte[4096]; int total = 0, read; + while ((read = input.read(buffer)) >= 0) { total += read; if (total > maximum) throw new IllegalArgumentException("HTTP body is too large"); output.write(buffer, 0, read); } + return output.toByteArray(); + } + private static void reply(HttpsExchange exchange, int status, byte[] body) throws IOException { + Headers headers = exchange.getResponseHeaders(); headers.set("Cache-Control", "no-store"); headers.set("Content-Type", "application/json; charset=utf-8"); + exchange.sendResponseHeaders(status, body.length); try (var output = exchange.getResponseBody()) { output.write(body); } + } + private static boolean json(HttpsExchange exchange) { + String contentType = exchange.getRequestHeaders().getFirst("Content-Type"); + return contentType != null && contentType.toLowerCase(java.util.Locale.ROOT).matches("application/json(?:\\s*;.*)?"); + } + private static boolean boundedFixedBody(HttpsExchange exchange, int maximum) { + if (exchange.getRequestHeaders().getFirst("Transfer-Encoding") != null) return false; + String value = exchange.getRequestHeaders().getFirst("Content-Length"); + try { long length = Long.parseLong(value); return length > 0L && length <= maximum; } + catch (RuntimeException invalid) { return false; } + } + private static ThreadPoolExecutor executor(String name, int threads, int queue) { + ThreadFactory factory = task -> { Thread thread = new Thread(task, name); thread.setDaemon(true); return thread; }; + return new ThreadPoolExecutor(threads, threads, 0L, TimeUnit.MILLISECONDS, new ArrayBlockingQueue<>(queue), factory, new ThreadPoolExecutor.AbortPolicy()); + } + private static void setDefault(String name, String value) { if (System.getProperty(name) == null) System.setProperty(name, value); } + private static void shutdown(ExecutorService executor) { executor.shutdown(); try { if (!executor.awaitTermination(5, TimeUnit.SECONDS)) executor.shutdownNow(); } catch (InterruptedException interrupted) { Thread.currentThread().interrupt(); executor.shutdownNow(); } } + + public record ReceivedEnvelope(String serverId, String messageId, JsonEnvelope envelope) { } + + @FunctionalInterface + public interface DeliveryAcknowledgement { + void confirm(String serverId, String deliveryId) throws IOException; + } + static record Response(Collection acks, Collection messages) { } + static final class BackendState { + private final String serverId; + private final DurableOutgoingQueue durableOutgoing; + private final DeliveryAcknowledgement onAcknowledged; + private final LongSupplier nanoTime; + private String session; private long sequence = -1L; + private final LinkedHashMap outgoing = new LinkedHashMap<>(); + private final Set seen = new LinkedHashSet<>(); private final Set processing = new LinkedHashSet<>(); + private final ArrayDeque acknowledgements = new ArrayDeque<>(); + private final Map deliveredAtNanos = new HashMap<>(); + private double requestTokens = 24.0d; + private long lastTokenNanos = System.nanoTime(); + private boolean activePoll; + BackendState() { this(null, null, (serverId, deliveryId) -> { }, System::nanoTime); } + BackendState(LongSupplier nanoTime) { this(null, null, (serverId, deliveryId) -> { }, nanoTime); } + private BackendState(String serverId, DurableOutgoingQueue durableOutgoing) { + this(serverId, durableOutgoing, (ignoredServer, ignoredDelivery) -> { }, System::nanoTime); + } + BackendState(String serverId, DurableOutgoingQueue durableOutgoing, + DeliveryAcknowledgement onAcknowledged) { + this(serverId, durableOutgoing, onAcknowledged, System::nanoTime); + } + private BackendState(String serverId, DurableOutgoingQueue durableOutgoing, + DeliveryAcknowledgement onAcknowledged, LongSupplier nanoTime) { + this.serverId = serverId; this.durableOutgoing = durableOutgoing; + this.onAcknowledged = onAcknowledged; this.nanoTime = nanoTime; + } + private synchronized void restore(Collection deliveries) { + for (HttpTransportProtocol.Delivery delivery : deliveries) outgoing.put(delivery.id(), delivery); + } + private boolean beginPoll(String requestedSession) { synchronized (this) { if (activePoll) return false; activePoll = true; return true; } } + private void endPoll() { synchronized (this) { activePoll = false; notifyAll(); } } + private boolean allowRequest() { + long now = System.nanoTime(); requestTokens = Math.min(24.0d, requestTokens + ((now - lastTokenNanos) / 1_000_000_000.0d) * 2.0d); + lastTokenNanos = now; if (requestTokens < 1.0d) return false; requestTokens -= 1.0d; return true; + } + boolean acceptSession(String requested, long requestedSequence) { + if (!requested.equals(session)) { session = requested; sequence = -1L; deliveredAtNanos.clear(); } + // The connector allocates a fresh monotonic sequence for every attempt. Rejecting equality + // prevents a captured request from being replayed with altered ACKs or a new payload. + if (requestedSequence <= sequence) return false; sequence = requestedSequence; return true; + } + synchronized boolean enqueue(HttpTransportProtocol.Delivery delivery) { + HttpTransportProtocol.Delivery existing = outgoing.get(delivery.id()); + if (existing != null) return Arrays.equals(HttpTransportProtocol.storedDelivery(existing), + HttpTransportProtocol.storedDelivery(delivery)); + if (outgoing.size() >= HttpTransportProtocol.MAX_QUEUE) return false; + if (durableOutgoing != null) try { durableOutgoing.persist(serverId, delivery); } + catch (IOException failure) { return false; } + outgoing.put(delivery.id(), delivery); signal(); return true; + } + void acknowledge(Collection acks) throws IOException { + for (String id : acks) { + synchronized (this) { if (!outgoing.containsKey(id)) continue; } + onAcknowledged.confirm(serverId, id); + synchronized (this) { + if (!outgoing.containsKey(id)) continue; + // A 200 response is the backend's proof that its durable replay fence may + // be deleted. Never return success while the proxy delivery still exists. + if (durableOutgoing != null) durableOutgoing.remove(serverId, id); + outgoing.remove(id); deliveredAtNanos.remove(id); + } + } + } + List acceptIncoming(List received) { + List accepted = new java.util.ArrayList<>(); + for (HttpTransportProtocol.Delivery delivery : received) { + if (seen.contains(delivery.id())) { queueAck(delivery.id()); continue; } + if (!processing.contains(delivery.id())) { + processing.add(delivery.id()); accepted.add(delivery); + } + } + return accepted; + } + synchronized void completeIncoming(String id, boolean success) { processing.remove(id); if (success) { seen.add(id); while (seen.size() > HttpTransportProtocol.MAX_QUEUE) seen.remove(seen.iterator().next()); queueAck(id); signal(); } } + private void queueAck(String id) { if (acknowledgements.size() < HttpTransportProtocol.MAX_QUEUE && !acknowledgements.contains(id)) acknowledgements.add(id); } + synchronized Response await(String serverId, String requestedSession, long requestedSequence) { + long deadline = System.nanoTime() + LONG_POLL.toNanos(); + while (acknowledgements.isEmpty() && !hasUndelivered()) { + long retryRemaining = nanosUntilRedelivery(nanoTime.getAsLong()); + if (retryRemaining <= 0L) break; + long requestRemaining = deadline - System.nanoTime(); if (requestRemaining <= 0L) break; + long wait = Math.min(requestRemaining, retryRemaining); + try { TimeUnit.NANOSECONDS.timedWait(this, wait); } catch (InterruptedException interrupted) { Thread.currentThread().interrupt(); break; } + } + List acks = new java.util.ArrayList<>(); while (!acknowledgements.isEmpty() && acks.size() < HttpTransportProtocol.MAX_BATCH) acks.add(acknowledgements.remove()); + List candidates = new java.util.ArrayList<>(); + long now = nanoTime.getAsLong(); + if (hasUndelivered() || redeliveryDue(now)) for (HttpTransportProtocol.Delivery delivery : outgoing.values()) { + if (!deliveredAtNanos.containsKey(delivery.id()) || redeliveryDue(delivery.id(), now)) candidates.add(delivery); + if (candidates.size() == HttpTransportProtocol.MAX_BATCH) break; + } + List messages = HttpTransportProtocol.fittingMessages(serverId, requestedSession, + requestedSequence, acks, candidates); + long deliveredAt = nanoTime.getAsLong(); + for (HttpTransportProtocol.Delivery delivery : messages) deliveredAtNanos.put(delivery.id(), deliveredAt); + return new Response(acks, messages); + } + private boolean hasUndelivered() { for (String id : outgoing.keySet()) if (!deliveredAtNanos.containsKey(id)) return true; return false; } + private boolean redeliveryDue(long now) { + for (String id : outgoing.keySet()) if (redeliveryDue(id, now)) return true; + return false; + } + private boolean redeliveryDue(String id, long now) { + Long deliveredAt = deliveredAtNanos.get(id); + return deliveredAt != null && now - deliveredAt >= LONG_POLL.toNanos(); + } + private long nanosUntilRedelivery(long now) { + long remaining = Long.MAX_VALUE; + for (String id : outgoing.keySet()) { + Long deliveredAt = deliveredAtNanos.get(id); + if (deliveredAt == null) continue; + long candidate = LONG_POLL.toNanos() - (now - deliveredAt); + if (candidate <= 0L) return 0L; + remaining = Math.min(remaining, candidate); + } + return remaining; + } + private synchronized void signal() { notifyAll(); } + } + + private static final class DurableOutgoingQueue { + private static final String FILE_PATTERN = "[0-9]{20}-[0-9a-f-]{36}\\.json"; + private final Path root; + private final Map> files = new HashMap<>(); + private long sequence; + + private DurableOutgoingQueue(Path root) throws IOException { + this.root = root.toAbsolutePath().normalize(); + boolean created = false; + try { Files.createDirectory(this.root); created = true; } + catch (java.nio.file.FileAlreadyExistsException existing) { } + try { + if (Files.isSymbolicLink(this.root) || !Files.isDirectory(this.root, LinkOption.NOFOLLOW_LINKS)) + throw new IOException("HTTP outgoing queue directory is invalid"); + ownerOnlyDirectory(this.root); + } finally { + if (created) DurableFiles.forceDirectory(this.root.getParent()); + } + } + + private synchronized Map> load() throws IOException { + Map> loaded = new LinkedHashMap<>(); + try (DirectoryStream servers = Files.newDirectoryStream(root)) { + for (Path directory : servers) { + if (Files.isSymbolicLink(directory) || !Files.isDirectory(directory, LinkOption.NOFOLLOW_LINKS)) + throw new IOException("HTTP outgoing queue contains an invalid entry"); + String serverId; + try { serverId = HttpTlsIdentity.canonicalServerId(directory.getFileName().toString()); } + catch (IllegalArgumentException invalid) { throw new IOException("HTTP outgoing queue server is invalid", invalid); } + if (!serverId.equals(directory.getFileName().toString())) + throw new IOException("HTTP outgoing queue server is not canonical"); + List entries = new java.util.ArrayList<>(); + try (DirectoryStream messages = Files.newDirectoryStream(directory)) { + for (Path message : messages) entries.add(message); + } + entries.sort(java.util.Comparator.comparing(path -> path.getFileName().toString())); + List deliveries = new java.util.ArrayList<>(); + Map serverFiles = files.computeIfAbsent(serverId, ignored -> new HashMap<>()); + for (Path message : entries) { + String name = message.getFileName().toString(); + if (name.startsWith(".pending-") && name.endsWith(".tmp") + && !Files.isSymbolicLink(message) && Files.isRegularFile(message, LinkOption.NOFOLLOW_LINKS)) { + DurableFiles.deleteIfExists(message); + continue; + } + if (Files.isSymbolicLink(message) || !Files.isRegularFile(message, LinkOption.NOFOLLOW_LINKS) + || !name.matches(FILE_PATTERN) || Files.size(message) > HttpTransportProtocol.MAX_ENVELOPE_BYTES * 2L) + throw new IOException("HTTP outgoing queue message is invalid"); + HttpTransportProtocol.Delivery delivery; + try { delivery = HttpTransportProtocol.parseStoredDelivery(Files.readAllBytes(message)); } + catch (IllegalArgumentException invalid) { throw new IOException("HTTP outgoing queue message is invalid", invalid); } + if (!name.endsWith("-" + delivery.id() + ".json") || serverFiles.put(delivery.id(), message) != null) + throw new IOException("HTTP outgoing queue message id is invalid"); + deliveries.add(delivery); + if (deliveries.size() > HttpTransportProtocol.MAX_QUEUE) + throw new IOException("HTTP outgoing queue exceeds its bound"); + sequence = Math.max(sequence, Long.parseLong(name.substring(0, 20))); + } + if (!deliveries.isEmpty()) loaded.put(serverId, deliveries); + } + } + return loaded; + } + + private synchronized void persist(String serverId, HttpTransportProtocol.Delivery delivery) throws IOException { + Path directory = root.resolve(serverId).normalize(); + if (!directory.getParent().equals(root)) throw new IOException("HTTP outgoing queue server is invalid"); + boolean created = false; + try { Files.createDirectory(directory); created = true; } + catch (java.nio.file.FileAlreadyExistsException existing) { } + try { + if (Files.isSymbolicLink(directory) || !Files.isDirectory(directory, LinkOption.NOFOLLOW_LINKS)) + throw new IOException("HTTP outgoing queue server directory is invalid"); + ownerOnlyDirectory(directory); + } finally { + // The child fsync below cannot make this newly published name durable in + // its parent. Persist the root entry before accepting the first message. + if (created) DurableFiles.forceDirectory(root); + } + if (sequence == Long.MAX_VALUE) throw new IOException("HTTP outgoing queue sequence is exhausted"); + String name = String.format(java.util.Locale.ROOT, "%020d-%s.json", ++sequence, delivery.id()); + Path target = directory.resolve(name); + Path temporary = Files.createTempFile(directory, ".pending-", ".tmp"); + try { + ownerOnlyFile(temporary); + Files.write(temporary, HttpTransportProtocol.storedDelivery(delivery), StandardOpenOption.TRUNCATE_EXISTING); + DurableFiles.forceFile(temporary); + try { Files.move(temporary, target, StandardCopyOption.ATOMIC_MOVE); } + catch (java.nio.file.AtomicMoveNotSupportedException unsupported) { Files.move(temporary, target); } + ownerOnlyFile(target); DurableFiles.forceDirectory(directory); + files.computeIfAbsent(serverId, ignored -> new HashMap<>()).put(delivery.id(), target); + } finally { Files.deleteIfExists(temporary); } + } + + private synchronized void remove(String serverId, String id) throws IOException { + Map serverFiles = files.get(serverId); + Path file = serverFiles == null ? null : serverFiles.get(id); + if (file == null) throw new IOException("HTTP outgoing queue acknowledgement is unknown"); + DurableFiles.deleteIfExists(file); + serverFiles.remove(id); + } + + private static void ownerOnlyFile(Path path) throws IOException { + try { Files.setPosixFilePermissions(path, java.util.EnumSet.of( + java.nio.file.attribute.PosixFilePermission.OWNER_READ, + java.nio.file.attribute.PosixFilePermission.OWNER_WRITE)); } + catch (UnsupportedOperationException ignored) { } + } + private static void ownerOnlyDirectory(Path path) throws IOException { + try { Files.setPosixFilePermissions(path, java.util.EnumSet.of( + java.nio.file.attribute.PosixFilePermission.OWNER_READ, + java.nio.file.attribute.PosixFilePermission.OWNER_WRITE, + java.nio.file.attribute.PosixFilePermission.OWNER_EXECUTE)); } + catch (UnsupportedOperationException ignored) { } + } + } +} diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpTlsIdentity.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpTlsIdentity.java new file mode 100644 index 0000000..edc21f6 --- /dev/null +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpTlsIdentity.java @@ -0,0 +1,467 @@ +package com.bencodez.simpleapi.servercomm.http; + +import java.io.IOException; +import java.io.OutputStream; +import java.math.BigInteger; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.nio.file.attribute.PosixFilePermission; +import java.security.KeyPair; +import java.security.KeyPairGenerator; +import java.security.KeyStore; +import java.security.PrivateKey; +import java.security.Principal; +import java.security.Security; +import java.security.cert.Certificate; +import java.security.cert.X509Certificate; +import java.time.Instant; +import java.time.Clock; +import java.time.Duration; +import java.util.Date; +import java.util.EnumSet; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.net.Socket; +import javax.net.ssl.KeyManager; +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLEngine; +import javax.net.ssl.TrustManager; +import javax.net.ssl.TrustManagerFactory; +import javax.net.ssl.X509ExtendedKeyManager; +import org.bouncycastle.asn1.x500.X500Name; +import org.bouncycastle.asn1.x509.BasicConstraints; +import org.bouncycastle.asn1.x509.Extension; +import org.bouncycastle.asn1.x509.GeneralName; +import org.bouncycastle.asn1.x509.GeneralNames; +import org.bouncycastle.asn1.x509.KeyUsage; +import org.bouncycastle.asn1.x509.ExtendedKeyUsage; +import org.bouncycastle.asn1.x509.KeyPurposeId; +import org.bouncycastle.cert.X509CertificateHolder; +import org.bouncycastle.cert.X509v3CertificateBuilder; +import org.bouncycastle.cert.jcajce.JcaX509CertificateConverter; +import org.bouncycastle.cert.jcajce.JcaX509v3CertificateBuilder; +import org.bouncycastle.jce.provider.BouncyCastleProvider; +import org.bouncycastle.operator.ContentSigner; +import org.bouncycastle.operator.jcajce.JcaContentSignerBuilder; +import com.bencodez.simpleapi.file.DurableFiles; + +/** Durable private CA plus server identity used by the proxy HTTP listener. */ +public final class HttpTlsIdentity { + private static final String CA_FILE = "http-transport-ca.p12"; + private static final String SERVER_FILE = "http-transport-server.p12"; + private static final String PASSWORD_FILE = "http-transport-password"; + private static final String INITIALIZING_FILE = "http-transport-initializing"; + private static final String ENROLLMENT_STATE_FILE = "http-transport-clients.properties"; + private static final String OUTGOING_DIRECTORY = "outgoing-v1"; + private static final char[] EMPTY_PASSWORD = new char[0]; + static final Duration RENEW_BEFORE = Duration.ofDays(30); + static final Duration CA_RENEW_BEFORE = Duration.ofDays(365); + private final PrivateKey caKey; + private volatile X509Certificate caCertificate; + private volatile PrivateKey serverKey; + private volatile X509Certificate serverCertificate; + private final char[] password; + private final Path caFile; + private final Path serverFile; + private final String advertisedHost; + + private HttpTlsIdentity(PrivateKey caKey, X509Certificate caCertificate, PrivateKey serverKey, + X509Certificate serverCertificate, char[] password, Path caFile, Path serverFile, String advertisedHost) { + this.caKey = caKey; + this.caCertificate = caCertificate; + this.serverKey = serverKey; + this.serverCertificate = serverCertificate; + this.password = password.clone(); + this.caFile = caFile; + this.serverFile = serverFile; + this.advertisedHost = advertisedHost; + } + + public static HttpTlsIdentity loadOrCreate(Path directory, String advertisedHost) throws Exception { + return loadOrCreate(directory, advertisedHost, Clock.systemUTC()); + } + + static HttpTlsIdentity loadOrCreate(Path directory, String advertisedHost, Clock clock) throws Exception { + if (advertisedHost == null || advertisedHost.isBlank() || advertisedHost.length() > 253) + throw new IllegalArgumentException("Advertised HTTPS host is invalid"); + if (clock == null) throw new IllegalArgumentException("Clock is required"); + Path identityDirectory = directory.toAbsolutePath().normalize(); + boolean created = !Files.exists(identityDirectory, LinkOption.NOFOLLOW_LINKS); + Files.createDirectories(identityDirectory); + // The identity files cannot make the newly created directory entry durable. + // Persist its parent before the TLS identity is returned for listener use. + if (created) DurableFiles.forceDirectory(identityDirectory.getParent()); + directory = identityDirectory; + Path caFile = safe(directory.resolve(CA_FILE)); + Path serverFile = safe(directory.resolve(SERVER_FILE)); + Path passwordFile = safe(directory.resolve(PASSWORD_FILE)); + Path initializingFile = safe(directory.resolve(INITIALIZING_FILE)); + boolean caExists = Files.exists(caFile, LinkOption.NOFOLLOW_LINKS); + boolean serverExists = Files.exists(serverFile, LinkOption.NOFOLLOW_LINKS); + boolean passwordExists = Files.exists(passwordFile, LinkOption.NOFOLLOW_LINKS); + boolean initializing = Files.exists(initializingFile, LinkOption.NOFOLLOW_LINKS); + boolean anyIdentityFile = caExists || serverExists || passwordExists; + boolean completeIdentity = caExists && serverExists && passwordExists; + boolean persistentTransportState = hasPersistentTransportState(directory); + if (initializing || (anyIdentityFile && !completeIdentity)) { + if (persistentTransportState) + throw new IOException("HTTP TLS identity files are incomplete"); + if (!initializing) writeInitializationMarker(initializingFile); + discardUncommittedIdentity(caFile, serverFile, passwordFile); + caExists = false; + serverExists = false; + passwordExists = false; + initializing = true; + } + if (caExists || serverExists || passwordExists) { + if (!completeIdentity) throw new IOException("HTTP TLS identity files are incomplete"); + char[] password = readPassword(passwordFile); + try { + KeyStore ca = load(caFile, password); + KeyStore server = load(serverFile, password); + PrivateKey caKey = (PrivateKey) ca.getKey("ca", password); + X509Certificate caCertificate = (X509Certificate) ca.getCertificate("ca"); + PrivateKey serverKey = (PrivateKey) server.getKey("server", password); + X509Certificate serverCertificate = (X509Certificate) server.getCertificate("server"); + if (caKey == null || caCertificate == null || serverKey == null || serverCertificate == null) + throw new IOException("HTTP TLS identity files are invalid"); + boolean caRenewed = needsCaRenewal(caCertificate, clock); + if (caRenewed) { + ensureBouncyCastle(); + KeyPair caPair = new KeyPair(caCertificate.getPublicKey(), caKey); + caCertificate = certificate("CN=VotingPlugin HTTP private CA", caPair, null, null, + CertificateRole.CA, null, clock.instant()); + ca = KeyStore.getInstance("PKCS12"); + ca.load(null, EMPTY_PASSWORD); + ca.setKeyEntry("ca", caKey, password, new Certificate[] { caCertificate }); + writeStore(caFile, ca, password); + } + if (caRenewed || !hasServerName(serverCertificate, advertisedHost) || needsRenewal(serverCertificate, clock)) { + ensureBouncyCastle(); + KeyPair serverPair = keyPair(); + serverCertificate = certificate("CN=" + certificateName(advertisedHost), serverPair, caCertificate, caKey, + CertificateRole.SERVER, advertisedHost, clock.instant()); + serverKey = serverPair.getPrivate(); + server = KeyStore.getInstance("PKCS12"); + server.load(null, EMPTY_PASSWORD); + server.setKeyEntry("server", serverKey, password, new Certificate[] { serverCertificate, caCertificate }); + writeStore(serverFile, server, password); + } + return new HttpTlsIdentity(caKey, caCertificate, serverKey, serverCertificate, password, caFile, serverFile, + advertisedHost); + } finally { Arrays.fill(password, '\0'); } + } + if (persistentTransportState) + throw new IOException("HTTP TLS identity files are missing"); + if (!initializing) writeInitializationMarker(initializingFile); + ensureBouncyCastle(); + char[] password = HttpTransportSecrets.randomToken().toCharArray(); + try { + KeyPair caPair = keyPair(); + X509Certificate caCertificate = certificate("CN=VotingPlugin HTTP private CA", caPair, null, null, CertificateRole.CA, null, + clock.instant()); + KeyPair serverPair = keyPair(); + X509Certificate serverCertificate = certificate("CN=" + certificateName(advertisedHost), serverPair, caCertificate, + caPair.getPrivate(), CertificateRole.SERVER, advertisedHost, clock.instant()); + KeyStore ca = KeyStore.getInstance("PKCS12"); + ca.load(null, EMPTY_PASSWORD); + ca.setKeyEntry("ca", caPair.getPrivate(), password, new Certificate[] { caCertificate }); + KeyStore server = KeyStore.getInstance("PKCS12"); + server.load(null, EMPTY_PASSWORD); + server.setKeyEntry("server", serverPair.getPrivate(), password, new Certificate[] { serverCertificate, caCertificate }); + writeStore(caFile, ca, password); + writeStore(serverFile, server, password); + byte[] passwordBytes = asciiBytes(password); + try { writePrivate(passwordFile, passwordBytes); } + finally { Arrays.fill(passwordBytes, (byte) 0); } + DurableFiles.deleteIfExists(initializingFile); + return new HttpTlsIdentity(caPair.getPrivate(), caCertificate, serverPair.getPrivate(), serverCertificate, password, + caFile, serverFile, advertisedHost); + } finally { Arrays.fill(password, '\0'); } + } + + public String serverCertificatePin() { + refreshIdentity(); + return HttpTransportSecrets.certificatePin(serverCertificate); + } + public String caCertificatePin() { refreshIdentity(); return HttpTransportSecrets.certificatePin(caCertificate); } + public X509Certificate caCertificate() { return caCertificate; } + public X509Certificate serverCertificate() { return serverCertificate; } + + /** + * The listener requests an optional client certificate so enrollment can share the same port. + * Any certificate that is presented must chain to this transport's private CA; normal requests + * additionally validate the certificate's persisted backend binding in the HTTP handler. + */ + public SSLContext serverContext() throws Exception { + renewIdentityIfNeeded(); + SSLContext context = SSLContext.getInstance("TLS"); + context.init(new KeyManager[] { new RotatingServerKeyManager() }, trustManagers(caCertificate), null); + return context; + } + + static TrustManager[] trustManagers(X509Certificate caCertificate) throws Exception { + KeyStore trustStore = KeyStore.getInstance(KeyStore.getDefaultType()); + trustStore.load(null, EMPTY_PASSWORD); + trustStore.setCertificateEntry("http-transport-ca", caCertificate); + TrustManagerFactory factory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); + factory.init(trustStore); + return factory.getTrustManagers(); + } + + private void refreshIdentity() { + try { renewIdentityIfNeeded(); } + catch (Exception failure) { throw new IllegalStateException("Could not renew HTTP TLS identity", failure); } + } + + private synchronized void renewIdentityIfNeeded() throws Exception { + Clock clock = Clock.systemUTC(); + boolean renewCa = needsCaRenewal(caCertificate, clock); + if (!renewCa && !needsRenewal(serverCertificate, clock)) return; + ensureBouncyCastle(); + X509Certificate replacementCa = caCertificate; + if (renewCa) { + KeyPair caPair = new KeyPair(caCertificate.getPublicKey(), caKey); + replacementCa = certificate("CN=VotingPlugin HTTP private CA", caPair, null, null, + CertificateRole.CA, null, clock.instant()); + KeyStore caStore = KeyStore.getInstance("PKCS12"); + caStore.load(null, EMPTY_PASSWORD); + caStore.setKeyEntry("ca", caKey, password, new Certificate[] { replacementCa }); + writeStore(caFile, caStore, password); + } + KeyPair pair = keyPair(); + X509Certificate replacement = certificate("CN=" + certificateName(advertisedHost), pair, replacementCa, caKey, + CertificateRole.SERVER, advertisedHost, clock.instant()); + KeyStore store = KeyStore.getInstance("PKCS12"); + store.load(null, EMPTY_PASSWORD); + store.setKeyEntry("server", pair.getPrivate(), password, new Certificate[] { replacement, replacementCa }); + writeStore(serverFile, store, password); + caCertificate = replacementCa; + serverKey = pair.getPrivate(); + serverCertificate = replacement; + } + + public IssuedClientCertificate issueClientCertificate(String serverId) throws Exception { + return issueClientCertificate(serverId, Instant.now()); + } + + IssuedClientCertificate issueClientCertificate(String serverId, Instant issuedAt) throws Exception { + serverId = canonicalServerId(serverId); + if (issuedAt == null) throw new IllegalArgumentException("Certificate issuance time is required"); + ensureBouncyCastle(); + KeyPair pair = keyPair(); + X509Certificate certificate = certificate("CN=" + serverId, pair, caCertificate, caKey, CertificateRole.CLIENT, + "urn:votingplugin:http-backend:" + serverId, issuedAt); + char[] clientPassword = HttpTransportSecrets.randomToken().toCharArray(); + try { + KeyStore store = KeyStore.getInstance("PKCS12"); + store.load(null, EMPTY_PASSWORD); + store.setKeyEntry("client", pair.getPrivate(), clientPassword, new Certificate[] { certificate, caCertificate }); + java.io.ByteArrayOutputStream bytes = new java.io.ByteArrayOutputStream(); + store.store(bytes, clientPassword); + return new IssuedClientCertificate(serverId, certificate, bytes.toByteArray(), clientPassword); + } finally { Arrays.fill(clientPassword, '\0'); } + } + + public static String canonicalServerId(String serverId) { + if (serverId == null || !serverId.matches("[A-Za-z0-9][A-Za-z0-9._-]{0,63}")) + throw new IllegalArgumentException("Server id is invalid"); + return serverId.toLowerCase(java.util.Locale.ROOT); + } + + public boolean issuedByThisCa(X509Certificate certificate) { + if (certificate == null) return false; + try { + certificate.checkValidity(); + certificate.verify(caCertificate.getPublicKey()); + return true; + } catch (Exception failure) { + return false; + } + } + + public boolean validClientCertificate(String expectedServerId, X509Certificate certificate) { + if (!issuedByThisCa(certificate)) return false; + try { + List usage = certificate.getExtendedKeyUsage(); + boolean[] keyUsage = certificate.getKeyUsage(); + if (usage == null || !usage.contains(KeyPurposeId.id_kp_clientAuth.getId()) || keyUsage == null || !keyUsage[0]) return false; + String expectedUri = "urn:votingplugin:http-backend:" + canonicalServerId(expectedServerId); + Collection> names = certificate.getSubjectAlternativeNames(); + if (names == null) return false; + for (List name : names) { + if (name.size() == 2 && Integer.valueOf(GeneralName.uniformResourceIdentifier).equals(name.get(0)) + && expectedUri.equals(name.get(1))) return true; + } + return false; + } catch (Exception failure) { return false; } + } + + public record IssuedClientCertificate(String serverId, X509Certificate certificate, byte[] pkcs12, char[] password) { + public IssuedClientCertificate { + pkcs12 = pkcs12.clone(); + password = password.clone(); + } + @Override public byte[] pkcs12() { return pkcs12.clone(); } + @Override public char[] password() { return password.clone(); } + } + + private static KeyPair keyPair() throws Exception { + KeyPairGenerator generator = KeyPairGenerator.getInstance("EC"); + generator.initialize(new java.security.spec.ECGenParameterSpec("secp256r1")); + return generator.generateKeyPair(); + } + + private static X509Certificate certificate(String subject, KeyPair subjectKey, X509Certificate issuer, PrivateKey issuerKey, + CertificateRole role, String subjectAlternativeName, Instant now) throws Exception { + X500Name issuerName = issuer == null ? new X500Name(subject) : new X500Name(issuer.getSubjectX500Principal().getName()); + X509v3CertificateBuilder builder = new JcaX509v3CertificateBuilder(issuerName, + new BigInteger(160, new java.security.SecureRandom()).setBit(159), Date.from(now.minusSeconds(300)), + Date.from(now.plusSeconds(role == CertificateRole.CA ? 315360000L : 31536000L)), new X500Name(subject), subjectKey.getPublic()); + builder.addExtension(Extension.basicConstraints, true, new BasicConstraints(role == CertificateRole.CA)); + builder.addExtension(Extension.keyUsage, true, new KeyUsage(role == CertificateRole.CA ? KeyUsage.keyCertSign | KeyUsage.cRLSign + : KeyUsage.digitalSignature)); + if (role == CertificateRole.SERVER) builder.addExtension(Extension.extendedKeyUsage, false, + new ExtendedKeyUsage(KeyPurposeId.id_kp_serverAuth)); + if (role == CertificateRole.CLIENT) builder.addExtension(Extension.extendedKeyUsage, false, + new ExtendedKeyUsage(KeyPurposeId.id_kp_clientAuth)); + if (role == CertificateRole.SERVER && subjectAlternativeName != null) { + GeneralName name; + if (subjectAlternativeName.matches("(?:\\d{1,3}\\.){3}\\d{1,3}") || subjectAlternativeName.indexOf(':') >= 0) + name = new GeneralName(GeneralName.iPAddress, subjectAlternativeName); + else name = new GeneralName(GeneralName.dNSName, subjectAlternativeName); + builder.addExtension(Extension.subjectAlternativeName, false, new GeneralNames(name)); + } + if (role == CertificateRole.CLIENT) builder.addExtension(Extension.subjectAlternativeName, false, + new GeneralNames(new GeneralName(GeneralName.uniformResourceIdentifier, subjectAlternativeName))); + ContentSigner signer = new JcaContentSignerBuilder("SHA256withECDSA").setProvider("BC") + .build(issuerKey == null ? subjectKey.getPrivate() : issuerKey); + X509CertificateHolder holder = builder.build(signer); + return new JcaX509CertificateConverter().setProvider("BC").getCertificate(holder); + } + + static boolean needsRenewal(X509Certificate certificate, Clock clock) { + return certificate == null || !certificate.getNotAfter().toInstant().isAfter(clock.instant().plus(RENEW_BEFORE)); + } + + static boolean needsCaRenewal(X509Certificate certificate, Clock clock) { + return certificate == null || !certificate.getNotAfter().toInstant().isAfter(clock.instant().plus(CA_RENEW_BEFORE)); + } + + private static void ensureBouncyCastle() { + if (Security.getProvider("BC") == null) Security.addProvider(new BouncyCastleProvider()); + } + + private static String certificateName(String host) { + return host.replaceAll("[^A-Za-z0-9 ._-]", "_"); + } + + private static boolean hasServerName(X509Certificate certificate, String advertisedHost) { + try { + Collection> names = certificate.getSubjectAlternativeNames(); + if (names == null) return false; + for (List name : names) { + if (name.size() != 2 || !(name.get(1) instanceof String value)) continue; + if ((Integer.valueOf(GeneralName.dNSName).equals(name.get(0)) || Integer.valueOf(GeneralName.iPAddress).equals(name.get(0))) + && advertisedHost.equalsIgnoreCase(value)) return true; + } + return false; + } catch (Exception failure) { return false; } + } + + private static Path safe(Path file) throws IOException { + Path parent = file.toAbsolutePath().normalize().getParent(); + if (parent == null || Files.isSymbolicLink(file)) throw new IOException("Refusing unsafe HTTP TLS identity path"); + return file.toAbsolutePath().normalize(); + } + + private static KeyStore load(Path path, char[] password) throws Exception { + KeyStore store = KeyStore.getInstance("PKCS12"); + try (var input = Files.newInputStream(path, LinkOption.NOFOLLOW_LINKS)) { store.load(input, password); } + return store; + } + + private static char[] readPassword(Path path) throws IOException { + byte[] bytes = Files.readAllBytes(path); + if (bytes.length < 40 || bytes.length > 128) throw new IOException("HTTP TLS password file is invalid"); + try { return new String(bytes, java.nio.charset.StandardCharsets.US_ASCII).toCharArray(); } + finally { Arrays.fill(bytes, (byte) 0); } + } + + private static void writeStore(Path file, KeyStore store, char[] password) throws Exception { + java.io.ByteArrayOutputStream bytes = new java.io.ByteArrayOutputStream(); + store.store(bytes, password); + byte[] contents = bytes.toByteArray(); + try { writePrivate(file, contents); } + finally { Arrays.fill(contents, (byte) 0); } + } + + private static void writePrivate(Path file, byte[] contents) throws IOException { + Path temporary = Files.createTempFile(file.getParent(), file.getFileName().toString(), ".tmp"); + try { + setOwnerOnly(temporary); + Files.write(temporary, contents, StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING); + DurableFiles.forceFile(temporary); + try { Files.move(temporary, file, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); } + catch (java.nio.file.AtomicMoveNotSupportedException unsupported) { Files.move(temporary, file, StandardCopyOption.REPLACE_EXISTING); } + setOwnerOnly(file); + DurableFiles.forceDirectory(file.getParent()); + } finally { Files.deleteIfExists(temporary); } + } + + private static void writeInitializationMarker(Path file) throws IOException { + writePrivate(file, "initializing\n".getBytes(java.nio.charset.StandardCharsets.US_ASCII)); + } + + private static void discardUncommittedIdentity(Path caFile, Path serverFile, Path passwordFile) throws IOException { + DurableFiles.deleteIfExists(caFile); + DurableFiles.deleteIfExists(serverFile); + DurableFiles.deleteIfExists(passwordFile); + } + + private static boolean hasPersistentTransportState(Path directory) { + return Files.exists(directory.resolve(ENROLLMENT_STATE_FILE), LinkOption.NOFOLLOW_LINKS) + || Files.exists(directory.resolve(OUTGOING_DIRECTORY), LinkOption.NOFOLLOW_LINKS); + } + + private static byte[] asciiBytes(char[] characters) { + byte[] output = new byte[characters.length]; + for (int index = 0; index < characters.length; index++) output[index] = (byte) characters[index]; + return output; + } + + private static void setOwnerOnly(Path path) throws IOException { + try { Files.setPosixFilePermissions(path, EnumSet.of(PosixFilePermission.OWNER_READ, PosixFilePermission.OWNER_WRITE)); } + catch (UnsupportedOperationException ignored) { /* Windows ACLs are inherited; never make the file world-readable. */ } + } + + private final class RotatingServerKeyManager extends X509ExtendedKeyManager { + private static final String ALIAS = "server"; + private void refresh() { + refreshIdentity(); + } + private String alias(String keyType) { + refresh(); + return keyType != null && ("EC".equalsIgnoreCase(keyType) || keyType.toUpperCase(java.util.Locale.ROOT).startsWith("EC_")) + ? ALIAS : null; + } + @Override public String[] getClientAliases(String keyType, Principal[] issuers) { return null; } + @Override public String chooseClientAlias(String[] keyTypes, Principal[] issuers, Socket socket) { return null; } + @Override public String[] getServerAliases(String keyType, Principal[] issuers) { + return alias(keyType) == null ? null : new String[] { ALIAS }; + } + @Override public String chooseServerAlias(String keyType, Principal[] issuers, Socket socket) { return alias(keyType); } + @Override public String chooseEngineServerAlias(String keyType, Principal[] issuers, SSLEngine engine) { return alias(keyType); } + @Override public X509Certificate[] getCertificateChain(String alias) { + refresh(); + return ALIAS.equals(alias) ? new X509Certificate[] { serverCertificate, caCertificate } : null; + } + @Override public PrivateKey getPrivateKey(String alias) { refresh(); return ALIAS.equals(alias) ? serverKey : null; } + } + + private enum CertificateRole { CA, SERVER, CLIENT } +} diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpTransportProtocol.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpTransportProtocol.java new file mode 100644 index 0000000..d6030e3 --- /dev/null +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpTransportProtocol.java @@ -0,0 +1,231 @@ +package com.bencodez.simpleapi.servercomm.http; + +import com.bencodez.simpleapi.servercomm.codec.JsonEnvelope; +import com.bencodez.simpleapi.servercomm.codec.JsonEnvelopeCodec; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Base64; +import java.util.Collection; +import java.util.List; +import java.util.UUID; + +/** Strict, versioned HTTP transport envelope. Payloads remain canonical JsonEnvelopeCodec values. */ +final class HttpTransportProtocol { + static final int VERSION = 1; + static final int MAX_BODY_BYTES = 256 * 1024; + static final int MAX_BATCH = 64; + static final int MAX_ENVELOPE_BYTES = 48 * 1024; + static final int MAX_QUEUE = 1024; + static final long MAX_CLOCK_SKEW_MILLIS = 90_000L; + + private HttpTransportProtocol() { } + + static void validateEnvelope(JsonEnvelope envelope) { + if (envelope == null || JsonEnvelopeCodec.encode(envelope).getBytes(StandardCharsets.UTF_8).length > MAX_ENVELOPE_BYTES) throw bad(); + } + + static byte[] request(String server, String session, long sequence, Collection acks, + Collection messages) { + JsonObject root = base(server, session, sequence); + root.add("acks", ids(acks)); + root.add("messages", messages(messages)); + byte[] encoded = root.toString().getBytes(StandardCharsets.UTF_8); + if (encoded.length > MAX_BODY_BYTES) throw bad(); + return encoded; + } + + static List fittingMessages(String server, String session, long sequence, Collection acks, + Collection candidates) { + List output = new ArrayList<>(); + for (Delivery candidate : candidates) { + if (output.size() == MAX_BATCH) break; + output.add(candidate); + try { request(server, session, sequence, acks, output); } + catch (IllegalArgumentException tooLarge) { output.remove(output.size() - 1); break; } + } + return output; + } + + static byte[] storedDelivery(Delivery delivery) { + validId(delivery.id()); + validateEnvelope(delivery.envelope()); + JsonObject root = new JsonObject(); + root.addProperty("v", VERSION); + root.addProperty("id", delivery.id()); + root.addProperty("payload", Base64.getUrlEncoder().withoutPadding().encodeToString( + JsonEnvelopeCodec.encode(delivery.envelope()).getBytes(StandardCharsets.UTF_8))); + return root.toString().getBytes(StandardCharsets.UTF_8); + } + + static Delivery parseStoredDelivery(byte[] body) { + if (body == null || body.length == 0 || body.length > MAX_ENVELOPE_BYTES * 2) throw bad(); + try { + JsonObject root = JsonParser.parseString(new String(body, StandardCharsets.UTF_8)).getAsJsonObject(); + requireOnly(root, "v", "id", "payload"); + if (integer(root, "v") != VERSION) throw bad(); + String id = string(root, "id", 64); validId(id); + byte[] payload = Base64.getUrlDecoder().decode(string(root, "payload", MAX_ENVELOPE_BYTES * 2)); + if (payload.length == 0 || payload.length > MAX_ENVELOPE_BYTES) throw bad(); + return new Delivery(id, JsonEnvelopeCodec.decode(new String(payload, StandardCharsets.UTF_8))); + } catch (RuntimeException invalid) { throw bad(); } + } + + static byte[] response(String server, String session, long sequence, Collection acks, + Collection messages) { + return request(server, session, sequence, acks, messages); + } + + static Packet parsePacket(byte[] body) { + if (body == null || body.length == 0 || body.length > MAX_BODY_BYTES) throw bad(); + try { + JsonElement parsed = JsonParser.parseString(new String(body, StandardCharsets.UTF_8)); + if (!parsed.isJsonObject()) throw bad(); + JsonObject root = parsed.getAsJsonObject(); + requireOnly(root, "v", "server", "session", "sequence", "timestamp", "acks", "messages"); + if (integer(root, "v") != VERSION) throw bad(); + String server = HttpTlsIdentity.canonicalServerId(string(root, "server", 64)); + String session = uuid(root, "session"); + long sequence = nonNegative(root, "sequence"); + long timestamp = integer(root, "timestamp"); + long now = Instant.now().toEpochMilli(); + if (timestamp < now - MAX_CLOCK_SKEW_MILLIS || timestamp > now + MAX_CLOCK_SKEW_MILLIS) throw bad(); + List acks = parseIds(root.get("acks")); + List messages = parseMessages(root.get("messages")); + return new Packet(server, session, sequence, acks, messages); + } catch (RuntimeException invalid) { throw bad(); } + } + + static byte[] enrollmentResponse(HttpTlsIdentity.IssuedClientCertificate certificate) { + JsonObject output = new JsonObject(); + byte[] bundle = certificate.pkcs12(); + try { output.addProperty("bundle", Base64.getUrlEncoder().withoutPadding().encodeToString(bundle)); } + finally { java.util.Arrays.fill(bundle, (byte) 0); } + char[] password = certificate.password(); + try { output.addProperty("password", new String(password)); } + finally { java.util.Arrays.fill(password, '\0'); } + return output.toString().getBytes(StandardCharsets.UTF_8); + } + + static Enrollment parseEnrollment(byte[] body) { + if (body == null || body.length == 0 || body.length > 8192) throw bad(); + try { + JsonElement parsed = JsonParser.parseString(new String(body, StandardCharsets.UTF_8)); + if (!parsed.isJsonObject()) throw bad(); + JsonObject root = parsed.getAsJsonObject(); + requireOnly(root, "server", "token"); + String server = HttpTlsIdentity.canonicalServerId(string(root, "server", 64)); + String token = string(root, "token", 128); + if (!token.matches("[A-Za-z0-9_-]{43,128}")) throw bad(); + return new Enrollment(server, token); + } catch (RuntimeException invalid) { throw bad(); } + } + + static byte[] renewalRequest(String server) { + JsonObject root = new JsonObject(); + root.addProperty("server", HttpTlsIdentity.canonicalServerId(server)); + return root.toString().getBytes(StandardCharsets.UTF_8); + } + + static String parseRenewal(byte[] body) { + if (body == null || body.length == 0 || body.length > 1024) throw bad(); + try { + JsonElement parsed = JsonParser.parseString(new String(body, StandardCharsets.UTF_8)); + if (!parsed.isJsonObject()) throw bad(); + JsonObject root = parsed.getAsJsonObject(); + requireOnly(root, "server"); + return HttpTlsIdentity.canonicalServerId(string(root, "server", 64)); + } catch (RuntimeException invalid) { throw bad(); } + } + + static HttpTlsIdentity.IssuedClientCertificate parseEnrollmentResponse(String server, byte[] body) { + if (body == null || body.length == 0 || body.length > MAX_BODY_BYTES) throw bad(); + try { + JsonObject root = JsonParser.parseString(new String(body, StandardCharsets.UTF_8)).getAsJsonObject(); + requireOnly(root, "bundle", "password"); + byte[] bundle = Base64.getUrlDecoder().decode(string(root, "bundle", MAX_BODY_BYTES * 2)); + char[] password = string(root, "password", 128).toCharArray(); + if (bundle.length == 0 || password.length < 40) throw bad(); + try { return new HttpTlsIdentity.IssuedClientCertificate(HttpTlsIdentity.canonicalServerId(server), null, bundle, password); } + finally { java.util.Arrays.fill(bundle, (byte) 0); java.util.Arrays.fill(password, '\0'); } + } catch (RuntimeException invalid) { throw bad(); } + } + + private static JsonObject base(String server, String session, long sequence) { + JsonObject root = new JsonObject(); + root.addProperty("v", VERSION); root.addProperty("server", server); root.addProperty("session", session); + root.addProperty("sequence", sequence); root.addProperty("timestamp", Instant.now().toEpochMilli()); + return root; + } + private static JsonArray ids(Collection values) { + if (values == null || values.size() > MAX_BATCH) throw bad(); + JsonArray output = new JsonArray(); + for (String value : values) { validId(value); output.add(value); } + return output; + } + private static JsonArray messages(Collection values) { + if (values == null || values.size() > MAX_BATCH) throw bad(); + JsonArray output = new JsonArray(); + for (Delivery delivery : values) { + validId(delivery.id()); + String encoded = JsonEnvelopeCodec.encode(delivery.envelope()); + byte[] bytes = encoded.getBytes(StandardCharsets.UTF_8); + if (bytes.length > MAX_ENVELOPE_BYTES) throw bad(); + JsonObject item = new JsonObject(); item.addProperty("id", delivery.id()); + item.addProperty("payload", Base64.getUrlEncoder().withoutPadding().encodeToString(bytes)); output.add(item); + } + return output; + } + private static List parseIds(JsonElement value) { + if (value == null || !value.isJsonArray() || value.getAsJsonArray().size() > MAX_BATCH) throw bad(); + List output = new ArrayList<>(); + for (JsonElement item : value.getAsJsonArray()) { if (!item.isJsonPrimitive()) throw bad(); String id = item.getAsString(); validId(id); output.add(id); } + return output; + } + private static List parseMessages(JsonElement value) { + if (value == null || !value.isJsonArray() || value.getAsJsonArray().size() > MAX_BATCH) throw bad(); + List output = new ArrayList<>(); + for (JsonElement item : value.getAsJsonArray()) { + if (!item.isJsonObject()) throw bad(); JsonObject object = item.getAsJsonObject(); requireOnly(object, "id", "payload"); + String id = string(object, "id", 64); validId(id); + byte[] payload = Base64.getUrlDecoder().decode(string(object, "payload", MAX_ENVELOPE_BYTES * 2)); + if (payload.length == 0 || payload.length > MAX_ENVELOPE_BYTES) throw bad(); + JsonEnvelope envelope = JsonEnvelopeCodec.decode(new String(payload, StandardCharsets.UTF_8)); + output.add(new Delivery(id, envelope)); + } + return output; + } + private static void requireOnly(JsonObject object, String... names) { + for (String name : object.keySet()) { boolean found = false; for (String allowed : names) if (allowed.equals(name)) { found = true; break; } if (!found) throw bad(); } + for (String name : names) if (!object.has(name) || object.get(name).isJsonNull()) throw bad(); + } + private static String string(JsonObject object, String name, int max) { JsonElement v = object.get(name); if (!v.isJsonPrimitive() || !v.getAsJsonPrimitive().isString()) throw bad(); String value = v.getAsString(); if (value.isEmpty() || value.length() > max) throw bad(); return value; } + private static long integer(JsonObject object, String name) { + try { + JsonElement value = object.get(name); + if (!value.isJsonPrimitive() || !value.getAsJsonPrimitive().isNumber()) throw bad(); + String token = value.getAsString(); + if (!token.matches("-?(?:0|[1-9][0-9]*)")) throw bad(); + return Long.parseLong(token); + } catch (RuntimeException failure) { throw bad(); } + } + private static long nonNegative(JsonObject object, String name) { long n = integer(object, name); if (n < 0) throw bad(); return n; } + private static String uuid(JsonObject object, String name) { return canonicalUuid(string(object, name, 64)); } + static void validId(String id) { if (id == null || id.length() > 64) throw bad(); canonicalUuid(id); } + private static String canonicalUuid(String value) { + try { + UUID parsed = UUID.fromString(value); + if (!parsed.toString().equals(value)) throw bad(); + return value; + } catch (IllegalArgumentException invalid) { throw bad(); } + } + private static IllegalArgumentException bad() { return new IllegalArgumentException("Invalid HTTP transport message"); } + + record Delivery(String id, JsonEnvelope envelope) { } + record Packet(String server, String session, long sequence, List acks, List messages) { } + record Enrollment(String server, String token) { } +} diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpTransportSecrets.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpTransportSecrets.java new file mode 100644 index 0000000..efd0b3b --- /dev/null +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpTransportSecrets.java @@ -0,0 +1,64 @@ +package com.bencodez.simpleapi.servercomm.http; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.security.SecureRandom; +import java.security.cert.X509Certificate; +import java.util.Base64; +import javax.crypto.Mac; +import javax.crypto.spec.SecretKeySpec; + +/** Small, deliberately dependency-free cryptographic helpers for the HTTP transport. */ +final class HttpTransportSecrets { + private static final SecureRandom RANDOM = new SecureRandom(); + + private HttpTransportSecrets() { } + + static byte[] randomBytes(int length) { + if (length < 16) throw new IllegalArgumentException("Secret length is too small"); + byte[] value = new byte[length]; + RANDOM.nextBytes(value); + return value; + } + + static String randomToken() { + return Base64.getUrlEncoder().withoutPadding().encodeToString(randomBytes(32)); + } + + static byte[] sha256(byte[] value) { + try { + return MessageDigest.getInstance("SHA-256").digest(value); + } catch (NoSuchAlgorithmException impossible) { + throw new IllegalStateException("SHA-256 is unavailable", impossible); + } + } + + static String sha256Hex(byte[] value) { + StringBuilder output = new StringBuilder(64); + for (byte part : sha256(value)) output.append(String.format("%02x", part & 0xff)); + return output.toString(); + } + + static String certificatePin(X509Certificate certificate) { + try { + return sha256Hex(certificate.getEncoded()); + } catch (Exception failure) { + throw new IllegalArgumentException("Could not encode certificate", failure); + } + } + + static boolean constantTimeEquals(byte[] first, byte[] second) { + return first != null && second != null && MessageDigest.isEqual(first, second); + } + + static String hmacSha256Url(byte[] key, String value) { + try { + Mac mac = Mac.getInstance("HmacSHA256"); + mac.init(new SecretKeySpec(key, "HmacSHA256")); + return Base64.getUrlEncoder().withoutPadding().encodeToString(mac.doFinal(value.getBytes(StandardCharsets.US_ASCII))); + } catch (Exception failure) { + throw new IllegalStateException("HMAC-SHA-256 is unavailable", failure); + } + } +} diff --git a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportRuntimeTest.java b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportRuntimeTest.java new file mode 100644 index 0000000..d93030f --- /dev/null +++ b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportRuntimeTest.java @@ -0,0 +1,759 @@ +package com.bencodez.simpleapi.servercomm.http; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTimeoutPreemptively; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.bencodez.simpleapi.servercomm.codec.JsonEnvelope; +import java.net.InetSocketAddress; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.time.Instant; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class HttpTransportRuntimeTest { + @TempDir Path directory; + + @Test + void enrollsThenDeliversBothDirectionsWithAuthenticatedIdentity() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, directory.resolve("authority")); + CountDownLatch proxyReceived = new CountDownLatch(1), backendReceived = new CountDownLatch(1); + AtomicReference received = new AtomicReference<>(); + try (HttpProxyTransportServer server = new HttpProxyTransportServer(new InetSocketAddress("localhost", 0), identity, authority, + message -> { received.set(message); proxyReceived.countDown(); })) { + server.start(); + HttpConnectionCode code = authority.createConnectionCode("lobby-1", server.endpoint("localhost"), Duration.ofMinutes(5)); + HttpBackendTransportConnector.enroll(code, "lobby-1", directory.resolve("client")); + try (HttpBackendTransportConnector connector = new HttpBackendTransportConnector(directory.resolve("client"), + envelope -> backendReceived.countDown())) { + connector.start(); + assertTrue(connector.awaitFirstResponse(System.nanoTime() + TimeUnit.SECONDS.toNanos(8)), + "an authenticated transport response must make the connector ready"); + assertTrue(connector.send(JsonEnvelope.builder("to-proxy").put("server", "forged").build())); + assertTrue(proxyReceived.await(8, TimeUnit.SECONDS)); + assertEquals("lobby-1", received.get().serverId()); + assertEquals("lobby-1", received.get().envelope().getFields().get("server")); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(3); + while (connector.queuedOutgoing() != 0 && System.nanoTime() < deadline) Thread.sleep(10); + assertEquals(0, connector.queuedOutgoing(), "proxy ACK must remove the exact outbound delivery ID"); + assertTrue(server.send("lobby-1", JsonEnvelope.builder("to-backend").build())); + assertTrue(backendReceived.await(8, TimeUnit.SECONDS)); + Path inboundFence = directory.resolve("client").resolve("http-transport-inbound-deliveries"); + long fenceDeadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (countRegularFiles(inboundFence) != 0L && System.nanoTime() < fenceDeadline) Thread.sleep(20); + assertEquals(0L, countRegularFiles(inboundFence), "a confirmed ACK must remove the backend replay fence"); + } + } + } + + @Test + void responseBodyConsumptionRemainsBoundedByRequestTimeout() throws Exception { + com.sun.net.httpserver.HttpServer server = com.sun.net.httpserver.HttpServer.create( + new InetSocketAddress("localhost", 0), 1); + CountDownLatch release = new CountDownLatch(1); + server.createContext("/stall", exchange -> { + exchange.sendResponseHeaders(200, 8); + try (var output = exchange.getResponseBody()) { + output.write(1); + output.flush(); + try { release.await(5, TimeUnit.SECONDS); } + catch (InterruptedException interrupted) { Thread.currentThread().interrupt(); } + } + }); + server.start(); + try { + HttpRequest request = HttpRequest.newBuilder(URI.create("http://localhost:" + server.getAddress().getPort() + + "/stall")).timeout(Duration.ofMillis(250)).GET().build(); + assertTimeoutPreemptively(Duration.ofSeconds(2), () -> assertThrows(java.io.IOException.class, + () -> HttpBackendTransportConnector.sendLimited(HttpClient.newHttpClient(), request))); + } finally { + release.countDown(); + server.stop(0); + } + } + + @Test + void stableProxyDeliveryIdsAreIdempotentAndAcknowledgedBeforeRemoval() throws Exception { + AtomicLong acknowledged = new AtomicLong(); + HttpProxyTransportServer.BackendState state = new HttpProxyTransportServer.BackendState("lobby-1", null, + (server, deliveryId) -> acknowledged.incrementAndGet()); + String deliveryId = java.util.UUID.randomUUID().toString(); + HttpTransportProtocol.Delivery delivery = new HttpTransportProtocol.Delivery(deliveryId, + JsonEnvelope.builder("vote-party").build()); + assertTrue(state.enqueue(delivery)); + assertTrue(state.enqueue(delivery)); + assertFalse(state.enqueue(new HttpTransportProtocol.Delivery(deliveryId, + JsonEnvelope.builder("different").build()))); + state.acknowledge(java.util.List.of(deliveryId)); + assertEquals(1L, acknowledged.get()); + assertTrue(state.await("lobby-1", java.util.UUID.randomUUID().toString(), 0).messages().isEmpty()); + } + + @Test + void failedAcknowledgementCallbackRetainsProxyDelivery() throws Exception { + HttpProxyTransportServer.BackendState state = new HttpProxyTransportServer.BackendState("lobby-1", null, + (server, deliveryId) -> { throw new java.io.IOException("cache save failed"); }); + String deliveryId = java.util.UUID.randomUUID().toString(); + assertTrue(state.enqueue(new HttpTransportProtocol.Delivery(deliveryId, + JsonEnvelope.builder("vote-party").build()))); + assertThrows(java.io.IOException.class, () -> state.acknowledge(java.util.List.of(deliveryId))); + assertEquals(deliveryId, state.await("lobby-1", java.util.UUID.randomUUID().toString(), 0) + .messages().iterator().next().id()); + } + + @Test + void closeWaitsForTheCredentialOwningPollerToStop() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("close-proxy"), "localhost"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, directory.resolve("close-authority")); + try (HttpProxyTransportServer server = new HttpProxyTransportServer(new InetSocketAddress("localhost", 0), + identity, authority, ignored -> { })) { + server.start(); + Path clientDirectory = directory.resolve("close-client"); + HttpConnectionCode code = authority.createConnectionCode("lobby-1", server.endpoint("localhost"), + Duration.ofMinutes(5)); + HttpBackendTransportConnector.enroll(code, "lobby-1", clientDirectory); + try (HttpBackendTransportConnector connector = new HttpBackendTransportConnector(clientDirectory, ignored -> { })) { + connector.start(); + assertTrue(connector.awaitFirstResponse(System.nanoTime() + TimeUnit.SECONDS.toNanos(8))); + connector.close(); + assertFalse(connector.pollerAlive(), + "credential-directory ownership must outlive every poller filesystem mutation"); + } + } + } + + @Test + void finalFlushDeliversMessagesQueuedBehindAnActiveLongPoll() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("flush-proxy"), "localhost"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, directory.resolve("flush-authority")); + CountDownLatch received = new CountDownLatch(1); + AtomicReference envelope = new AtomicReference<>(); + try (HttpProxyTransportServer server = new HttpProxyTransportServer(new InetSocketAddress("localhost", 0), + identity, authority, message -> { + envelope.set(message.envelope()); + received.countDown(); + })) { + server.start(); + Path clientDirectory = directory.resolve("flush-client"); + HttpConnectionCode code = authority.createConnectionCode("lobby-1", server.endpoint("localhost"), + Duration.ofMinutes(5)); + HttpBackendTransportConnector.enroll(code, "lobby-1", clientDirectory); + HttpBackendTransportConnector connector = new HttpBackendTransportConnector(clientDirectory, ignored -> { }); + connector.start(); + assertTrue(connector.awaitFirstResponse(System.nanoTime() + TimeUnit.SECONDS.toNanos(8))); + assertTrue(connector.send(JsonEnvelope.builder("backend-stopped").build())); + try { + assertTrue(connector.flushOutgoing(System.nanoTime() + TimeUnit.SECONDS.toNanos(5))); + assertTrue(received.await(1, TimeUnit.SECONDS)); + assertEquals("backend-stopped", envelope.get().getSubChannel()); + assertEquals(0, connector.queuedOutgoing()); + } finally { connector.close(); } + } + } + + @Test + void normalTransportRejectsAClientWithoutCertificate() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, directory.resolve("authority")); + try (HttpProxyTransportServer server = new HttpProxyTransportServer(new InetSocketAddress("localhost", 0), identity, authority, ignored -> { })) { + server.start(); + HttpConnectionCode code = authority.createConnectionCode("lobby-1", server.endpoint("localhost"), Duration.ofMinutes(5)); + HttpClient client = HttpClient.newBuilder().sslContext(HttpPinnedTls.clientContext(code)).build(); + byte[] body = HttpTransportProtocol.request("lobby-1", java.util.UUID.randomUUID().toString(), 0, java.util.List.of(), java.util.List.of()); + HttpResponse response = client.send(HttpRequest.newBuilder(code.endpoint().resolve("v1/transport")) + .timeout(Duration.ofSeconds(5)).header("Content-Type", "application/json").POST(HttpRequest.BodyPublishers.ofByteArray(body)).build(), HttpResponse.BodyHandlers.ofByteArray()); + assertEquals(401, response.statusCode()); + } + } + + @Test + void boundedQueuesFailClosed() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, directory.resolve("authority")); + try (HttpProxyTransportServer server = new HttpProxyTransportServer(new InetSocketAddress("localhost", 0), identity, authority, ignored -> { })) { + for (int i = 0; i < HttpTransportProtocol.MAX_QUEUE; i++) assertTrue(server.send("lobby-1", JsonEnvelope.builder("x").build())); + assertFalse(server.send("lobby-1", JsonEnvelope.builder("x").build())); + assertFalse(server.send("lobby-1", JsonEnvelope.builder("x").put("large", "x".repeat(HttpTransportProtocol.MAX_ENVELOPE_BYTES)).build())); + } + } + + @Test + void proxyOutgoingQueueSurvivesRestartUntilBackendAcknowledges() throws Exception { + Path proxyDirectory = directory.resolve("proxy"); + Path authorityDirectory = directory.resolve("authority"); + Path queueDirectory = directory.resolve("outgoing"); + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(proxyDirectory, "localhost"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, authorityDirectory); + HttpProxyTransportServer first = new HttpProxyTransportServer(new InetSocketAddress("localhost", 0), + identity, authority, queueDirectory, ignored -> { }); + assertTrue(first.send("lobby-1", JsonEnvelope.builder("durable").build())); + first.close(); + + CountDownLatch received = new CountDownLatch(1); + try (HttpProxyTransportServer restarted = new HttpProxyTransportServer( + new InetSocketAddress("localhost", 0), identity, authority, queueDirectory, ignored -> { })) { + restarted.start(); + HttpConnectionCode code = authority.createConnectionCode("lobby-1", restarted.endpoint("localhost"), + Duration.ofMinutes(5)); + HttpClientCredentialStore.ClientCredential credential = HttpBackendTransportConnector.enroll(code, + "lobby-1", directory.resolve("durable-client")); + try (HttpBackendTransportConnector connector = new HttpBackendTransportConnector(code, "lobby-1", + credential, envelope -> received.countDown())) { + connector.start(); + assertTrue(received.await(8, TimeUnit.SECONDS)); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (countRegularFiles(queueDirectory) != 0L && System.nanoTime() < deadline) Thread.sleep(20); + assertEquals(0L, countRegularFiles(queueDirectory), "backend ACK must durably remove the delivery"); + } + } + } + + @Test + void pollCreatedBackendStateUsesDurableOutgoingQueue() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("poll-proxy"), "localhost"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, directory.resolve("poll-authority")); + Path queueDirectory = directory.resolve("poll-outgoing"); + CountDownLatch proxyReceived = new CountDownLatch(1), backendReceived = new CountDownLatch(1); + CountDownLatch releaseBackendCallback = new CountDownLatch(1); + try (HttpProxyTransportServer server = new HttpProxyTransportServer(new InetSocketAddress("localhost", 0), + identity, authority, queueDirectory, ignored -> proxyReceived.countDown())) { + server.start(); + HttpConnectionCode code = authority.createConnectionCode("lobby-1", server.endpoint("localhost"), + Duration.ofMinutes(5)); + HttpClientCredentialStore.ClientCredential credential = HttpBackendTransportConnector.enroll(code, + "lobby-1", directory.resolve("poll-client")); + try (HttpBackendTransportConnector connector = new HttpBackendTransportConnector(code, "lobby-1", + credential, envelope -> { + backendReceived.countDown(); + try { releaseBackendCallback.await(5, TimeUnit.SECONDS); } + catch (InterruptedException interrupted) { Thread.currentThread().interrupt(); } + })) { + connector.start(); + assertTrue(connector.send(JsonEnvelope.builder("establish-poll").build())); + assertTrue(proxyReceived.await(8, TimeUnit.SECONDS)); + assertTrue(server.send("lobby-1", JsonEnvelope.builder("durable-after-poll").build())); + assertTrue(backendReceived.await(8, TimeUnit.SECONDS)); + assertEquals(1L, countRegularFiles(queueDirectory), + "a poll-created backend state must persist before reporting acceptance"); + releaseBackendCallback.countDown(); + } + } finally { + releaseBackendCallback.countDown(); + } + } + + private static long countRegularFiles(Path root) throws Exception { + try (java.util.stream.Stream paths = java.nio.file.Files.walk(root)) { + return paths.filter(path -> java.nio.file.Files.isRegularFile(path, java.nio.file.LinkOption.NOFOLLOW_LINKS)).count(); + } + } + + @Test + void aggregatePacketBudgetSplitsLargeValidEnvelopes() { + java.util.List candidates = new java.util.ArrayList<>(); + for (int index = 0; index < 12; index++) candidates.add(new HttpTransportProtocol.Delivery( + java.util.UUID.randomUUID().toString(), JsonEnvelope.builder("large").put("value", "x".repeat(40_000)).build())); + String session = java.util.UUID.randomUUID().toString(); + java.util.List fitted = HttpTransportProtocol.fittingMessages( + "lobby-1", session, 0, java.util.List.of(), candidates); + assertTrue(fitted.size() > 0 && fitted.size() < candidates.size()); + assertTrue(HttpTransportProtocol.request("lobby-1", session, 0, java.util.List.of(), fitted).length + <= HttpTransportProtocol.MAX_BODY_BYTES); + } + + @Test + void packetNumbersMustUseCanonicalJsonIntegerTokens() { + com.google.gson.JsonObject packet = com.google.gson.JsonParser.parseString(new String(HttpTransportProtocol.request( + "lobby-1", java.util.UUID.randomUUID().toString(), 0, java.util.List.of(), java.util.List.of()), + java.nio.charset.StandardCharsets.UTF_8)).getAsJsonObject(); + assertDoesNotThrow(() -> HttpTransportProtocol.parsePacket(packet.toString() + .getBytes(java.nio.charset.StandardCharsets.UTF_8))); + String timestamp = packet.get("timestamp").getAsString(); + java.util.Map> invalid = java.util.Map.of( + "v", java.util.List.of("\"1\"", "1.0", "1e0"), + "sequence", java.util.List.of("\"0\"", "0.0", "0e0"), + "timestamp", java.util.List.of("\"" + timestamp + "\"", timestamp + ".0", timestamp + "e0")); + for (var field : invalid.entrySet()) for (String token : field.getValue()) { + com.google.gson.JsonObject rejected = packet.deepCopy(); + rejected.add(field.getKey(), com.google.gson.JsonParser.parseString(token)); + assertThrows(IllegalArgumentException.class, () -> HttpTransportProtocol.parsePacket( + rejected.toString().getBytes(java.nio.charset.StandardCharsets.UTF_8)), field.getKey() + "=" + token); + } + } + + @Test + void packetParsingRejectsNoncanonicalUuidForms() { + String deliveryId = java.util.UUID.randomUUID().toString(); + com.google.gson.JsonObject packet = com.google.gson.JsonParser.parseString(new String(HttpTransportProtocol.request( + "lobby-1", java.util.UUID.randomUUID().toString(), 0, java.util.List.of(deliveryId), + java.util.List.of(new HttpTransportProtocol.Delivery(deliveryId, JsonEnvelope.builder("payload").build()))), + java.nio.charset.StandardCharsets.UTF_8)).getAsJsonObject(); + String abbreviated = "1-1-1-1-1"; + + com.google.gson.JsonObject invalidSession = packet.deepCopy(); + invalidSession.addProperty("session", abbreviated); + assertThrows(IllegalArgumentException.class, () -> HttpTransportProtocol.parsePacket( + invalidSession.toString().getBytes(java.nio.charset.StandardCharsets.UTF_8))); + + com.google.gson.JsonObject invalidAck = packet.deepCopy(); + invalidAck.getAsJsonArray("acks").set(0, new com.google.gson.JsonPrimitive(abbreviated)); + assertThrows(IllegalArgumentException.class, () -> HttpTransportProtocol.parsePacket( + invalidAck.toString().getBytes(java.nio.charset.StandardCharsets.UTF_8))); + + com.google.gson.JsonObject invalidMessage = packet.deepCopy(); + invalidMessage.getAsJsonArray("messages").get(0).getAsJsonObject().addProperty("id", abbreviated); + assertThrows(IllegalArgumentException.class, () -> HttpTransportProtocol.parsePacket( + invalidMessage.toString().getBytes(java.nio.charset.StandardCharsets.UTF_8))); + } + + @Test + void persistedProfileStartsAfterTheEnrollmentCodeExpires() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, directory.resolve("authority")); + try (HttpProxyTransportServer server = new HttpProxyTransportServer(new InetSocketAddress("localhost", 0), identity, authority, ignored -> { })) { + server.start(); + HttpConnectionCode active = authority.createConnectionCode("lobby-1", server.endpoint("localhost"), Duration.ofMinutes(5)); + HttpBackendTransportConnector.enroll(active, "lobby-1", directory.resolve("client")); + HttpConnectionCode expired = new HttpConnectionCode(active.serverId(), active.endpoint(), active.serverCertificatePin(), active.caCertificatePin(), + java.time.Instant.now().minusSeconds(1), active.enrollmentToken()); + try (HttpBackendTransportConnector ignored = new HttpBackendTransportConnector(expired, "lobby-1", directory.resolve("client"), message -> { })) { + assertTrue(true); + } + try (HttpBackendTransportConnector ignored = new HttpBackendTransportConnector(directory.resolve("client"), message -> { })) { + assertTrue(true); + } + } + } + + @Test + void persistedBackendConnectsAfterAutomaticServerLeafRotation() throws Exception { + Instant now = Instant.now(); + Path proxyDirectory = directory.resolve("proxy"); + HttpTlsIdentity original = HttpTlsIdentity.loadOrCreate(proxyDirectory, "localhost", + java.time.Clock.fixed(now.minus(Duration.ofDays(340)), java.time.ZoneOffset.UTC)); + String originalServerPin = HttpTransportSecrets.certificatePin(original.serverCertificate()); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(original, directory.resolve("authority")); + HttpTlsIdentity rotated = HttpTlsIdentity.loadOrCreate(proxyDirectory, "localhost"); + CountDownLatch received = new CountDownLatch(1); + try (HttpProxyTransportServer server = new HttpProxyTransportServer(new InetSocketAddress("localhost", 0), rotated, + authority, ignored -> received.countDown())) { + HttpConnectionCode activeCode = authority.createConnectionCode("lobby-1", server.endpoint("localhost"), Duration.ofMinutes(5)); + HttpTlsIdentity.IssuedClientCertificate issued = authority.enroll("lobby-1", activeCode.enrollmentToken()); + HttpConnectionCode oldProfileCode = new HttpConnectionCode(activeCode.serverId(), activeCode.endpoint(), originalServerPin, + activeCode.caCertificatePin(), activeCode.expiresAt(), activeCode.enrollmentToken()); + HttpClientCredentialStore.saveEnrolled(directory.resolve("client"), oldProfileCode, issued); + assertFalse(oldProfileCode.serverCertificatePin().equals(rotated.serverCertificatePin())); + server.start(); + try (HttpBackendTransportConnector connector = new HttpBackendTransportConnector(directory.resolve("client"), ignored -> { })) { + connector.start(); + assertTrue(connector.send(JsonEnvelope.builder("after-rotation").build())); + assertTrue(received.await(8, TimeUnit.SECONDS)); + } + } + } + + @Test + void backendRenewsClientCertificateBeforeExpiryWithoutNewConnectionCode() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); + HttpTlsIdentity.IssuedClientCertificate expiring = identity.issueClientCertificate("lobby-1", + Instant.now().minus(Duration.ofDays(340))); + String originalPin = HttpTransportSecrets.certificatePin(expiring.certificate()); + Path authorityDirectory = directory.resolve("authority"); + java.nio.file.Files.createDirectories(authorityDirectory); + String key = java.util.Base64.getUrlEncoder().withoutPadding() + .encodeToString("lobby-1".getBytes(java.nio.charset.StandardCharsets.UTF_8)); + java.nio.file.Files.writeString(authorityDirectory.resolve("http-transport-clients.properties"), + "version=2\nbinding." + key + "=" + originalPin + ":-:0\n"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, authorityDirectory); + try (HttpProxyTransportServer server = new HttpProxyTransportServer(new InetSocketAddress("localhost", 0), identity, + authority, ignored -> { })) { + HttpConnectionCode profileCode = new HttpConnectionCode("lobby-1", server.endpoint("localhost"), + identity.serverCertificatePin(), identity.caCertificatePin(), Instant.now().plusSeconds(60), "A".repeat(43)); + Path clientDirectory = directory.resolve("client"); + HttpClientCredentialStore.saveEnrolled(clientDirectory, profileCode, expiring); + server.start(); + try (HttpBackendTransportConnector connector = new HttpBackendTransportConnector(clientDirectory, ignored -> { })) { + connector.start(); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(8); + String renewedPin = originalPin; + while (renewedPin.equals(originalPin) && System.nanoTime() < deadline) { + Thread.sleep(25); + renewedPin = HttpTransportSecrets.certificatePin(HttpClientCredentialStore.load(clientDirectory).certificate()); + } + assertFalse(renewedPin.equals(originalPin)); + HttpClientCredentialStore.ClientCredential renewed = HttpClientCredentialStore.load(clientDirectory); + long promotionDeadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (authority.authenticate("lobby-1", expiring.certificate()) && System.nanoTime() < promotionDeadline) Thread.sleep(25); + assertTrue(authority.authenticate("lobby-1", renewed.certificate())); + assertFalse(authority.authenticate("lobby-1", expiring.certificate())); + } + } + } + + @Test + void duplicateInboundDeliveryIsReAcknowledgedWithoutSecondDispatch() { + HttpProxyTransportServer.BackendState state = new HttpProxyTransportServer.BackendState(); + String session = java.util.UUID.randomUUID().toString(); + String id = java.util.UUID.randomUUID().toString(); + HttpTransportProtocol.Delivery delivery = new HttpTransportProtocol.Delivery(id, JsonEnvelope.builder("x").build()); + assertTrue(state.acceptSession(session, 0)); + assertEquals(1, state.acceptIncoming(java.util.List.of(delivery)).size()); + state.completeIncoming(id, true); + assertEquals(java.util.List.of(id), state.await("lobby-1", session, 0).acks()); + assertTrue(state.acceptSession(session, 1)); + assertTrue(state.acceptIncoming(java.util.List.of(delivery)).isEmpty()); + assertEquals(java.util.List.of(id), state.await("lobby-1", session, 1).acks()); + String replacementSession = java.util.UUID.randomUUID().toString(); + assertTrue(state.acceptSession(replacementSession, 0)); + assertTrue(state.acceptIncoming(java.util.List.of(delivery)).isEmpty()); + assertEquals(java.util.List.of(id), state.await("lobby-1", replacementSession, 0).acks()); + } + + @Test + void newerDeliveriesDoNotPostponeRetryOfOlderUnacknowledgedDelivery() { + AtomicLong nanoTime = new AtomicLong(1L); + HttpProxyTransportServer.BackendState state = new HttpProxyTransportServer.BackendState(nanoTime::get); + String session = java.util.UUID.randomUUID().toString(); + HttpTransportProtocol.Delivery first = new HttpTransportProtocol.Delivery( + java.util.UUID.randomUUID().toString(), JsonEnvelope.builder("first").build()); + HttpTransportProtocol.Delivery second = new HttpTransportProtocol.Delivery( + java.util.UUID.randomUUID().toString(), JsonEnvelope.builder("second").build()); + + assertTrue(state.acceptSession(session, 0)); + assertTrue(state.enqueue(first)); + assertEquals(java.util.List.of(first), state.await("lobby-1", session, 0).messages()); + + nanoTime.addAndGet(TimeUnit.SECONDS.toNanos(1)); + assertTrue(state.acceptSession(session, 1)); + assertTrue(state.enqueue(second)); + assertEquals(java.util.List.of(second), state.await("lobby-1", session, 1).messages()); + + nanoTime.addAndGet(TimeUnit.MILLISECONDS.toNanos(1100)); + assertTrue(state.acceptSession(session, 2)); + assertEquals(java.util.List.of(first), state.await("lobby-1", session, 2).messages(), + "sending newer traffic must not reset an older delivery's retry age"); + } + + @Test + void longPollWakesAtTheOldestDeliveryRetryDeadline() throws Exception { + AtomicLong nanoTime = new AtomicLong(1L); + HttpProxyTransportServer.BackendState state = new HttpProxyTransportServer.BackendState(nanoTime::get); + String session = java.util.UUID.randomUUID().toString(); + HttpTransportProtocol.Delivery first = new HttpTransportProtocol.Delivery( + java.util.UUID.randomUUID().toString(), JsonEnvelope.builder("first").build()); + HttpTransportProtocol.Delivery second = new HttpTransportProtocol.Delivery( + java.util.UUID.randomUUID().toString(), JsonEnvelope.builder("second").build()); + assertTrue(state.acceptSession(session, 0)); + assertTrue(state.enqueue(first)); + assertEquals(java.util.List.of(first), state.await("lobby-1", session, 0).messages()); + + nanoTime.addAndGet(HttpProxyTransportServer.LONG_POLL.minusMillis(100).toNanos()); + assertTrue(state.acceptSession(session, 1)); + assertTrue(state.enqueue(second)); + assertEquals(java.util.List.of(second), state.await("lobby-1", session, 1).messages()); + assertTrue(state.acceptSession(session, 2)); + Thread clock = new Thread(() -> { + try { Thread.sleep(50L); } + catch (InterruptedException interrupted) { Thread.currentThread().interrupt(); } + nanoTime.addAndGet(TimeUnit.MILLISECONDS.toNanos(100)); + }, "HTTP-retry-test-clock"); + clock.setDaemon(true); + long started = System.nanoTime(); + clock.start(); + assertEquals(java.util.List.of(first), state.await("lobby-1", session, 2).messages()); + clock.join(); + assertTrue(System.nanoTime() - started < TimeUnit.SECONDS.toNanos(1), + "the poll must wake at the oldest delivery deadline, not a fresh long-poll deadline"); + } + + @Test + void proxyDedupWindowEvictsOldestCompletedDeliveryAtCapacity() { + HttpProxyTransportServer.BackendState state = new HttpProxyTransportServer.BackendState(); + String oldest = null, newest = null; + for (int index = 0; index < HttpTransportProtocol.MAX_QUEUE + 1; index++) { + String id = java.util.UUID.randomUUID().toString(); + if (index == 0) oldest = id; + newest = id; + HttpTransportProtocol.Delivery delivery = new HttpTransportProtocol.Delivery(id, JsonEnvelope.builder("x").build()); + assertEquals(1, state.acceptIncoming(java.util.List.of(delivery)).size()); + state.completeIncoming(id, true); + } + HttpTransportProtocol.Delivery evicted = new HttpTransportProtocol.Delivery(oldest, JsonEnvelope.builder("x").build()); + HttpTransportProtocol.Delivery retained = new HttpTransportProtocol.Delivery(newest, JsonEnvelope.builder("x").build()); + assertEquals(1, state.acceptIncoming(java.util.List.of(evicted)).size()); + assertTrue(state.acceptIncoming(java.util.List.of(retained)).isEmpty()); + } + + @Test + void backendReAcknowledgesLostAckDuplicateWithoutSecondCallback() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); + HttpTlsIdentity.IssuedClientCertificate issued = identity.issueClientCertificate("lobby-1"); + HttpClientCredentialStore.save(directory.resolve("client"), issued); + HttpClientCredentialStore.HttpClientProfile profile = new HttpClientCredentialStore.HttpClientProfile("lobby-1", + java.net.URI.create("https://localhost:8443/"), identity.serverCertificatePin(), identity.caCertificatePin()); + CountDownLatch callback = new CountDownLatch(1); + try (HttpBackendTransportConnector connector = new HttpBackendTransportConnector(profile, + HttpClientCredentialStore.load(directory.resolve("client")), envelope -> callback.countDown())) { + String id = java.util.UUID.randomUUID().toString(); + HttpTransportProtocol.Delivery delivery = new HttpTransportProtocol.Delivery(id, JsonEnvelope.builder("x").build()); + connector.dispatch(delivery); + assertTrue(callback.await(2, TimeUnit.SECONDS)); + java.util.List acknowledgements = java.util.List.of(); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(2); + while (acknowledgements.isEmpty() && System.nanoTime() < deadline) { + acknowledgements = connector.drainAcknowledgements(); + if (acknowledgements.isEmpty()) Thread.sleep(5); + } + assertEquals(java.util.List.of(id), acknowledgements); + assertTrue(connector.accept(java.util.List.of(delivery)).isEmpty()); + assertEquals(java.util.List.of(id), connector.drainAcknowledgements()); + } + } + + @Test + void backendCallbacksAreSerializedInDeliveryOrder() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("ordered-proxy"), "localhost"); + HttpTlsIdentity.IssuedClientCertificate issued = identity.issueClientCertificate("lobby-1"); + HttpClientCredentialStore.save(directory.resolve("ordered-client"), issued); + HttpClientCredentialStore.HttpClientProfile profile = new HttpClientCredentialStore.HttpClientProfile("lobby-1", + java.net.URI.create("https://localhost:8443/"), identity.serverCertificatePin(), identity.caCertificatePin()); + CountDownLatch firstStarted = new CountDownLatch(1), releaseFirst = new CountDownLatch(1), secondStarted = new CountDownLatch(1); + java.util.List order = new java.util.concurrent.CopyOnWriteArrayList<>(); + try (HttpBackendTransportConnector connector = new HttpBackendTransportConnector(profile, + HttpClientCredentialStore.load(directory.resolve("ordered-client")), envelope -> { + String marker = String.valueOf(envelope.getFields().get("marker")); + order.add(marker); + if ("first".equals(marker)) { + firstStarted.countDown(); + try { releaseFirst.await(5, TimeUnit.SECONDS); } + catch (InterruptedException interrupted) { Thread.currentThread().interrupt(); } + } else secondStarted.countDown(); + })) { + connector.dispatch(new HttpTransportProtocol.Delivery(java.util.UUID.randomUUID().toString(), + JsonEnvelope.builder("x").put("marker", "first").build())); + connector.dispatch(new HttpTransportProtocol.Delivery(java.util.UUID.randomUUID().toString(), + JsonEnvelope.builder("x").put("marker", "second").build())); + assertTrue(firstStarted.await(2, TimeUnit.SECONDS)); + assertFalse(secondStarted.await(150, TimeUnit.MILLISECONDS)); + releaseFirst.countDown(); + assertTrue(secondStarted.await(2, TimeUnit.SECONDS)); + assertEquals(java.util.List.of("first", "second"), order); + } finally { releaseFirst.countDown(); } + } + + @Test + void backendCallbackQueueBackpressuresWithoutBreakingFifo() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("backpressure-proxy"), "localhost"); + HttpTlsIdentity.IssuedClientCertificate issued = identity.issueClientCertificate("lobby-1"); + HttpClientCredentialStore.save(directory.resolve("backpressure-client"), issued); + HttpClientCredentialStore.HttpClientProfile profile = new HttpClientCredentialStore.HttpClientProfile("lobby-1", + java.net.URI.create("https://localhost:8443/"), identity.serverCertificatePin(), identity.caCertificatePin()); + CountDownLatch firstStarted = new CountDownLatch(1), releaseFirst = new CountDownLatch(1); + int deliveries = HttpBackendTransportConnector.CALLBACK_QUEUE_CAPACITY + 2; + CountDownLatch completed = new CountDownLatch(deliveries), overflowSubmitted = new CountDownLatch(1); + java.util.List order = new java.util.concurrent.CopyOnWriteArrayList<>(); + try (HttpBackendTransportConnector connector = new HttpBackendTransportConnector(profile, + HttpClientCredentialStore.load(directory.resolve("backpressure-client")), envelope -> { + int marker = Integer.parseInt(envelope.getFields().get("marker")); + order.add(marker); + if (marker == 0) { + firstStarted.countDown(); + try { releaseFirst.await(5, TimeUnit.SECONDS); } + catch (InterruptedException interrupted) { Thread.currentThread().interrupt(); } + } + completed.countDown(); + })) { + connector.dispatch(delivery(0)); + assertTrue(firstStarted.await(2, TimeUnit.SECONDS)); + for (int marker = 1; marker <= HttpBackendTransportConnector.CALLBACK_QUEUE_CAPACITY; marker++) + connector.dispatch(delivery(marker)); + Thread overflow = new Thread(() -> { + connector.dispatch(delivery(deliveries - 1)); + overflowSubmitted.countDown(); + }, "HTTP-overflow-submitter"); + overflow.start(); + assertFalse(overflowSubmitted.await(150, TimeUnit.MILLISECONDS), "a full ordered lane must backpressure its producer"); + releaseFirst.countDown(); + assertTrue(overflowSubmitted.await(2, TimeUnit.SECONDS)); + assertTrue(completed.await(5, TimeUnit.SECONDS)); + assertEquals(java.util.stream.IntStream.range(0, deliveries).boxed().toList(), order); + } finally { releaseFirst.countDown(); } + } + + @Test + void durableBackendFencePreventsCallbackReplayAfterRestartBeforeAck() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("fence-proxy"), "localhost"); + HttpTlsIdentity.IssuedClientCertificate issued = identity.issueClientCertificate("lobby-1"); + HttpConnectionCode code = new HttpConnectionCode("lobby-1", java.net.URI.create("https://localhost:8443/"), + identity.serverCertificatePin(), identity.caCertificatePin(), Instant.now().plusSeconds(300), "A".repeat(43)); + Path clientDirectory = directory.resolve("fence-client"); + HttpClientCredentialStore.saveEnrolled(clientDirectory, code, issued); + java.util.concurrent.atomic.AtomicInteger callbacks = new java.util.concurrent.atomic.AtomicInteger(); + String id = java.util.UUID.randomUUID().toString(); + HttpTransportProtocol.Delivery delivery = new HttpTransportProtocol.Delivery(id, JsonEnvelope.builder("vote").build()); + try (HttpBackendTransportConnector first = new HttpBackendTransportConnector(clientDirectory, + ignored -> callbacks.incrementAndGet())) { + first.dispatch(delivery); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(2); + while (callbacks.get() != 1 && System.nanoTime() < deadline) Thread.sleep(5); + assertEquals(1, callbacks.get()); + } + try (HttpBackendTransportConnector restarted = new HttpBackendTransportConnector(clientDirectory, + ignored -> callbacks.incrementAndGet())) { + assertEquals(java.util.List.of(id), restarted.drainAcknowledgements(), + "restart must retain and acknowledge the pre-callback delivery fence"); + assertTrue(restarted.accept(java.util.List.of(delivery)).isEmpty()); + assertEquals(1, callbacks.get(), "a durable proxy replay must not award twice"); + } + } + + @Test + void failedBackendCallbackRemainsUnacknowledgedAndIsNotReplayed() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("retry-fence-proxy"), "localhost"); + HttpTlsIdentity.IssuedClientCertificate issued = identity.issueClientCertificate("lobby-1"); + HttpConnectionCode code = new HttpConnectionCode("lobby-1", java.net.URI.create("https://localhost:8443/"), + identity.serverCertificatePin(), identity.caCertificatePin(), Instant.now().plusSeconds(300), "B".repeat(43)); + Path clientDirectory = directory.resolve("retry-fence-client"); + HttpClientCredentialStore.saveEnrolled(clientDirectory, code, issued); + java.util.concurrent.atomic.AtomicInteger attempts = new java.util.concurrent.atomic.AtomicInteger(); + CountDownLatch failed = new CountDownLatch(1); + HttpTransportProtocol.Delivery delivery = new HttpTransportProtocol.Delivery(java.util.UUID.randomUUID().toString(), + JsonEnvelope.builder("vote").build()); + try (HttpBackendTransportConnector first = new HttpBackendTransportConnector(clientDirectory, ignored -> { + attempts.incrementAndGet(); failed.countDown(); throw new IllegalStateException("retry"); + })) { + first.dispatch(delivery); + assertTrue(failed.await(2, TimeUnit.SECONDS)); + assertTrue(first.drainAcknowledgements().isEmpty()); + } + try (HttpBackendTransportConnector restarted = new HttpBackendTransportConnector(clientDirectory, + ignored -> attempts.incrementAndGet())) { + assertTrue(restarted.drainAcknowledgements().isEmpty()); + java.util.List accepted = restarted.accept(java.util.List.of(delivery)); + assertTrue(accepted.isEmpty(), "an ambiguous callback must not be awarded twice"); + assertEquals(1, attempts.get()); + } + } + + @Test + void reservedButNotStartedDeliveryResumesAfterRestart() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("reserved-proxy"), "localhost"); + HttpTlsIdentity.IssuedClientCertificate issued = identity.issueClientCertificate("lobby-1"); + HttpConnectionCode code = new HttpConnectionCode("lobby-1", java.net.URI.create("https://localhost:8443/"), + identity.serverCertificatePin(), identity.caCertificatePin(), Instant.now().plusSeconds(300), "C".repeat(43)); + Path clientDirectory = directory.resolve("reserved-client"); + HttpClientCredentialStore.saveEnrolled(clientDirectory, code, issued); + String id = java.util.UUID.randomUUID().toString(); + new HttpInboundDeliveryStore(clientDirectory).reserve(id); + CountDownLatch completed = new CountDownLatch(1); + HttpTransportProtocol.Delivery delivery = new HttpTransportProtocol.Delivery(id, JsonEnvelope.builder("vote").build()); + try (HttpBackendTransportConnector restarted = new HttpBackendTransportConnector(clientDirectory, ignored -> completed.countDown())) { + assertTrue(restarted.drainAcknowledgements().isEmpty(), "a reservation alone must never be acknowledged"); + java.util.List accepted = restarted.accept(java.util.List.of(delivery)); + assertEquals(1, accepted.size()); + restarted.dispatch(accepted.get(0)); + assertTrue(completed.await(2, TimeUnit.SECONDS)); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(2); + java.util.List acknowledgements = java.util.List.of(); + while (acknowledgements.isEmpty() && System.nanoTime() < deadline) { + acknowledgements = restarted.drainAcknowledgements(); + if (acknowledgements.isEmpty()) Thread.sleep(5); + } + assertEquals(java.util.List.of(id), acknowledgements); + } + } + + @Test + void interruptedStateRenameRetainsTheFurthestSafeState() throws Exception { + Path clientDirectory = directory.resolve("interrupted-state-client"); + String id = java.util.UUID.randomUUID().toString(); + String completedId = java.util.UUID.randomUUID().toString(); + Path states = clientDirectory.resolve("http-transport-inbound-deliveries"); + Files.createDirectories(states); + Files.writeString(states.resolve(id + ".reserved"), id); + Files.writeString(states.resolve(id + ".running"), id); + Files.writeString(states.resolve(completedId + ".running"), completedId); + Files.writeString(states.resolve(completedId + ".completed"), completedId); + HttpInboundDeliveryStore store = new HttpInboundDeliveryStore(clientDirectory); + assertEquals(HttpInboundDeliveryStore.State.RUNNING, store.state(id)); + assertEquals(HttpInboundDeliveryStore.State.COMPLETED, store.state(completedId)); + assertFalse(Files.exists(states.resolve(id + ".reserved"))); + assertTrue(Files.exists(states.resolve(id + ".running"))); + assertFalse(Files.exists(states.resolve(completedId + ".running"))); + assertTrue(Files.exists(states.resolve(completedId + ".completed"))); + } + + private static HttpTransportProtocol.Delivery delivery(int marker) { + return new HttpTransportProtocol.Delivery(java.util.UUID.randomUUID().toString(), + JsonEnvelope.builder("x").put("marker", marker).build()); + } + + @Test + void proxyCallbacksAreSerializedInDeliveryOrder() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("ordered-proxy-server"), "localhost"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, directory.resolve("ordered-authority")); + CountDownLatch firstStarted = new CountDownLatch(1), releaseFirst = new CountDownLatch(1), secondStarted = new CountDownLatch(1); + java.util.List order = new java.util.concurrent.CopyOnWriteArrayList<>(); + try (HttpProxyTransportServer server = new HttpProxyTransportServer(new InetSocketAddress("localhost", 0), identity, + authority, received -> { + String marker = String.valueOf(received.envelope().getFields().get("marker")); + order.add(marker); + if ("first".equals(marker)) { + firstStarted.countDown(); + try { releaseFirst.await(5, TimeUnit.SECONDS); } + catch (InterruptedException interrupted) { Thread.currentThread().interrupt(); } + } else secondStarted.countDown(); + })) { + server.start(); + HttpConnectionCode code = authority.createConnectionCode("lobby-1", server.endpoint("localhost"), Duration.ofMinutes(5)); + HttpClientCredentialStore.ClientCredential credential = HttpBackendTransportConnector.enroll(code, "lobby-1", + directory.resolve("ordered-proxy-client")); + try (HttpBackendTransportConnector connector = new HttpBackendTransportConnector(code, "lobby-1", credential, + ignored -> { })) { + connector.start(); + assertTrue(connector.send(JsonEnvelope.builder("x").put("marker", "first").build())); + assertTrue(connector.send(JsonEnvelope.builder("x").put("marker", "second").build())); + assertTrue(firstStarted.await(3, TimeUnit.SECONDS)); + assertFalse(secondStarted.await(150, TimeUnit.MILLISECONDS)); + releaseFirst.countDown(); + assertTrue(secondStarted.await(3, TimeUnit.SECONDS)); + assertEquals(java.util.List.of("first", "second"), order); + } + } finally { releaseFirst.countDown(); } + } + + @Test + void backendDedupWindowContinuesAfterCapacity() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); + HttpTlsIdentity.IssuedClientCertificate issued = identity.issueClientCertificate("lobby-1"); + HttpClientCredentialStore.save(directory.resolve("client"), issued); + HttpClientCredentialStore.HttpClientProfile profile = new HttpClientCredentialStore.HttpClientProfile("lobby-1", + java.net.URI.create("https://localhost:8443/"), identity.serverCertificatePin(), identity.caCertificatePin()); + try (HttpBackendTransportConnector connector = new HttpBackendTransportConnector(profile, + HttpClientCredentialStore.load(directory.resolve("client")), envelope -> { })) { + for (int index = 0; index < HttpTransportProtocol.MAX_QUEUE; index++) { + String id = java.util.UUID.randomUUID().toString(); + HttpTransportProtocol.Delivery delivery = new HttpTransportProtocol.Delivery(id, JsonEnvelope.builder("x").build()); + assertEquals(1, connector.accept(java.util.List.of(delivery)).size()); + connector.completeIncoming(id, true); + } + HttpTransportProtocol.Delivery next = new HttpTransportProtocol.Delivery(java.util.UUID.randomUUID().toString(), + JsonEnvelope.builder("x").build()); + assertEquals(1, connector.accept(java.util.List.of(next)).size()); + } + } +} diff --git a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportSecurityTest.java b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportSecurityTest.java new file mode 100644 index 0000000..11a095a --- /dev/null +++ b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportSecurityTest.java @@ -0,0 +1,401 @@ +package com.bencodez.simpleapi.servercomm.http; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.net.URI; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.cert.X509Certificate; +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.time.ZoneOffset; +import java.util.Arrays; +import javax.net.ssl.X509TrustManager; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class HttpTransportSecurityTest { + @Test + void connectionCodeRejectsExplicitZeroPort() { + assertThrows(IllegalArgumentException.class, + () -> new HttpConnectionCode("lobby", URI.create("https://proxy.example.test:0/"), pin('a'), + pin('b'), Instant.now().plusSeconds(60), "token")); + } + + @TempDir Path directory; + + @Test + void backendResponseReaderRejectsBodiesBeyondTheWireLimit() throws Exception { + byte[] maximum = new byte[HttpTransportProtocol.MAX_BODY_BYTES]; + assertEquals(maximum.length, HttpBackendTransportConnector.readLimited( + new java.io.ByteArrayInputStream(maximum)).length); + assertThrows(java.io.IOException.class, () -> HttpBackendTransportConnector.readLimited( + new java.io.ByteArrayInputStream(new byte[HttpTransportProtocol.MAX_BODY_BYTES + 1]))); + } + + @Test + void connectionCodeRoundTripsAndRejectsAccidentalCorruption() { + HttpConnectionCode original = new HttpConnectionCode("lobby.eu", URI.create("https://Proxy.Example.test:8443/http"), pin('a'), pin('b'), + Instant.parse("2030-01-01T00:00:00Z"), HttpTransportSecrets.randomToken()); + String encoded = original.encode(); + HttpConnectionCode parsed = HttpConnectionCode.parse(encoded); + assertEquals("lobby.eu", parsed.serverId()); + assertEquals(URI.create("https://proxy.example.test:8443/http/"), parsed.endpoint()); + assertEquals(original.serverCertificatePin(), parsed.serverCertificatePin()); + char last = encoded.charAt(encoded.length() - 1); + assertThrows(IllegalArgumentException.class, () -> HttpConnectionCode.parse(encoded.substring(0, encoded.length() - 1) + + (last == 'A' ? 'B' : 'A'))); + assertThrows(IllegalArgumentException.class, () -> HttpConnectionCode.parse("http://not-a-code")); + } + + @Test + void legacyConnectionCodesAndConsumedMarkersRemainCompatible() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("legacy-code-proxy"), "proxy.example.test"); + HttpConnectionCode legacy = new HttpConnectionCode("lobby", URI.create("https://proxy.example.test:8443/"), + identity.serverCertificatePin(), identity.caCertificatePin(), + Instant.now().plusSeconds(60).truncatedTo(java.time.temporal.ChronoUnit.SECONDS), + HttpTransportSecrets.randomToken()); + assertEquals(legacy, HttpConnectionCode.parse(legacy.encodeLegacy())); + + Path client = directory.resolve("legacy-code-client"); + HttpClientCredentialStore.saveEnrolled(client, legacy, identity.issueClientCertificate("lobby")); + Path active = client.resolve("http-transport-client-generations") + .resolve(Files.readString(client.resolve("http-transport-client-current"))); + Files.writeString(active.resolve("http-transport-connection-code.sha256"), + HttpTransportSecrets.sha256Hex(legacy.encodeLegacy().getBytes(java.nio.charset.StandardCharsets.US_ASCII))); + assertTrue(HttpClientCredentialStore.matchesEnrollmentCode(client, + HttpConnectionCode.parse(legacy.encodeLegacy()))); + } + + @Test + void expiredCodesAreNotActive() { + HttpConnectionCode code = new HttpConnectionCode("lobby", URI.create("https://proxy.example.test/"), pin('a'), pin('b'), + Instant.parse("2029-12-31T23:59:59Z"), HttpTransportSecrets.randomToken()); + assertFalse(code.isActive(Clock.fixed(Instant.parse("2030-01-01T00:00:00Z"), ZoneOffset.UTC))); + assertThrows(IllegalArgumentException.class, () -> code.requireActive(Clock.fixed(Instant.parse("2030-01-01T00:00:00Z"), ZoneOffset.UTC))); + } + + @Test + void inboundDeliveryFenceRejectsCorruptionAndPathReplacement() throws Exception { + Path corruptCredentials = directory.resolve("corrupt-client"); + Path corruptFence = corruptCredentials.resolve("http-transport-inbound-deliveries"); + Files.createDirectories(corruptFence); + Files.writeString(corruptFence.resolve("not-a-delivery.seen"), "not-a-delivery"); + assertThrows(java.io.IOException.class, () -> new HttpInboundDeliveryStore(corruptCredentials)); + + Path replacedCredentials = directory.resolve("replaced-client"); + Files.createDirectories(replacedCredentials); + HttpInboundDeliveryStore store = new HttpInboundDeliveryStore(replacedCredentials); + Path fence = replacedCredentials.resolve("http-transport-inbound-deliveries"); + Path outside = directory.resolve("outside-fence"); + Files.createDirectory(outside); + Files.delete(fence); + Files.createSymbolicLink(fence, outside); + assertThrows(java.io.IOException.class, () -> store.reserve(java.util.UUID.randomUUID().toString())); + } + + @Test + void sealedInboundStoreCannotChangeAfterOwnershipHandoff() throws Exception { + Path credentials = directory.resolve("sealed-client"); + Files.createDirectories(credentials); + String id = java.util.UUID.randomUUID().toString(); + HttpInboundDeliveryStore store = new HttpInboundDeliveryStore(credentials); + store.reserve(id); + store.markRunning(id); + store.seal(); + assertThrows(java.io.IOException.class, () -> store.markCompleted(id)); + assertEquals(HttpInboundDeliveryStore.State.RUNNING, new HttpInboundDeliveryStore(credentials).state(id)); + } + + @Test + void identityIsDurableAndPinsRejectTheWrongServer() throws Exception { + HttpTlsIdentity created = HttpTlsIdentity.loadOrCreate(directory, "localhost"); + HttpTlsIdentity loaded = HttpTlsIdentity.loadOrCreate(directory, "localhost"); + assertEquals(created.serverCertificatePin(), loaded.serverCertificatePin()); + assertEquals(created.caCertificatePin(), loaded.caCertificatePin()); + HttpConnectionCode correct = new HttpConnectionCode("lobby", URI.create("https://localhost:8443/"), created.serverCertificatePin(), + created.caCertificatePin(), Instant.now().plusSeconds(60), HttpTransportSecrets.randomToken()); + HttpConnectionCode incorrect = new HttpConnectionCode("lobby", URI.create("https://localhost:8443/"), pin('0'), created.caCertificatePin(), + Instant.now().plusSeconds(60), HttpTransportSecrets.randomToken()); + assertTrue(HttpPinnedTls.matchesServerPin(correct, created.serverCertificate())); + assertFalse(HttpPinnedTls.matchesServerPin(incorrect, created.serverCertificate())); + assertTrue(Files.exists(directory.resolve("http-transport-ca.p12"))); + HttpTlsIdentity rotated = HttpTlsIdentity.loadOrCreate(directory, "127.0.0.1"); + assertEquals(created.caCertificatePin(), rotated.caCertificatePin()); + assertNotEquals(created.serverCertificatePin(), rotated.serverCertificatePin()); + } + + @Test + void incompleteFirstRunTlsProvisioningRecoversWithoutManualCleanup() throws Exception { + Path source = directory.resolve("complete-identity"); + HttpTlsIdentity original = HttpTlsIdentity.loadOrCreate(source, "localhost"); + Path interrupted = directory.resolve("interrupted-identity"); + Files.createDirectories(interrupted); + Files.copy(source.resolve("http-transport-ca.p12"), interrupted.resolve("http-transport-ca.p12")); + Files.copy(source.resolve("http-transport-server.p12"), interrupted.resolve("http-transport-server.p12")); + + HttpTlsIdentity recovered = HttpTlsIdentity.loadOrCreate(interrupted, "localhost"); + + assertNotEquals(original.caCertificatePin(), recovered.caCertificatePin()); + assertTrue(Files.exists(interrupted.resolve("http-transport-ca.p12"))); + assertTrue(Files.exists(interrupted.resolve("http-transport-server.p12"))); + assertTrue(Files.exists(interrupted.resolve("http-transport-password"))); + assertFalse(Files.exists(interrupted.resolve("http-transport-initializing"))); + } + + @Test + void completedFirstRunFilesRecoverWhenInitializationMarkerSurvives() throws Exception { + Path interrupted = directory.resolve("marked-identity"); + HttpTlsIdentity original = HttpTlsIdentity.loadOrCreate(interrupted, "localhost"); + String originalCaPin = original.caCertificatePin(); + Files.writeString(interrupted.resolve("http-transport-initializing"), "initializing\n"); + + HttpTlsIdentity recovered = HttpTlsIdentity.loadOrCreate(interrupted, "localhost"); + + assertNotEquals(originalCaPin, recovered.caCertificatePin()); + assertFalse(Files.exists(interrupted.resolve("http-transport-initializing"))); + } + + @Test + void incompleteEstablishedTlsIdentityFailsClosed() throws Exception { + Path established = directory.resolve("established-identity"); + HttpTlsIdentity.loadOrCreate(established, "localhost"); + Files.writeString(established.resolve("http-transport-clients.properties"), "version=2\n"); + Files.delete(established.resolve("http-transport-server.p12")); + + assertThrows(java.io.IOException.class, () -> HttpTlsIdentity.loadOrCreate(established, "localhost")); + assertTrue(Files.exists(established.resolve("http-transport-ca.p12"))); + assertTrue(Files.exists(established.resolve("http-transport-password"))); + } + + @Test + void enrollmentIsSingleUseBoundToServerAndRevocable() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory, "localhost"); + Clock clock = Clock.fixed(Instant.parse("2030-01-01T00:00:00Z"), ZoneOffset.UTC); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, clock); + HttpConnectionCode wrongTargetCode = authority.createConnectionCode("lobby-1", URI.create("https://localhost:8443/"), Duration.ofMinutes(5)); + assertThrows(IllegalArgumentException.class, () -> authority.enroll("attacker", wrongTargetCode.enrollmentToken())); + assertTrue(authority.authenticate("lobby-1", authority.enroll("lobby-1", wrongTargetCode.enrollmentToken()).certificate()), + "a wrong backend must not consume another backend's connection code"); + authority.revoke("lobby-1"); + HttpConnectionCode code = authority.createConnectionCode("lobby-1", URI.create("https://localhost:8443/"), Duration.ofMinutes(5)); + HttpTlsIdentity.IssuedClientCertificate issued = authority.enroll("lobby-1", code.enrollmentToken()); + assertTrue(authority.authenticate("lobby-1", issued.certificate())); + assertTrue(identity.validClientCertificate("LOBBY-1", issued.certificate())); + assertFalse(authority.authenticate("lobby-2", issued.certificate())); + assertThrows(IllegalArgumentException.class, () -> authority.enroll("lobby-2", code.enrollmentToken())); + authority.revoke("lobby-1"); + assertFalse(authority.authenticate("lobby-1", issued.certificate())); + HttpConnectionCode replacementCode = authority.createConnectionCode("lobby-1", URI.create("https://localhost:8443/"), Duration.ofMinutes(5)); + HttpTlsIdentity.IssuedClientCertificate replacement = authority.enroll("lobby-1", replacementCode.enrollmentToken()); + assertTrue(authority.authenticate("lobby-1", replacement.certificate())); + assertFalse(authority.authenticate("lobby-1", issued.certificate())); + HttpClientCredentialStore.saveEnrolled(directory.resolve("client"), code, issued); + HttpClientCredentialStore.ClientCredential restored = HttpClientCredentialStore.load(directory.resolve("client")); + assertEquals(HttpTransportSecrets.certificatePin(issued.certificate()), HttpTransportSecrets.certificatePin(restored.certificate())); + HttpClientCredentialStore.HttpClientProfile profile = HttpClientCredentialStore.loadProfile(directory.resolve("client")); + assertEquals("lobby-1", profile.serverId()); + assertEquals(code.endpoint(), profile.endpoint()); + assertEquals("lobby-1", HttpClientCredentialStore.loadEnrolled(directory.resolve("client")).profile().serverId()); + assertNotEquals(null, HttpPinnedTls.mutualTlsContext(code, restored)); + Path clientDirectory = directory.resolve("client"); + String generation = Files.readString(clientDirectory.resolve("http-transport-client-current")); + Files.writeString(clientDirectory.resolve("http-transport-client-generations").resolve(generation) + .resolve("http-transport-profile.properties"), "version=1\nserverId=lobby-1\n"); + assertThrows(java.io.IOException.class, () -> HttpClientCredentialStore.loadProfile(directory.resolve("client"))); + HttpEnrollmentAuthority durable = new HttpEnrollmentAuthority(identity, directory.resolve("state")); + HttpConnectionCode durableCode = durable.createConnectionCode("survival", URI.create("https://localhost:8443/"), Duration.ofMinutes(5)); + HttpTlsIdentity.IssuedClientCertificate durableIssued = durable.enroll("survival", durableCode.enrollmentToken()); + assertTrue(new HttpEnrollmentAuthority(identity, directory.resolve("state")).authenticate("survival", durableIssued.certificate())); + durable.revoke("survival"); + assertFalse(new HttpEnrollmentAuthority(identity, directory.resolve("state")).authenticate("survival", durableIssued.certificate())); + } + + @Test + void revocationInvalidatesEveryPendingCodeForTheBackend() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("revoke-pending"), "localhost"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, + Clock.fixed(Instant.parse("2030-01-01T00:00:00Z"), ZoneOffset.UTC)); + URI endpoint = URI.create("https://localhost:8443/"); + HttpConnectionCode beforeEnrollment = authority.createConnectionCode("lobby-1", endpoint, Duration.ofMinutes(5)); + authority.revoke("LOBBY-1"); + assertThrows(IllegalArgumentException.class, + () -> authority.enroll("lobby-1", beforeEnrollment.enrollmentToken())); + + HttpConnectionCode active = authority.createConnectionCode("lobby-1", endpoint, Duration.ofMinutes(5)); + authority.enroll("lobby-1", active.enrollmentToken()); + HttpConnectionCode firstPending = authority.createConnectionCode("lobby-1", endpoint, Duration.ofMinutes(5)); + HttpConnectionCode secondPending = authority.createConnectionCode("lobby-1", endpoint, Duration.ofMinutes(5)); + authority.revoke("lobby-1"); + assertThrows(IllegalArgumentException.class, + () -> authority.enroll("lobby-1", firstPending.enrollmentToken())); + assertThrows(IllegalArgumentException.class, + () -> authority.enroll("lobby-1", secondPending.enrollmentToken())); + } + + @Test + void renewalKeepsOldCredentialUntilReplacementAuthenticates() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, directory.resolve("state")); + HttpConnectionCode code = authority.createConnectionCode("lobby-1", URI.create("https://localhost:8443/"), Duration.ofMinutes(5)); + HttpTlsIdentity.IssuedClientCertificate original = authority.enroll("lobby-1", code.enrollmentToken()); + HttpTlsIdentity.IssuedClientCertificate replacement = authority.renew("lobby-1", original.certificate()); + + assertTrue(authority.authenticate("lobby-1", original.certificate()), "lost renewal responses must leave the old credential usable"); + assertTrue(authority.authenticate("lobby-1", replacement.certificate()), "first replacement request promotes the pending binding"); + assertFalse(authority.authenticate("lobby-1", original.certificate()), "promotion revokes the superseded credential"); + assertTrue(new HttpEnrollmentAuthority(identity, directory.resolve("state")) + .authenticate("lobby-1", replacement.certificate()), "promoted renewal must survive restart"); + } + + @Test + void serverLeafRotatesInsideRenewalWindowAndPreservesAuthority() throws Exception { + Instant now = Instant.now(); + HttpTlsIdentity original = HttpTlsIdentity.loadOrCreate(directory, "localhost", + Clock.fixed(now.minus(Duration.ofDays(340)), ZoneOffset.UTC)); + String originalPin = HttpTransportSecrets.certificatePin(original.serverCertificate()); + HttpTlsIdentity renewed = HttpTlsIdentity.loadOrCreate(directory, "localhost", Clock.fixed(now, ZoneOffset.UTC)); + assertNotEquals(originalPin, renewed.serverCertificatePin()); + assertEquals(original.caCertificatePin(), renewed.caCertificatePin()); + assertFalse(HttpTlsIdentity.needsRenewal(renewed.serverCertificate(), Clock.fixed(now, ZoneOffset.UTC))); + } + + @Test + void runningPrivateCaRollsOverBeforeExpiryWithoutStrandingExistingClients() throws Exception { + Instant now = Instant.now(); + Clock originalClock = Clock.fixed(now.minus(Duration.ofDays(9 * 365L + 30L)), ZoneOffset.UTC); + HttpTlsIdentity original = HttpTlsIdentity.loadOrCreate(directory, "localhost", originalClock); + X509Certificate originalCa = original.caCertificate(); + HttpTlsIdentity.IssuedClientCertificate existingClient = original.issueClientCertificate("lobby-1", now); + Path client = directory.resolve("client"); + HttpConnectionCode oldCode = new HttpConnectionCode("lobby-1", URI.create("https://localhost:8443/"), + HttpTransportSecrets.certificatePin(original.serverCertificate()), HttpTransportSecrets.certificatePin(originalCa), + now.plusSeconds(60), "A".repeat(43)); + HttpClientCredentialStore.saveEnrolled(client, oldCode, existingClient); + + String renewedPin = original.caCertificatePin(); + assertNotEquals(HttpTransportSecrets.certificatePin(originalCa), renewedPin); + assertEquals(originalCa.getPublicKey(), original.caCertificate().getPublicKey(), + "certificate rollover keeps the private authority key so old and new trust anchors overlap"); + assertFalse(HttpTlsIdentity.needsCaRenewal(original.caCertificate(), Clock.fixed(now, ZoneOffset.UTC))); + assertTrue(original.validClientCertificate("lobby-1", existingClient.certificate())); + + X509TrustManager oldClientTrust = Arrays.stream(HttpTlsIdentity.trustManagers(originalCa)) + .filter(X509TrustManager.class::isInstance).map(X509TrustManager.class::cast).findFirst().orElseThrow(); + assertDoesNotThrow(() -> oldClientTrust.checkServerTrusted( + new X509Certificate[] { original.serverCertificate(), original.caCertificate() }, "ECDHE_ECDSA")); + HttpClientCredentialStore.StagedCredential staged = HttpClientCredentialStore.stageReplacement(client, + original.issueClientCertificate("lobby-1", now)); + assertEquals(HttpTransportSecrets.certificatePin(originalCa), HttpClientCredentialStore.loadProfile(client).caCertificatePin()); + assertEquals(renewedPin, staged.profile().caCertificatePin()); + HttpClientCredentialStore.activateReplacement(client, staged); + assertEquals(renewedPin, HttpClientCredentialStore.loadProfile(client).caCertificatePin()); + assertEquals(renewedPin, HttpTlsIdentity.loadOrCreate(directory, "localhost").caCertificatePin(), + "live CA rollover must survive restart"); + } + + @Test + void activeTlsContextRotatesServerLeafInsideRenewalWindow() throws Exception { + Instant now = Instant.now(); + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory, "localhost", + Clock.fixed(now.minus(Duration.ofDays(340)), ZoneOffset.UTC)); + String expiringPin = HttpTransportSecrets.certificatePin(identity.serverCertificate()); + identity.serverContext(); + assertNotEquals(expiringPin, identity.serverCertificatePin()); + assertFalse(HttpTlsIdentity.needsRenewal(identity.serverCertificate(), Clock.systemUTC())); + } + + @Test + void serverTlsUsesPrivateCaTrustAndRejectsForeignClients() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); + HttpTlsIdentity foreign = HttpTlsIdentity.loadOrCreate(directory.resolve("foreign"), "localhost"); + X509TrustManager trust = Arrays.stream(HttpTlsIdentity.trustManagers(identity.caCertificate())) + .filter(X509TrustManager.class::isInstance).map(X509TrustManager.class::cast).findFirst().orElseThrow(); + HttpTlsIdentity.IssuedClientCertificate accepted = identity.issueClientCertificate("lobby-1"); + HttpTlsIdentity.IssuedClientCertificate rejected = foreign.issueClientCertificate("lobby-1"); + assertDoesNotThrow(() -> trust.checkClientTrusted( + new java.security.cert.X509Certificate[] { accepted.certificate(), identity.caCertificate() }, "EC")); + assertThrows(java.security.cert.CertificateException.class, () -> trust.checkClientTrusted( + new java.security.cert.X509Certificate[] { rejected.certificate(), foreign.caCertificate() }, "EC")); + } + + @Test + void stagedCredentialDoesNotReplaceActiveGenerationUntilAtomicActivation() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); + Path client = directory.resolve("client"); + HttpTlsIdentity.IssuedClientCertificate original = identity.issueClientCertificate("lobby-1"); + HttpConnectionCode code = new HttpConnectionCode("lobby-1", URI.create("https://localhost:8443/"), + identity.serverCertificatePin(), identity.caCertificatePin(), Instant.now().plusSeconds(60), "A".repeat(43)); + HttpClientCredentialStore.saveEnrolled(client, code, original); + String originalPin = HttpTransportSecrets.certificatePin(HttpClientCredentialStore.load(client).certificate()); + HttpTlsIdentity.IssuedClientCertificate replacement = identity.issueClientCertificate("lobby-1"); + HttpClientCredentialStore.StagedCredential staged = HttpClientCredentialStore.stageReplacement(client, replacement); + assertEquals(originalPin, HttpTransportSecrets.certificatePin(HttpClientCredentialStore.load(client).certificate())); + HttpClientCredentialStore.activateReplacement(client, staged); + assertNotEquals(originalPin, HttpTransportSecrets.certificatePin(HttpClientCredentialStore.load(client).certificate())); + assertTrue(HttpClientCredentialStore.matchesEnrollmentCode(client, code), + "automatic certificate renewal must retain the consumed-code marker"); + HttpTlsIdentity.IssuedClientCertificate manuallyReenrolled = identity.issueClientCertificate("lobby-1"); + HttpClientCredentialStore.saveEnrolled(client, code, manuallyReenrolled); + assertEquals(HttpTransportSecrets.certificatePin(manuallyReenrolled.certificate()), + HttpTransportSecrets.certificatePin(HttpClientCredentialStore.loadEnrolled(client).credential().certificate())); + } + + @Test + void restoresCredentialGenerationAfterFailedReenrollment() throws Exception { + Path client = directory.resolve("client-rollback"); + HttpTlsIdentity oldIdentity = HttpTlsIdentity.loadOrCreate(directory.resolve("old-proxy"), "old.example.test"); + HttpConnectionCode oldCode = new HttpConnectionCode("lobby-1", URI.create("https://old.example.test:1297/"), + oldIdentity.serverCertificatePin(), oldIdentity.caCertificatePin(), Instant.now().plusSeconds(60), + "R".repeat(43)); + HttpClientCredentialStore.saveEnrolled(client, oldCode, oldIdentity.issueClientCertificate("lobby-1")); + HttpClientCredentialStore.ActiveCredentialGeneration previous = + HttpClientCredentialStore.snapshotActiveGeneration(client); + + HttpTlsIdentity replacementIdentity = HttpTlsIdentity.loadOrCreate(directory.resolve("new-proxy"), "new.example.test"); + HttpConnectionCode replacementCode = new HttpConnectionCode("lobby-1", URI.create("https://new.example.test:1297/"), + replacementIdentity.serverCertificatePin(), replacementIdentity.caCertificatePin(), + Instant.now().plusSeconds(60), "S".repeat(43)); + HttpClientCredentialStore.saveEnrolled(client, replacementCode, + replacementIdentity.issueClientCertificate("lobby-1")); + assertEquals(replacementCode.endpoint(), HttpClientCredentialStore.loadProfile(client).endpoint()); + + HttpClientCredentialStore.restoreActiveGeneration(client, previous); + assertEquals(oldCode.endpoint(), HttpClientCredentialStore.loadProfile(client).endpoint()); + assertTrue(HttpClientCredentialStore.matchesEnrollmentCode(client, oldCode)); + } + + @Test + void rollbackRetainsNewerCredentialForTheSameEndpoint() throws Exception { + Path client = directory.resolve("client-renewal-rollback"); + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("renewal-proxy"), "renew.example.test"); + HttpConnectionCode code = new HttpConnectionCode("lobby-1", URI.create("https://renew.example.test:1297/"), + identity.serverCertificatePin(), identity.caCertificatePin(), Instant.now().plusSeconds(60), + "T".repeat(43)); + HttpTlsIdentity.IssuedClientCertificate original = identity.issueClientCertificate("lobby-1"); + HttpClientCredentialStore.saveEnrolled(client, code, original); + HttpClientCredentialStore.ActiveCredentialGeneration previous = + HttpClientCredentialStore.snapshotActiveGeneration(client); + + HttpConnectionCode replacementCode = new HttpConnectionCode("lobby-1", code.endpoint(), + identity.serverCertificatePin(), identity.caCertificatePin(), Instant.now().plusSeconds(60), + "U".repeat(43)); + HttpTlsIdentity.IssuedClientCertificate renewed = identity.issueClientCertificate("lobby-1"); + HttpClientCredentialStore.saveEnrolled(client, replacementCode, renewed); + HttpClientCredentialStore.restoreActiveGenerationAfterReplacement(client, previous); + + assertEquals(HttpTransportSecrets.certificatePin(renewed.certificate()), + HttpTransportSecrets.certificatePin(HttpClientCredentialStore.load(client).certificate()), + "rollback must not reactivate a same-endpoint certificate that renewal may have revoked"); + assertTrue(HttpClientCredentialStore.matchesEnrollmentCode(client, code), + "the retained credential must recognize the connection code restored in YAML"); + } + + private static String pin(char character) { return String.valueOf(character).repeat(64); } +} From 2951868d7fc7d01ee2108899a4dbc4fd2198202f Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Fri, 4 Sep 2026 18:36:01 -0600 Subject: [PATCH 02/38] fix(servercomm): persist HTTP replay state --- .../http/HttpEnrollmentAuthority.java | 55 +++++++++--- .../http/HttpInboundDeliveryStore.java | 29 ++++++- .../http/HttpProxyTransportServer.java | 85 +++++++++++++++---- .../http/HttpTransportRuntimeTest.java | 26 ++++++ .../http/HttpTransportSecurityTest.java | 23 +++++ 5 files changed, 191 insertions(+), 27 deletions(-) diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentAuthority.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentAuthority.java index b223acd..a4b3941 100644 --- a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentAuthority.java +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentAuthority.java @@ -26,6 +26,7 @@ */ public final class HttpEnrollmentAuthority { private static final Duration MAX_ENROLLMENT_LIFETIME = Duration.ofMinutes(15); + private static final int MAX_PENDING_ENROLLMENTS = 128; private final HttpTlsIdentity identity; private final Clock clock; private final Path stateFile; @@ -34,7 +35,7 @@ public final class HttpEnrollmentAuthority { private final Set revokedCertificatePins = new HashSet<>(); private boolean persistenceFailure; - /** Creates a restart-safe authority. State contains only public certificate pins and revocations. */ + /** Creates a restart-safe authority. State contains public certificate pins plus bounded hashes of pending tokens. */ public HttpEnrollmentAuthority(HttpTlsIdentity identity, Path stateDirectory) throws java.io.IOException { this(identity, Clock.systemUTC(), stateFile(stateDirectory)); loadState(); @@ -56,11 +57,18 @@ public synchronized HttpConnectionCode createConnectionCode(String serverId, URI if (lifetime == null || lifetime.isNegative() || lifetime.isZero() || lifetime.compareTo(MAX_ENROLLMENT_LIFETIME) > 0) throw new IllegalArgumentException("Enrollment lifetime must be between one second and fifteen minutes"); expireEnrollments(); + if (enrollments.size() >= MAX_PENDING_ENROLLMENTS) + throw new IllegalStateException("Too many pending HTTP enrollments"); Instant expiresAt = clock.instant().plus(lifetime); String token = HttpTransportSecrets.randomToken(); byte[] tokenHash = HttpTransportSecrets.sha256(token.getBytes(StandardCharsets.US_ASCII)); String lookup = java.util.Base64.getUrlEncoder().withoutPadding().encodeToString(tokenHash); enrollments.put(lookup, new Enrollment(tokenHash, expiresAt, serverId)); + try { persistState(); } + catch (java.io.IOException failure) { + enrollments.remove(lookup); + throw new IllegalStateException("Could not persist HTTP enrollment", failure); + } return new HttpConnectionCode(serverId, endpoint, identity.serverCertificatePin(), identity.caCertificatePin(), expiresAt, token); } @@ -119,15 +127,15 @@ public synchronized void revoke(String serverId) { try { serverId = HttpTlsIdentity.canonicalServerId(serverId); } catch (IllegalArgumentException invalid) { return; } final String revokedServer = serverId; - enrollments.entrySet().removeIf(entry -> revokedServer.equals(entry.getValue().serverId())); + boolean pendingRemoved = enrollments.entrySet().removeIf(entry -> revokedServer.equals(entry.getValue().serverId())); ClientBinding binding = bindings.get(serverId); if (binding != null) { bindings.put(serverId, new ClientBinding(binding.certificatePin(), binding.pendingCertificatePin(), true)); revokedCertificatePins.add(binding.certificatePin()); if (binding.pendingCertificatePin() != null) revokedCertificatePins.add(binding.pendingCertificatePin()); - try { persistState(); } - catch (java.io.IOException failure) { persistenceFailure = true; throw new IllegalStateException("Could not persist HTTP certificate revocation", failure); } } + if (binding != null || pendingRemoved) try { persistState(); } + catch (java.io.IOException failure) { persistenceFailure = true; throw new IllegalStateException("Could not persist HTTP certificate revocation", failure); } } private synchronized void loadState() throws java.io.IOException { @@ -136,13 +144,16 @@ private synchronized void loadState() throws java.io.IOException { throw new java.io.IOException("HTTP enrollment state is invalid"); Properties properties = new Properties(); try (var input = Files.newInputStream(stateFile, LinkOption.NOFOLLOW_LINKS)) { properties.load(input); } + String version = properties.getProperty("version"); + if (!("1".equals(version) || "2".equals(version) || "3".equals(version))) + throw new java.io.IOException("HTTP enrollment state is invalid"); for (String key : properties.stringPropertyNames()) { if (key.startsWith("binding.")) { String serverId = new String(Base64.getUrlDecoder().decode(key.substring("binding.".length())), StandardCharsets.UTF_8); serverId = HttpTlsIdentity.canonicalServerId(serverId); String[] value = properties.getProperty(key, "").split(":", -1); - if (!((value.length == 2 && "1".equals(properties.getProperty("version"))) - || (value.length == 3 && "2".equals(properties.getProperty("version")))) + if (!((value.length == 2 && "1".equals(version)) + || (value.length == 3 && ("2".equals(version) || "3".equals(version)))) || !value[0].matches("[0-9a-f]{64}")) throw new java.io.IOException("HTTP enrollment state is invalid"); String pending = value.length == 3 && !"-".equals(value[1]) ? value[1] : null; @@ -151,24 +162,48 @@ private synchronized void loadState() throws java.io.IOException { throw new java.io.IOException("HTTP enrollment state is invalid"); bindings.put(serverId, new ClientBinding(value[0], pending, "1".equals(revoked))); if ("1".equals(revoked)) { revokedCertificatePins.add(value[0]); if (pending != null) revokedCertificatePins.add(pending); } + } else if (key.startsWith("enrollment.") && "3".equals(version)) { + String lookup = key.substring("enrollment.".length()); + if (!lookup.matches("[A-Za-z0-9_-]{43}")) throw new java.io.IOException("HTTP enrollment state is invalid"); + byte[] tokenHash; + try { tokenHash = Base64.getUrlDecoder().decode(lookup); } + catch (IllegalArgumentException invalid) { throw new java.io.IOException("HTTP enrollment state is invalid", invalid); } + if (tokenHash.length != 32) throw new java.io.IOException("HTTP enrollment state is invalid"); + String[] value = properties.getProperty(key, "").split(":", -1); + if (value.length != 2) throw new java.io.IOException("HTTP enrollment state is invalid"); + Instant expiresAt; + String serverId; + try { + expiresAt = Instant.ofEpochMilli(Long.parseLong(value[0])); + serverId = HttpTlsIdentity.canonicalServerId(new String(Base64.getUrlDecoder().decode(value[1]), StandardCharsets.UTF_8)); + } catch (RuntimeException invalid) { throw new java.io.IOException("HTTP enrollment state is invalid", invalid); } + if (expiresAt.isAfter(clock.instant())) { + if (enrollments.size() >= MAX_PENDING_ENROLLMENTS) + throw new java.io.IOException("HTTP enrollment state exceeds its bound"); + enrollments.put(lookup, new Enrollment(tokenHash, expiresAt, serverId)); + } } else if (!"version".equals(key)) throw new java.io.IOException("HTTP enrollment state is invalid"); } - if (!("1".equals(properties.getProperty("version")) || "2".equals(properties.getProperty("version")))) - throw new java.io.IOException("HTTP enrollment state is invalid"); } private synchronized void persistState() throws java.io.IOException { if (stateFile == null) return; Properties properties = new Properties(); - properties.setProperty("version", "2"); + properties.setProperty("version", "3"); for (Map.Entry entry : bindings.entrySet()) { String key = Base64.getUrlEncoder().withoutPadding().encodeToString(entry.getKey().getBytes(StandardCharsets.UTF_8)); properties.setProperty("binding." + key, entry.getValue().certificatePin() + ":" + (entry.getValue().pendingCertificatePin() == null ? "-" : entry.getValue().pendingCertificatePin()) + ":" + (entry.getValue().revoked() ? "1" : "0")); } + for (Map.Entry entry : enrollments.entrySet()) { + String server = Base64.getUrlEncoder().withoutPadding().encodeToString( + entry.getValue().serverId().getBytes(StandardCharsets.UTF_8)); + properties.setProperty("enrollment." + entry.getKey(), + entry.getValue().expiresAt().toEpochMilli() + ":" + server); + } java.io.ByteArrayOutputStream bytes = new java.io.ByteArrayOutputStream(); - properties.store(bytes, "VotingPlugin HTTP certificate bindings"); + properties.store(bytes, "VotingPlugin HTTP transport authority state"); Path temporary = Files.createTempFile(stateFile.getParent(), stateFile.getFileName().toString(), ".tmp"); try { setOwnerOnly(temporary); diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpInboundDeliveryStore.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpInboundDeliveryStore.java index 22022d2..0c1477e 100644 --- a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpInboundDeliveryStore.java +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpInboundDeliveryStore.java @@ -24,11 +24,21 @@ final class HttpInboundDeliveryStore { private boolean sealed; HttpInboundDeliveryStore(Path credentialDirectory) throws IOException { - Path credentials = credentialDirectory.toAbsolutePath().normalize(); + this(credentialDirectory, DIRECTORY); + } + + static HttpInboundDeliveryStore open(Path parent, String directoryName) throws IOException { + return new HttpInboundDeliveryStore(parent, directoryName); + } + + private HttpInboundDeliveryStore(Path parent, String directoryName) throws IOException { + Path credentials = parent.toAbsolutePath().normalize(); if (Files.isSymbolicLink(credentials) || !Files.isDirectory(credentials, LinkOption.NOFOLLOW_LINKS)) throw new IOException("HTTP credential directory is unsafe"); + if (directoryName == null || !directoryName.matches("[A-Za-z0-9][A-Za-z0-9._-]{0,63}")) + throw new IOException("HTTP inbound delivery directory name is invalid"); ownerOnlyDirectory(credentials); - root = credentials.resolve(DIRECTORY).normalize(); + root = credentials.resolve(directoryName).normalize(); if (!root.getParent().equals(credentials)) throw new IOException("HTTP inbound delivery directory is invalid"); boolean created = false; try { Files.createDirectory(root); created = true; } @@ -67,6 +77,21 @@ synchronized void reserve(String id) throws IOException { } finally { Files.deleteIfExists(temporary); } } + /** Admits a new proxy-side fence by retiring one completed bounded-window entry when necessary. */ + synchronized void reserveReplacingCompleted(String id) throws IOException { + id = canonical(id); + if (entries.get(id) != null) return; + if (entries.size() >= MAX_ENTRIES) { + String completed = null; + for (Map.Entry entry : entries.entrySet()) { + if (entry.getValue() == State.COMPLETED) { completed = entry.getKey(); break; } + } + if (completed == null) throw new IOException("HTTP inbound delivery fence is full"); + remove(completed); + } + reserve(id); + } + synchronized void markRunning(String id) throws IOException { transition(id, State.RESERVED, State.RUNNING); } synchronized void markCompleted(String id) throws IOException { transition(id, State.RUNNING, State.COMPLETED); } synchronized void seal() { sealed = true; } diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java index cc45cc4..cdf370e 100644 --- a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java @@ -67,6 +67,7 @@ public final class HttpProxyTransportServer implements AutoCloseable { private final Semaphore admission = new Semaphore(64); private final Map backends = new HashMap<>(); private final DurableOutgoingQueue durableOutgoing; + private final Path durableIncomingRoot; private final Consumer onEnvelope; private final DeliveryAcknowledgement onAcknowledged; private volatile boolean closed; @@ -89,11 +90,11 @@ public HttpProxyTransportServer(InetSocketAddress bind, HttpTlsIdentity identity this.identity = identity; this.authority = authority; this.onEnvelope = onEnvelope; this.onAcknowledged = onAcknowledged; durableOutgoing = outgoingDirectory == null ? null : new DurableOutgoingQueue(outgoingDirectory); + durableIncomingRoot = outgoingDirectory == null ? null : incomingRoot(outgoingDirectory); if (durableOutgoing != null) for (Map.Entry> pending : durableOutgoing.load().entrySet()) { - BackendState state = new BackendState(pending.getKey(), durableOutgoing, onAcknowledged); + BackendState state = backendState(pending.getKey()); state.restore(pending.getValue()); - backends.put(pending.getKey(), state); } server = HttpsServer.create(bind, 32); server.setHttpsConfigurator(new HttpsConfigurator(identity.serverContext()) { @@ -150,15 +151,15 @@ public boolean send(String serverId, String deliveryId, JsonEnvelope envelope) { catch (IllegalArgumentException invalid) { return false; } BackendState backend; final String canonicalServerId = serverId; - synchronized (backends) { backend = backends.computeIfAbsent(serverId, - ignored -> new BackendState(canonicalServerId, durableOutgoing, onAcknowledged)); } + try { backend = backendState(canonicalServerId); } + catch (IOException persistenceFailure) { return false; } return backend.enqueue(new HttpTransportProtocol.Delivery(deliveryId, envelope)); } @Override public void close() { if (closed) return; closed = true; server.stop(1); shutdown(handlerExecutor); shutdown(listenerExecutor); - synchronized (backends) { for (BackendState backend : backends.values()) backend.signal(); backends.clear(); } + synchronized (backends) { for (BackendState backend : backends.values()) { backend.seal(); backend.signal(); } backends.clear(); } } private void enroll(HttpsExchange exchange) throws IOException { @@ -184,8 +185,7 @@ private void transport(HttpsExchange exchange) throws IOException { X509Certificate certificate = peerCertificate(exchange); if (certificate == null || !authority.authenticate(packet.server(), certificate)) { reply(exchange, 401, new byte[0]); return; } BackendState backend; - synchronized (backends) { backend = backends.computeIfAbsent(packet.server(), - ignored -> new BackendState(packet.server(), durableOutgoing, onAcknowledged)); } + backend = backendState(packet.server()); if (!backend.beginPoll(packet.session())) { reply(exchange, 409, new byte[0]); return; } try { handlePacket(packet, backend); @@ -210,13 +210,44 @@ private void handlePacket(HttpTransportProtocol.Packet packet, BackendState back private void dispatch(String serverId, BackendState backend, HttpTransportProtocol.Delivery delivery) { Runnable callback = () -> { boolean success = false; - try { onEnvelope.accept(new ReceivedEnvelope(serverId, delivery.id(), normalizeBackendIdentity(serverId, delivery.envelope()))); success = true; } + try { + backend.beginIncoming(delivery.id()); + onEnvelope.accept(new ReceivedEnvelope(serverId, delivery.id(), normalizeBackendIdentity(serverId, delivery.envelope()))); + backend.completeIncomingDurably(delivery.id()); + success = true; + } + catch (IOException persistenceFailure) { } catch (RuntimeException ignored) { } synchronized (backend) { backend.completeIncoming(delivery.id(), success); } }; if (!HttpBackendTransportConnector.executeOrdered(handlerExecutor, callback)) synchronized (backend) { backend.completeIncoming(delivery.id(), false); } } + private BackendState backendState(String serverId) throws IOException { + synchronized (backends) { + BackendState existing = backends.get(serverId); + if (existing != null) return existing; + HttpInboundDeliveryStore inbound = durableIncomingRoot == null ? null + : HttpInboundDeliveryStore.open(durableIncomingRoot, serverId); + BackendState created = new BackendState(serverId, durableOutgoing, inbound, onAcknowledged); + backends.put(serverId, created); + return created; + } + } + private static Path incomingRoot(Path outgoingDirectory) throws IOException { + Path outgoing = outgoingDirectory.toAbsolutePath().normalize(); + Path parent = outgoing.getParent(); + if (parent == null || outgoing.getFileName() == null) throw new IOException("HTTP incoming queue path is invalid"); + Path root = parent.resolve(outgoing.getFileName().toString() + "-incoming"); + boolean created = false; + try { Files.createDirectory(root); created = true; } + catch (java.nio.file.FileAlreadyExistsException existing) { } + if (Files.isSymbolicLink(root) || !Files.isDirectory(root, LinkOption.NOFOLLOW_LINKS)) + throw new IOException("HTTP incoming queue directory is invalid"); + DurableOutgoingQueue.ownerOnlyDirectory(root); + if (created) DurableFiles.forceDirectory(parent); + return root; + } private static JsonEnvelope normalizeBackendIdentity(String serverId, JsonEnvelope envelope) { // The authenticated TLS identity is authoritative; never forward a forged `server` field. return envelope.toBuilder().put("server", serverId).build(); @@ -262,6 +293,7 @@ static record Response(Collection acks, Collection { }, System::nanoTime); } - BackendState(LongSupplier nanoTime) { this(null, null, (serverId, deliveryId) -> { }, nanoTime); } + BackendState() { this(null, null, null, (serverId, deliveryId) -> { }, System::nanoTime); } + BackendState(LongSupplier nanoTime) { this(null, null, null, (serverId, deliveryId) -> { }, nanoTime); } private BackendState(String serverId, DurableOutgoingQueue durableOutgoing) { - this(serverId, durableOutgoing, (ignoredServer, ignoredDelivery) -> { }, System::nanoTime); + this(serverId, durableOutgoing, null, (ignoredServer, ignoredDelivery) -> { }, System::nanoTime); } BackendState(String serverId, DurableOutgoingQueue durableOutgoing, DeliveryAcknowledgement onAcknowledged) { - this(serverId, durableOutgoing, onAcknowledged, System::nanoTime); + this(serverId, durableOutgoing, null, onAcknowledged, System::nanoTime); + } + BackendState(String serverId, DurableOutgoingQueue durableOutgoing, HttpInboundDeliveryStore durableIncoming, + DeliveryAcknowledgement onAcknowledged) { + this(serverId, durableOutgoing, durableIncoming, onAcknowledged, System::nanoTime); } private BackendState(String serverId, DurableOutgoingQueue durableOutgoing, - DeliveryAcknowledgement onAcknowledged, LongSupplier nanoTime) { + HttpInboundDeliveryStore durableIncoming, DeliveryAcknowledgement onAcknowledged, LongSupplier nanoTime) { this.serverId = serverId; this.durableOutgoing = durableOutgoing; - this.onAcknowledged = onAcknowledged; this.nanoTime = nanoTime; + this.durableIncoming = durableIncoming; this.onAcknowledged = onAcknowledged; this.nanoTime = nanoTime; + if (durableIncoming != null) for (Map.Entry entry + : durableIncoming.snapshot().entrySet()) { + if (entry.getValue() == HttpInboundDeliveryStore.State.COMPLETED) { + seen.add(entry.getKey()); queueAck(entry.getKey()); + } + } } private synchronized void restore(Collection deliveries) { for (HttpTransportProtocol.Delivery delivery : deliveries) outgoing.put(delivery.id(), delivery); @@ -326,14 +368,27 @@ void acknowledge(Collection acks) throws IOException { List acceptIncoming(List received) { List accepted = new java.util.ArrayList<>(); for (HttpTransportProtocol.Delivery delivery : received) { - if (seen.contains(delivery.id())) { queueAck(delivery.id()); continue; } + HttpInboundDeliveryStore.State persisted = durableIncoming == null ? null : durableIncoming.state(delivery.id()); + if (seen.contains(delivery.id()) || persisted == HttpInboundDeliveryStore.State.COMPLETED) { + seen.add(delivery.id()); queueAck(delivery.id()); continue; + } + if (persisted == HttpInboundDeliveryStore.State.RUNNING) continue; if (!processing.contains(delivery.id())) { processing.add(delivery.id()); accepted.add(delivery); } } return accepted; } + void beginIncoming(String id) throws IOException { + if (durableIncoming == null) return; + if (durableIncoming.state(id) == null) durableIncoming.reserveReplacingCompleted(id); + durableIncoming.markRunning(id); + } + void completeIncomingDurably(String id) throws IOException { + if (durableIncoming != null) durableIncoming.markCompleted(id); + } synchronized void completeIncoming(String id, boolean success) { processing.remove(id); if (success) { seen.add(id); while (seen.size() > HttpTransportProtocol.MAX_QUEUE) seen.remove(seen.iterator().next()); queueAck(id); signal(); } } + private void seal() { if (durableIncoming != null) durableIncoming.seal(); } private void queueAck(String id) { if (acknowledgements.size() < HttpTransportProtocol.MAX_QUEUE && !acknowledgements.contains(id)) acknowledgements.add(id); } synchronized Response await(String serverId, String requestedSession, long requestedSequence) { long deadline = System.nanoTime() + LONG_POLL.toNanos(); diff --git a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportRuntimeTest.java b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportRuntimeTest.java index d93030f..7a87961 100644 --- a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportRuntimeTest.java +++ b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportRuntimeTest.java @@ -103,6 +103,32 @@ void stableProxyDeliveryIdsAreIdempotentAndAcknowledgedBeforeRemoval() throws Ex assertTrue(state.await("lobby-1", java.util.UUID.randomUUID().toString(), 0).messages().isEmpty()); } + @Test + void proxyInboundCompletionSurvivesRestartBeforeAcknowledgement() throws Exception { + Path root = directory.resolve("proxy-incoming"); + Files.createDirectory(root); + String deliveryId = java.util.UUID.randomUUID().toString(); + HttpTransportProtocol.Delivery delivery = new HttpTransportProtocol.Delivery(deliveryId, + JsonEnvelope.builder("backend-event").build()); + HttpInboundDeliveryStore firstStore = HttpInboundDeliveryStore.open(root, "lobby-1"); + HttpProxyTransportServer.BackendState first = new HttpProxyTransportServer.BackendState( + "lobby-1", null, firstStore, (server, id) -> { }); + assertEquals(java.util.List.of(delivery), first.acceptIncoming(java.util.List.of(delivery))); + first.beginIncoming(deliveryId); + first.completeIncomingDurably(deliveryId); + first.completeIncoming(deliveryId, true); + firstStore.seal(); + + HttpInboundDeliveryStore restartedStore = HttpInboundDeliveryStore.open(root, "lobby-1"); + HttpProxyTransportServer.BackendState restarted = new HttpProxyTransportServer.BackendState( + "lobby-1", null, restartedStore, (server, id) -> { }); + assertTrue(restarted.acceptIncoming(java.util.List.of(delivery)).isEmpty(), + "a completed callback must not run again after a lost response and proxy restart"); + HttpProxyTransportServer.Response response = restarted.await("lobby-1", + java.util.UUID.randomUUID().toString(), 0); + assertEquals(java.util.List.of(deliveryId), response.acks()); + } + @Test void failedAcknowledgementCallbackRetainsProxyDelivery() throws Exception { HttpProxyTransportServer.BackendState state = new HttpProxyTransportServer.BackendState("lobby-1", null, diff --git a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportSecurityTest.java b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportSecurityTest.java index 11a095a..d4878a3 100644 --- a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportSecurityTest.java +++ b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportSecurityTest.java @@ -239,6 +239,29 @@ void revocationInvalidatesEveryPendingCodeForTheBackend() throws Exception { () -> authority.enroll("lobby-1", secondPending.enrollmentToken())); } + @Test + void pendingEnrollmentSurvivesRestartAndRevocationRemainsDurable() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("pending-restart-proxy"), "localhost"); + Path state = directory.resolve("pending-restart-state"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, state); + HttpConnectionCode code = authority.createConnectionCode("lobby-1", + URI.create("https://localhost:8443/"), Duration.ofMinutes(5)); + assertFalse(Files.readString(state.resolve("http-transport-clients.properties")) + .contains(code.enrollmentToken()), "raw enrollment tokens must never be persisted"); + + HttpEnrollmentAuthority restarted = new HttpEnrollmentAuthority(identity, state); + HttpTlsIdentity.IssuedClientCertificate issued = restarted.enroll("lobby-1", code.enrollmentToken()); + assertTrue(restarted.authenticate("lobby-1", issued.certificate())); + + restarted.revoke("lobby-1"); + HttpConnectionCode revokedPending = restarted.createConnectionCode("lobby-1", + URI.create("https://localhost:8443/"), Duration.ofMinutes(5)); + restarted.revoke("lobby-1"); + HttpEnrollmentAuthority afterRevocation = new HttpEnrollmentAuthority(identity, state); + assertThrows(IllegalArgumentException.class, + () -> afterRevocation.enroll("lobby-1", revokedPending.enrollmentToken())); + } + @Test void renewalKeepsOldCredentialUntilReplacementAuthenticates() throws Exception { HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); From f62bf69998fef40e35a369416b0c89e636edc7bf Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Fri, 4 Sep 2026 18:45:28 -0600 Subject: [PATCH 03/38] fix(servercomm): confirm HTTP delivery acknowledgements --- .../http/HttpBackendTransportConnector.java | 34 +++++++++++++++---- .../http/HttpInboundDeliveryStore.java | 30 +++++++++------- .../http/HttpProxyTransportServer.java | 14 ++++++-- .../http/HttpTransportRuntimeTest.java | 14 ++++++++ 4 files changed, 72 insertions(+), 20 deletions(-) diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpBackendTransportConnector.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpBackendTransportConnector.java index aa9997d..6bddf30 100644 --- a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpBackendTransportConnector.java +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpBackendTransportConnector.java @@ -51,6 +51,7 @@ public final class HttpBackendTransportConnector implements AutoCloseable { private volatile HttpClientCredentialStore.ClientCredential credential; private final Path credentialDirectory; private final HttpInboundDeliveryStore inboundDeliveries; + private final HttpInboundDeliveryStore acknowledgementConfirmations; private final URI transportEndpoint; private final ThreadPoolExecutor callbackExecutor; private final AtomicBoolean running = new AtomicBoolean(); @@ -95,12 +96,19 @@ private HttpBackendTransportConnector(HttpClientCredentialStore.HttpClientProfil this.credential = credential; this.credentialDirectory = credentialDirectory; inboundDeliveries = credentialDirectory == null ? null : new HttpInboundDeliveryStore(credentialDirectory); + acknowledgementConfirmations = credentialDirectory == null ? null + : HttpInboundDeliveryStore.open(credentialDirectory, "http-transport-ack-confirmations"); if (inboundDeliveries != null) for (var entry : inboundDeliveries.snapshot().entrySet()) { if (entry.getValue() == HttpInboundDeliveryStore.State.COMPLETED) { received.add(entry.getKey()); queueAck(entry.getKey()); } } + if (acknowledgementConfirmations != null) for (var entry : acknowledgementConfirmations.snapshot().entrySet()) { + if (entry.getValue() != HttpInboundDeliveryStore.State.COMPLETED) + throw new IOException("HTTP acknowledgement confirmation state is invalid"); + queueAck(entry.getKey()); + } client = client(profile, credential); transportEndpoint = profile.endpoint().resolve("v1/transport"); // GlobalMessageHandler routes mutate backend vote state and must observe the @@ -183,7 +191,8 @@ private boolean pollOnce(Duration timeout, boolean requireRunning, boolean accep if (!serverId.equals(packet.server()) || !session.equals(packet.session()) || packet.sequence() != requestSequence) return false; confirmAcknowledgements(acks); acknowledgementsConfirmed = true; - synchronized (state) { for (String ack : packet.acks()) outgoing.remove(ack); } + if (!recordAcknowledgementConfirmations(packet.acks())) return false; + synchronized (state) { for (String ack : packet.acks()) { outgoing.remove(ack); queueAck(ack); } } if (acceptIncoming) for (HttpTransportProtocol.Delivery delivery : accept(packet.messages())) dispatch(delivery); firstResponse.countDown(); return true; @@ -197,7 +206,7 @@ public boolean flushOutgoing(long deadlineNanos) { Thread current = poller; if (current != null) current.interrupt(); if (!joinPoller(current, deadlineNanos)) return false; - while (queuedOutgoing() != 0) { + while (queuedOutgoing() != 0 || queuedAcknowledgements() != 0) { long remaining = deadlineNanos - System.nanoTime(); if (remaining <= 0L) return false; Duration timeout = Duration.ofNanos(Math.min(CLIENT_TIMEOUT.toNanos(), remaining)); @@ -221,6 +230,7 @@ public boolean flushOutgoing(long deadlineNanos) { // In-flight transitions serialize with seal(): either COMPLETED is already // durable, or the delivery remains durably RUNNING and fail-closed. if (inboundDeliveries != null) inboundDeliveries.seal(); + if (acknowledgementConfirmations != null) acknowledgementConfirmations.seal(); Thread current = poller; if (current != null) current.interrupt(); callbackExecutor.shutdown(); try { if (!callbackExecutor.awaitTermination(5, TimeUnit.SECONDS)) callbackExecutor.shutdownNow(); } catch (InterruptedException interrupted) { Thread.currentThread().interrupt(); callbackExecutor.shutdownNow(); } @@ -302,24 +312,36 @@ void completeIncoming(String id, boolean success) { synchronized (state) { processing.remove(id); if (success) { received.add(id); while (received.size() > HttpTransportProtocol.MAX_QUEUE) received.remove(received.iterator().next()); queueAck(id); } } } - private void queueAck(String id) { if (acknowledgements.size() < HttpTransportProtocol.MAX_QUEUE && !acknowledgements.contains(id)) acknowledgements.add(id); } + private void queueAck(String id) { if (acknowledgements.size() < HttpTransportProtocol.MAX_QUEUE * 2 && !acknowledgements.contains(id)) acknowledgements.add(id); } private void requeueAcknowledgements(List ids) { synchronized (state) { for (int index = ids.size() - 1; index >= 0; index--) { String id = ids.get(index); if (!acknowledgements.contains(id)) { - while (acknowledgements.size() >= HttpTransportProtocol.MAX_QUEUE) acknowledgements.removeLast(); + while (acknowledgements.size() >= HttpTransportProtocol.MAX_QUEUE * 2) acknowledgements.removeLast(); acknowledgements.addFirst(id); } } } } private void confirmAcknowledgements(Collection ids) { for (String id : ids) { + boolean removed = true; if (inboundDeliveries != null) try { inboundDeliveries.remove(id); } - catch (IOException cleanupFailure) { continue; } - synchronized (state) { received.remove(id); } + catch (IOException cleanupFailure) { removed = false; } + if (acknowledgementConfirmations != null) try { acknowledgementConfirmations.remove(id); } + catch (IOException cleanupFailure) { removed = false; } + synchronized (state) { + if (removed) received.remove(id); + else queueAck(id); + } } } + private boolean recordAcknowledgementConfirmations(Collection ids) { + if (acknowledgementConfirmations == null) return true; + try { for (String id : ids) acknowledgementConfirmations.recordCompleted(id); return true; } + catch (IOException persistenceFailure) { return false; } + } int queuedOutgoing() { synchronized (state) { return outgoing.size(); } } + int queuedAcknowledgements() { synchronized (state) { return acknowledgements.size(); } } List drainAcknowledgements() { synchronized (state) { return drain(acknowledgements); } } private static List first(Collection values) { List output = new java.util.ArrayList<>(); for (T value : values) { output.add(value); if (output.size() == HttpTransportProtocol.MAX_BATCH) break; } return output; } private static List drain(ArrayDeque values) { List output = new java.util.ArrayList<>(); while (!values.isEmpty() && output.size() < HttpTransportProtocol.MAX_BATCH) output.add(values.remove()); return output; } diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpInboundDeliveryStore.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpInboundDeliveryStore.java index 0c1477e..420ae02 100644 --- a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpInboundDeliveryStore.java +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpInboundDeliveryStore.java @@ -77,19 +77,25 @@ synchronized void reserve(String id) throws IOException { } finally { Files.deleteIfExists(temporary); } } - /** Admits a new proxy-side fence by retiring one completed bounded-window entry when necessary. */ - synchronized void reserveReplacingCompleted(String id) throws IOException { + /** Records receipt of a remote acknowledgement before its confirmation is sent. */ + synchronized void recordCompleted(String id) throws IOException { + requireWritable(); id = canonical(id); - if (entries.get(id) != null) return; - if (entries.size() >= MAX_ENTRIES) { - String completed = null; - for (Map.Entry entry : entries.entrySet()) { - if (entry.getValue() == State.COMPLETED) { completed = entry.getKey(); break; } - } - if (completed == null) throw new IOException("HTTP inbound delivery fence is full"); - remove(completed); - } - reserve(id); + if (entries.get(id) == State.COMPLETED) return; + if (entries.containsKey(id) || entries.size() >= MAX_ENTRIES) + throw new IOException("HTTP acknowledgement confirmation fence is full"); + requireRoot(); + Path target = file(id, State.COMPLETED); + Path temporary = Files.createTempFile(root, ".pending-", ".tmp"); + try { + ownerOnlyFile(temporary); + Files.writeString(temporary, id, StandardCharsets.US_ASCII, StandardOpenOption.TRUNCATE_EXISTING); + DurableFiles.forceFile(temporary); + move(temporary, target); + ownerOnlyFile(target); + DurableFiles.forceDirectory(root); + entries.put(id, State.COMPLETED); + } finally { Files.deleteIfExists(temporary); } } synchronized void markRunning(String id) throws IOException { transition(id, State.RESERVED, State.RUNNING); } diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java index cdf370e..15eecae 100644 --- a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java @@ -72,7 +72,8 @@ public final class HttpProxyTransportServer implements AutoCloseable { private final DeliveryAcknowledgement onAcknowledged; private volatile boolean closed; - public HttpProxyTransportServer(InetSocketAddress bind, HttpTlsIdentity identity, HttpEnrollmentAuthority authority, + /** In-memory constructor for tests; production callers must supply a durable state directory. */ + HttpProxyTransportServer(InetSocketAddress bind, HttpTlsIdentity identity, HttpEnrollmentAuthority authority, Consumer onEnvelope) throws Exception { this(bind, identity, authority, null, onEnvelope, (serverId, deliveryId) -> { }); } @@ -203,6 +204,7 @@ private void handlePacket(HttpTransportProtocol.Packet packet, BackendState back if (!backend.allowRequest()) throw new IllegalArgumentException("transport rate limited"); if (!backend.acceptSession(packet.session(), packet.sequence())) throw new IllegalArgumentException("stale session request"); } + backend.confirmIncoming(packet.acks()); backend.acknowledge(packet.acks()); synchronized (backend) { accepted = backend.acceptIncoming(packet.messages()); } for (HttpTransportProtocol.Delivery delivery : accepted) dispatch(packet.server(), backend, delivery); @@ -381,13 +383,21 @@ List acceptIncoming(List HttpTransportProtocol.MAX_QUEUE) seen.remove(seen.iterator().next()); queueAck(id); signal(); } } + synchronized void confirmIncoming(Collection ids) throws IOException { + for (String id : ids) { + if (durableIncoming != null && durableIncoming.state(id) == HttpInboundDeliveryStore.State.COMPLETED) + durableIncoming.remove(id); + seen.remove(id); + acknowledgements.removeIf(id::equals); + } + } private void seal() { if (durableIncoming != null) durableIncoming.seal(); } private void queueAck(String id) { if (acknowledgements.size() < HttpTransportProtocol.MAX_QUEUE && !acknowledgements.contains(id)) acknowledgements.add(id); } synchronized Response await(String serverId, String requestedSession, long requestedSequence) { diff --git a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportRuntimeTest.java b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportRuntimeTest.java index 7a87961..66d1575 100644 --- a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportRuntimeTest.java +++ b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportRuntimeTest.java @@ -33,7 +33,9 @@ void enrollsThenDeliversBothDirectionsWithAuthenticatedIdentity() throws Excepti HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, directory.resolve("authority")); CountDownLatch proxyReceived = new CountDownLatch(1), backendReceived = new CountDownLatch(1); AtomicReference received = new AtomicReference<>(); + Path proxyOutgoing = directory.resolve("proxy-outgoing"); try (HttpProxyTransportServer server = new HttpProxyTransportServer(new InetSocketAddress("localhost", 0), identity, authority, + proxyOutgoing, message -> { received.set(message); proxyReceived.countDown(); })) { server.start(); HttpConnectionCode code = authority.createConnectionCode("lobby-1", server.endpoint("localhost"), Duration.ofMinutes(5)); @@ -50,6 +52,11 @@ void enrollsThenDeliversBothDirectionsWithAuthenticatedIdentity() throws Excepti long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(3); while (connector.queuedOutgoing() != 0 && System.nanoTime() < deadline) Thread.sleep(10); assertEquals(0, connector.queuedOutgoing(), "proxy ACK must remove the exact outbound delivery ID"); + Path proxyInboundFence = directory.resolve("proxy-outgoing-incoming"); + long confirmationDeadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(3); + while (countRegularFiles(proxyInboundFence) != 0L && System.nanoTime() < confirmationDeadline) Thread.sleep(10); + assertEquals(0L, countRegularFiles(proxyInboundFence), + "the backend must durably confirm receipt of the proxy ACK"); assertTrue(server.send("lobby-1", JsonEnvelope.builder("to-backend").build())); assertTrue(backendReceived.await(8, TimeUnit.SECONDS)); Path inboundFence = directory.resolve("client").resolve("http-transport-inbound-deliveries"); @@ -127,6 +134,13 @@ void proxyInboundCompletionSurvivesRestartBeforeAcknowledgement() throws Excepti HttpProxyTransportServer.Response response = restarted.await("lobby-1", java.util.UUID.randomUUID().toString(), 0); assertEquals(java.util.List.of(deliveryId), response.acks()); + restarted.confirmIncoming(response.acks()); + restartedStore.seal(); + HttpInboundDeliveryStore confirmedStore = HttpInboundDeliveryStore.open(root, "lobby-1"); + HttpProxyTransportServer.BackendState confirmed = new HttpProxyTransportServer.BackendState( + "lobby-1", null, confirmedStore, (server, id) -> { }); + assertEquals(java.util.List.of(delivery), confirmed.acceptIncoming(java.util.List.of(delivery)), + "only an acknowledgement confirmation may retire the durable replay fence"); } @Test From e9159c1ee8d97fa762ec8aa1b8bf9985dfda6e46 Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Fri, 4 Sep 2026 18:52:42 -0600 Subject: [PATCH 04/38] fix(servercomm): fail closed on partial HTTP state --- .../http/HttpEnrollmentAuthority.java | 16 +++++++++++++-- .../servercomm/http/HttpTlsIdentity.java | 5 +++-- .../http/HttpTransportSecurityTest.java | 20 ++++++++++++++++++- 3 files changed, 36 insertions(+), 5 deletions(-) diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentAuthority.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentAuthority.java index a4b3941..b152f5d 100644 --- a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentAuthority.java +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentAuthority.java @@ -218,8 +218,14 @@ private synchronized void persistState() throws java.io.IOException { private static Path stateFile(Path directory) throws java.io.IOException { if (directory == null) throw new IllegalArgumentException("State directory is required"); - Files.createDirectories(directory); - Path file = directory.toAbsolutePath().normalize().resolve("http-transport-clients.properties"); + Path stateDirectory = directory.toAbsolutePath().normalize(); + boolean created = !Files.exists(stateDirectory, LinkOption.NOFOLLOW_LINKS); + Files.createDirectories(stateDirectory); + if (Files.isSymbolicLink(stateDirectory) || !Files.isDirectory(stateDirectory, LinkOption.NOFOLLOW_LINKS)) + throw new java.io.IOException("HTTP enrollment state directory is unsafe"); + setOwnerOnlyDirectory(stateDirectory); + if (created) DurableFiles.forceDirectory(stateDirectory.getParent()); + Path file = stateDirectory.resolve("http-transport-clients.properties"); if (Files.isSymbolicLink(file)) throw new java.io.IOException("Refusing unsafe HTTP enrollment state path"); return file; } @@ -229,6 +235,12 @@ private static void setOwnerOnly(Path path) throws java.io.IOException { catch (UnsupportedOperationException ignored) { } } + private static void setOwnerOnlyDirectory(Path path) throws java.io.IOException { + try { Files.setPosixFilePermissions(path, EnumSet.of(PosixFilePermission.OWNER_READ, + PosixFilePermission.OWNER_WRITE, PosixFilePermission.OWNER_EXECUTE)); } + catch (UnsupportedOperationException ignored) { } + } + private void expireEnrollments() { Instant now = clock.instant(); enrollments.entrySet().removeIf(entry -> !entry.getValue().expiresAt().isAfter(now)); diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpTlsIdentity.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpTlsIdentity.java index edc21f6..dc76169 100644 --- a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpTlsIdentity.java +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpTlsIdentity.java @@ -107,10 +107,11 @@ static HttpTlsIdentity loadOrCreate(Path directory, String advertisedHost, Clock boolean anyIdentityFile = caExists || serverExists || passwordExists; boolean completeIdentity = caExists && serverExists && passwordExists; boolean persistentTransportState = hasPersistentTransportState(directory); - if (initializing || (anyIdentityFile && !completeIdentity)) { + if (anyIdentityFile && !completeIdentity && !initializing) + throw new IOException("HTTP TLS identity files are incomplete"); + if (initializing) { if (persistentTransportState) throw new IOException("HTTP TLS identity files are incomplete"); - if (!initializing) writeInitializationMarker(initializingFile); discardUncommittedIdentity(caFile, serverFile, passwordFile); caExists = false; serverExists = false; diff --git a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportSecurityTest.java b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportSecurityTest.java index d4878a3..a8e971f 100644 --- a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportSecurityTest.java +++ b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportSecurityTest.java @@ -132,11 +132,12 @@ void identityIsDurableAndPinsRejectTheWrongServer() throws Exception { } @Test - void incompleteFirstRunTlsProvisioningRecoversWithoutManualCleanup() throws Exception { + void markedIncompleteFirstRunTlsProvisioningRecoversWithoutManualCleanup() throws Exception { Path source = directory.resolve("complete-identity"); HttpTlsIdentity original = HttpTlsIdentity.loadOrCreate(source, "localhost"); Path interrupted = directory.resolve("interrupted-identity"); Files.createDirectories(interrupted); + Files.writeString(interrupted.resolve("http-transport-initializing"), "initializing\n"); Files.copy(source.resolve("http-transport-ca.p12"), interrupted.resolve("http-transport-ca.p12")); Files.copy(source.resolve("http-transport-server.p12"), interrupted.resolve("http-transport-server.p12")); @@ -149,6 +150,23 @@ void incompleteFirstRunTlsProvisioningRecoversWithoutManualCleanup() throws Exce assertFalse(Files.exists(interrupted.resolve("http-transport-initializing"))); } + @Test + void unmarkedPartialIdentityFailsClosedWithExternalTransportState() throws Exception { + Path source = directory.resolve("external-state-source"); + HttpTlsIdentity.loadOrCreate(source, "localhost"); + Path partial = directory.resolve("external-state-partial"); + Files.createDirectories(partial); + Files.copy(source.resolve("http-transport-ca.p12"), partial.resolve("http-transport-ca.p12")); + byte[] retainedCa = Files.readAllBytes(partial.resolve("http-transport-ca.p12")); + Path externalState = directory.resolve("external-authority"); + Files.createDirectories(externalState); + Files.writeString(externalState.resolve("http-transport-clients.properties"), "version=3\n"); + + assertThrows(java.io.IOException.class, () -> HttpTlsIdentity.loadOrCreate(partial, "localhost")); + assertTrue(Arrays.equals(retainedCa, Files.readAllBytes(partial.resolve("http-transport-ca.p12"))), + "fail-closed recovery must preserve the surviving CA bytes"); + } + @Test void completedFirstRunFilesRecoverWhenInitializationMarkerSurvives() throws Exception { Path interrupted = directory.resolve("marked-identity"); From e3f08ccf4da495308efa68e4cfb52ee8daa5f289 Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Fri, 4 Sep 2026 19:11:59 -0600 Subject: [PATCH 05/38] fix(servercomm): bound authority state and support IPv6 --- .../http/HttpEnrollmentAuthority.java | 33 +++++++------- .../http/HttpProxyTransportServer.java | 11 ++++- .../http/HttpTransportRuntimeTest.java | 14 ++++++ .../http/HttpTransportSecurityTest.java | 44 +++++++++++++++++++ 4 files changed, 85 insertions(+), 17 deletions(-) diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentAuthority.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentAuthority.java index b152f5d..c544d27 100644 --- a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentAuthority.java +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentAuthority.java @@ -12,9 +12,7 @@ import java.time.Duration; import java.time.Instant; import java.util.HashMap; -import java.util.HashSet; import java.util.Map; -import java.util.Set; import java.util.Properties; import java.util.Base64; import java.util.EnumSet; @@ -27,12 +25,13 @@ public final class HttpEnrollmentAuthority { private static final Duration MAX_ENROLLMENT_LIFETIME = Duration.ofMinutes(15); private static final int MAX_PENDING_ENROLLMENTS = 128; + private static final int MAX_BINDINGS = 128; + private static final long MAX_STATE_BYTES = 65536; private final HttpTlsIdentity identity; private final Clock clock; private final Path stateFile; private final Map enrollments = new HashMap<>(); private final Map bindings = new HashMap<>(); - private final Set revokedCertificatePins = new HashSet<>(); private boolean persistenceFailure; /** Creates a restart-safe authority. State contains public certificate pins plus bounded hashes of pending tokens. */ @@ -82,9 +81,11 @@ public synchronized HttpTlsIdentity.IssuedClientCertificate enroll(String server if (enrollment == null || !HttpTransportSecrets.constantTimeEquals(enrollment.tokenHash(), suppliedHash)) throw new IllegalArgumentException("Enrollment was rejected"); if (!serverId.equals(enrollment.serverId())) throw new IllegalArgumentException("Enrollment was rejected"); - enrollments.remove(lookup); // consume only after the token and its intended backend both match. ClientBinding existing = bindings.get(serverId); if (existing != null && !existing.revoked()) throw new IllegalStateException("Server id is already enrolled"); + if (existing == null && bindings.size() >= MAX_BINDINGS) + throw new IllegalStateException("Too many enrolled HTTP backends"); + enrollments.remove(lookup); // consume only after all checks for the token and its intended backend pass. HttpTlsIdentity.IssuedClientCertificate issued = identity.issueClientCertificate(serverId); bindings.put(serverId, new ClientBinding(HttpTransportSecrets.certificatePin(issued.certificate()), null, false)); try { persistState(); } @@ -99,11 +100,10 @@ public synchronized boolean authenticate(String serverId, java.security.cert.X50 if (!identity.validClientCertificate(serverId, certificate)) return false; ClientBinding binding = bindings.get(serverId); String pin = HttpTransportSecrets.certificatePin(certificate); - if (binding == null || binding.revoked() || revokedCertificatePins.contains(pin)) return false; + if (binding == null || binding.revoked()) return false; if (samePin(binding.certificatePin(), pin)) return true; if (!samePin(binding.pendingCertificatePin(), pin)) return false; bindings.put(serverId, new ClientBinding(pin, null, false)); - revokedCertificatePins.add(binding.certificatePin()); try { persistState(); return true; } catch (java.io.IOException failure) { persistenceFailure = true; return false; } } @@ -128,19 +128,15 @@ public synchronized void revoke(String serverId) { catch (IllegalArgumentException invalid) { return; } final String revokedServer = serverId; boolean pendingRemoved = enrollments.entrySet().removeIf(entry -> revokedServer.equals(entry.getValue().serverId())); - ClientBinding binding = bindings.get(serverId); - if (binding != null) { - bindings.put(serverId, new ClientBinding(binding.certificatePin(), binding.pendingCertificatePin(), true)); - revokedCertificatePins.add(binding.certificatePin()); - if (binding.pendingCertificatePin() != null) revokedCertificatePins.add(binding.pendingCertificatePin()); - } - if (binding != null || pendingRemoved) try { persistState(); } + // Absence is the durable revocation fence: authentication always requires an exact active binding. + boolean bindingRemoved = bindings.remove(serverId) != null; + if (bindingRemoved || pendingRemoved) try { persistState(); } catch (java.io.IOException failure) { persistenceFailure = true; throw new IllegalStateException("Could not persist HTTP certificate revocation", failure); } } private synchronized void loadState() throws java.io.IOException { if (stateFile == null || !Files.exists(stateFile, LinkOption.NOFOLLOW_LINKS)) return; - if (!Files.isRegularFile(stateFile, LinkOption.NOFOLLOW_LINKS) || Files.size(stateFile) > 65536) + if (!Files.isRegularFile(stateFile, LinkOption.NOFOLLOW_LINKS) || Files.size(stateFile) > MAX_STATE_BYTES) throw new java.io.IOException("HTTP enrollment state is invalid"); Properties properties = new Properties(); try (var input = Files.newInputStream(stateFile, LinkOption.NOFOLLOW_LINKS)) { properties.load(input); } @@ -160,8 +156,10 @@ private synchronized void loadState() throws java.io.IOException { String revoked = value[value.length - 1]; if ((pending != null && !pending.matches("[0-9a-f]{64}")) || !("0".equals(revoked) || "1".equals(revoked))) throw new java.io.IOException("HTTP enrollment state is invalid"); - bindings.put(serverId, new ClientBinding(value[0], pending, "1".equals(revoked))); - if ("1".equals(revoked)) { revokedCertificatePins.add(value[0]); if (pending != null) revokedCertificatePins.add(pending); } + if ("0".equals(revoked)) { + if (bindings.size() >= MAX_BINDINGS) throw new java.io.IOException("HTTP enrollment state exceeds its bound"); + bindings.put(serverId, new ClientBinding(value[0], pending, false)); + } } else if (key.startsWith("enrollment.") && "3".equals(version)) { String lookup = key.substring("enrollment.".length()); if (!lookup.matches("[A-Za-z0-9_-]{43}")) throw new java.io.IOException("HTTP enrollment state is invalid"); @@ -188,6 +186,8 @@ private synchronized void loadState() throws java.io.IOException { private synchronized void persistState() throws java.io.IOException { if (stateFile == null) return; + if (bindings.size() > MAX_BINDINGS || enrollments.size() > MAX_PENDING_ENROLLMENTS) + throw new java.io.IOException("HTTP enrollment state exceeds its bound"); Properties properties = new Properties(); properties.setProperty("version", "3"); for (Map.Entry entry : bindings.entrySet()) { @@ -204,6 +204,7 @@ private synchronized void persistState() throws java.io.IOException { } java.io.ByteArrayOutputStream bytes = new java.io.ByteArrayOutputStream(); properties.store(bytes, "VotingPlugin HTTP transport authority state"); + if (bytes.size() > MAX_STATE_BYTES) throw new java.io.IOException("HTTP enrollment state exceeds its byte bound"); Path temporary = Files.createTempFile(stateFile.getParent(), stateFile.getFileName().toString(), ".tmp"); try { setOwnerOnly(temporary); diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java index 15eecae..04756d5 100644 --- a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java @@ -134,7 +134,16 @@ private void renew(HttpsExchange exchange) throws IOException { public void start() { if (closed) throw new IllegalStateException("HTTP transport is closed"); server.start(); } public int port() { return server.getAddress().getPort(); } - public URI endpoint(String host) { return URI.create("https://" + host + ":" + port() + "/"); } + public URI endpoint(String host) { + if (host == null || host.isBlank()) throw new IllegalArgumentException("Advertised host is required"); + try { + URI endpoint = new URI("https", null, host, port(), "/", null, null); + if (endpoint.getHost() == null) throw new IllegalArgumentException("Advertised host is invalid"); + return endpoint; + } catch (java.net.URISyntaxException invalid) { + throw new IllegalArgumentException("Advertised host is invalid", invalid); + } + } /** Queues a proxy-origin envelope durably before reporting acceptance. */ public boolean send(String serverId, JsonEnvelope envelope) { diff --git a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportRuntimeTest.java b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportRuntimeTest.java index 66d1575..d2ef0a7 100644 --- a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportRuntimeTest.java +++ b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportRuntimeTest.java @@ -27,6 +27,20 @@ class HttpTransportRuntimeTest { @TempDir Path directory; + @Test + void endpointHelperSupportsIpv6Literals() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("ipv6-proxy"), "::1"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, directory.resolve("ipv6-authority")); + try (HttpProxyTransportServer server = new HttpProxyTransportServer(new InetSocketAddress("localhost", 0), + identity, authority, directory.resolve("ipv6-outgoing"), ignored -> { })) { + URI endpoint = server.endpoint("::1"); + assertTrue(endpoint.getHost() != null); + assertTrue(endpoint.toASCIIString().startsWith("https://[::1]:")); + assertDoesNotThrow(() -> new HttpConnectionCode("lobby-1", endpoint, identity.serverCertificatePin(), + identity.caCertificatePin(), Instant.now().plusSeconds(60), "A".repeat(43))); + } + } + @Test void enrollsThenDeliversBothDirectionsWithAuthenticatedIdentity() throws Exception { HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); diff --git a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportSecurityTest.java b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportSecurityTest.java index a8e971f..ab67e8d 100644 --- a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportSecurityTest.java +++ b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportSecurityTest.java @@ -280,6 +280,49 @@ void pendingEnrollmentSurvivesRestartAndRevocationRemainsDurable() throws Except () -> afterRevocation.enroll("lobby-1", revokedPending.enrollmentToken())); } + @Test + void authorityStatePrunesRevocationsAndBoundsActiveBindings() throws Exception { + Path proxy = directory.resolve("bounded-proxy"); + Path state = directory.resolve("bounded-state"); + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(proxy, "localhost"); + Files.createDirectories(state); + java.util.Properties properties = new java.util.Properties(); + properties.setProperty("version", "3"); + String firstServer = boundedServerId(0); + for (int index = 0; index < 128; index++) { + String serverId = boundedServerId(index); + String encodedServer = java.util.Base64.getUrlEncoder().withoutPadding() + .encodeToString(serverId.getBytes(java.nio.charset.StandardCharsets.UTF_8)); + properties.setProperty("binding." + encodedServer, pin('a') + ":-:0"); + byte[] hash = new byte[32]; + java.nio.ByteBuffer.wrap(hash).putInt(index); + properties.setProperty("enrollment." + java.util.Base64.getUrlEncoder().withoutPadding().encodeToString(hash), + Instant.parse("2099-01-01T00:00:00Z").toEpochMilli() + ":" + java.util.Base64.getUrlEncoder() + .withoutPadding().encodeToString(firstServer.getBytes(java.nio.charset.StandardCharsets.UTF_8))); + } + Path stateFile = state.resolve("http-transport-clients.properties"); + try (var output = Files.newOutputStream(stateFile)) { properties.store(output, "bounded authority state"); } + assertTrue(Files.size(stateFile) < 65536, "the maximum supported state must fit the read bound"); + + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, state); + authority.revoke(firstServer); + assertFalse(Files.readString(stateFile).contains("binding." + java.util.Base64.getUrlEncoder().withoutPadding() + .encodeToString(firstServer.getBytes(java.nio.charset.StandardCharsets.UTF_8))), + "revoked bindings must not accumulate in durable state"); + HttpEnrollmentAuthority restarted = new HttpEnrollmentAuthority(identity, state); + HttpConnectionCode replacement = restarted.createConnectionCode("replacement", URI.create("https://localhost:8443/"), + Duration.ofMinutes(5)); + restarted.enroll("replacement", replacement.enrollmentToken()); + HttpConnectionCode overflow = restarted.createConnectionCode("overflow", URI.create("https://localhost:8443/"), + Duration.ofMinutes(5)); + assertThrows(IllegalStateException.class, () -> restarted.enroll("overflow", overflow.enrollmentToken())); + restarted.revoke(boundedServerId(1)); + assertDoesNotThrow(() -> restarted.enroll("overflow", overflow.enrollmentToken()), + "a capacity rejection must not consume the enrollment token"); + assertTrue(Files.size(stateFile) <= 65536); + assertDoesNotThrow(() -> new HttpEnrollmentAuthority(identity, state)); + } + @Test void renewalKeepsOldCredentialUntilReplacementAuthenticates() throws Exception { HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); @@ -439,4 +482,5 @@ void rollbackRetainsNewerCredentialForTheSameEndpoint() throws Exception { } private static String pin(char character) { return String.valueOf(character).repeat(64); } + private static String boundedServerId(int index) { return String.format("s%03d", index) + "x".repeat(60); } } From 4b92ea397ba6c9d386352265c88a2e51051b0c6a Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Fri, 4 Sep 2026 19:20:52 -0600 Subject: [PATCH 06/38] fix(servercomm): bound proxy state and secure credential roots --- .../http/HttpClientCredentialStore.java | 41 +++++++++++++++---- .../http/HttpProxyTransportServer.java | 4 ++ .../servercomm/http/HttpTlsIdentity.java | 4 ++ .../http/HttpTransportRuntimeTest.java | 15 +++++++ .../http/HttpTransportSecurityTest.java | 24 +++++++++++ 5 files changed, 79 insertions(+), 9 deletions(-) diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpClientCredentialStore.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpClientCredentialStore.java index 9b42bc4..f61900e 100644 --- a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpClientCredentialStore.java +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpClientCredentialStore.java @@ -29,7 +29,7 @@ private HttpClientCredentialStore() { } public static void save(Path directory, HttpTlsIdentity.IssuedClientCertificate issued) throws IOException { if (issued == null) throw new IllegalArgumentException("Issued credential is required"); - Files.createDirectories(directory); + directory = credentialRoot(directory, true); byte[] bundle = issued.pkcs12(); try { writePrivate(safe(directory.resolve(BUNDLE_FILE)), bundle); } finally { java.util.Arrays.fill(bundle, (byte) 0); } @@ -97,15 +97,15 @@ static StagedCredential stageReplacement(Path directory, HttpTlsIdentity.IssuedC private static StagedCredential stage(Path directory, HttpTlsIdentity.IssuedClientCertificate issued, HttpClientProfile profile, String connectionCodeDigest) throws Exception { if (directory == null || issued == null) throw new IllegalArgumentException("Credential replacement is required"); - Path credentialDirectory = directory.toAbsolutePath().normalize(); - boolean created = !Files.exists(credentialDirectory, LinkOption.NOFOLLOW_LINKS); + Path credentialDirectory = credentialRoot(directory, true); Path generations = credentialDirectory.resolve(GENERATIONS_DIRECTORY); + if (Files.isSymbolicLink(generations)) throw new IOException("HTTP credential generation directory is unsafe"); + boolean generationsCreated = !Files.exists(generations, LinkOption.NOFOLLOW_LINKS); Files.createDirectories(generations); - // Credential files cannot make the newly created credential-root entry durable. - // Persist its parent before an enrolled transport can activate this root. - if (created) DurableFiles.forceDirectory(credentialDirectory.getParent()); if (Files.isSymbolicLink(generations) || !Files.isDirectory(generations, LinkOption.NOFOLLOW_LINKS)) throw new IOException("HTTP credential generation directory is unsafe"); + setOwnerOnlyDirectory(generations); + if (generationsCreated) DurableFiles.forceDirectory(credentialDirectory); String name = java.util.UUID.randomUUID().toString(); Path generation = generations.resolve(name); Files.createDirectory(generation); @@ -136,7 +136,10 @@ private static StagedCredential stage(Path directory, HttpTlsIdentity.IssuedClie static void activateReplacement(Path directory, StagedCredential staged) throws IOException { if (directory == null || staged == null || !staged.name().matches("[0-9a-f-]{36}")) throw new IllegalArgumentException("Staged credential is invalid"); - Path generations = directory.toAbsolutePath().normalize().resolve(GENERATIONS_DIRECTORY); + directory = credentialRoot(directory, false); + Path generations = directory.resolve(GENERATIONS_DIRECTORY); + if (Files.isSymbolicLink(generations) || !Files.isDirectory(generations, LinkOption.NOFOLLOW_LINKS)) + throw new IOException("HTTP credential generation directory is unsafe"); Path generation = generations.resolve(staged.name()).normalize(); if (!generation.getParent().equals(generations) || Files.isSymbolicLink(generation) || !Files.isRegularFile(generation.resolve(BUNDLE_FILE), LinkOption.NOFOLLOW_LINKS) @@ -232,8 +235,11 @@ public static void restoreActiveGenerationAfterReplacement(Path directory, /** Atomically restores a previously validated credential generation after replacement rollback. */ public static void restoreActiveGeneration(Path directory, ActiveCredentialGeneration snapshot) throws Exception { if (directory == null || snapshot == null) throw new IllegalArgumentException("Credential rollback is required"); - Path root = directory.toAbsolutePath().normalize(); + Path root = credentialRoot(directory, false); Path generations = root.resolve(GENERATIONS_DIRECTORY); + if (!snapshot.name().isEmpty() && (Files.isSymbolicLink(generations) + || !Files.isDirectory(generations, LinkOption.NOFOLLOW_LINKS))) + throw new IOException("HTTP credential generation directory is unsafe"); Path target = snapshot.name().isEmpty() ? root : generations.resolve(snapshot.name()).normalize(); if (!snapshot.name().isEmpty() && (!target.getParent().equals(generations) || Files.isSymbolicLink(target) || !Files.isDirectory(target, LinkOption.NOFOLLOW_LINKS))) @@ -302,7 +308,7 @@ private static Path safe(Path file) throws IOException { } private static Path activeDirectory(Path directory) throws IOException { - Path root = directory.toAbsolutePath().normalize(); + Path root = credentialRoot(directory, false); Path current = safe(root.resolve(CURRENT_FILE)); if (!Files.exists(current, LinkOption.NOFOLLOW_LINKS)) return root; if (!Files.isRegularFile(current, LinkOption.NOFOLLOW_LINKS) || Files.size(current) > 64) @@ -310,6 +316,8 @@ private static Path activeDirectory(Path directory) throws IOException { String name = Files.readString(current, StandardCharsets.US_ASCII); if (!name.matches("[0-9a-f-]{36}")) throw new IOException("HTTP client credential pointer is invalid"); Path generations = root.resolve(GENERATIONS_DIRECTORY); + if (Files.isSymbolicLink(generations) || !Files.isDirectory(generations, LinkOption.NOFOLLOW_LINKS)) + throw new IOException("HTTP credential generation directory is unsafe"); Path generation = generations.resolve(name).normalize(); if (!generation.getParent().equals(generations) || Files.isSymbolicLink(generation) || !Files.isDirectory(generation, LinkOption.NOFOLLOW_LINKS)) throw new IOException("HTTP client credential generation is invalid"); @@ -326,6 +334,21 @@ private static String readConnectionCodeDigest(Path directory) throws IOExceptio return value; } + private static Path credentialRoot(Path directory, boolean create) throws IOException { + if (directory == null) throw new IllegalArgumentException("Credential directory is required"); + Path root = directory.toAbsolutePath().normalize(); + if (Files.isSymbolicLink(root)) throw new IOException("HTTP credential directory is unsafe"); + boolean created = !Files.exists(root, LinkOption.NOFOLLOW_LINKS); + if (create) Files.createDirectories(root); + if (Files.isSymbolicLink(root) || !Files.isDirectory(root, LinkOption.NOFOLLOW_LINKS)) + throw new IOException("HTTP credential directory is unsafe"); + if (create) { + setOwnerOnlyDirectory(root); + if (created) DurableFiles.forceDirectory(root.getParent()); + } + return root; + } + private static String connectionCodeDigest(HttpConnectionCode code) { return connectionCodeDigest(code.encode()); } diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java index 04756d5..59babed 100644 --- a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java @@ -47,6 +47,7 @@ * Every normal request is certificate-authenticated in the handler, rather than relying on TLS WANT auth. */ public final class HttpProxyTransportServer implements AutoCloseable { + private static final int MAX_BACKENDS = 128; static { // JDK HttpServer reads these once when its internal server configuration is initialized. // Set conservative process-wide bounds before this transport creates its listener. @@ -238,6 +239,7 @@ private BackendState backendState(String serverId) throws IOException { synchronized (backends) { BackendState existing = backends.get(serverId); if (existing != null) return existing; + if (backends.size() >= MAX_BACKENDS) throw new IOException("HTTP backend state exceeds its bound"); HttpInboundDeliveryStore inbound = durableIncomingRoot == null ? null : HttpInboundDeliveryStore.open(durableIncomingRoot, serverId); BackendState created = new BackendState(serverId, durableOutgoing, inbound, onAcknowledged); @@ -476,8 +478,10 @@ private DurableOutgoingQueue(Path root) throws IOException { private synchronized Map> load() throws IOException { Map> loaded = new LinkedHashMap<>(); + int serverDirectories = 0; try (DirectoryStream servers = Files.newDirectoryStream(root)) { for (Path directory : servers) { + if (++serverDirectories > MAX_BACKENDS) throw new IOException("HTTP outgoing queue exceeds its backend bound"); if (Files.isSymbolicLink(directory) || !Files.isDirectory(directory, LinkOption.NOFOLLOW_LINKS)) throw new IOException("HTTP outgoing queue contains an invalid entry"); String serverId; diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpTlsIdentity.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpTlsIdentity.java index dc76169..3ba11fc 100644 --- a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpTlsIdentity.java +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpTlsIdentity.java @@ -86,12 +86,16 @@ public static HttpTlsIdentity loadOrCreate(Path directory, String advertisedHost } static HttpTlsIdentity loadOrCreate(Path directory, String advertisedHost, Clock clock) throws Exception { + if (directory == null) throw new IllegalArgumentException("Identity directory is required"); if (advertisedHost == null || advertisedHost.isBlank() || advertisedHost.length() > 253) throw new IllegalArgumentException("Advertised HTTPS host is invalid"); if (clock == null) throw new IllegalArgumentException("Clock is required"); Path identityDirectory = directory.toAbsolutePath().normalize(); + if (Files.isSymbolicLink(identityDirectory)) throw new IOException("HTTP TLS identity directory is unsafe"); boolean created = !Files.exists(identityDirectory, LinkOption.NOFOLLOW_LINKS); Files.createDirectories(identityDirectory); + if (Files.isSymbolicLink(identityDirectory) || !Files.isDirectory(identityDirectory, LinkOption.NOFOLLOW_LINKS)) + throw new IOException("HTTP TLS identity directory is unsafe"); // The identity files cannot make the newly created directory entry durable. // Persist its parent before the TLS identity is returned for listener use. if (created) DurableFiles.forceDirectory(identityDirectory.getParent()); diff --git a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportRuntimeTest.java b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportRuntimeTest.java index d2ef0a7..0459349 100644 --- a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportRuntimeTest.java +++ b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportRuntimeTest.java @@ -245,6 +245,21 @@ void boundedQueuesFailClosed() throws Exception { } } + @Test + void proxyBackendStateIsGloballyBounded() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("bounded-backend-proxy"), "localhost"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, + directory.resolve("bounded-backend-authority")); + try (HttpProxyTransportServer server = new HttpProxyTransportServer(new InetSocketAddress("localhost", 0), + identity, authority, ignored -> { })) { + for (int index = 0; index < 128; index++) + assertTrue(server.send("server-" + index, JsonEnvelope.builder("x").build())); + assertFalse(server.send("server-overflow", JsonEnvelope.builder("x").build())); + assertTrue(server.send("server-0", JsonEnvelope.builder("existing").build()), + "the global bound must not reject an existing backend state"); + } + } + @Test void proxyOutgoingQueueSurvivesRestartUntilBackendAcknowledges() throws Exception { Path proxyDirectory = directory.resolve("proxy"); diff --git a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportSecurityTest.java b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportSecurityTest.java index ab67e8d..f6092fa 100644 --- a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportSecurityTest.java +++ b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportSecurityTest.java @@ -131,6 +131,30 @@ void identityIsDurableAndPinsRejectTheWrongServer() throws Exception { assertNotEquals(created.serverCertificatePin(), rotated.serverCertificatePin()); } + @Test + void privateCredentialRootsRejectSymbolicLinks() throws Exception { + Path identityTarget = directory.resolve("identity-target"); + Path identityLink = directory.resolve("identity-link"); + Files.createDirectory(identityTarget); + Files.createSymbolicLink(identityLink, identityTarget); + assertThrows(java.io.IOException.class, () -> HttpTlsIdentity.loadOrCreate(identityLink, "localhost")); + assertFalse(Files.exists(identityTarget.resolve("http-transport-ca.p12"))); + + HttpTlsIdentity authority = HttpTlsIdentity.loadOrCreate(directory.resolve("safe-identity"), "localhost"); + HttpTlsIdentity.IssuedClientCertificate issued = authority.issueClientCertificate("lobby-1"); + Path credentialTarget = directory.resolve("credential-target"); + Path credentialLink = directory.resolve("credential-link"); + Files.createDirectory(credentialTarget); + Files.createSymbolicLink(credentialLink, credentialTarget); + assertThrows(java.io.IOException.class, () -> HttpClientCredentialStore.save(credentialLink, issued)); + HttpConnectionCode code = new HttpConnectionCode("lobby-1", URI.create("https://localhost:8443/"), + authority.serverCertificatePin(), authority.caCertificatePin(), Instant.now().plusSeconds(60), "A".repeat(43)); + assertThrows(java.io.IOException.class, () -> HttpClientCredentialStore.saveEnrolled(credentialLink, code, issued)); + try (var files = Files.list(credentialTarget)) { + assertTrue(files.findAny().isEmpty(), "a symlinked credential root must receive no private files"); + } + } + @Test void markedIncompleteFirstRunTlsProvisioningRecoversWithoutManualCleanup() throws Exception { Path source = directory.resolve("complete-identity"); From 1faa2d2159d0d9337fd5913d8c430364a4c84ecd Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Fri, 4 Sep 2026 19:38:22 -0600 Subject: [PATCH 07/38] fix(servercomm): separate HTTP acknowledgement directions --- .../http/HttpBackendTransportConnector.java | 66 ++++++++++++------- .../http/HttpProxyTransportServer.java | 38 ++++++++--- .../http/HttpTransportProtocol.java | 19 +++--- .../http/HttpTransportRuntimeTest.java | 46 +++++++++++-- 4 files changed, 127 insertions(+), 42 deletions(-) diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpBackendTransportConnector.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpBackendTransportConnector.java index 6bddf30..8c40fec 100644 --- a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpBackendTransportConnector.java +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpBackendTransportConnector.java @@ -51,7 +51,7 @@ public final class HttpBackendTransportConnector implements AutoCloseable { private volatile HttpClientCredentialStore.ClientCredential credential; private final Path credentialDirectory; private final HttpInboundDeliveryStore inboundDeliveries; - private final HttpInboundDeliveryStore acknowledgementConfirmations; + private final HttpInboundDeliveryStore acknowledgementConfirmationStore; private final URI transportEndpoint; private final ThreadPoolExecutor callbackExecutor; private final AtomicBoolean running = new AtomicBoolean(); @@ -61,6 +61,7 @@ public final class HttpBackendTransportConnector implements AutoCloseable { private final LinkedHashMap outgoing = new LinkedHashMap<>(); private final Set received = new LinkedHashSet<>(), processing = new LinkedHashSet<>(); private final ArrayDeque acknowledgements = new ArrayDeque<>(); + private final ArrayDeque acknowledgementConfirmations = new ArrayDeque<>(); private final String session = UUID.randomUUID().toString(); private volatile Thread poller; private long sequence; @@ -96,7 +97,7 @@ private HttpBackendTransportConnector(HttpClientCredentialStore.HttpClientProfil this.credential = credential; this.credentialDirectory = credentialDirectory; inboundDeliveries = credentialDirectory == null ? null : new HttpInboundDeliveryStore(credentialDirectory); - acknowledgementConfirmations = credentialDirectory == null ? null + acknowledgementConfirmationStore = credentialDirectory == null ? null : HttpInboundDeliveryStore.open(credentialDirectory, "http-transport-ack-confirmations"); if (inboundDeliveries != null) for (var entry : inboundDeliveries.snapshot().entrySet()) { if (entry.getValue() == HttpInboundDeliveryStore.State.COMPLETED) { @@ -104,10 +105,10 @@ private HttpBackendTransportConnector(HttpClientCredentialStore.HttpClientProfil queueAck(entry.getKey()); } } - if (acknowledgementConfirmations != null) for (var entry : acknowledgementConfirmations.snapshot().entrySet()) { + if (acknowledgementConfirmationStore != null) for (var entry : acknowledgementConfirmationStore.snapshot().entrySet()) { if (entry.getValue() != HttpInboundDeliveryStore.State.COMPLETED) throw new IOException("HTTP acknowledgement confirmation state is invalid"); - queueAck(entry.getKey()); + queueAcknowledgementConfirmation(entry.getKey()); } client = client(profile, credential); transportEndpoint = profile.endpoint().resolve("v1/transport"); @@ -174,30 +175,44 @@ public synchronized boolean pollOnce() { } private boolean pollOnce(Duration timeout, boolean requireRunning, boolean acceptIncoming) { if (requireRunning && !running.get()) return false; - List acks = List.of(); boolean acknowledgementsConfirmed = false; + List acks = List.of(), ackConfirmations = List.of(); + boolean acknowledgementsConfirmed = false, confirmationsConfirmed = false; try { if (requireRunning) maybeRenewCredential(); List messages; long requestSequence; synchronized (state) { - acks = first(acknowledgements); requestSequence = sequence++; - messages = HttpTransportProtocol.fittingMessages(serverId, session, requestSequence, acks, outgoing.values()); + acks = first(acknowledgements); + ackConfirmations = first(acknowledgementConfirmations); + requestSequence = sequence++; + messages = HttpTransportProtocol.fittingMessages(serverId, session, requestSequence, acks, + ackConfirmations, outgoing.values()); for (int index = 0; index < acks.size(); index++) acknowledgements.removeFirst(); + for (int index = 0; index < ackConfirmations.size(); index++) acknowledgementConfirmations.removeFirst(); } HttpRequest request = HttpRequest.newBuilder(transportEndpoint).timeout(timeout).header("Content-Type", "application/json") - .header("Cache-Control", "no-store").POST(HttpRequest.BodyPublishers.ofByteArray(HttpTransportProtocol.request(serverId, session, requestSequence, acks, messages))).build(); + .header("Cache-Control", "no-store").POST(HttpRequest.BodyPublishers.ofByteArray(HttpTransportProtocol.request( + serverId, session, requestSequence, acks, ackConfirmations, messages))).build(); LimitedResponse response = sendLimited(client, request); if (response.statusCode() != 200) return false; HttpTransportProtocol.Packet packet = HttpTransportProtocol.parsePacket(response.body()); if (!serverId.equals(packet.server()) || !session.equals(packet.session()) || packet.sequence() != requestSequence) return false; - confirmAcknowledgements(acks); + if (!acks.equals(packet.ackConfirmations())) return false; + confirmAcknowledgements(packet.ackConfirmations()); acknowledgementsConfirmed = true; + if (!confirmSentAcknowledgementConfirmations(ackConfirmations)) return false; + confirmationsConfirmed = true; if (!recordAcknowledgementConfirmations(packet.acks())) return false; - synchronized (state) { for (String ack : packet.acks()) { outgoing.remove(ack); queueAck(ack); } } + synchronized (state) { for (String ack : packet.acks()) { + outgoing.remove(ack); queueAcknowledgementConfirmation(ack); + } } if (acceptIncoming) for (HttpTransportProtocol.Delivery delivery : accept(packet.messages())) dispatch(delivery); firstResponse.countDown(); return true; } catch (Exception failure) { return false; - } finally { if (!acknowledgementsConfirmed) requeueAcknowledgements(acks); } + } finally { + if (!acknowledgementsConfirmed) requeue(acknowledgements, acks); + if (!confirmationsConfirmed) requeue(acknowledgementConfirmations, ackConfirmations); + } } /** Stops normal polling and gives already-queued outbound messages a bounded final delivery attempt. */ public boolean flushOutgoing(long deadlineNanos) { @@ -230,7 +245,7 @@ public boolean flushOutgoing(long deadlineNanos) { // In-flight transitions serialize with seal(): either COMPLETED is already // durable, or the delivery remains durably RUNNING and fail-closed. if (inboundDeliveries != null) inboundDeliveries.seal(); - if (acknowledgementConfirmations != null) acknowledgementConfirmations.seal(); + if (acknowledgementConfirmationStore != null) acknowledgementConfirmationStore.seal(); Thread current = poller; if (current != null) current.interrupt(); callbackExecutor.shutdown(); try { if (!callbackExecutor.awaitTermination(5, TimeUnit.SECONDS)) callbackExecutor.shutdownNow(); } catch (InterruptedException interrupted) { Thread.currentThread().interrupt(); callbackExecutor.shutdownNow(); } @@ -312,13 +327,17 @@ void completeIncoming(String id, boolean success) { synchronized (state) { processing.remove(id); if (success) { received.add(id); while (received.size() > HttpTransportProtocol.MAX_QUEUE) received.remove(received.iterator().next()); queueAck(id); } } } - private void queueAck(String id) { if (acknowledgements.size() < HttpTransportProtocol.MAX_QUEUE * 2 && !acknowledgements.contains(id)) acknowledgements.add(id); } - private void requeueAcknowledgements(List ids) { synchronized (state) { + private void queueAck(String id) { if (acknowledgements.size() < HttpTransportProtocol.MAX_QUEUE && !acknowledgements.contains(id)) acknowledgements.add(id); } + private void queueAcknowledgementConfirmation(String id) { + if (acknowledgementConfirmations.size() < HttpTransportProtocol.MAX_QUEUE + && !acknowledgementConfirmations.contains(id)) acknowledgementConfirmations.add(id); + } + private void requeue(ArrayDeque queue, List ids) { synchronized (state) { for (int index = ids.size() - 1; index >= 0; index--) { String id = ids.get(index); - if (!acknowledgements.contains(id)) { - while (acknowledgements.size() >= HttpTransportProtocol.MAX_QUEUE * 2) acknowledgements.removeLast(); - acknowledgements.addFirst(id); + if (!queue.contains(id)) { + while (queue.size() >= HttpTransportProtocol.MAX_QUEUE) queue.removeLast(); + queue.addFirst(id); } } } } @@ -327,8 +346,6 @@ private void confirmAcknowledgements(Collection ids) { boolean removed = true; if (inboundDeliveries != null) try { inboundDeliveries.remove(id); } catch (IOException cleanupFailure) { removed = false; } - if (acknowledgementConfirmations != null) try { acknowledgementConfirmations.remove(id); } - catch (IOException cleanupFailure) { removed = false; } synchronized (state) { if (removed) received.remove(id); else queueAck(id); @@ -336,12 +353,17 @@ private void confirmAcknowledgements(Collection ids) { } } private boolean recordAcknowledgementConfirmations(Collection ids) { - if (acknowledgementConfirmations == null) return true; - try { for (String id : ids) acknowledgementConfirmations.recordCompleted(id); return true; } + if (acknowledgementConfirmationStore == null) return true; + try { for (String id : ids) acknowledgementConfirmationStore.recordCompleted(id); return true; } catch (IOException persistenceFailure) { return false; } } + private boolean confirmSentAcknowledgementConfirmations(Collection ids) { + if (acknowledgementConfirmationStore == null) return true; + try { for (String id : ids) acknowledgementConfirmationStore.remove(id); return true; } + catch (IOException cleanupFailure) { return false; } + } int queuedOutgoing() { synchronized (state) { return outgoing.size(); } } - int queuedAcknowledgements() { synchronized (state) { return acknowledgements.size(); } } + int queuedAcknowledgements() { synchronized (state) { return acknowledgements.size() + acknowledgementConfirmations.size(); } } List drainAcknowledgements() { synchronized (state) { return drain(acknowledgements); } } private static List first(Collection values) { List output = new java.util.ArrayList<>(); for (T value : values) { output.add(value); if (output.size() == HttpTransportProtocol.MAX_BATCH) break; } return output; } private static List drain(ArrayDeque values) { List output = new java.util.ArrayList<>(); while (!values.isEmpty() && output.size() < HttpTransportProtocol.MAX_BATCH) output.add(values.remove()); return output; } diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java index 59babed..9238c78 100644 --- a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java @@ -200,8 +200,9 @@ private void transport(HttpsExchange exchange) throws IOException { if (!backend.beginPoll(packet.session())) { reply(exchange, 409, new byte[0]); return; } try { handlePacket(packet, backend); - Response response = backend.await(packet.server(), packet.session(), packet.sequence()); - reply(exchange, 200, HttpTransportProtocol.response(packet.server(), packet.session(), packet.sequence(), response.acks(), response.messages())); + Response response = backend.await(packet.server(), packet.session(), packet.sequence(), packet.acks()); + reply(exchange, 200, HttpTransportProtocol.response(packet.server(), packet.session(), packet.sequence(), + response.acks(), packet.acks(), response.messages())); } finally { backend.endPoll(); } } catch (IllegalArgumentException rejected) { reply(exchange, 400, new byte[0]); } catch (Exception failure) { reply(exchange, 503, new byte[0]); @@ -214,7 +215,7 @@ private void handlePacket(HttpTransportProtocol.Packet packet, BackendState back if (!backend.allowRequest()) throw new IllegalArgumentException("transport rate limited"); if (!backend.acceptSession(packet.session(), packet.sequence())) throw new IllegalArgumentException("stale session request"); } - backend.confirmIncoming(packet.acks()); + backend.confirmIncoming(packet.ackConfirmations()); backend.acknowledge(packet.acks()); synchronized (backend) { accepted = backend.acceptIncoming(packet.messages()); } for (HttpTransportProtocol.Delivery delivery : accepted) dispatch(packet.server(), backend, delivery); @@ -412,6 +413,10 @@ synchronized void confirmIncoming(Collection ids) throws IOException { private void seal() { if (durableIncoming != null) durableIncoming.seal(); } private void queueAck(String id) { if (acknowledgements.size() < HttpTransportProtocol.MAX_QUEUE && !acknowledgements.contains(id)) acknowledgements.add(id); } synchronized Response await(String serverId, String requestedSession, long requestedSequence) { + return await(serverId, requestedSession, requestedSequence, List.of()); + } + synchronized Response await(String serverId, String requestedSession, long requestedSequence, + Collection ackConfirmations) { long deadline = System.nanoTime() + LONG_POLL.toNanos(); while (acknowledgements.isEmpty() && !hasUndelivered()) { long retryRemaining = nanosUntilRedelivery(nanoTime.getAsLong()); @@ -428,7 +433,7 @@ synchronized Response await(String serverId, String requestedSession, long reque if (candidates.size() == HttpTransportProtocol.MAX_BATCH) break; } List messages = HttpTransportProtocol.fittingMessages(serverId, requestedSession, - requestedSequence, acks, candidates); + requestedSequence, acks, ackConfirmations, candidates); long deliveredAt = nanoTime.getAsLong(); for (HttpTransportProtocol.Delivery delivery : messages) deliveredAtNanos.put(delivery.id(), deliveredAt); return new Response(acks, messages); @@ -554,10 +559,27 @@ private synchronized void persist(String serverId, HttpTransportProtocol.Deliver private synchronized void remove(String serverId, String id) throws IOException { Map serverFiles = files.get(serverId); - Path file = serverFiles == null ? null : serverFiles.get(id); - if (file == null) throw new IOException("HTTP outgoing queue acknowledgement is unknown"); - DurableFiles.deleteIfExists(file); - serverFiles.remove(id); + if (serverFiles == null) throw new IOException("HTTP outgoing queue acknowledgement is unknown"); + Path file = serverFiles.get(id); + if (file != null) { + Files.deleteIfExists(file); + // Also makes a retried deletion durable if an earlier directory force failed after unlinking the file. + DurableFiles.forceDirectory(file.getParent()); + serverFiles.remove(id); + } + if (serverFiles.isEmpty()) { + Path directory = root.resolve(serverId).normalize(); + if (!directory.getParent().equals(root) || Files.isSymbolicLink(directory)) + throw new IOException("HTTP outgoing queue server directory is invalid"); + try { + Files.deleteIfExists(directory); + DurableFiles.forceDirectory(root); + files.remove(serverId); + } catch (java.nio.file.DirectoryNotEmptyException unexpectedEntry) { + // The acknowledged delivery is already durably removed; unrelated/tampered entries + // must not make its acknowledgement permanently unprocessable. + } + } } private static void ownerOnlyFile(Path path) throws IOException { diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpTransportProtocol.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpTransportProtocol.java index d6030e3..e1aae28 100644 --- a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpTransportProtocol.java +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpTransportProtocol.java @@ -30,9 +30,10 @@ static void validateEnvelope(JsonEnvelope envelope) { } static byte[] request(String server, String session, long sequence, Collection acks, - Collection messages) { + Collection ackConfirmations, Collection messages) { JsonObject root = base(server, session, sequence); root.add("acks", ids(acks)); + root.add("ackConfirmations", ids(ackConfirmations)); root.add("messages", messages(messages)); byte[] encoded = root.toString().getBytes(StandardCharsets.UTF_8); if (encoded.length > MAX_BODY_BYTES) throw bad(); @@ -40,12 +41,12 @@ static byte[] request(String server, String session, long sequence, Collection fittingMessages(String server, String session, long sequence, Collection acks, - Collection candidates) { + Collection ackConfirmations, Collection candidates) { List output = new ArrayList<>(); for (Delivery candidate : candidates) { if (output.size() == MAX_BATCH) break; output.add(candidate); - try { request(server, session, sequence, acks, output); } + try { request(server, session, sequence, acks, ackConfirmations, output); } catch (IllegalArgumentException tooLarge) { output.remove(output.size() - 1); break; } } return output; @@ -76,8 +77,8 @@ static Delivery parseStoredDelivery(byte[] body) { } static byte[] response(String server, String session, long sequence, Collection acks, - Collection messages) { - return request(server, session, sequence, acks, messages); + Collection ackConfirmations, Collection messages) { + return request(server, session, sequence, acks, ackConfirmations, messages); } static Packet parsePacket(byte[] body) { @@ -86,7 +87,7 @@ static Packet parsePacket(byte[] body) { JsonElement parsed = JsonParser.parseString(new String(body, StandardCharsets.UTF_8)); if (!parsed.isJsonObject()) throw bad(); JsonObject root = parsed.getAsJsonObject(); - requireOnly(root, "v", "server", "session", "sequence", "timestamp", "acks", "messages"); + requireOnly(root, "v", "server", "session", "sequence", "timestamp", "acks", "ackConfirmations", "messages"); if (integer(root, "v") != VERSION) throw bad(); String server = HttpTlsIdentity.canonicalServerId(string(root, "server", 64)); String session = uuid(root, "session"); @@ -95,8 +96,9 @@ static Packet parsePacket(byte[] body) { long now = Instant.now().toEpochMilli(); if (timestamp < now - MAX_CLOCK_SKEW_MILLIS || timestamp > now + MAX_CLOCK_SKEW_MILLIS) throw bad(); List acks = parseIds(root.get("acks")); + List ackConfirmations = parseIds(root.get("ackConfirmations")); List messages = parseMessages(root.get("messages")); - return new Packet(server, session, sequence, acks, messages); + return new Packet(server, session, sequence, acks, ackConfirmations, messages); } catch (RuntimeException invalid) { throw bad(); } } @@ -226,6 +228,7 @@ private static String canonicalUuid(String value) { private static IllegalArgumentException bad() { return new IllegalArgumentException("Invalid HTTP transport message"); } record Delivery(String id, JsonEnvelope envelope) { } - record Packet(String server, String session, long sequence, List acks, List messages) { } + record Packet(String server, String session, long sequence, List acks, + List ackConfirmations, List messages) { } record Enrollment(String server, String token) { } } diff --git a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportRuntimeTest.java b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportRuntimeTest.java index 0459349..b2f4587 100644 --- a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportRuntimeTest.java +++ b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportRuntimeTest.java @@ -73,6 +73,11 @@ void enrollsThenDeliversBothDirectionsWithAuthenticatedIdentity() throws Excepti "the backend must durably confirm receipt of the proxy ACK"); assertTrue(server.send("lobby-1", JsonEnvelope.builder("to-backend").build())); assertTrue(backendReceived.await(8, TimeUnit.SECONDS)); + long outgoingCleanupDeadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(3); + while (Files.exists(proxyOutgoing.resolve("lobby-1")) && System.nanoTime() < outgoingCleanupDeadline) + Thread.sleep(10); + assertFalse(Files.exists(proxyOutgoing.resolve("lobby-1")), + "acknowledging the final delivery must remove its empty backend directory"); Path inboundFence = directory.resolve("client").resolve("http-transport-inbound-deliveries"); long fenceDeadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); while (countRegularFiles(inboundFence) != 0L && System.nanoTime() < fenceDeadline) Thread.sleep(20); @@ -169,6 +174,33 @@ void failedAcknowledgementCallbackRetainsProxyDelivery() throws Exception { .messages().iterator().next().id()); } + @Test + void oppositeDirectionIdsUseSeparateAcknowledgementNamespaces() throws Exception { + String deliveryId = java.util.UUID.randomUUID().toString(); + Path incomingRoot = directory.resolve("separate-ack-incoming"); + Files.createDirectory(incomingRoot); + HttpInboundDeliveryStore incoming = HttpInboundDeliveryStore.open(incomingRoot, "lobby-1"); + HttpProxyTransportServer.BackendState state = new HttpProxyTransportServer.BackendState( + "lobby-1", null, incoming, (server, id) -> { }); + assertTrue(state.enqueue(new HttpTransportProtocol.Delivery(deliveryId, + JsonEnvelope.builder("proxy-reply").build()))); + HttpTransportProtocol.Delivery backendMessage = new HttpTransportProtocol.Delivery(deliveryId, + JsonEnvelope.builder("backend-request").build()); + assertEquals(java.util.List.of(backendMessage), state.acceptIncoming(java.util.List.of(backendMessage))); + state.beginIncoming(deliveryId); + state.completeIncomingDurably(deliveryId); + state.completeIncoming(deliveryId, true); + + byte[] request = HttpTransportProtocol.request("lobby-1", java.util.UUID.randomUUID().toString(), 0, + java.util.List.of(), java.util.List.of(deliveryId), java.util.List.of()); + HttpTransportProtocol.Packet packet = HttpTransportProtocol.parsePacket(request); + state.confirmIncoming(packet.ackConfirmations()); + state.acknowledge(packet.acks()); + assertEquals(deliveryId, state.await("lobby-1", java.util.UUID.randomUUID().toString(), 0) + .messages().iterator().next().id(), + "confirming the backend-origin acknowledgement must not acknowledge a same-ID proxy reply"); + } + @Test void closeWaitsForTheCredentialOwningPollerToStop() throws Exception { HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("close-proxy"), "localhost"); @@ -227,7 +259,8 @@ void normalTransportRejectsAClientWithoutCertificate() throws Exception { server.start(); HttpConnectionCode code = authority.createConnectionCode("lobby-1", server.endpoint("localhost"), Duration.ofMinutes(5)); HttpClient client = HttpClient.newBuilder().sslContext(HttpPinnedTls.clientContext(code)).build(); - byte[] body = HttpTransportProtocol.request("lobby-1", java.util.UUID.randomUUID().toString(), 0, java.util.List.of(), java.util.List.of()); + byte[] body = HttpTransportProtocol.request("lobby-1", java.util.UUID.randomUUID().toString(), 0, + java.util.List.of(), java.util.List.of(), java.util.List.of()); HttpResponse response = client.send(HttpRequest.newBuilder(code.endpoint().resolve("v1/transport")) .timeout(Duration.ofSeconds(5)).header("Content-Type", "application/json").POST(HttpRequest.BodyPublishers.ofByteArray(body)).build(), HttpResponse.BodyHandlers.ofByteArray()); assertEquals(401, response.statusCode()); @@ -338,16 +371,16 @@ void aggregatePacketBudgetSplitsLargeValidEnvelopes() { java.util.UUID.randomUUID().toString(), JsonEnvelope.builder("large").put("value", "x".repeat(40_000)).build())); String session = java.util.UUID.randomUUID().toString(); java.util.List fitted = HttpTransportProtocol.fittingMessages( - "lobby-1", session, 0, java.util.List.of(), candidates); + "lobby-1", session, 0, java.util.List.of(), java.util.List.of(), candidates); assertTrue(fitted.size() > 0 && fitted.size() < candidates.size()); - assertTrue(HttpTransportProtocol.request("lobby-1", session, 0, java.util.List.of(), fitted).length + assertTrue(HttpTransportProtocol.request("lobby-1", session, 0, java.util.List.of(), java.util.List.of(), fitted).length <= HttpTransportProtocol.MAX_BODY_BYTES); } @Test void packetNumbersMustUseCanonicalJsonIntegerTokens() { com.google.gson.JsonObject packet = com.google.gson.JsonParser.parseString(new String(HttpTransportProtocol.request( - "lobby-1", java.util.UUID.randomUUID().toString(), 0, java.util.List.of(), java.util.List.of()), + "lobby-1", java.util.UUID.randomUUID().toString(), 0, java.util.List.of(), java.util.List.of(), java.util.List.of()), java.nio.charset.StandardCharsets.UTF_8)).getAsJsonObject(); assertDoesNotThrow(() -> HttpTransportProtocol.parsePacket(packet.toString() .getBytes(java.nio.charset.StandardCharsets.UTF_8))); @@ -369,6 +402,7 @@ void packetParsingRejectsNoncanonicalUuidForms() { String deliveryId = java.util.UUID.randomUUID().toString(); com.google.gson.JsonObject packet = com.google.gson.JsonParser.parseString(new String(HttpTransportProtocol.request( "lobby-1", java.util.UUID.randomUUID().toString(), 0, java.util.List.of(deliveryId), + java.util.List.of(deliveryId), java.util.List.of(new HttpTransportProtocol.Delivery(deliveryId, JsonEnvelope.builder("payload").build()))), java.nio.charset.StandardCharsets.UTF_8)).getAsJsonObject(); String abbreviated = "1-1-1-1-1"; @@ -382,6 +416,10 @@ void packetParsingRejectsNoncanonicalUuidForms() { invalidAck.getAsJsonArray("acks").set(0, new com.google.gson.JsonPrimitive(abbreviated)); assertThrows(IllegalArgumentException.class, () -> HttpTransportProtocol.parsePacket( invalidAck.toString().getBytes(java.nio.charset.StandardCharsets.UTF_8))); + com.google.gson.JsonObject invalidConfirmation = packet.deepCopy(); + invalidConfirmation.getAsJsonArray("ackConfirmations").set(0, new com.google.gson.JsonPrimitive(abbreviated)); + assertThrows(IllegalArgumentException.class, () -> HttpTransportProtocol.parsePacket( + invalidConfirmation.toString().getBytes(java.nio.charset.StandardCharsets.UTF_8))); com.google.gson.JsonObject invalidMessage = packet.deepCopy(); invalidMessage.getAsJsonArray("messages").get(0).getAsJsonObject().addProperty("id", abbreviated); From c01f2ccea2d68cd1909f8f9ea675cd6c15625575 Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Fri, 4 Sep 2026 20:44:40 -0600 Subject: [PATCH 08/38] Handle HTTP renewal and backend state churn --- .../http/HttpBackendTransportConnector.java | 30 +++++++++- .../http/HttpInboundDeliveryStore.java | 11 ++++ .../http/HttpProxyTransportServer.java | 55 ++++++++++++++++--- .../http/HttpTransportRuntimeTest.java | 22 ++++++++ .../http/HttpTransportSecurityTest.java | 10 ++++ 5 files changed, 116 insertions(+), 12 deletions(-) diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpBackendTransportConnector.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpBackendTransportConnector.java index 8c40fec..1c5fb4d 100644 --- a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpBackendTransportConnector.java +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpBackendTransportConnector.java @@ -16,6 +16,7 @@ import java.security.cert.X509Certificate; import java.time.Clock; import java.time.Duration; +import java.time.Instant; import java.util.ArrayDeque; import java.util.Collection; import java.util.LinkedHashMap; @@ -43,6 +44,8 @@ /** Backend-side, persistent HTTP/1.1 long-poll connector. */ public final class HttpBackendTransportConnector implements AutoCloseable { public static final Duration CLIENT_TIMEOUT = Duration.ofSeconds(35); + private static final Duration RENEWAL_SUCCESS_CHECK = Duration.ofHours(6); + private static final Duration RENEWAL_FAILURE_RETRY = Duration.ofMinutes(5); static final int CALLBACK_QUEUE_CAPACITY = 128; private volatile HttpClientCredentialStore.HttpClientProfile profile; private final String serverId; @@ -403,14 +406,18 @@ private void maybeRenewCredential() { if (directory == null || !HttpTlsIdentity.needsRenewal(credential.certificate(), Clock.systemUTC())) return; long now = System.nanoTime(); if (nextRenewalCheckNanos != 0L && now - nextRenewalCheckNanos < 0L) return; - nextRenewalCheckNanos = now + Duration.ofHours(6).toNanos(); + Duration retry = renewalRetryDelay(Duration.between(Instant.now(), + credential.certificate().getNotAfter().toInstant())); try { byte[] body = HttpTransportProtocol.renewalRequest(serverId); HttpRequest request = HttpRequest.newBuilder(profile.endpoint().resolve("v1/renew")).timeout(CLIENT_TIMEOUT) .header("Content-Type", "application/json").header("Cache-Control", "no-store") .POST(HttpRequest.BodyPublishers.ofByteArray(body)).build(); LimitedResponse response = sendLimited(client, request); - if (response.statusCode() != 201) return; + if (response.statusCode() != 201) { + nextRenewalCheckNanos = renewalDeadline(now, retry); + return; + } HttpTlsIdentity.IssuedClientCertificate issued = HttpTransportProtocol.parseEnrollmentResponse(serverId, response.body()); HttpClientCredentialStore.StagedCredential staged = HttpClientCredentialStore.stageReplacement(directory, issued); HttpClientCredentialStore.ClientCredential replacement = staged.credential(); @@ -421,9 +428,26 @@ private void maybeRenewCredential() { profile = replacementProfile; client = replacementClient; credential = replacement; - } catch (Exception ignored) { /* The active generation is unchanged; retry on the bounded schedule. */ } + nextRenewalCheckNanos = renewalDeadline(now, RENEWAL_SUCCESS_CHECK); + } catch (Exception ignored) { + // Keep the active generation and retry well before its remaining validity is consumed. + nextRenewalCheckNanos = renewalDeadline(now, retry); + } } } + static Duration renewalRetryDelay(Duration remainingValidity) { + if (remainingValidity == null || remainingValidity.isNegative() || remainingValidity.isZero()) + return Duration.ofSeconds(1); + Duration beforeExpiry = remainingValidity.dividedBy(4L); + if (beforeExpiry.isZero()) beforeExpiry = Duration.ofNanos(1L); + return beforeExpiry.compareTo(RENEWAL_FAILURE_RETRY) < 0 ? beforeExpiry : RENEWAL_FAILURE_RETRY; + } + private static long renewalDeadline(long now, Duration delay) { + long nanos; + try { nanos = delay.toNanos(); } + catch (ArithmeticException overflow) { nanos = Long.MAX_VALUE; } + return nanos > Long.MAX_VALUE - now ? Long.MAX_VALUE : now + nanos; + } private static HttpClient client(HttpClientCredentialStore.HttpClientProfile profile, HttpClientCredentialStore.ClientCredential credential) throws Exception { return HttpClient.newBuilder().version(HttpClient.Version.HTTP_1_1).followRedirects(HttpClient.Redirect.NEVER) diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpInboundDeliveryStore.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpInboundDeliveryStore.java index 420ae02..3166d2c 100644 --- a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpInboundDeliveryStore.java +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpInboundDeliveryStore.java @@ -101,6 +101,17 @@ synchronized void recordCompleted(String id) throws IOException { synchronized void markRunning(String id) throws IOException { transition(id, State.RESERVED, State.RUNNING); } synchronized void markCompleted(String id) throws IOException { transition(id, State.RUNNING, State.COMPLETED); } synchronized void seal() { sealed = true; } + synchronized void sealAndDeleteIfEmpty() throws IOException { + if (!entries.isEmpty()) throw new IOException("HTTP inbound delivery store is not empty"); + requireRoot(); + try (DirectoryStream files = Files.newDirectoryStream(root)) { + if (files.iterator().hasNext()) throw new IOException("HTTP inbound delivery directory is not empty"); + } + sealed = true; + Path parent = root.getParent(); + Files.delete(root); + DurableFiles.forceDirectory(parent); + } synchronized void remove(String id) throws IOException { requireWritable(); diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java index 9238c78..e289a44 100644 --- a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java @@ -25,6 +25,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.HashMap; +import java.util.Iterator; import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; @@ -48,6 +49,8 @@ */ public final class HttpProxyTransportServer implements AutoCloseable { private static final int MAX_BACKENDS = 128; + private static final long BACKEND_REPLAY_RETENTION_NANOS = + TimeUnit.MILLISECONDS.toNanos(HttpTransportProtocol.MAX_CLOCK_SKEW_MILLIS) + 1L; static { // JDK HttpServer reads these once when its internal server configuration is initialized. // Set conservative process-wide bounds before this transport creates its listener. @@ -71,6 +74,7 @@ public final class HttpProxyTransportServer implements AutoCloseable { private final Path durableIncomingRoot; private final Consumer onEnvelope; private final DeliveryAcknowledgement onAcknowledged; + private final LongSupplier nanoTime; private volatile boolean closed; /** In-memory constructor for tests; production callers must supply a durable state directory. */ @@ -87,10 +91,17 @@ public HttpProxyTransportServer(InetSocketAddress bind, HttpTlsIdentity identity public HttpProxyTransportServer(InetSocketAddress bind, HttpTlsIdentity identity, HttpEnrollmentAuthority authority, Path outgoingDirectory, Consumer onEnvelope, DeliveryAcknowledgement onAcknowledged) throws Exception { + this(bind, identity, authority, outgoingDirectory, onEnvelope, onAcknowledged, System::nanoTime); + } + + HttpProxyTransportServer(InetSocketAddress bind, HttpTlsIdentity identity, HttpEnrollmentAuthority authority, + Path outgoingDirectory, Consumer onEnvelope, + DeliveryAcknowledgement onAcknowledged, LongSupplier nanoTime) throws Exception { if (bind == null || identity == null || authority == null || onEnvelope == null || onAcknowledged == null) throw new IllegalArgumentException("HTTP transport configuration is required"); + if (nanoTime == null) throw new IllegalArgumentException("HTTP transport clock is required"); this.identity = identity; this.authority = authority; this.onEnvelope = onEnvelope; - this.onAcknowledged = onAcknowledged; + this.onAcknowledged = onAcknowledged; this.nanoTime = nanoTime; durableOutgoing = outgoingDirectory == null ? null : new DurableOutgoingQueue(outgoingDirectory); durableIncomingRoot = outgoingDirectory == null ? null : incomingRoot(outgoingDirectory); if (durableOutgoing != null) for (Map.Entry> pending @@ -240,14 +251,26 @@ private BackendState backendState(String serverId) throws IOException { synchronized (backends) { BackendState existing = backends.get(serverId); if (existing != null) return existing; + if (backends.size() >= MAX_BACKENDS) reclaimInactiveBackend(); if (backends.size() >= MAX_BACKENDS) throw new IOException("HTTP backend state exceeds its bound"); HttpInboundDeliveryStore inbound = durableIncomingRoot == null ? null : HttpInboundDeliveryStore.open(durableIncomingRoot, serverId); - BackendState created = new BackendState(serverId, durableOutgoing, inbound, onAcknowledged); + BackendState created = new BackendState(serverId, durableOutgoing, inbound, onAcknowledged, nanoTime); backends.put(serverId, created); return created; } } + private void reclaimInactiveBackend() throws IOException { + long now = nanoTime.getAsLong(); + for (Iterator> iterator = backends.entrySet().iterator(); iterator.hasNext();) { + BackendState state = iterator.next().getValue(); + if (!state.retireIfQuiescent(now, BACKEND_REPLAY_RETENTION_NANOS)) continue; + iterator.remove(); + return; + } + } + BackendState backendStateForTest(String serverId) throws IOException { return backendState(serverId); } + int backendCountForTest() { synchronized (backends) { return backends.size(); } } private static Path incomingRoot(Path outgoingDirectory) throws IOException { Path outgoing = outgoingDirectory.toAbsolutePath().normalize(); Path parent = outgoing.getParent(); @@ -316,8 +339,9 @@ static final class BackendState { private final ArrayDeque acknowledgements = new ArrayDeque<>(); private final Map deliveredAtNanos = new HashMap<>(); private double requestTokens = 24.0d; - private long lastTokenNanos = System.nanoTime(); - private boolean activePoll; + private long lastTokenNanos; + private long lastActivityNanos; + private boolean activePoll, retired; BackendState() { this(null, null, null, (serverId, deliveryId) -> { }, System::nanoTime); } BackendState(LongSupplier nanoTime) { this(null, null, null, (serverId, deliveryId) -> { }, nanoTime); } private BackendState(String serverId, DurableOutgoingQueue durableOutgoing) { @@ -335,6 +359,7 @@ private BackendState(String serverId, DurableOutgoingQueue durableOutgoing, HttpInboundDeliveryStore durableIncoming, DeliveryAcknowledgement onAcknowledged, LongSupplier nanoTime) { this.serverId = serverId; this.durableOutgoing = durableOutgoing; this.durableIncoming = durableIncoming; this.onAcknowledged = onAcknowledged; this.nanoTime = nanoTime; + lastTokenNanos = lastActivityNanos = nanoTime.getAsLong(); if (durableIncoming != null) for (Map.Entry entry : durableIncoming.snapshot().entrySet()) { if (entry.getValue() == HttpInboundDeliveryStore.State.COMPLETED) { @@ -345,11 +370,13 @@ private BackendState(String serverId, DurableOutgoingQueue durableOutgoing, private synchronized void restore(Collection deliveries) { for (HttpTransportProtocol.Delivery delivery : deliveries) outgoing.put(delivery.id(), delivery); } - private boolean beginPoll(String requestedSession) { synchronized (this) { if (activePoll) return false; activePoll = true; return true; } } - private void endPoll() { synchronized (this) { activePoll = false; notifyAll(); } } + private boolean beginPoll(String requestedSession) { synchronized (this) { if (retired || activePoll) return false; activePoll = true; touch(); return true; } } + private void endPoll() { synchronized (this) { activePoll = false; touch(); notifyAll(); } } + boolean beginPollForTest() { return beginPoll("test"); } + void endPollForTest() { endPoll(); } private boolean allowRequest() { - long now = System.nanoTime(); requestTokens = Math.min(24.0d, requestTokens + ((now - lastTokenNanos) / 1_000_000_000.0d) * 2.0d); - lastTokenNanos = now; if (requestTokens < 1.0d) return false; requestTokens -= 1.0d; return true; + long now = nanoTime.getAsLong(); requestTokens = Math.min(24.0d, requestTokens + ((now - lastTokenNanos) / 1_000_000_000.0d) * 2.0d); + lastTokenNanos = now; touch(); if (requestTokens < 1.0d) return false; requestTokens -= 1.0d; return true; } boolean acceptSession(String requested, long requestedSequence) { if (!requested.equals(session)) { session = requested; sequence = -1L; deliveredAtNanos.clear(); } @@ -358,13 +385,14 @@ boolean acceptSession(String requested, long requestedSequence) { if (requestedSequence <= sequence) return false; sequence = requestedSequence; return true; } synchronized boolean enqueue(HttpTransportProtocol.Delivery delivery) { + if (retired) return false; HttpTransportProtocol.Delivery existing = outgoing.get(delivery.id()); if (existing != null) return Arrays.equals(HttpTransportProtocol.storedDelivery(existing), HttpTransportProtocol.storedDelivery(delivery)); if (outgoing.size() >= HttpTransportProtocol.MAX_QUEUE) return false; if (durableOutgoing != null) try { durableOutgoing.persist(serverId, delivery); } catch (IOException failure) { return false; } - outgoing.put(delivery.id(), delivery); signal(); return true; + outgoing.put(delivery.id(), delivery); touch(); signal(); return true; } void acknowledge(Collection acks) throws IOException { for (String id : acks) { @@ -411,6 +439,15 @@ synchronized void confirmIncoming(Collection ids) throws IOException { } } private void seal() { if (durableIncoming != null) durableIncoming.seal(); } + private synchronized boolean retireIfQuiescent(long now, long retentionNanos) throws IOException { + if (retired || activePoll || !outgoing.isEmpty() || !deliveredAtNanos.isEmpty() || !seen.isEmpty() + || !processing.isEmpty() || !acknowledgements.isEmpty() || now - lastActivityNanos < retentionNanos + || durableIncoming != null && !durableIncoming.snapshot().isEmpty()) return false; + if (durableIncoming != null) durableIncoming.sealAndDeleteIfEmpty(); + retired = true; + return true; + } + private void touch() { lastActivityNanos = nanoTime.getAsLong(); } private void queueAck(String id) { if (acknowledgements.size() < HttpTransportProtocol.MAX_QUEUE && !acknowledgements.contains(id)) acknowledgements.add(id); } synchronized Response await(String serverId, String requestedSession, long requestedSequence) { return await(serverId, requestedSession, requestedSequence, List.of()); diff --git a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportRuntimeTest.java b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportRuntimeTest.java index b2f4587..0f6c816 100644 --- a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportRuntimeTest.java +++ b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportRuntimeTest.java @@ -293,6 +293,28 @@ void proxyBackendStateIsGloballyBounded() throws Exception { } } + @Test + void proxyReclaimsOnlyQuiescentBackendStateAfterReplayWindow() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("reclaim-proxy"), "localhost"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, + directory.resolve("reclaim-authority")); + AtomicLong nanoTime = new AtomicLong(); + try (HttpProxyTransportServer server = new HttpProxyTransportServer(new InetSocketAddress("localhost", 0), + identity, authority, null, ignored -> { }, (serverId, deliveryId) -> { }, nanoTime::get)) { + for (int index = 0; index < 128; index++) { + HttpProxyTransportServer.BackendState state = server.backendStateForTest("server-" + index); + assertTrue(state.beginPollForTest()); + state.endPollForTest(); + } + assertFalse(server.send("replacement", JsonEnvelope.builder("x").build()), + "fresh state must retain its replay fence"); + nanoTime.addAndGet(TimeUnit.MILLISECONDS.toNanos(HttpTransportProtocol.MAX_CLOCK_SKEW_MILLIS) + 1L); + assertTrue(server.send("replacement", JsonEnvelope.builder("x").build()), + "a quiescent state must be reclaimable after captured requests expire"); + assertEquals(128, server.backendCountForTest()); + } + } + @Test void proxyOutgoingQueueSurvivesRestartUntilBackendAcknowledges() throws Exception { Path proxyDirectory = directory.resolve("proxy"); diff --git a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportSecurityTest.java b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportSecurityTest.java index f6092fa..d37cdb2 100644 --- a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportSecurityTest.java +++ b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportSecurityTest.java @@ -21,6 +21,16 @@ import org.junit.jupiter.api.io.TempDir; class HttpTransportSecurityTest { + @Test + void failedRenewalRetriesBeforeActiveCertificateExpires() { + assertEquals(Duration.ofMinutes(5), + HttpBackendTransportConnector.renewalRetryDelay(Duration.ofHours(6))); + assertEquals(Duration.ofMinutes(1), + HttpBackendTransportConnector.renewalRetryDelay(Duration.ofMinutes(4))); + assertTrue(HttpBackendTransportConnector.renewalRetryDelay(Duration.ofSeconds(3)) + .compareTo(Duration.ofSeconds(3)) < 0); + } + @Test void connectionCodeRejectsExplicitZeroPort() { assertThrows(IllegalArgumentException.class, From 80a55dada17fa489d851c46ebb0d90b223687ccd Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Fri, 4 Sep 2026 20:55:05 -0600 Subject: [PATCH 09/38] Make HTTP revocation persistence retryable --- .../http/HttpEnrollmentAuthority.java | 35 +++++++++++++++---- .../http/HttpTransportSecurityTest.java | 25 +++++++++++++ 2 files changed, 54 insertions(+), 6 deletions(-) diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentAuthority.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentAuthority.java index c544d27..00ecfc3 100644 --- a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentAuthority.java +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentAuthority.java @@ -33,6 +33,7 @@ public final class HttpEnrollmentAuthority { private final Map enrollments = new HashMap<>(); private final Map bindings = new HashMap<>(); private boolean persistenceFailure; + private boolean revocationRetryRequired; /** Creates a restart-safe authority. State contains public certificate pins plus bounded hashes of pending tokens. */ public HttpEnrollmentAuthority(HttpTlsIdentity identity, Path stateDirectory) throws java.io.IOException { @@ -127,11 +128,29 @@ public synchronized void revoke(String serverId) { try { serverId = HttpTlsIdentity.canonicalServerId(serverId); } catch (IllegalArgumentException invalid) { return; } final String revokedServer = serverId; - boolean pendingRemoved = enrollments.entrySet().removeIf(entry -> revokedServer.equals(entry.getValue().serverId())); + Map removedEnrollments = new HashMap<>(); + enrollments.entrySet().removeIf(entry -> { + if (!revokedServer.equals(entry.getValue().serverId())) return false; + removedEnrollments.put(entry.getKey(), entry.getValue()); + return true; + }); // Absence is the durable revocation fence: authentication always requires an exact active binding. - boolean bindingRemoved = bindings.remove(serverId) != null; - if (bindingRemoved || pendingRemoved) try { persistState(); } - catch (java.io.IOException failure) { persistenceFailure = true; throw new IllegalStateException("Could not persist HTTP certificate revocation", failure); } + ClientBinding removedBinding = bindings.remove(serverId); + if (removedBinding != null || !removedEnrollments.isEmpty() || revocationRetryRequired) try { + persistState(); + revocationRetryRequired = false; + } + catch (java.io.IOException failure) { + // Before publication, restore the exact disk-backed state so a retry still has work to persist. + // After publication, retain the fail-closed new state and let a retry force it durably again. + if (!(failure instanceof DurableFiles.PublishedException)) { + if (removedBinding != null) bindings.put(serverId, removedBinding); + enrollments.putAll(removedEnrollments); + } + persistenceFailure = true; + revocationRetryRequired = true; + throw new IllegalStateException("Could not persist HTTP certificate revocation", failure); + } } private synchronized void loadState() throws java.io.IOException { @@ -212,8 +231,12 @@ private synchronized void persistState() throws java.io.IOException { DurableFiles.forceFile(temporary); try { Files.move(temporary, stateFile, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); } catch (java.nio.file.AtomicMoveNotSupportedException unsupported) { Files.move(temporary, stateFile, StandardCopyOption.REPLACE_EXISTING); } - setOwnerOnly(stateFile); - DurableFiles.forceDirectory(stateFile.getParent()); + try { + setOwnerOnly(stateFile); + DurableFiles.forceDirectory(stateFile.getParent()); + } catch (java.io.IOException postPublicationFailure) { + throw new DurableFiles.PublishedException(postPublicationFailure); + } } finally { Files.deleteIfExists(temporary); } } diff --git a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportSecurityTest.java b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportSecurityTest.java index d37cdb2..c3e196b 100644 --- a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportSecurityTest.java +++ b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportSecurityTest.java @@ -291,6 +291,31 @@ void revocationInvalidatesEveryPendingCodeForTheBackend() throws Exception { () -> authority.enroll("lobby-1", secondPending.enrollmentToken())); } + @Test + void failedRevocationPersistenceCanBeRetriedWithoutLosingState() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("retry-revoke-proxy"), "localhost"); + Path stateDirectory = directory.resolve("retry-revoke-state"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, stateDirectory); + URI endpoint = URI.create("https://localhost:8443/"); + HttpConnectionCode active = authority.createConnectionCode("lobby-1", endpoint, Duration.ofMinutes(5)); + HttpTlsIdentity.IssuedClientCertificate issued = authority.enroll("lobby-1", active.enrollmentToken()); + HttpConnectionCode pending = authority.createConnectionCode("lobby-1", endpoint, Duration.ofMinutes(5)); + Path stateFile = stateDirectory.resolve("http-transport-clients.properties"); + Files.delete(stateFile); + Files.createDirectory(stateFile); + + assertThrows(IllegalStateException.class, () -> authority.revoke("lobby-1")); + assertFalse(authority.authenticate("lobby-1", issued.certificate()), + "an unpersisted revocation must fail authentication closed"); + Files.delete(stateFile); + authority.revoke("lobby-1"); + + HttpEnrollmentAuthority restarted = new HttpEnrollmentAuthority(identity, stateDirectory); + assertFalse(restarted.authenticate("lobby-1", issued.certificate())); + assertThrows(IllegalArgumentException.class, + () -> restarted.enroll("lobby-1", pending.enrollmentToken())); + } + @Test void pendingEnrollmentSurvivesRestartAndRevocationRemainsDurable() throws Exception { HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("pending-restart-proxy"), "localhost"); From 9d0166f19c4f703e6f9e01cfa441afe88afba71a Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Sat, 5 Sep 2026 16:55:18 -0600 Subject: [PATCH 10/38] Fix remaining HTTP transport review findings --- .../servercomm/http/HttpConnectionCode.java | 4 +++- .../servercomm/http/HttpEnrollmentAuthority.java | 1 + .../http/HttpTransportSecurityTest.java | 16 ++++++++++++++++ 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpConnectionCode.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpConnectionCode.java index 872bf4b..dfad475 100644 --- a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpConnectionCode.java +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpConnectionCode.java @@ -88,7 +88,9 @@ private static URI validateEndpoint(URI value) { if (path == null || path.isEmpty()) path = "/"; if (!path.endsWith("/")) path += "/"; try { - return new URI("https", null, value.getHost().toLowerCase(Locale.ROOT), value.getPort(), path, null, null); + URI authority = new URI("https", null, value.getHost().toLowerCase(Locale.ROOT), + value.getPort(), null, null, null); + return new URI(authority.toASCIIString() + path); } catch (URISyntaxException failure) { throw new IllegalArgumentException("Endpoint is invalid", failure); } diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentAuthority.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentAuthority.java index 00ecfc3..0409d03 100644 --- a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentAuthority.java +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentAuthority.java @@ -138,6 +138,7 @@ public synchronized void revoke(String serverId) { ClientBinding removedBinding = bindings.remove(serverId); if (removedBinding != null || !removedEnrollments.isEmpty() || revocationRetryRequired) try { persistState(); + persistenceFailure = false; revocationRetryRequired = false; } catch (java.io.IOException failure) { diff --git a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportSecurityTest.java b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportSecurityTest.java index c3e196b..d61fbff 100644 --- a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportSecurityTest.java +++ b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportSecurityTest.java @@ -64,6 +64,16 @@ void connectionCodeRoundTripsAndRejectsAccidentalCorruption() { assertThrows(IllegalArgumentException.class, () -> HttpConnectionCode.parse("http://not-a-code")); } + @Test + void connectionCodePreservesEscapedEndpointPaths() { + HttpConnectionCode original = new HttpConnectionCode("lobby.eu", + URI.create("https://Proxy.Example.test:8443/api%20root/%2F"), pin('a'), pin('b'), + Instant.parse("2030-01-01T00:00:00Z"), HttpTransportSecrets.randomToken()); + URI expected = URI.create("https://proxy.example.test:8443/api%20root/%2F/"); + assertEquals(expected, original.endpoint()); + assertEquals(expected, HttpConnectionCode.parse(original.encode()).endpoint()); + } + @Test void legacyConnectionCodesAndConsumedMarkersRemainCompatible() throws Exception { HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("legacy-code-proxy"), "proxy.example.test"); @@ -300,6 +310,8 @@ void failedRevocationPersistenceCanBeRetriedWithoutLosingState() throws Exceptio HttpConnectionCode active = authority.createConnectionCode("lobby-1", endpoint, Duration.ofMinutes(5)); HttpTlsIdentity.IssuedClientCertificate issued = authority.enroll("lobby-1", active.enrollmentToken()); HttpConnectionCode pending = authority.createConnectionCode("lobby-1", endpoint, Duration.ofMinutes(5)); + HttpConnectionCode unaffected = authority.createConnectionCode("lobby-2", endpoint, Duration.ofMinutes(5)); + HttpTlsIdentity.IssuedClientCertificate unaffectedIssued = authority.enroll("lobby-2", unaffected.enrollmentToken()); Path stateFile = stateDirectory.resolve("http-transport-clients.properties"); Files.delete(stateFile); Files.createDirectory(stateFile); @@ -307,8 +319,12 @@ void failedRevocationPersistenceCanBeRetriedWithoutLosingState() throws Exceptio assertThrows(IllegalStateException.class, () -> authority.revoke("lobby-1")); assertFalse(authority.authenticate("lobby-1", issued.certificate()), "an unpersisted revocation must fail authentication closed"); + assertFalse(authority.authenticate("lobby-2", unaffectedIssued.certificate()), + "all authentication must fail closed while persistence is unresolved"); Files.delete(stateFile); authority.revoke("lobby-1"); + assertTrue(authority.authenticate("lobby-2", unaffectedIssued.certificate()), + "a successful full-state retry must restore authentication availability"); HttpEnrollmentAuthority restarted = new HttpEnrollmentAuthority(identity, stateDirectory); assertFalse(restarted.authenticate("lobby-1", issued.certificate())); From ea26fde6ea396dcaa9c6d4c1c978f772e283fe4b Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Sat, 5 Sep 2026 17:03:47 -0600 Subject: [PATCH 11/38] Track uncertain HTTP queue publications --- .../http/HttpProxyTransportServer.java | 46 ++++++++++++++++--- .../http/HttpTransportRuntimeTest.java | 24 ++++++++++ 2 files changed, 63 insertions(+), 7 deletions(-) diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java index e289a44..02c6fe2 100644 --- a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java @@ -387,10 +387,20 @@ boolean acceptSession(String requested, long requestedSequence) { synchronized boolean enqueue(HttpTransportProtocol.Delivery delivery) { if (retired) return false; HttpTransportProtocol.Delivery existing = outgoing.get(delivery.id()); - if (existing != null) return Arrays.equals(HttpTransportProtocol.storedDelivery(existing), - HttpTransportProtocol.storedDelivery(delivery)); + if (existing != null) { + if (!Arrays.equals(HttpTransportProtocol.storedDelivery(existing), + HttpTransportProtocol.storedDelivery(delivery))) return false; + if (durableOutgoing != null) try { durableOutgoing.confirm(serverId, delivery.id()); } + catch (IOException failure) { return false; } + return true; + } if (outgoing.size() >= HttpTransportProtocol.MAX_QUEUE) return false; if (durableOutgoing != null) try { durableOutgoing.persist(serverId, delivery); } + catch (DurableFiles.PublishedException uncertain) { + // Keep the published entry reachable for delivery, acknowledgement, and a + // same-ID durability retry even though acceptance cannot yet be confirmed. + outgoing.put(delivery.id(), delivery); touch(); signal(); return false; + } catch (IOException failure) { return false; } outgoing.put(delivery.id(), delivery); touch(); signal(); return true; } @@ -498,13 +508,23 @@ private long nanosUntilRedelivery(long now) { private synchronized void signal() { notifyAll(); } } - private static final class DurableOutgoingQueue { + static final class DurableOutgoingQueue { + @FunctionalInterface + interface DirectoryForcer { void force(Path directory) throws IOException; } private static final String FILE_PATTERN = "[0-9]{20}-[0-9a-f-]{36}\\.json"; private final Path root; + private final DirectoryForcer directoryForcer; private final Map> files = new HashMap<>(); private long sequence; private DurableOutgoingQueue(Path root) throws IOException { + this(root, DurableFiles::forceDirectory); + } + + DurableOutgoingQueue(Path root, DirectoryForcer directoryForcer) throws IOException { + if (root == null || directoryForcer == null) + throw new IllegalArgumentException("HTTP outgoing queue configuration is required"); + this.directoryForcer = directoryForcer; this.root = root.toAbsolutePath().normalize(); boolean created = false; try { Files.createDirectory(this.root); created = true; } @@ -514,7 +534,7 @@ private DurableOutgoingQueue(Path root) throws IOException { throw new IOException("HTTP outgoing queue directory is invalid"); ownerOnlyDirectory(this.root); } finally { - if (created) DurableFiles.forceDirectory(this.root.getParent()); + if (created) directoryForcer.force(this.root.getParent()); } } @@ -577,7 +597,7 @@ private synchronized void persist(String serverId, HttpTransportProtocol.Deliver } finally { // The child fsync below cannot make this newly published name durable in // its parent. Persist the root entry before accepting the first message. - if (created) DurableFiles.forceDirectory(root); + if (created) directoryForcer.force(root); } if (sequence == Long.MAX_VALUE) throw new IOException("HTTP outgoing queue sequence is exhausted"); String name = String.format(java.util.Locale.ROOT, "%020d-%s.json", ++sequence, delivery.id()); @@ -589,11 +609,23 @@ private synchronized void persist(String serverId, HttpTransportProtocol.Deliver DurableFiles.forceFile(temporary); try { Files.move(temporary, target, StandardCopyOption.ATOMIC_MOVE); } catch (java.nio.file.AtomicMoveNotSupportedException unsupported) { Files.move(temporary, target); } - ownerOnlyFile(target); DurableFiles.forceDirectory(directory); files.computeIfAbsent(serverId, ignored -> new HashMap<>()).put(delivery.id(), target); + try { ownerOnlyFile(target); directoryForcer.force(directory); } + catch (IOException postPublicationFailure) { + throw new DurableFiles.PublishedException(postPublicationFailure); + } } finally { Files.deleteIfExists(temporary); } } + private synchronized void confirm(String serverId, String id) throws IOException { + Map serverFiles = files.get(serverId); + Path file = serverFiles == null ? null : serverFiles.get(id); + if (file == null || Files.isSymbolicLink(file) || !Files.isRegularFile(file, LinkOption.NOFOLLOW_LINKS)) + throw new IOException("HTTP outgoing queue delivery is unavailable"); + ownerOnlyFile(file); + directoryForcer.force(file.getParent()); + } + private synchronized void remove(String serverId, String id) throws IOException { Map serverFiles = files.get(serverId); if (serverFiles == null) throw new IOException("HTTP outgoing queue acknowledgement is unknown"); @@ -610,7 +642,7 @@ private synchronized void remove(String serverId, String id) throws IOException throw new IOException("HTTP outgoing queue server directory is invalid"); try { Files.deleteIfExists(directory); - DurableFiles.forceDirectory(root); + directoryForcer.force(root); files.remove(serverId); } catch (java.nio.file.DirectoryNotEmptyException unexpectedEntry) { // The acknowledged delivery is already durably removed; unrelated/tampered entries diff --git a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportRuntimeTest.java b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportRuntimeTest.java index 0f6c816..53bd8b2 100644 --- a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportRuntimeTest.java +++ b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportRuntimeTest.java @@ -129,6 +129,30 @@ void stableProxyDeliveryIdsAreIdempotentAndAcknowledgedBeforeRemoval() throws Ex assertTrue(state.await("lobby-1", java.util.UUID.randomUUID().toString(), 0).messages().isEmpty()); } + @Test + void publishedOutgoingDeliveryRemainsTrackedUntilDurabilityCanBeConfirmed() throws Exception { + AtomicLong forceCalls = new AtomicLong(); + Path queueRoot = directory.resolve("uncertain-outgoing"); + HttpProxyTransportServer.DurableOutgoingQueue queue = new HttpProxyTransportServer.DurableOutgoingQueue( + queueRoot, ignored -> { + if (forceCalls.incrementAndGet() == 3L) throw new java.io.IOException("injected directory force failure"); + }); + HttpProxyTransportServer.BackendState state = new HttpProxyTransportServer.BackendState( + "lobby-1", queue, (server, id) -> { }); + String deliveryId = java.util.UUID.randomUUID().toString(); + HttpTransportProtocol.Delivery delivery = new HttpTransportProtocol.Delivery(deliveryId, + JsonEnvelope.builder("durable").build()); + + assertFalse(state.enqueue(delivery), "post-publication failure must not confirm durable acceptance"); + assertEquals(java.util.List.of(delivery), + state.await("lobby-1", java.util.UUID.randomUUID().toString(), 0).messages()); + assertEquals(1L, countRegularFiles(queueRoot)); + assertTrue(state.enqueue(delivery), "same-ID retry must confirm the existing published file"); + assertEquals(1L, countRegularFiles(queueRoot), "durability retry must not create a duplicate file"); + state.acknowledge(java.util.List.of(deliveryId)); + assertEquals(0L, countRegularFiles(queueRoot), "the tracked published file must be removable by ACK"); + } + @Test void proxyInboundCompletionSurvivesRestartBeforeAcknowledgement() throws Exception { Path root = directory.resolve("proxy-incoming"); From 8d8e9c8d11f2e3ac7df39bbff53cb4e40adf5bda Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Sat, 5 Sep 2026 17:22:48 -0600 Subject: [PATCH 12/38] Make HTTP enrollment and queue retries safe --- .../http/HttpEnrollmentAuthority.java | 91 ++++++++++++++----- .../http/HttpProxyTransportServer.java | 18 ++-- .../http/HttpTransportRuntimeTest.java | 6 +- .../http/HttpTransportSecurityTest.java | 41 +++++++++ 4 files changed, 127 insertions(+), 29 deletions(-) diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentAuthority.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentAuthority.java index 0409d03..48dc86d 100644 --- a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentAuthority.java +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentAuthority.java @@ -19,8 +19,9 @@ import com.bencodez.simpleapi.file.DurableFiles; /** - * Single-use enrollment tokens and client-certificate binding. Token material is never retained; - * only SHA-256 hashes are kept until expiry. This type is thread-safe. + * Single-activation enrollment tokens and client-certificate binding. A token may retry certificate + * issuance until possession is proved, but can activate only one binding. Token material is never + * retained; only SHA-256 hashes are kept until expiry. This type is thread-safe. */ public final class HttpEnrollmentAuthority { private static final Duration MAX_ENROLLMENT_LIFETIME = Duration.ofMinutes(15); @@ -63,7 +64,7 @@ public synchronized HttpConnectionCode createConnectionCode(String serverId, URI String token = HttpTransportSecrets.randomToken(); byte[] tokenHash = HttpTransportSecrets.sha256(token.getBytes(StandardCharsets.US_ASCII)); String lookup = java.util.Base64.getUrlEncoder().withoutPadding().encodeToString(tokenHash); - enrollments.put(lookup, new Enrollment(tokenHash, expiresAt, serverId)); + enrollments.put(lookup, new Enrollment(tokenHash, expiresAt, serverId, null)); try { persistState(); } catch (java.io.IOException failure) { enrollments.remove(lookup); @@ -84,13 +85,17 @@ public synchronized HttpTlsIdentity.IssuedClientCertificate enroll(String server if (!serverId.equals(enrollment.serverId())) throw new IllegalArgumentException("Enrollment was rejected"); ClientBinding existing = bindings.get(serverId); if (existing != null && !existing.revoked()) throw new IllegalStateException("Server id is already enrolled"); - if (existing == null && bindings.size() >= MAX_BINDINGS) + if (existing == null && !hasPendingCertificate(serverId) && reservedBindingCount() >= MAX_BINDINGS) throw new IllegalStateException("Too many enrolled HTTP backends"); - enrollments.remove(lookup); // consume only after all checks for the token and its intended backend pass. HttpTlsIdentity.IssuedClientCertificate issued = identity.issueClientCertificate(serverId); - bindings.put(serverId, new ClientBinding(HttpTransportSecrets.certificatePin(issued.certificate()), null, false)); - try { persistState(); } - catch (java.io.IOException failure) { persistenceFailure = true; throw failure; } + enrollments.put(lookup, new Enrollment(enrollment.tokenHash(), enrollment.expiresAt(), serverId, + HttpTransportSecrets.certificatePin(issued.certificate()))); + try { persistState(); persistenceFailure = false; } + catch (java.io.IOException failure) { + if (!(failure instanceof DurableFiles.PublishedException)) enrollments.put(lookup, enrollment); + else persistenceFailure = true; + throw failure; + } return issued; } @@ -101,12 +106,28 @@ public synchronized boolean authenticate(String serverId, java.security.cert.X50 if (!identity.validClientCertificate(serverId, certificate)) return false; ClientBinding binding = bindings.get(serverId); String pin = HttpTransportSecrets.certificatePin(certificate); - if (binding == null || binding.revoked()) return false; - if (samePin(binding.certificatePin(), pin)) return true; - if (!samePin(binding.pendingCertificatePin(), pin)) return false; + if (binding != null && !binding.revoked()) { + if (samePin(binding.certificatePin(), pin)) return true; + if (!samePin(binding.pendingCertificatePin(), pin)) return false; + bindings.put(serverId, new ClientBinding(pin, null, false)); + try { persistState(); return true; } + catch (java.io.IOException failure) { persistenceFailure = true; return false; } + } + Map.Entry pending = pendingCertificate(serverId, pin); + if (pending == null || bindings.size() >= MAX_BINDINGS) return false; + enrollments.remove(pending.getKey()); bindings.put(serverId, new ClientBinding(pin, null, false)); try { persistState(); return true; } - catch (java.io.IOException failure) { persistenceFailure = true; return false; } + catch (DurableFiles.PublishedException published) { + // The visible state is active. If its directory entry is lost on a crash, the + // prior pending state can promote this same certificate again after restart. + return true; + } + catch (java.io.IOException failure) { + bindings.remove(serverId); + enrollments.put(pending.getKey(), pending.getValue()); + return false; + } } /** Issues a replacement while the currently bound certificate is still valid. The old binding remains active @@ -161,7 +182,7 @@ private synchronized void loadState() throws java.io.IOException { Properties properties = new Properties(); try (var input = Files.newInputStream(stateFile, LinkOption.NOFOLLOW_LINKS)) { properties.load(input); } String version = properties.getProperty("version"); - if (!("1".equals(version) || "2".equals(version) || "3".equals(version))) + if (!("1".equals(version) || "2".equals(version) || "3".equals(version) || "4".equals(version))) throw new java.io.IOException("HTTP enrollment state is invalid"); for (String key : properties.stringPropertyNames()) { if (key.startsWith("binding.")) { @@ -169,7 +190,7 @@ private synchronized void loadState() throws java.io.IOException { serverId = HttpTlsIdentity.canonicalServerId(serverId); String[] value = properties.getProperty(key, "").split(":", -1); if (!((value.length == 2 && "1".equals(version)) - || (value.length == 3 && ("2".equals(version) || "3".equals(version)))) + || (value.length == 3 && ("2".equals(version) || "3".equals(version) || "4".equals(version)))) || !value[0].matches("[0-9a-f]{64}")) throw new java.io.IOException("HTTP enrollment state is invalid"); String pending = value.length == 3 && !"-".equals(value[1]) ? value[1] : null; @@ -180,7 +201,7 @@ private synchronized void loadState() throws java.io.IOException { if (bindings.size() >= MAX_BINDINGS) throw new java.io.IOException("HTTP enrollment state exceeds its bound"); bindings.put(serverId, new ClientBinding(value[0], pending, false)); } - } else if (key.startsWith("enrollment.") && "3".equals(version)) { + } else if (key.startsWith("enrollment.") && ("3".equals(version) || "4".equals(version))) { String lookup = key.substring("enrollment.".length()); if (!lookup.matches("[A-Za-z0-9_-]{43}")) throw new java.io.IOException("HTTP enrollment state is invalid"); byte[] tokenHash; @@ -188,28 +209,34 @@ private synchronized void loadState() throws java.io.IOException { catch (IllegalArgumentException invalid) { throw new java.io.IOException("HTTP enrollment state is invalid", invalid); } if (tokenHash.length != 32) throw new java.io.IOException("HTTP enrollment state is invalid"); String[] value = properties.getProperty(key, "").split(":", -1); - if (value.length != 2) throw new java.io.IOException("HTTP enrollment state is invalid"); + if (!(value.length == 2 && "3".equals(version)) && !(value.length == 3 && "4".equals(version))) + throw new java.io.IOException("HTTP enrollment state is invalid"); Instant expiresAt; String serverId; + String pendingPin = value.length == 3 && !"-".equals(value[2]) ? value[2] : null; try { expiresAt = Instant.ofEpochMilli(Long.parseLong(value[0])); serverId = HttpTlsIdentity.canonicalServerId(new String(Base64.getUrlDecoder().decode(value[1]), StandardCharsets.UTF_8)); } catch (RuntimeException invalid) { throw new java.io.IOException("HTTP enrollment state is invalid", invalid); } + if (pendingPin != null && !pendingPin.matches("[0-9a-f]{64}")) + throw new java.io.IOException("HTTP enrollment state is invalid"); if (expiresAt.isAfter(clock.instant())) { if (enrollments.size() >= MAX_PENDING_ENROLLMENTS) throw new java.io.IOException("HTTP enrollment state exceeds its bound"); - enrollments.put(lookup, new Enrollment(tokenHash, expiresAt, serverId)); + enrollments.put(lookup, new Enrollment(tokenHash, expiresAt, serverId, pendingPin)); } } else if (!"version".equals(key)) throw new java.io.IOException("HTTP enrollment state is invalid"); } + if (reservedBindingCount() > MAX_BINDINGS) + throw new java.io.IOException("HTTP enrollment state exceeds its bound"); } private synchronized void persistState() throws java.io.IOException { if (stateFile == null) return; - if (bindings.size() > MAX_BINDINGS || enrollments.size() > MAX_PENDING_ENROLLMENTS) + if (reservedBindingCount() > MAX_BINDINGS || enrollments.size() > MAX_PENDING_ENROLLMENTS) throw new java.io.IOException("HTTP enrollment state exceeds its bound"); Properties properties = new Properties(); - properties.setProperty("version", "3"); + properties.setProperty("version", "4"); for (Map.Entry entry : bindings.entrySet()) { String key = Base64.getUrlEncoder().withoutPadding().encodeToString(entry.getKey().getBytes(StandardCharsets.UTF_8)); properties.setProperty("binding." + key, entry.getValue().certificatePin() + ":" @@ -220,7 +247,8 @@ private synchronized void persistState() throws java.io.IOException { String server = Base64.getUrlEncoder().withoutPadding().encodeToString( entry.getValue().serverId().getBytes(StandardCharsets.UTF_8)); properties.setProperty("enrollment." + entry.getKey(), - entry.getValue().expiresAt().toEpochMilli() + ":" + server); + entry.getValue().expiresAt().toEpochMilli() + ":" + server + ":" + + (entry.getValue().pendingCertificatePin() == null ? "-" : entry.getValue().pendingCertificatePin())); } java.io.ByteArrayOutputStream bytes = new java.io.ByteArrayOutputStream(); properties.store(bytes, "VotingPlugin HTTP transport authority state"); @@ -271,7 +299,28 @@ private void expireEnrollments() { enrollments.entrySet().removeIf(entry -> !entry.getValue().expiresAt().isAfter(now)); } - private record Enrollment(byte[] tokenHash, Instant expiresAt, String serverId) { + private int reservedBindingCount() { + java.util.Set reserved = new java.util.HashSet<>(bindings.keySet()); + for (Enrollment enrollment : enrollments.values()) + if (enrollment.pendingCertificatePin() != null) reserved.add(enrollment.serverId()); + return reserved.size(); + } + + private boolean hasPendingCertificate(String serverId) { + for (Enrollment enrollment : enrollments.values()) + if (serverId.equals(enrollment.serverId()) && enrollment.pendingCertificatePin() != null) return true; + return false; + } + + private Map.Entry pendingCertificate(String serverId, String pin) { + for (Map.Entry entry : enrollments.entrySet()) { + Enrollment enrollment = entry.getValue(); + if (serverId.equals(enrollment.serverId()) && samePin(enrollment.pendingCertificatePin(), pin)) return entry; + } + return null; + } + + private record Enrollment(byte[] tokenHash, Instant expiresAt, String serverId, String pendingCertificatePin) { private Enrollment { tokenHash = tokenHash.clone(); } @Override public byte[] tokenHash() { return tokenHash.clone(); } } diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java index 02c6fe2..e67a66c 100644 --- a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java @@ -396,11 +396,6 @@ synchronized boolean enqueue(HttpTransportProtocol.Delivery delivery) { } if (outgoing.size() >= HttpTransportProtocol.MAX_QUEUE) return false; if (durableOutgoing != null) try { durableOutgoing.persist(serverId, delivery); } - catch (DurableFiles.PublishedException uncertain) { - // Keep the published entry reachable for delivery, acknowledgement, and a - // same-ID durability retry even though acceptance cannot yet be confirmed. - outgoing.put(delivery.id(), delivery); touch(); signal(); return false; - } catch (IOException failure) { return false; } outgoing.put(delivery.id(), delivery); touch(); signal(); return true; } @@ -599,6 +594,17 @@ private synchronized void persist(String serverId, HttpTransportProtocol.Deliver // its parent. Persist the root entry before accepting the first message. if (created) directoryForcer.force(root); } + Map serverFiles = files.computeIfAbsent(serverId, ignored -> new HashMap<>()); + Path existing = serverFiles.get(delivery.id()); + if (existing != null) { + if (Files.isSymbolicLink(existing) || !Files.isRegularFile(existing, LinkOption.NOFOLLOW_LINKS) + || Files.size(existing) > HttpTransportProtocol.MAX_ENVELOPE_BYTES * 2L + || !Arrays.equals(Files.readAllBytes(existing), HttpTransportProtocol.storedDelivery(delivery))) + throw new IOException("HTTP outgoing queue delivery id conflicts with persisted data"); + ownerOnlyFile(existing); + directoryForcer.force(directory); + return; + } if (sequence == Long.MAX_VALUE) throw new IOException("HTTP outgoing queue sequence is exhausted"); String name = String.format(java.util.Locale.ROOT, "%020d-%s.json", ++sequence, delivery.id()); Path target = directory.resolve(name); @@ -609,7 +615,7 @@ private synchronized void persist(String serverId, HttpTransportProtocol.Deliver DurableFiles.forceFile(temporary); try { Files.move(temporary, target, StandardCopyOption.ATOMIC_MOVE); } catch (java.nio.file.AtomicMoveNotSupportedException unsupported) { Files.move(temporary, target); } - files.computeIfAbsent(serverId, ignored -> new HashMap<>()).put(delivery.id(), target); + serverFiles.put(delivery.id(), target); try { ownerOnlyFile(target); directoryForcer.force(directory); } catch (IOException postPublicationFailure) { throw new DurableFiles.PublishedException(postPublicationFailure); diff --git a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportRuntimeTest.java b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportRuntimeTest.java index 53bd8b2..a3ef6d9 100644 --- a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportRuntimeTest.java +++ b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportRuntimeTest.java @@ -144,11 +144,13 @@ void publishedOutgoingDeliveryRemainsTrackedUntilDurabilityCanBeConfirmed() thro JsonEnvelope.builder("durable").build()); assertFalse(state.enqueue(delivery), "post-publication failure must not confirm durable acceptance"); - assertEquals(java.util.List.of(delivery), - state.await("lobby-1", java.util.UUID.randomUUID().toString(), 0).messages()); + assertTrue(state.await("lobby-1", java.util.UUID.randomUUID().toString(), 0).messages().isEmpty(), + "an uncertain publication must remain hidden until its durability retry succeeds"); assertEquals(1L, countRegularFiles(queueRoot)); assertTrue(state.enqueue(delivery), "same-ID retry must confirm the existing published file"); assertEquals(1L, countRegularFiles(queueRoot), "durability retry must not create a duplicate file"); + assertEquals(java.util.List.of(delivery), + state.await("lobby-1", java.util.UUID.randomUUID().toString(), 0).messages()); state.acknowledge(java.util.List.of(deliveryId)); assertEquals(0L, countRegularFiles(queueRoot), "the tracked published file must be removable by ACK"); } diff --git a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportSecurityTest.java b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportSecurityTest.java index d61fbff..8d3eaa7 100644 --- a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportSecurityTest.java +++ b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportSecurityTest.java @@ -355,6 +355,47 @@ void pendingEnrollmentSurvivesRestartAndRevocationRemainsDurable() throws Except () -> afterRevocation.enroll("lobby-1", revokedPending.enrollmentToken())); } + @Test + void lostEnrollmentResponseCanRetryUntilCertificatePossessionIsProved() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("retry-enrollment-proxy"), "localhost"); + Path state = directory.resolve("retry-enrollment-state"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, state); + HttpConnectionCode code = authority.createConnectionCode("lobby-1", + URI.create("https://localhost:8443/"), Duration.ofMinutes(5)); + HttpTlsIdentity.IssuedClientCertificate lost = authority.enroll("lobby-1", code.enrollmentToken()); + + HttpEnrollmentAuthority restarted = new HttpEnrollmentAuthority(identity, state); + HttpTlsIdentity.IssuedClientCertificate retried = restarted.enroll("lobby-1", code.enrollmentToken()); + HttpEnrollmentAuthority beforeProof = new HttpEnrollmentAuthority(identity, state); + assertFalse(beforeProof.authenticate("lobby-1", lost.certificate()), + "retrying enrollment must supersede the certificate from the lost response"); + assertTrue(beforeProof.authenticate("lobby-1", retried.certificate()), + "the first authenticated request must promote the certificate durably"); + + HttpEnrollmentAuthority afterProof = new HttpEnrollmentAuthority(identity, state); + assertTrue(afterProof.authenticate("lobby-1", retried.certificate())); + assertThrows(IllegalArgumentException.class, + () -> afterProof.enroll("lobby-1", code.enrollmentToken()), + "proof of possession must consume the one-time enrollment token"); + } + + @Test + void failedPendingCertificateWriteLeavesEnrollmentTokenRetryable() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("failed-enrollment-proxy"), "localhost"); + Path state = directory.resolve("failed-enrollment-state"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, state); + HttpConnectionCode code = authority.createConnectionCode("lobby-1", + URI.create("https://localhost:8443/"), Duration.ofMinutes(5)); + Path stateFile = state.resolve("http-transport-clients.properties"); + Files.delete(stateFile); + Files.createDirectory(stateFile); + + assertThrows(java.io.IOException.class, () -> authority.enroll("lobby-1", code.enrollmentToken())); + Files.delete(stateFile); + HttpTlsIdentity.IssuedClientCertificate retried = authority.enroll("lobby-1", code.enrollmentToken()); + assertTrue(authority.authenticate("lobby-1", retried.certificate())); + } + @Test void authorityStatePrunesRevocationsAndBoundsActiveBindings() throws Exception { Path proxy = directory.resolve("bounded-proxy"); From 989ec3f28b8c4d1903e99e73ba4ba8d7cbb4a4f7 Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Sat, 5 Sep 2026 17:37:50 -0600 Subject: [PATCH 13/38] Fix remaining HTTP transport review findings --- .../http/HttpBackendTransportConnector.java | 28 +++- .../http/HttpClientCredentialStore.java | 15 ++- .../http/HttpEnrollmentAuthority.java | 14 +- .../http/HttpProxyTransportServer.java | 46 +++++-- .../servercomm/http/HttpTlsIdentity.java | 15 ++- .../http/HttpTransportRuntimeTest.java | 124 ++++++++++++++++++ .../http/HttpTransportSecurityTest.java | 35 +++++ 7 files changed, 251 insertions(+), 26 deletions(-) diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpBackendTransportConnector.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpBackendTransportConnector.java index 1c5fb4d..2216db7 100644 --- a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpBackendTransportConnector.java +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpBackendTransportConnector.java @@ -244,18 +244,32 @@ public boolean flushOutgoing(long deadlineNanos) { @Override public void close() { running.getAndSet(false); firstResponse.countDown(); - // Revoke this connector's journal writer before a replacement snapshots it. - // In-flight transitions serialize with seal(): either COMPLETED is already - // durable, or the delivery remains durably RUNNING and fail-closed. - if (inboundDeliveries != null) inboundDeliveries.seal(); - if (acknowledgementConfirmationStore != null) acknowledgementConfirmationStore.seal(); Thread current = poller; if (current != null) current.interrupt(); - callbackExecutor.shutdown(); try { if (!callbackExecutor.awaitTermination(5, TimeUnit.SECONDS)) callbackExecutor.shutdownNow(); } - catch (InterruptedException interrupted) { Thread.currentThread().interrupt(); callbackExecutor.shutdownNow(); } // The owning transport may release the credential-directory semaphore as soon // as close returns. Wait for the interrupted poller so an in-flight renewal // cannot activate an old credential generation after that ownership handoff. joinPoller(current); + shutdownCallbacks(); + // No poller can enqueue more callbacks and every running journal transition has + // finished, so ownership can now be revoked without stranding completed work. + if (inboundDeliveries != null) inboundDeliveries.seal(); + if (acknowledgementConfirmationStore != null) acknowledgementConfirmationStore.seal(); + } + private void shutdownCallbacks() { + callbackExecutor.shutdown(); + boolean interrupted = false; + try { + if (!callbackExecutor.awaitTermination(5, TimeUnit.SECONDS)) { + callbackExecutor.shutdownNow(); + callbackExecutor.awaitTermination(1, TimeUnit.SECONDS); + } + } catch (InterruptedException stopRequested) { + interrupted = true; + callbackExecutor.shutdownNow(); + try { callbackExecutor.awaitTermination(1, TimeUnit.SECONDS); } + catch (InterruptedException repeated) { interrupted = true; } + } + if (interrupted) Thread.currentThread().interrupt(); } boolean pollerAlive() { Thread current = poller; return current != null && current.isAlive(); } private static boolean joinPoller(Thread poller, long deadlineNanos) { diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpClientCredentialStore.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpClientCredentialStore.java index f61900e..46ab04e 100644 --- a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpClientCredentialStore.java +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpClientCredentialStore.java @@ -72,10 +72,17 @@ private static ClientCredential loadCredential(Path directory) throws Exception Path passwordFile = safe(directory.resolve(PASSWORD_FILE)); if (!Files.isRegularFile(bundle, LinkOption.NOFOLLOW_LINKS) || !Files.isRegularFile(passwordFile, LinkOption.NOFOLLOW_LINKS)) throw new IOException("HTTP client certificate has not been enrolled"); - byte[] passwordBytes = Files.readAllBytes(passwordFile); - if (passwordBytes.length < 40 || passwordBytes.length > 128) throw new IOException("HTTP client password is invalid"); - char[] password = new String(passwordBytes, StandardCharsets.US_ASCII).toCharArray(); - java.util.Arrays.fill(passwordBytes, (byte) 0); + long passwordSize = Files.size(passwordFile); + if (passwordSize < 40L || passwordSize > 128L) throw new IOException("HTTP client password is invalid"); + byte[] passwordBytes; + try (var input = Files.newInputStream(passwordFile, LinkOption.NOFOLLOW_LINKS)) { + passwordBytes = input.readNBytes(129); + } + char[] password; + try { + if (passwordBytes.length < 40 || passwordBytes.length > 128) throw new IOException("HTTP client password is invalid"); + password = new String(passwordBytes, StandardCharsets.US_ASCII).toCharArray(); + } finally { java.util.Arrays.fill(passwordBytes, (byte) 0); } try { KeyStore store = KeyStore.getInstance("PKCS12"); try (var input = Files.newInputStream(bundle, LinkOption.NOFOLLOW_LINKS)) { store.load(input, password); } diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentAuthority.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentAuthority.java index 48dc86d..3a48c21 100644 --- a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentAuthority.java +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentAuthority.java @@ -111,7 +111,12 @@ public synchronized boolean authenticate(String serverId, java.security.cert.X50 if (!samePin(binding.pendingCertificatePin(), pin)) return false; bindings.put(serverId, new ClientBinding(pin, null, false)); try { persistState(); return true; } - catch (java.io.IOException failure) { persistenceFailure = true; return false; } + catch (DurableFiles.PublishedException published) { + // The replacement is visible. If publication is lost on a crash, the + // previous durable pending binding can promote this certificate again. + return true; + } + catch (java.io.IOException failure) { bindings.put(serverId, binding); return false; } } Map.Entry pending = pendingCertificate(serverId, pin); if (pending == null || bindings.size() >= MAX_BINDINGS) return false; @@ -141,7 +146,12 @@ public synchronized HttpTlsIdentity.IssuedClientCertificate renew(String serverI bindings.put(serverId, new ClientBinding(binding.certificatePin(), HttpTransportSecrets.certificatePin(issued.certificate()), false)); try { persistState(); } - catch (java.io.IOException failure) { persistenceFailure = true; throw failure; } + catch (DurableFiles.PublishedException published) { + // The old certificate remains active in both the old and newly visible state, + // so a lost response can safely retry and republish the complete state. + throw published; + } + catch (java.io.IOException failure) { bindings.put(serverId, binding); throw failure; } return issued; } diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java index e67a66c..746cdb4 100644 --- a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java @@ -132,7 +132,9 @@ private void renew(HttpsExchange exchange) throws IOException { if (!"/v1/renew".equals(exchange.getRequestURI().getPath()) || exchange.getRequestURI().getRawQuery() != null) { reply(exchange, 404, new byte[0]); return; } if (!"POST".equals(exchange.getRequestMethod())) { reply(exchange, 405, new byte[0]); return; } if (!json(exchange)) { reply(exchange, 415, new byte[0]); return; } - if (!boundedFixedBody(exchange, 1024) || !admission.tryAcquire()) { reply(exchange, 429, new byte[0]); return; } + int bodyError = fixedBodyErrorStatus(exchange.getRequestHeaders(), 1024); + if (bodyError != 0) { reply(exchange, bodyError, new byte[0]); return; } + if (!admission.tryAcquire()) { reply(exchange, 429, new byte[0]); return; } try { String serverId = HttpTransportProtocol.parseRenewal(read(exchange.getRequestBody(), 1024)); X509Certificate certificate = peerCertificate(exchange); @@ -188,12 +190,15 @@ private void enroll(HttpsExchange exchange) throws IOException { if (!"/v1/enroll".equals(exchange.getRequestURI().getPath()) || exchange.getRequestURI().getRawQuery() != null) { reply(exchange, 404, new byte[0]); return; } if (!"POST".equals(exchange.getRequestMethod())) { reply(exchange, 405, new byte[0]); return; } if (!json(exchange)) { reply(exchange, 415, new byte[0]); return; } - if (!boundedFixedBody(exchange, 8192) || !admission.tryAcquire()) { reply(exchange, 429, new byte[0]); return; } + int bodyError = fixedBodyErrorStatus(exchange.getRequestHeaders(), 8192); + if (bodyError != 0) { reply(exchange, bodyError, new byte[0]); return; } + if (!admission.tryAcquire()) { reply(exchange, 429, new byte[0]); return; } try { HttpTransportProtocol.Enrollment request = HttpTransportProtocol.parseEnrollment(read(exchange.getRequestBody(), 8192)); HttpTlsIdentity.IssuedClientCertificate issued = authority.enroll(request.server(), request.token()); reply(exchange, 201, HttpTransportProtocol.enrollmentResponse(issued)); - } catch (Exception rejected) { reply(exchange, 403, new byte[0]); } + } catch (IllegalArgumentException rejected) { reply(exchange, 403, new byte[0]); + } catch (Exception failure) { reply(exchange, 503, new byte[0]); } finally { admission.release(); } } @@ -201,7 +206,9 @@ private void transport(HttpsExchange exchange) throws IOException { if (!"/v1/transport".equals(exchange.getRequestURI().getPath()) || exchange.getRequestURI().getRawQuery() != null) { reply(exchange, 404, new byte[0]); return; } if (!"POST".equals(exchange.getRequestMethod())) { reply(exchange, 405, new byte[0]); return; } if (!json(exchange)) { reply(exchange, 415, new byte[0]); return; } - if (!boundedFixedBody(exchange, HttpTransportProtocol.MAX_BODY_BYTES) || !admission.tryAcquire()) { reply(exchange, 429, new byte[0]); return; } + int bodyError = fixedBodyErrorStatus(exchange.getRequestHeaders(), HttpTransportProtocol.MAX_BODY_BYTES); + if (bodyError != 0) { reply(exchange, bodyError, new byte[0]); return; } + if (!admission.tryAcquire()) { reply(exchange, 429, new byte[0]); return; } try { HttpTransportProtocol.Packet packet = HttpTransportProtocol.parsePacket(read(exchange.getRequestBody(), HttpTransportProtocol.MAX_BODY_BYTES)); X509Certificate certificate = peerCertificate(exchange); @@ -307,18 +314,37 @@ private static boolean json(HttpsExchange exchange) { String contentType = exchange.getRequestHeaders().getFirst("Content-Type"); return contentType != null && contentType.toLowerCase(java.util.Locale.ROOT).matches("application/json(?:\\s*;.*)?"); } - private static boolean boundedFixedBody(HttpsExchange exchange, int maximum) { - if (exchange.getRequestHeaders().getFirst("Transfer-Encoding") != null) return false; - String value = exchange.getRequestHeaders().getFirst("Content-Length"); - try { long length = Long.parseLong(value); return length > 0L && length <= maximum; } - catch (RuntimeException invalid) { return false; } + static int fixedBodyErrorStatus(Headers headers, int maximum) { + if (headers.getFirst("Transfer-Encoding") != null) return 400; + String value = headers.getFirst("Content-Length"); + if (value == null) return 411; + try { + long length = Long.parseLong(value); + if (length <= 0L) return 400; + return length <= maximum ? 0 : 413; + } catch (NumberFormatException invalid) { return 400; } } private static ThreadPoolExecutor executor(String name, int threads, int queue) { ThreadFactory factory = task -> { Thread thread = new Thread(task, name); thread.setDaemon(true); return thread; }; return new ThreadPoolExecutor(threads, threads, 0L, TimeUnit.MILLISECONDS, new ArrayBlockingQueue<>(queue), factory, new ThreadPoolExecutor.AbortPolicy()); } private static void setDefault(String name, String value) { if (System.getProperty(name) == null) System.setProperty(name, value); } - private static void shutdown(ExecutorService executor) { executor.shutdown(); try { if (!executor.awaitTermination(5, TimeUnit.SECONDS)) executor.shutdownNow(); } catch (InterruptedException interrupted) { Thread.currentThread().interrupt(); executor.shutdownNow(); } } + static void shutdown(ExecutorService executor) { + executor.shutdown(); + boolean interrupted = false; + try { + if (!executor.awaitTermination(5, TimeUnit.SECONDS)) { + executor.shutdownNow(); + executor.awaitTermination(1, TimeUnit.SECONDS); + } + } catch (InterruptedException stopRequested) { + interrupted = true; + executor.shutdownNow(); + try { executor.awaitTermination(1, TimeUnit.SECONDS); } + catch (InterruptedException repeated) { interrupted = true; } + } + if (interrupted) Thread.currentThread().interrupt(); + } public record ReceivedEnvelope(String serverId, String messageId, JsonEnvelope envelope) { } diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpTlsIdentity.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpTlsIdentity.java index 3ba11fc..17c7865 100644 --- a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpTlsIdentity.java +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpTlsIdentity.java @@ -391,9 +391,18 @@ private static KeyStore load(Path path, char[] password) throws Exception { } private static char[] readPassword(Path path) throws IOException { - byte[] bytes = Files.readAllBytes(path); - if (bytes.length < 40 || bytes.length > 128) throw new IOException("HTTP TLS password file is invalid"); - try { return new String(bytes, java.nio.charset.StandardCharsets.US_ASCII).toCharArray(); } + if (!Files.isRegularFile(path, LinkOption.NOFOLLOW_LINKS)) + throw new IOException("HTTP TLS password file is invalid"); + long size = Files.size(path); + if (size < 40L || size > 128L) throw new IOException("HTTP TLS password file is invalid"); + byte[] bytes; + try (var input = Files.newInputStream(path, LinkOption.NOFOLLOW_LINKS)) { + bytes = input.readNBytes(129); + } + try { + if (bytes.length < 40 || bytes.length > 128) throw new IOException("HTTP TLS password file is invalid"); + return new String(bytes, java.nio.charset.StandardCharsets.US_ASCII).toCharArray(); + } finally { Arrays.fill(bytes, (byte) 0); } } diff --git a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportRuntimeTest.java b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportRuntimeTest.java index a3ef6d9..a760465 100644 --- a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportRuntimeTest.java +++ b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportRuntimeTest.java @@ -8,6 +8,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import com.bencodez.simpleapi.servercomm.codec.JsonEnvelope; +import com.sun.net.httpserver.Headers; import java.net.InetSocketAddress; import java.net.URI; import java.net.http.HttpClient; @@ -248,6 +249,90 @@ void closeWaitsForTheCredentialOwningPollerToStop() throws Exception { } } + @Test + void closeDrainsRunningCallbacksBeforeSealingTheirJournal() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("close-callback-proxy"), "localhost"); + HttpTlsIdentity.IssuedClientCertificate issued = identity.issueClientCertificate("lobby-1"); + HttpConnectionCode code = new HttpConnectionCode("lobby-1", URI.create("https://localhost:8443/"), + identity.serverCertificatePin(), identity.caCertificatePin(), Instant.now().plusSeconds(300), "A".repeat(43)); + Path clientDirectory = directory.resolve("close-callback-client"); + HttpClientCredentialStore.saveEnrolled(clientDirectory, code, issued); + CountDownLatch started = new CountDownLatch(1), release = new CountDownLatch(1); + HttpBackendTransportConnector connector = new HttpBackendTransportConnector(clientDirectory, ignored -> { + started.countDown(); + try { release.await(); } + catch (InterruptedException interrupted) { Thread.currentThread().interrupt(); } + }); + String id = java.util.UUID.randomUUID().toString(); + connector.dispatch(new HttpTransportProtocol.Delivery(id, JsonEnvelope.builder("close-callback").build())); + assertTrue(started.await(2, TimeUnit.SECONDS)); + Thread closer = new Thread(connector::close, "close-callback-test"); + closer.start(); + try { + Thread.sleep(100); + assertTrue(closer.isAlive(), "close must wait for a running callback to finish its journal transition"); + } finally { release.countDown(); } + closer.join(3000); + assertFalse(closer.isAlive()); + assertEquals(HttpInboundDeliveryStore.State.COMPLETED, + new HttpInboundDeliveryStore(clientDirectory).state(id)); + } + + @Test + void interruptedCloseRemainsBoundedWhenACallbackIgnoresInterruption() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("bounded-close-proxy"), "localhost"); + HttpTlsIdentity.IssuedClientCertificate issued = identity.issueClientCertificate("lobby-1"); + HttpConnectionCode code = new HttpConnectionCode("lobby-1", URI.create("https://localhost:8443/"), + identity.serverCertificatePin(), identity.caCertificatePin(), Instant.now().plusSeconds(300), "A".repeat(43)); + Path clientDirectory = directory.resolve("bounded-close-client"); + HttpClientCredentialStore.saveEnrolled(clientDirectory, code, issued); + CountDownLatch started = new CountDownLatch(1), release = new CountDownLatch(1), stopped = new CountDownLatch(1); + HttpBackendTransportConnector connector = new HttpBackendTransportConnector(clientDirectory, ignored -> { + started.countDown(); + while (release.getCount() != 0L) try { release.await(); } + catch (InterruptedException ignoredInterrupt) { } + stopped.countDown(); + }); + String id = java.util.UUID.randomUUID().toString(); + connector.dispatch(new HttpTransportProtocol.Delivery(id, JsonEnvelope.builder("bounded-close").build())); + assertTrue(started.await(2, TimeUnit.SECONDS)); + Thread closer = new Thread(connector::close, "bounded-close-test"); + closer.start(); + closer.interrupt(); + closer.join(2500); + assertFalse(closer.isAlive(), "a non-cooperative application callback must not hang connector shutdown"); + try { + release.countDown(); + assertTrue(stopped.await(2, TimeUnit.SECONDS)); + } finally { release.countDown(); } + assertEquals(HttpInboundDeliveryStore.State.RUNNING, + new HttpInboundDeliveryStore(clientDirectory).state(id), + "an ambiguous callback must remain fail-closed after bounded shutdown"); + } + + @Test + void proxyShutdownGivesInterruptedCallbacksTimeToFinish() throws Exception { + java.util.concurrent.ThreadPoolExecutor executor = new java.util.concurrent.ThreadPoolExecutor(1, 1, 0L, + TimeUnit.MILLISECONDS, new java.util.concurrent.ArrayBlockingQueue<>(1)); + CountDownLatch started = new CountDownLatch(1), finished = new CountDownLatch(1); + executor.execute(() -> { + started.countDown(); + try { new CountDownLatch(1).await(); } + catch (InterruptedException stopRequested) { + try { Thread.sleep(100); } + catch (InterruptedException repeated) { Thread.currentThread().interrupt(); } + } finally { finished.countDown(); } + }); + assertTrue(started.await(2, TimeUnit.SECONDS)); + Thread closer = new Thread(() -> HttpProxyTransportServer.shutdown(executor), "proxy-shutdown-test"); + closer.start(); + closer.interrupt(); + closer.join(2500); + assertFalse(closer.isAlive()); + assertEquals(0L, finished.getCount(), + "forced shutdown must give a cooperative callback time to finish before journals are sealed"); + } + @Test void finalFlushDeliversMessagesQueuedBehindAnActiveLongPoll() throws Exception { HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("flush-proxy"), "localhost"); @@ -293,6 +378,45 @@ void normalTransportRejectsAClientWithoutCertificate() throws Exception { } } + @Test + void fixedRequestBodiesReportProtocolErrorsBeforeAdmissionErrors() { + Headers headers = new Headers(); + assertEquals(411, HttpProxyTransportServer.fixedBodyErrorStatus(headers, 128)); + headers.set("Content-Length", "invalid"); + assertEquals(400, HttpProxyTransportServer.fixedBodyErrorStatus(headers, 128)); + headers.set("Content-Length", "0"); + assertEquals(400, HttpProxyTransportServer.fixedBodyErrorStatus(headers, 128)); + headers.set("Content-Length", "129"); + assertEquals(413, HttpProxyTransportServer.fixedBodyErrorStatus(headers, 128)); + headers.set("Content-Length", "128"); + assertEquals(0, HttpProxyTransportServer.fixedBodyErrorStatus(headers, 128)); + headers.set("Transfer-Encoding", "chunked"); + assertEquals(400, HttpProxyTransportServer.fixedBodyErrorStatus(headers, 128)); + } + + @Test + void enrollmentPersistenceFailuresReturnServiceUnavailable() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("failed-enroll-proxy"), "localhost"); + Path authorityDirectory = directory.resolve("failed-enroll-authority"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, authorityDirectory); + try (HttpProxyTransportServer server = new HttpProxyTransportServer(new InetSocketAddress("localhost", 0), + identity, authority, ignored -> { })) { + server.start(); + HttpConnectionCode code = authority.createConnectionCode("lobby-1", server.endpoint("localhost"), + Duration.ofMinutes(5)); + Path state = authorityDirectory.resolve("http-transport-clients.properties"); + Files.delete(state); + Files.createDirectory(state); + byte[] body = ("{\"server\":\"lobby-1\",\"token\":\"" + code.enrollmentToken() + "\"}") + .getBytes(java.nio.charset.StandardCharsets.UTF_8); + HttpClient client = HttpClient.newBuilder().sslContext(HttpPinnedTls.clientContext(code)).build(); + HttpResponse response = client.send(HttpRequest.newBuilder(code.endpoint().resolve("v1/enroll")) + .timeout(Duration.ofSeconds(5)).header("Content-Type", "application/json") + .POST(HttpRequest.BodyPublishers.ofByteArray(body)).build(), HttpResponse.BodyHandlers.ofByteArray()); + assertEquals(503, response.statusCode()); + } + } + @Test void boundedQueuesFailClosed() throws Exception { HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); diff --git a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportSecurityTest.java b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportSecurityTest.java index 8d3eaa7..997aabb 100644 --- a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportSecurityTest.java +++ b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportSecurityTest.java @@ -151,6 +151,19 @@ void identityIsDurableAndPinsRejectTheWrongServer() throws Exception { assertNotEquals(created.serverCertificatePin(), rotated.serverCertificatePin()); } + @Test + void oversizedTlsAndClientPasswordFilesFailClosed() throws Exception { + Path identityDirectory = directory.resolve("oversized-password-proxy"); + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(identityDirectory, "localhost"); + Files.writeString(identityDirectory.resolve("http-transport-password"), "x".repeat(129)); + assertThrows(java.io.IOException.class, () -> HttpTlsIdentity.loadOrCreate(identityDirectory, "localhost")); + + Path clientDirectory = directory.resolve("oversized-password-client"); + HttpClientCredentialStore.save(clientDirectory, identity.issueClientCertificate("lobby-1")); + Files.writeString(clientDirectory.resolve("http-transport-client-password"), "x".repeat(129)); + assertThrows(java.io.IOException.class, () -> HttpClientCredentialStore.load(clientDirectory)); + } + @Test void privateCredentialRootsRejectSymbolicLinks() throws Exception { Path identityTarget = directory.resolve("identity-target"); @@ -454,6 +467,28 @@ void renewalKeepsOldCredentialUntilReplacementAuthenticates() throws Exception { .authenticate("lobby-1", replacement.certificate()), "promoted renewal must survive restart"); } + @Test + void failedRenewalPersistenceRestoresTheActiveBindingAndCanRetry() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("retry-renewal-proxy"), "localhost"); + Path stateDirectory = directory.resolve("retry-renewal-state"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, stateDirectory); + HttpConnectionCode code = authority.createConnectionCode("lobby-1", URI.create("https://localhost:8443/"), + Duration.ofMinutes(5)); + HttpTlsIdentity.IssuedClientCertificate original = authority.enroll("lobby-1", code.enrollmentToken()); + assertTrue(authority.authenticate("lobby-1", original.certificate())); + Path stateFile = stateDirectory.resolve("http-transport-clients.properties"); + Files.delete(stateFile); + Files.createDirectory(stateFile); + + assertThrows(java.io.IOException.class, () -> authority.renew("lobby-1", original.certificate())); + assertTrue(authority.authenticate("lobby-1", original.certificate()), + "a pre-publication renewal failure must leave the active credential usable"); + Files.delete(stateFile); + HttpTlsIdentity.IssuedClientCertificate retried = authority.renew("lobby-1", original.certificate()); + assertTrue(authority.authenticate("lobby-1", retried.certificate()), + "renewal must remain retryable after persistence recovers"); + } + @Test void serverLeafRotatesInsideRenewalWindowAndPreservesAuthority() throws Exception { Instant now = Instant.now(); From 831f2f15a75f76a07df7d0292674a7401299afaa Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sun, 6 Sep 2026 00:08:03 +0000 Subject: [PATCH 14/38] Fix transport queue uncertainty and 429 handling Co-authored-by: BenCodez <17074231+BenCodez@users.noreply.github.com> --- .../http/HttpProxyTransportServer.java | 35 +++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java index 746cdb4..b7217e4 100644 --- a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java @@ -222,6 +222,7 @@ private void transport(HttpsExchange exchange) throws IOException { reply(exchange, 200, HttpTransportProtocol.response(packet.server(), packet.session(), packet.sequence(), response.acks(), packet.acks(), response.messages())); } finally { backend.endPoll(); } + } catch (RateLimitException rateLimited) { reply(exchange, 429, new byte[0]); } catch (IllegalArgumentException rejected) { reply(exchange, 400, new byte[0]); } catch (Exception failure) { reply(exchange, 503, new byte[0]); } finally { admission.release(); } @@ -230,7 +231,7 @@ private void transport(HttpsExchange exchange) throws IOException { private void handlePacket(HttpTransportProtocol.Packet packet, BackendState backend) throws IOException { List accepted; synchronized (backend) { - if (!backend.allowRequest()) throw new IllegalArgumentException("transport rate limited"); + if (!backend.allowRequest()) throw new RateLimitException("transport rate limited"); if (!backend.acceptSession(packet.session(), packet.sequence())) throw new IllegalArgumentException("stale session request"); } backend.confirmIncoming(packet.ackConfirmations()); @@ -353,6 +354,9 @@ public interface DeliveryAcknowledgement { void confirm(String serverId, String deliveryId) throws IOException; } static record Response(Collection acks, Collection messages) { } + static final class RateLimitException extends IllegalArgumentException { + RateLimitException(String message) { super(message); } + } static final class BackendState { private final String serverId; private final DurableOutgoingQueue durableOutgoing; @@ -586,6 +590,9 @@ private synchronized Map> load() th DurableFiles.deleteIfExists(message); continue; } + if (name.startsWith(".pending-") && name.endsWith(".json") + && !Files.isSymbolicLink(message) && Files.isRegularFile(message, LinkOption.NOFOLLOW_LINKS)) + continue; if (Files.isSymbolicLink(message) || !Files.isRegularFile(message, LinkOption.NOFOLLOW_LINKS) || !name.matches(FILE_PATTERN) || Files.size(message) > HttpTransportProtocol.MAX_ENVELOPE_BYTES * 2L) throw new IOException("HTTP outgoing queue message is invalid"); @@ -631,6 +638,26 @@ private synchronized void persist(String serverId, HttpTransportProtocol.Deliver directoryForcer.force(directory); return; } + Path pending = directory.resolve(".pending-" + delivery.id() + ".json"); + if (Files.isRegularFile(pending, LinkOption.NOFOLLOW_LINKS)) { + if (Files.isSymbolicLink(pending) || Files.size(pending) > HttpTransportProtocol.MAX_ENVELOPE_BYTES * 2L + || !Arrays.equals(Files.readAllBytes(pending), HttpTransportProtocol.storedDelivery(delivery))) + throw new IOException("HTTP outgoing queue delivery id conflicts with persisted data"); + String name = String.format(java.util.Locale.ROOT, "%020d-%s.json", ++sequence, delivery.id()); + Path target = directory.resolve(name); + try { Files.move(pending, target, StandardCopyOption.ATOMIC_MOVE); } + catch (java.nio.file.AtomicMoveNotSupportedException unsupported) { Files.move(pending, target); } + try { ownerOnlyFile(target); directoryForcer.force(directory); } + catch (IOException postPublicationFailure) { + try { + Files.deleteIfExists(pending); + Files.move(target, pending, StandardCopyOption.ATOMIC_MOVE); + } catch (IOException ignored) { } + throw new DurableFiles.PublishedException(postPublicationFailure); + } + serverFiles.put(delivery.id(), target); + return; + } if (sequence == Long.MAX_VALUE) throw new IOException("HTTP outgoing queue sequence is exhausted"); String name = String.format(java.util.Locale.ROOT, "%020d-%s.json", ++sequence, delivery.id()); Path target = directory.resolve(name); @@ -641,11 +668,15 @@ private synchronized void persist(String serverId, HttpTransportProtocol.Deliver DurableFiles.forceFile(temporary); try { Files.move(temporary, target, StandardCopyOption.ATOMIC_MOVE); } catch (java.nio.file.AtomicMoveNotSupportedException unsupported) { Files.move(temporary, target); } - serverFiles.put(delivery.id(), target); try { ownerOnlyFile(target); directoryForcer.force(directory); } catch (IOException postPublicationFailure) { + try { + Files.deleteIfExists(pending); + Files.move(target, pending, StandardCopyOption.ATOMIC_MOVE); + } catch (IOException ignored) { } throw new DurableFiles.PublishedException(postPublicationFailure); } + serverFiles.put(delivery.id(), target); } finally { Files.deleteIfExists(temporary); } } From 6f316437a3bee34e75e4ae4bd37fda3cf6fea20e Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Sat, 5 Sep 2026 18:17:48 -0600 Subject: [PATCH 15/38] Keep uncertain queue entries hidden after restart --- .../http/HttpClientCredentialStore.java | 9 +- .../http/HttpEnrollmentAuthority.java | 4 +- .../http/HttpInboundDeliveryStore.java | 13 +-- .../http/HttpProxyTransportServer.java | 108 +++++++++++------- .../servercomm/http/HttpTlsIdentity.java | 3 +- .../http/HttpTransportRuntimeTest.java | 80 ++++++++++++- 6 files changed, 157 insertions(+), 60 deletions(-) diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpClientCredentialStore.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpClientCredentialStore.java index 46ab04e..dff5498 100644 --- a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpClientCredentialStore.java +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpClientCredentialStore.java @@ -107,12 +107,13 @@ private static StagedCredential stage(Path directory, HttpTlsIdentity.IssuedClie Path credentialDirectory = credentialRoot(directory, true); Path generations = credentialDirectory.resolve(GENERATIONS_DIRECTORY); if (Files.isSymbolicLink(generations)) throw new IOException("HTTP credential generation directory is unsafe"); - boolean generationsCreated = !Files.exists(generations, LinkOption.NOFOLLOW_LINKS); Files.createDirectories(generations); if (Files.isSymbolicLink(generations) || !Files.isDirectory(generations, LinkOption.NOFOLLOW_LINKS)) throw new IOException("HTTP credential generation directory is unsafe"); setOwnerOnlyDirectory(generations); - if (generationsCreated) DurableFiles.forceDirectory(credentialDirectory); + // Retry publication durability if an earlier staging attempt left this + // directory behind after its parent fsync failed. + DurableFiles.forceDirectory(credentialDirectory); String name = java.util.UUID.randomUUID().toString(); Path generation = generations.resolve(name); Files.createDirectory(generation); @@ -345,13 +346,13 @@ private static Path credentialRoot(Path directory, boolean create) throws IOExce if (directory == null) throw new IllegalArgumentException("Credential directory is required"); Path root = directory.toAbsolutePath().normalize(); if (Files.isSymbolicLink(root)) throw new IOException("HTTP credential directory is unsafe"); - boolean created = !Files.exists(root, LinkOption.NOFOLLOW_LINKS); if (create) Files.createDirectories(root); if (Files.isSymbolicLink(root) || !Files.isDirectory(root, LinkOption.NOFOLLOW_LINKS)) throw new IOException("HTTP credential directory is unsafe"); if (create) { setOwnerOnlyDirectory(root); - if (created) DurableFiles.forceDirectory(root.getParent()); + // Existing can mean create succeeded but publishing it durably did not. + DurableFiles.forceDirectory(root.getParent()); } return root; } diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentAuthority.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentAuthority.java index 3a48c21..e9a3813 100644 --- a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentAuthority.java +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentAuthority.java @@ -282,12 +282,12 @@ private synchronized void persistState() throws java.io.IOException { private static Path stateFile(Path directory) throws java.io.IOException { if (directory == null) throw new IllegalArgumentException("State directory is required"); Path stateDirectory = directory.toAbsolutePath().normalize(); - boolean created = !Files.exists(stateDirectory, LinkOption.NOFOLLOW_LINKS); Files.createDirectories(stateDirectory); if (Files.isSymbolicLink(stateDirectory) || !Files.isDirectory(stateDirectory, LinkOption.NOFOLLOW_LINKS)) throw new java.io.IOException("HTTP enrollment state directory is unsafe"); setOwnerOnlyDirectory(stateDirectory); - if (created) DurableFiles.forceDirectory(stateDirectory.getParent()); + // Existing can mean a previous create succeeded but its parent fsync did not. + DurableFiles.forceDirectory(stateDirectory.getParent()); Path file = stateDirectory.resolve("http-transport-clients.properties"); if (Files.isSymbolicLink(file)) throw new java.io.IOException("Refusing unsafe HTTP enrollment state path"); return file; diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpInboundDeliveryStore.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpInboundDeliveryStore.java index 3166d2c..56d876e 100644 --- a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpInboundDeliveryStore.java +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpInboundDeliveryStore.java @@ -40,15 +40,12 @@ private HttpInboundDeliveryStore(Path parent, String directoryName) throws IOExc ownerOnlyDirectory(credentials); root = credentials.resolve(directoryName).normalize(); if (!root.getParent().equals(credentials)) throw new IOException("HTTP inbound delivery directory is invalid"); - boolean created = false; - try { Files.createDirectory(root); created = true; } + try { Files.createDirectory(root); } catch (java.nio.file.FileAlreadyExistsException existing) { } - try { - requireRoot(); - ownerOnlyDirectory(root); - } finally { - if (created) DurableFiles.forceDirectory(credentials); - } + requireRoot(); + ownerOnlyDirectory(root); + // Retry a parent fsync that may have failed after creating this root. + DurableFiles.forceDirectory(credentials); load(); } diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java index b7217e4..9981568 100644 --- a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java @@ -284,13 +284,14 @@ private static Path incomingRoot(Path outgoingDirectory) throws IOException { Path parent = outgoing.getParent(); if (parent == null || outgoing.getFileName() == null) throw new IOException("HTTP incoming queue path is invalid"); Path root = parent.resolve(outgoing.getFileName().toString() + "-incoming"); - boolean created = false; - try { Files.createDirectory(root); created = true; } + try { Files.createDirectory(root); } catch (java.nio.file.FileAlreadyExistsException existing) { } if (Files.isSymbolicLink(root) || !Files.isDirectory(root, LinkOption.NOFOLLOW_LINKS)) throw new IOException("HTTP incoming queue directory is invalid"); DurableOutgoingQueue.ownerOnlyDirectory(root); - if (created) DurableFiles.forceDirectory(parent); + // Retry publication durability even when an earlier attempt created the + // directory but failed before its parent could be forced. + DurableFiles.forceDirectory(parent); return root; } private static JsonEnvelope normalizeBackendIdentity(String serverId, JsonEnvelope envelope) { @@ -540,6 +541,7 @@ interface DirectoryForcer { void force(Path directory) throws IOException; } private final Path root; private final DirectoryForcer directoryForcer; private final Map> files = new HashMap<>(); + private final Map> quarantinedFiles = new HashMap<>(); private long sequence; private DurableOutgoingQueue(Path root) throws IOException { @@ -551,19 +553,17 @@ private DurableOutgoingQueue(Path root) throws IOException { throw new IllegalArgumentException("HTTP outgoing queue configuration is required"); this.directoryForcer = directoryForcer; this.root = root.toAbsolutePath().normalize(); - boolean created = false; - try { Files.createDirectory(this.root); created = true; } + try { Files.createDirectory(this.root); } catch (java.nio.file.FileAlreadyExistsException existing) { } - try { - if (Files.isSymbolicLink(this.root) || !Files.isDirectory(this.root, LinkOption.NOFOLLOW_LINKS)) - throw new IOException("HTTP outgoing queue directory is invalid"); - ownerOnlyDirectory(this.root); - } finally { - if (created) directoryForcer.force(this.root.getParent()); - } + if (Files.isSymbolicLink(this.root) || !Files.isDirectory(this.root, LinkOption.NOFOLLOW_LINKS)) + throw new IOException("HTTP outgoing queue directory is invalid"); + ownerOnlyDirectory(this.root); + // A failed parent fsync can leave the directory present but not durable. + // Reopening must retry it before the queue can accept work. + directoryForcer.force(this.root.getParent()); } - private synchronized Map> load() throws IOException { + synchronized Map> load() throws IOException { Map> loaded = new LinkedHashMap<>(); int serverDirectories = 0; try (DirectoryStream servers = Files.newDirectoryStream(root)) { @@ -583,6 +583,8 @@ private synchronized Map> load() th entries.sort(java.util.Comparator.comparing(path -> path.getFileName().toString())); List deliveries = new java.util.ArrayList<>(); Map serverFiles = files.computeIfAbsent(serverId, ignored -> new HashMap<>()); + Map quarantined = quarantinedFiles.computeIfAbsent(serverId, ignored -> new HashMap<>()); + int durableEntries = 0; for (Path message : entries) { String name = message.getFileName().toString(); if (name.startsWith(".pending-") && name.endsWith(".tmp") @@ -590,19 +592,31 @@ private synchronized Map> load() th DurableFiles.deleteIfExists(message); continue; } - if (name.startsWith(".pending-") && name.endsWith(".json") - && !Files.isSymbolicLink(message) && Files.isRegularFile(message, LinkOption.NOFOLLOW_LINKS)) + if (name.startsWith(".pending-") && name.endsWith(".json")) { + if (Files.isSymbolicLink(message) || !Files.isRegularFile(message, LinkOption.NOFOLLOW_LINKS) + || Files.size(message) > HttpTransportProtocol.MAX_ENVELOPE_BYTES * 2L) + throw new IOException("HTTP outgoing queue quarantine is invalid"); + HttpTransportProtocol.Delivery delivery; + try { delivery = HttpTransportProtocol.parseStoredDelivery(Files.readAllBytes(message)); } + catch (IllegalArgumentException invalid) { throw new IOException("HTTP outgoing queue quarantine is invalid", invalid); } + if (!name.equals(".pending-" + delivery.id() + ".json") + || serverFiles.containsKey(delivery.id()) || quarantined.put(delivery.id(), message) != null) + throw new IOException("HTTP outgoing queue quarantine id is invalid"); + if (++durableEntries > HttpTransportProtocol.MAX_QUEUE) + throw new IOException("HTTP outgoing queue exceeds its bound"); continue; + } if (Files.isSymbolicLink(message) || !Files.isRegularFile(message, LinkOption.NOFOLLOW_LINKS) || !name.matches(FILE_PATTERN) || Files.size(message) > HttpTransportProtocol.MAX_ENVELOPE_BYTES * 2L) throw new IOException("HTTP outgoing queue message is invalid"); HttpTransportProtocol.Delivery delivery; try { delivery = HttpTransportProtocol.parseStoredDelivery(Files.readAllBytes(message)); } catch (IllegalArgumentException invalid) { throw new IOException("HTTP outgoing queue message is invalid", invalid); } - if (!name.endsWith("-" + delivery.id() + ".json") || serverFiles.put(delivery.id(), message) != null) + if (!name.endsWith("-" + delivery.id() + ".json") || quarantined.containsKey(delivery.id()) + || serverFiles.put(delivery.id(), message) != null) throw new IOException("HTTP outgoing queue message id is invalid"); deliveries.add(delivery); - if (deliveries.size() > HttpTransportProtocol.MAX_QUEUE) + if (++durableEntries > HttpTransportProtocol.MAX_QUEUE) throw new IOException("HTTP outgoing queue exceeds its bound"); sequence = Math.max(sequence, Long.parseLong(name.substring(0, 20))); } @@ -615,19 +629,16 @@ private synchronized Map> load() th private synchronized void persist(String serverId, HttpTransportProtocol.Delivery delivery) throws IOException { Path directory = root.resolve(serverId).normalize(); if (!directory.getParent().equals(root)) throw new IOException("HTTP outgoing queue server is invalid"); - boolean created = false; - try { Files.createDirectory(directory); created = true; } + try { Files.createDirectory(directory); } catch (java.nio.file.FileAlreadyExistsException existing) { } - try { - if (Files.isSymbolicLink(directory) || !Files.isDirectory(directory, LinkOption.NOFOLLOW_LINKS)) - throw new IOException("HTTP outgoing queue server directory is invalid"); - ownerOnlyDirectory(directory); - } finally { - // The child fsync below cannot make this newly published name durable in - // its parent. Persist the root entry before accepting the first message. - if (created) directoryForcer.force(root); - } + if (Files.isSymbolicLink(directory) || !Files.isDirectory(directory, LinkOption.NOFOLLOW_LINKS)) + throw new IOException("HTTP outgoing queue server directory is invalid"); + ownerOnlyDirectory(directory); + // The child fsync below cannot make this published name durable in its + // parent. Repeat it so a prior failed attempt is recoverable. + directoryForcer.force(root); Map serverFiles = files.computeIfAbsent(serverId, ignored -> new HashMap<>()); + Map quarantined = quarantinedFiles.computeIfAbsent(serverId, ignored -> new HashMap<>()); Path existing = serverFiles.get(delivery.id()); if (existing != null) { if (Files.isSymbolicLink(existing) || !Files.isRegularFile(existing, LinkOption.NOFOLLOW_LINKS) @@ -638,26 +649,27 @@ private synchronized void persist(String serverId, HttpTransportProtocol.Deliver directoryForcer.force(directory); return; } - Path pending = directory.resolve(".pending-" + delivery.id() + ".json"); - if (Files.isRegularFile(pending, LinkOption.NOFOLLOW_LINKS)) { - if (Files.isSymbolicLink(pending) || Files.size(pending) > HttpTransportProtocol.MAX_ENVELOPE_BYTES * 2L + Path pending = quarantined.get(delivery.id()); + if (pending != null) { + if (Files.isSymbolicLink(pending) || !Files.isRegularFile(pending, LinkOption.NOFOLLOW_LINKS) + || Files.size(pending) > HttpTransportProtocol.MAX_ENVELOPE_BYTES * 2L || !Arrays.equals(Files.readAllBytes(pending), HttpTransportProtocol.storedDelivery(delivery))) throw new IOException("HTTP outgoing queue delivery id conflicts with persisted data"); + if (sequence == Long.MAX_VALUE) throw new IOException("HTTP outgoing queue sequence is exhausted"); String name = String.format(java.util.Locale.ROOT, "%020d-%s.json", ++sequence, delivery.id()); Path target = directory.resolve(name); try { Files.move(pending, target, StandardCopyOption.ATOMIC_MOVE); } catch (java.nio.file.AtomicMoveNotSupportedException unsupported) { Files.move(pending, target); } try { ownerOnlyFile(target); directoryForcer.force(directory); } catch (IOException postPublicationFailure) { - try { - Files.deleteIfExists(pending); - Files.move(target, pending, StandardCopyOption.ATOMIC_MOVE); - } catch (IOException ignored) { } - throw new DurableFiles.PublishedException(postPublicationFailure); + quarantinePublished(directory, target, pending, quarantined, delivery.id(), postPublicationFailure); } + quarantined.remove(delivery.id()); serverFiles.put(delivery.id(), target); return; } + if (serverFiles.size() + quarantined.size() >= HttpTransportProtocol.MAX_QUEUE) + throw new IOException("HTTP outgoing queue exceeds its bound"); if (sequence == Long.MAX_VALUE) throw new IOException("HTTP outgoing queue sequence is exhausted"); String name = String.format(java.util.Locale.ROOT, "%020d-%s.json", ++sequence, delivery.id()); Path target = directory.resolve(name); @@ -670,16 +682,30 @@ private synchronized void persist(String serverId, HttpTransportProtocol.Deliver catch (java.nio.file.AtomicMoveNotSupportedException unsupported) { Files.move(temporary, target); } try { ownerOnlyFile(target); directoryForcer.force(directory); } catch (IOException postPublicationFailure) { - try { - Files.deleteIfExists(pending); - Files.move(target, pending, StandardCopyOption.ATOMIC_MOVE); - } catch (IOException ignored) { } - throw new DurableFiles.PublishedException(postPublicationFailure); + pending = directory.resolve(".pending-" + delivery.id() + ".json"); + quarantinePublished(directory, target, pending, quarantined, delivery.id(), postPublicationFailure); } serverFiles.put(delivery.id(), target); } finally { Files.deleteIfExists(temporary); } } + private void quarantinePublished(Path directory, Path target, Path pending, Map quarantined, + String id, IOException publicationFailure) throws IOException { + try { + try { Files.move(target, pending, StandardCopyOption.ATOMIC_MOVE); } + catch (java.nio.file.AtomicMoveNotSupportedException unsupported) { Files.move(target, pending); } + // Record the observed rename before metadata writeback. If the force is + // indeterminate, a same-process retry must still find this quarantine. + quarantined.put(id, pending); + ownerOnlyFile(pending); + directoryForcer.force(directory); + } catch (IOException quarantineFailure) { + quarantineFailure.addSuppressed(publicationFailure); + throw new IllegalStateException("HTTP outgoing queue publication could not be quarantined", quarantineFailure); + } + throw new DurableFiles.PublishedException(publicationFailure); + } + private synchronized void confirm(String serverId, String id) throws IOException { Map serverFiles = files.get(serverId); Path file = serverFiles == null ? null : serverFiles.get(id); diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpTlsIdentity.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpTlsIdentity.java index 17c7865..ba2babc 100644 --- a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpTlsIdentity.java +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpTlsIdentity.java @@ -92,13 +92,12 @@ static HttpTlsIdentity loadOrCreate(Path directory, String advertisedHost, Clock if (clock == null) throw new IllegalArgumentException("Clock is required"); Path identityDirectory = directory.toAbsolutePath().normalize(); if (Files.isSymbolicLink(identityDirectory)) throw new IOException("HTTP TLS identity directory is unsafe"); - boolean created = !Files.exists(identityDirectory, LinkOption.NOFOLLOW_LINKS); Files.createDirectories(identityDirectory); if (Files.isSymbolicLink(identityDirectory) || !Files.isDirectory(identityDirectory, LinkOption.NOFOLLOW_LINKS)) throw new IOException("HTTP TLS identity directory is unsafe"); // The identity files cannot make the newly created directory entry durable. // Persist its parent before the TLS identity is returned for listener use. - if (created) DurableFiles.forceDirectory(identityDirectory.getParent()); + DurableFiles.forceDirectory(identityDirectory.getParent()); directory = identityDirectory; Path caFile = safe(directory.resolve(CA_FILE)); Path serverFile = safe(directory.resolve(SERVER_FILE)); diff --git a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportRuntimeTest.java b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportRuntimeTest.java index a760465..908b1dc 100644 --- a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportRuntimeTest.java +++ b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportRuntimeTest.java @@ -130,6 +130,42 @@ void stableProxyDeliveryIdsAreIdempotentAndAcknowledgedBeforeRemoval() throws Ex assertTrue(state.await("lobby-1", java.util.UUID.randomUUID().toString(), 0).messages().isEmpty()); } + @Test + void outgoingQueueRetriesDirectoryPublicationAfterForceFailure() throws Exception { + Path queueRoot = directory.resolve("retry-outgoing-root"); + java.util.concurrent.atomic.AtomicBoolean failRootPublication = new java.util.concurrent.atomic.AtomicBoolean(true); + assertThrows(java.io.IOException.class, () -> new HttpProxyTransportServer.DurableOutgoingQueue(queueRoot, + ignored -> { + if (failRootPublication.getAndSet(false)) + throw new java.io.IOException("injected root publication failure"); + })); + assertTrue(Files.isDirectory(queueRoot), "the failed force occurs after the root name is published"); + AtomicLong parentForces = new AtomicLong(); + HttpProxyTransportServer.DurableOutgoingQueue queue = new HttpProxyTransportServer.DurableOutgoingQueue( + queueRoot, forced -> { + if (forced.equals(queueRoot.getParent())) parentForces.incrementAndGet(); + }); + assertEquals(1L, parentForces.get(), "reopening an existing root must retry its parent fsync"); + + AtomicLong serverRootForces = new AtomicLong(); + java.util.concurrent.atomic.AtomicBoolean failServerPublication = new java.util.concurrent.atomic.AtomicBoolean(true); + HttpProxyTransportServer.DurableOutgoingQueue serverQueue = new HttpProxyTransportServer.DurableOutgoingQueue( + directory.resolve("retry-outgoing-server"), forced -> { + if (forced.getFileName().toString().equals("retry-outgoing-server")) { + serverRootForces.incrementAndGet(); + if (failServerPublication.getAndSet(false)) + throw new java.io.IOException("injected backend publication failure"); + } + }); + HttpProxyTransportServer.BackendState state = new HttpProxyTransportServer.BackendState( + "lobby-1", serverQueue, (server, id) -> { }); + HttpTransportProtocol.Delivery delivery = new HttpTransportProtocol.Delivery( + java.util.UUID.randomUUID().toString(), JsonEnvelope.builder("retry-directory-force").build()); + assertFalse(state.enqueue(delivery)); + assertTrue(state.enqueue(delivery)); + assertEquals(2L, serverRootForces.get(), "retrying an existing backend directory must force its parent again"); + } + @Test void publishedOutgoingDeliveryRemainsTrackedUntilDurabilityCanBeConfirmed() throws Exception { AtomicLong forceCalls = new AtomicLong(); @@ -148,14 +184,52 @@ void publishedOutgoingDeliveryRemainsTrackedUntilDurabilityCanBeConfirmed() thro assertTrue(state.await("lobby-1", java.util.UUID.randomUUID().toString(), 0).messages().isEmpty(), "an uncertain publication must remain hidden until its durability retry succeeds"); assertEquals(1L, countRegularFiles(queueRoot)); - assertTrue(state.enqueue(delivery), "same-ID retry must confirm the existing published file"); + HttpProxyTransportServer.DurableOutgoingQueue restartedQueue = new HttpProxyTransportServer.DurableOutgoingQueue( + queueRoot, com.bencodez.simpleapi.file.DurableFiles::forceDirectory); + assertTrue(restartedQueue.load().isEmpty(), + "a restart must not expose an operation whose sender observed rejection"); + HttpProxyTransportServer.BackendState restarted = new HttpProxyTransportServer.BackendState( + "lobby-1", restartedQueue, (server, id) -> { }); + assertTrue(restarted.await("lobby-1", java.util.UUID.randomUUID().toString(), 0).messages().isEmpty()); + assertTrue(restarted.enqueue(delivery), "same-ID retry must confirm the quarantined file"); assertEquals(1L, countRegularFiles(queueRoot), "durability retry must not create a duplicate file"); assertEquals(java.util.List.of(delivery), - state.await("lobby-1", java.util.UUID.randomUUID().toString(), 0).messages()); - state.acknowledge(java.util.List.of(deliveryId)); + restarted.await("lobby-1", java.util.UUID.randomUUID().toString(), 0).messages()); + HttpProxyTransportServer.DurableOutgoingQueue confirmedQueue = new HttpProxyTransportServer.DurableOutgoingQueue( + queueRoot, com.bencodez.simpleapi.file.DurableFiles::forceDirectory); + java.util.List confirmed = confirmedQueue.load().get("lobby-1"); + assertEquals(1, confirmed.size()); + assertTrue(java.util.Arrays.equals(HttpTransportProtocol.storedDelivery(delivery), + HttpTransportProtocol.storedDelivery(confirmed.get(0))), + "a confirmed same-ID retry must become deliverable after restart"); + restarted.acknowledge(java.util.List.of(deliveryId)); assertEquals(0L, countRegularFiles(queueRoot), "the tracked published file must be removable by ACK"); } + @Test + void unresolvedOutgoingPublicationDoesNotReportAFalseRejection() throws Exception { + AtomicLong forceCalls = new AtomicLong(); + java.util.concurrent.atomic.AtomicBoolean failForces = new java.util.concurrent.atomic.AtomicBoolean(true); + Path queueRoot = directory.resolve("unresolved-outgoing"); + HttpProxyTransportServer.DurableOutgoingQueue queue = new HttpProxyTransportServer.DurableOutgoingQueue( + queueRoot, ignored -> { + if (forceCalls.incrementAndGet() >= 3L && failForces.get()) + throw new java.io.IOException("persistent directory force failure"); + }); + HttpProxyTransportServer.BackendState state = new HttpProxyTransportServer.BackendState( + "lobby-1", queue, (server, id) -> { }); + HttpTransportProtocol.Delivery delivery = new HttpTransportProtocol.Delivery( + java.util.UUID.randomUUID().toString(), JsonEnvelope.builder("unresolved").build()); + assertThrows(IllegalStateException.class, () -> state.enqueue(delivery), + "an indeterminate rollback must not be reported as a definitive false result"); + failForces.set(false); + assertTrue(state.enqueue(delivery), "a same-ID retry must recover the observed quarantine"); + assertEquals(1L, countRegularFiles(queueRoot), "recovery must not leave duplicate queue entries"); + HttpProxyTransportServer.DurableOutgoingQueue restarted = new HttpProxyTransportServer.DurableOutgoingQueue( + queueRoot, com.bencodez.simpleapi.file.DurableFiles::forceDirectory); + assertEquals(1, restarted.load().get("lobby-1").size()); + } + @Test void proxyInboundCompletionSurvivesRestartBeforeAcknowledgement() throws Exception { Path root = directory.resolve("proxy-incoming"); From a6967d83dd8bb8eabcf9e06a5ad22633b901d3c8 Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Sat, 5 Sep 2026 19:02:43 -0600 Subject: [PATCH 16/38] Recover generated sends and published credential replacements --- .../http/HttpBackendTransportConnector.java | 19 ++- .../http/HttpClientCredentialStore.java | 8 +- .../http/HttpEnrollmentAuthority.java | 6 + .../http/HttpProxyTransportServer.java | 61 +++++++- .../http/HttpTransportRuntimeTest.java | 142 ++++++++++++++++++ .../http/HttpTransportSecurityTest.java | 7 + 6 files changed, 231 insertions(+), 12 deletions(-) diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpBackendTransportConnector.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpBackendTransportConnector.java index 2216db7..dffad53 100644 --- a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpBackendTransportConnector.java +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpBackendTransportConnector.java @@ -69,6 +69,9 @@ public final class HttpBackendTransportConnector implements AutoCloseable { private volatile Thread poller; private long sequence; private volatile long nextRenewalCheckNanos; + // Guarded by renewal: do not supersede a visibly published credential until + // the pointer selecting it has been confirmed durable. + private HttpClientCredentialStore.StagedCredential pendingActivation; /** In-memory test constructor; production transport must use a directory-backed constructor. */ HttpBackendTransportConnector(HttpConnectionCode code, String serverId, @@ -417,7 +420,14 @@ private static boolean matchesCredential(HttpClientCredentialStore.HttpClientPro private void maybeRenewCredential() { synchronized (renewal) { Path directory = credentialDirectory; - if (directory == null || !HttpTlsIdentity.needsRenewal(credential.certificate(), Clock.systemUTC())) return; + if (directory == null) return; + if (pendingActivation != null) { + try { + HttpClientCredentialStore.activateReplacement(directory, pendingActivation); + pendingActivation = null; + } catch (IOException unconfirmed) { return; } + } + if (!HttpTlsIdentity.needsRenewal(credential.certificate(), Clock.systemUTC())) return; long now = System.nanoTime(); if (nextRenewalCheckNanos != 0L && now - nextRenewalCheckNanos < 0L) return; Duration retry = renewalRetryDelay(Duration.between(Instant.now(), @@ -438,7 +448,12 @@ private void maybeRenewCredential() { HttpClientCredentialStore.HttpClientProfile replacementProfile = staged.profile(); if (!matchesCredential(replacementProfile, replacement)) throw new IllegalArgumentException("Renewed HTTP certificate is invalid"); HttpClient replacementClient = client(replacementProfile, replacement); - HttpClientCredentialStore.activateReplacement(directory, staged); + try { HttpClientCredentialStore.activateReplacement(directory, staged); } + catch (com.bencodez.simpleapi.file.DurableFiles.PublishedException published) { + // CURRENT already selects this generation. Adopt it in memory as well, + // and retry this publication before requesting any newer certificate. + pendingActivation = staged; + } profile = replacementProfile; client = replacementClient; credential = replacement; diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpClientCredentialStore.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpClientCredentialStore.java index dff5498..2928461 100644 --- a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpClientCredentialStore.java +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpClientCredentialStore.java @@ -383,8 +383,12 @@ private static void writePrivate(Path file, byte[] contents) throws IOException DurableFiles.forceFile(temporary); try { Files.move(temporary, file, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); } catch (java.nio.file.AtomicMoveNotSupportedException unsupported) { Files.move(temporary, file, StandardCopyOption.REPLACE_EXISTING); } - setOwnerOnly(file); - DurableFiles.forceDirectory(file.getParent()); + try { + setOwnerOnly(file); + DurableFiles.forceDirectory(file.getParent()); + } catch (IOException postPublicationFailure) { + throw new DurableFiles.PublishedException(postPublicationFailure); + } } finally { Files.deleteIfExists(temporary); } } diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentAuthority.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentAuthority.java index e9a3813..3164096 100644 --- a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentAuthority.java +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentAuthority.java @@ -54,6 +54,8 @@ public HttpEnrollmentAuthority(HttpTlsIdentity identity, Path stateDirectory) th } public synchronized HttpConnectionCode createConnectionCode(String serverId, URI endpoint, Duration lifetime) { + if (revocationRetryRequired) + throw new IllegalStateException("HTTP certificate revocation durability must be retried"); serverId = HttpTlsIdentity.canonicalServerId(serverId); if (lifetime == null || lifetime.isNegative() || lifetime.isZero() || lifetime.compareTo(MAX_ENROLLMENT_LIFETIME) > 0) throw new IllegalArgumentException("Enrollment lifetime must be between one second and fifteen minutes"); @@ -74,6 +76,10 @@ public synchronized HttpConnectionCode createConnectionCode(String serverId, URI } public synchronized HttpTlsIdentity.IssuedClientCertificate enroll(String serverId, String enrollmentToken) throws Exception { + // A failed revoke may have restored a still-valid pending token in memory. + // Do not let enrollment persist that stale state before the revoke is retried. + if (revocationRetryRequired) + throw new java.io.IOException("HTTP certificate revocation durability must be retried"); serverId = HttpTlsIdentity.canonicalServerId(serverId); if (enrollmentToken == null || enrollmentToken.length() > 128) throw new IllegalArgumentException("Enrollment was rejected"); expireEnrollments(); diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java index 9981568..9f12d28 100644 --- a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java @@ -159,13 +159,36 @@ public URI endpoint(String host) { } } - /** Queues a proxy-origin envelope durably before reporting acceptance. */ + /** + * Queues a proxy-origin envelope durably before reporting acceptance. + * @throws DeliveryRetryException if publication needs recovery; persist its delivery ID + * and retry the same envelope through the stable-ID overload + */ public boolean send(String serverId, JsonEnvelope envelope) { - return send(serverId, UUID.randomUUID().toString(), envelope); + return send(serverId, UUID.randomUUID().toString(), envelope, true); } - /** Queues a proxy-origin envelope with a stable, caller-persisted delivery ID. */ + /** Carries the generated ID needed to recover a quarantined or indeterminate send. */ + @SuppressWarnings("serial") + public static final class DeliveryRetryException extends IllegalStateException { + private final String deliveryId; + private DeliveryRetryException(String deliveryId, Throwable cause) { + super("HTTP delivery requires a same-ID retry: " + deliveryId, cause); + this.deliveryId = deliveryId; + } + public String deliveryId() { return deliveryId; } + } + + /** + * Queues a proxy-origin envelope with a stable, caller-persisted delivery ID. + * Callers recovering {@link DeliveryRetryException} must persist its ID and retry + * the identical envelope through this overload until it returns {@code true}. + */ public boolean send(String serverId, String deliveryId, JsonEnvelope envelope) { + return send(serverId, deliveryId, envelope, false); + } + + private boolean send(String serverId, String deliveryId, JsonEnvelope envelope, boolean generatedId) { if (closed || serverId == null || envelope == null) return false; try { serverId = HttpTlsIdentity.canonicalServerId(serverId); @@ -177,7 +200,12 @@ public boolean send(String serverId, String deliveryId, JsonEnvelope envelope) { final String canonicalServerId = serverId; try { backend = backendState(canonicalServerId); } catch (IOException persistenceFailure) { return false; } - return backend.enqueue(new HttpTransportProtocol.Delivery(deliveryId, envelope)); + try { return backend.enqueue(new HttpTransportProtocol.Delivery(deliveryId, envelope), generatedId); } + catch (DeliveryRetryException failure) { throw failure; } + catch (IllegalStateException indeterminate) { + if (generatedId) throw new DeliveryRetryException(deliveryId, indeterminate); + throw indeterminate; + } } @Override public void close() { @@ -416,6 +444,9 @@ boolean acceptSession(String requested, long requestedSequence) { if (requestedSequence <= sequence) return false; sequence = requestedSequence; return true; } synchronized boolean enqueue(HttpTransportProtocol.Delivery delivery) { + return enqueue(delivery, false); + } + private synchronized boolean enqueue(HttpTransportProtocol.Delivery delivery, boolean generatedId) { if (retired) return false; HttpTransportProtocol.Delivery existing = outgoing.get(delivery.id()); if (existing != null) { @@ -427,6 +458,10 @@ synchronized boolean enqueue(HttpTransportProtocol.Delivery delivery) { } if (outgoing.size() >= HttpTransportProtocol.MAX_QUEUE) return false; if (durableOutgoing != null) try { durableOutgoing.persist(serverId, delivery); } + catch (DurableFiles.PublishedException quarantined) { + if (generatedId) throw new DeliveryRetryException(delivery.id(), quarantined); + return false; + } catch (IOException failure) { return false; } outgoing.put(delivery.id(), delivery); touch(); signal(); return true; } @@ -662,7 +697,7 @@ private synchronized void persist(String serverId, HttpTransportProtocol.Deliver catch (java.nio.file.AtomicMoveNotSupportedException unsupported) { Files.move(pending, target); } try { ownerOnlyFile(target); directoryForcer.force(directory); } catch (IOException postPublicationFailure) { - quarantinePublished(directory, target, pending, quarantined, delivery.id(), postPublicationFailure); + quarantinePublished(directory, target, pending, serverFiles, quarantined, delivery.id(), postPublicationFailure); } quarantined.remove(delivery.id()); serverFiles.put(delivery.id(), target); @@ -683,14 +718,14 @@ private synchronized void persist(String serverId, HttpTransportProtocol.Deliver try { ownerOnlyFile(target); directoryForcer.force(directory); } catch (IOException postPublicationFailure) { pending = directory.resolve(".pending-" + delivery.id() + ".json"); - quarantinePublished(directory, target, pending, quarantined, delivery.id(), postPublicationFailure); + quarantinePublished(directory, target, pending, serverFiles, quarantined, delivery.id(), postPublicationFailure); } serverFiles.put(delivery.id(), target); } finally { Files.deleteIfExists(temporary); } } - private void quarantinePublished(Path directory, Path target, Path pending, Map quarantined, - String id, IOException publicationFailure) throws IOException { + private void quarantinePublished(Path directory, Path target, Path pending, Map serverFiles, + Map quarantined, String id, IOException publicationFailure) throws IOException { try { try { Files.move(target, pending, StandardCopyOption.ATOMIC_MOVE); } catch (java.nio.file.AtomicMoveNotSupportedException unsupported) { Files.move(target, pending); } @@ -700,6 +735,16 @@ private void quarantinePublished(Path directory, Path target, Path pending, Map< ownerOnlyFile(pending); directoryForcer.force(directory); } catch (IOException quarantineFailure) { + // A failed quarantine rename leaves the original published name in place on + // ordinary filesystems. Preserve that observed target for a same-ID retry so + // persist() confirms it rather than publishing a second file for the ID. + if (Files.isRegularFile(target, LinkOption.NOFOLLOW_LINKS)) { + quarantined.remove(id); + serverFiles.put(id, target); + } else if (Files.isRegularFile(pending, LinkOption.NOFOLLOW_LINKS)) { + serverFiles.remove(id); + quarantined.put(id, pending); + } quarantineFailure.addSuppressed(publicationFailure); throw new IllegalStateException("HTTP outgoing queue publication could not be quarantined", quarantineFailure); } diff --git a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportRuntimeTest.java b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportRuntimeTest.java index 908b1dc..7a8ee0f 100644 --- a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportRuntimeTest.java +++ b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportRuntimeTest.java @@ -166,6 +166,84 @@ void outgoingQueueRetriesDirectoryPublicationAfterForceFailure() throws Exceptio assertEquals(2L, serverRootForces.get(), "retrying an existing backend directory must force its parent again"); } + @Test + void generatedSendExposesRecoverableIdAfterPublicationFailure() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("generated-proxy"), "localhost"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, directory.resolve("generated-authority")); + for (boolean persistentFailure : new boolean[] { false, true }) { + Path queueRoot = directory.resolve("generated-outgoing-" + persistentFailure); + JsonEnvelope envelope = JsonEnvelope.builder("generated-retry").build(); + String retryId; + try (HttpProxyTransportServer server = new HttpProxyTransportServer(new InetSocketAddress("localhost", 0), + identity, authority, queueRoot, ignored -> { })) { + AtomicLong publicationForces = new AtomicLong(); + try (var forces = org.mockito.Mockito.mockStatic(com.bencodez.simpleapi.file.DurableFiles.class, + org.mockito.Mockito.CALLS_REAL_METHODS)) { + forces.when(() -> com.bencodez.simpleapi.file.DurableFiles.forceDirectory(queueRoot.resolve("lobby-1"))) + .thenAnswer(call -> { + if (publicationForces.incrementAndGet() == 1 || persistentFailure) + throw new java.io.IOException("injected publication failure"); + return call.callRealMethod(); + }); + HttpProxyTransportServer.DeliveryRetryException retry = assertThrows( + HttpProxyTransportServer.DeliveryRetryException.class, () -> server.send("lobby-1", envelope)); + retryId = retry.deliveryId(); + assertEquals(retryId, java.util.UUID.fromString(retryId).toString()); + } + assertTrue(server.backendStateForTest("lobby-1") + .await("lobby-1", java.util.UUID.randomUUID().toString(), 0).messages().isEmpty()); + } + try (HttpProxyTransportServer restarted = new HttpProxyTransportServer(new InetSocketAddress("localhost", 0), + identity, authority, queueRoot, ignored -> { })) { + assertTrue(restarted.backendStateForTest("lobby-1") + .await("lobby-1", java.util.UUID.randomUUID().toString(), 0).messages().isEmpty()); + assertTrue(restarted.send("lobby-1", retryId, envelope)); + assertEquals(1L, countRegularFiles(queueRoot)); + var messages = restarted.backendStateForTest("lobby-1") + .await("lobby-1", java.util.UUID.randomUUID().toString(), 0).messages(); + assertEquals(1, messages.size()); + assertEquals(retryId, messages.iterator().next().id()); + restarted.backendStateForTest("lobby-1").acknowledge(java.util.List.of(retryId)); + assertEquals(0L, countRegularFiles(queueRoot)); + } + } + } + + @Test + void outgoingQueueRecoversWhenPromotionQuarantineRenameFails() throws Exception { + Path queueRoot = directory.resolve("promotion-rename-failure"); + String deliveryId = java.util.UUID.randomUUID().toString(); + Path serverDirectory = queueRoot.resolve("lobby-1"); + Path pending = serverDirectory.resolve(".pending-" + deliveryId + ".json"); + AtomicLong serverForces = new AtomicLong(); + HttpProxyTransportServer.DurableOutgoingQueue queue = new HttpProxyTransportServer.DurableOutgoingQueue(queueRoot, + forced -> { + if (!forced.equals(serverDirectory)) return; + long attempt = serverForces.incrementAndGet(); + if (attempt == 1L) throw new java.io.IOException("injected initial publication failure"); + if (attempt == 3L) { + Files.createDirectory(pending); + throw new java.io.IOException("injected promotion publication failure"); + } + }); + HttpProxyTransportServer.BackendState state = new HttpProxyTransportServer.BackendState("lobby-1", queue, + (server, id) -> { }); + HttpTransportProtocol.Delivery delivery = new HttpTransportProtocol.Delivery(deliveryId, + JsonEnvelope.builder("promotion-rename-failure").build()); + assertFalse(state.enqueue(delivery)); + assertThrows(IllegalStateException.class, () -> state.enqueue(delivery)); + Files.delete(pending); + assertTrue(state.enqueue(delivery), "same-ID retry must confirm the observed target instead of duplicating it"); + var quarantinedFiles = HttpProxyTransportServer.DurableOutgoingQueue.class.getDeclaredField("quarantinedFiles"); + quarantinedFiles.setAccessible(true); + @SuppressWarnings("unchecked") + var quarantined = (java.util.Map>) quarantinedFiles.get(queue); + assertFalse(quarantined.get("lobby-1").containsKey(deliveryId), + "target fallback must clear the stale quarantine index"); + state.acknowledge(java.util.List.of(deliveryId)); + assertFalse(Files.exists(serverDirectory)); + } + @Test void publishedOutgoingDeliveryRemainsTrackedUntilDurabilityCanBeConfirmed() throws Exception { AtomicLong forceCalls = new AtomicLong(); @@ -719,6 +797,70 @@ void persistedBackendConnectsAfterAutomaticServerLeafRotation() throws Exception } } + @Test + void renewalAdoptsPublishedPointerAndRetriesItBeforeAnotherRenewal() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("pointer-proxy"), "localhost"); + HttpTlsIdentity.IssuedClientCertificate expiring = identity.issueClientCertificate("lobby-1", + Instant.now().minus(Duration.ofDays(340))); + String originalPin = HttpTransportSecrets.certificatePin(expiring.certificate()); + Path authorityDirectory = Files.createDirectory(directory.resolve("pointer-authority")); + String key = java.util.Base64.getUrlEncoder().withoutPadding() + .encodeToString("lobby-1".getBytes(java.nio.charset.StandardCharsets.UTF_8)); + Files.writeString(authorityDirectory.resolve("http-transport-clients.properties"), + "version=2\nbinding." + key + "=" + originalPin + ":-:0\n"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, authorityDirectory); + CountDownLatch received = new CountDownLatch(1); + try (HttpProxyTransportServer server = new HttpProxyTransportServer(new InetSocketAddress("localhost", 0), + identity, authority, ignored -> received.countDown())) { + Path clientDirectory = directory.resolve("pointer-client"); + HttpConnectionCode code = new HttpConnectionCode("lobby-1", server.endpoint("localhost"), + identity.serverCertificatePin(), identity.caCertificatePin(), Instant.now().plusSeconds(60), "A".repeat(43)); + HttpClientCredentialStore.saveEnrolled(clientDirectory, code, expiring); + server.start(); + try (HttpBackendTransportConnector connector = new HttpBackendTransportConnector(clientDirectory, ignored -> { })) { + var renew = HttpBackendTransportConnector.class.getDeclaredMethod("maybeRenewCredential"); + renew.setAccessible(true); + var pending = HttpBackendTransportConnector.class.getDeclaredField("pendingActivation"); + pending.setAccessible(true); + var credential = HttpBackendTransportConnector.class.getDeclaredField("credential"); + credential.setAccessible(true); + Path pointer = clientDirectory.resolve("http-transport-client-current"); + String generation; + try (var forces = org.mockito.Mockito.mockStatic(com.bencodez.simpleapi.file.DurableFiles.class, + org.mockito.Mockito.CALLS_REAL_METHODS)) { + AtomicLong clientDirectoryForces = new AtomicLong(); + forces.when(() -> com.bencodez.simpleapi.file.DurableFiles.forceDirectory(clientDirectory)) + .thenAnswer(call -> { + // Staging first confirms the existing directory. Subsequent forces + // are the CURRENT pointer publication and its same-generation retry. + if (clientDirectoryForces.incrementAndGet() > 1L) + throw new java.io.IOException("injected pointer force failure"); + return call.callRealMethod(); + }); + renew.invoke(connector); + generation = Files.readString(pointer); + var selected = HttpClientCredentialStore.load(clientDirectory); + String selectedPin = HttpTransportSecrets.certificatePin(selected.certificate()); + assertFalse(originalPin.equals(selectedPin)); + assertEquals(selectedPin, HttpTransportSecrets.certificatePin( + ((HttpClientCredentialStore.ClientCredential) credential.get(connector)).certificate())); + assertTrue(pending.get(connector) != null); + assertTrue(authority.authenticate("lobby-1", selected.certificate())); + assertFalse(authority.authenticate("lobby-1", expiring.certificate())); + renew.invoke(connector); + assertEquals(generation, Files.readString(pointer)); + assertTrue(pending.get(connector) != null, "failed retry must retain the same pending activation"); + } + renew.invoke(connector); + assertTrue(pending.get(connector) == null); + assertEquals(generation, Files.readString(pointer)); + connector.start(); + assertTrue(connector.send(JsonEnvelope.builder("after-pointer-recovery").build())); + assertTrue(received.await(8, TimeUnit.SECONDS), "transport must use the adopted TLS client"); + } + } + } + @Test void backendRenewsClientCertificateBeforeExpiryWithoutNewConnectionCode() throws Exception { HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); diff --git a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportSecurityTest.java b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportSecurityTest.java index 997aabb..8713661 100644 --- a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportSecurityTest.java +++ b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportSecurityTest.java @@ -335,7 +335,14 @@ void failedRevocationPersistenceCanBeRetriedWithoutLosingState() throws Exceptio assertFalse(authority.authenticate("lobby-2", unaffectedIssued.certificate()), "all authentication must fail closed while persistence is unresolved"); Files.delete(stateFile); + assertThrows(java.io.IOException.class, () -> authority.enroll("lobby-1", pending.enrollmentToken()), + "a restored pending code must not republish an unresolved revocation"); + assertThrows(IllegalStateException.class, + () -> authority.createConnectionCode("lobby-3", endpoint, Duration.ofMinutes(5)), + "new codes must not persist the pre-revocation state either"); authority.revoke("lobby-1"); + assertThrows(IllegalArgumentException.class, () -> authority.enroll("lobby-1", pending.enrollmentToken()), + "the revocation retry must permanently consume every prior code"); assertTrue(authority.authenticate("lobby-2", unaffectedIssued.certificate()), "a successful full-state retry must restore authentication availability"); From cf155d977dc3426e2ebaea8f81cd125b1993081f Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Sat, 5 Sep 2026 19:37:32 -0600 Subject: [PATCH 17/38] Recover authority and completed delivery persistence failures --- .../http/HttpBackendTransportConnector.java | 9 +- .../http/HttpEnrollmentAuthority.java | 2 +- .../http/HttpInboundDeliveryStore.java | 82 +++++++++++++++++-- .../http/HttpProxyTransportServer.java | 12 ++- .../http/HttpTransportRuntimeTest.java | 80 ++++++++++++++++++ .../http/HttpTransportSecurityTest.java | 37 +++++++++ 6 files changed, 211 insertions(+), 11 deletions(-) diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpBackendTransportConnector.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpBackendTransportConnector.java index dffad53..dc72559 100644 --- a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpBackendTransportConnector.java +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpBackendTransportConnector.java @@ -107,6 +107,7 @@ private HttpBackendTransportConnector(HttpClientCredentialStore.HttpClientProfil : HttpInboundDeliveryStore.open(credentialDirectory, "http-transport-ack-confirmations"); if (inboundDeliveries != null) for (var entry : inboundDeliveries.snapshot().entrySet()) { if (entry.getValue() == HttpInboundDeliveryStore.State.COMPLETED) { + inboundDeliveries.confirmCompleted(entry.getKey()); received.add(entry.getKey()); queueAck(entry.getKey()); } @@ -114,6 +115,7 @@ private HttpBackendTransportConnector(HttpClientCredentialStore.HttpClientProfil if (acknowledgementConfirmationStore != null) for (var entry : acknowledgementConfirmationStore.snapshot().entrySet()) { if (entry.getValue() != HttpInboundDeliveryStore.State.COMPLETED) throw new IOException("HTTP acknowledgement confirmation state is invalid"); + acknowledgementConfirmationStore.confirmCompleted(entry.getKey()); queueAcknowledgementConfirmation(entry.getKey()); } client = client(profile, credential); @@ -307,7 +309,12 @@ List accept(List List accepted = new java.util.ArrayList<>(); for (HttpTransportProtocol.Delivery delivery : deliveries) { HttpInboundDeliveryStore.State persisted = inboundDeliveries == null ? null : inboundDeliveries.state(delivery.id()); - if (received.contains(delivery.id()) || persisted == HttpInboundDeliveryStore.State.COMPLETED) { + if (persisted == HttpInboundDeliveryStore.State.COMPLETED) { + try { inboundDeliveries.confirmCompleted(delivery.id()); } + catch (IOException unconfirmed) { continue; } + received.add(delivery.id()); queueAck(delivery.id()); continue; + } + if (received.contains(delivery.id())) { received.add(delivery.id()); queueAck(delivery.id()); continue; } // A callback that was running when the process stopped may already have diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentAuthority.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentAuthority.java index 3164096..912aafe 100644 --- a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentAuthority.java +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentAuthority.java @@ -67,7 +67,7 @@ public synchronized HttpConnectionCode createConnectionCode(String serverId, URI byte[] tokenHash = HttpTransportSecrets.sha256(token.getBytes(StandardCharsets.US_ASCII)); String lookup = java.util.Base64.getUrlEncoder().withoutPadding().encodeToString(tokenHash); enrollments.put(lookup, new Enrollment(tokenHash, expiresAt, serverId, null)); - try { persistState(); } + try { persistState(); persistenceFailure = false; } catch (java.io.IOException failure) { enrollments.remove(lookup); throw new IllegalStateException("Could not persist HTTP enrollment", failure); diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpInboundDeliveryStore.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpInboundDeliveryStore.java index 56d876e..9fbec0a 100644 --- a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpInboundDeliveryStore.java +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpInboundDeliveryStore.java @@ -11,8 +11,10 @@ import java.nio.file.StandardOpenOption; import java.nio.file.attribute.PosixFilePermission; import java.util.EnumSet; +import java.util.HashSet; import java.util.LinkedHashMap; import java.util.Map; +import java.util.Set; import java.util.UUID; /** Crash-durable state for proxy deliveries around a non-transactional application callback. */ @@ -21,6 +23,9 @@ final class HttpInboundDeliveryStore { private static final int MAX_ENTRIES = HttpTransportProtocol.MAX_QUEUE; private final Path root; private final Map entries = new LinkedHashMap<>(); + // A rename completed, but its directory entry still needs a successful fsync. + private final Set unconfirmedReservations = new HashSet<>(); + private final Set unconfirmedCompletions = new HashSet<>(); private boolean sealed; HttpInboundDeliveryStore(Path credentialDirectory) throws IOException { @@ -55,7 +60,10 @@ synchronized void reserve(String id) throws IOException { requireWritable(); id = canonical(id); State existing = entries.get(id); - if (existing == State.RESERVED) return; + if (existing == State.RESERVED) { + confirmReserved(id); + return; + } if (existing != null) throw new IOException("HTTP inbound delivery fence is already active"); if (entries.size() >= MAX_ENTRIES) throw new IOException("HTTP inbound delivery fence is full"); requireRoot(); @@ -68,8 +76,14 @@ synchronized void reserve(String id) throws IOException { Files.writeString(temporary, id, StandardCharsets.US_ASCII, StandardOpenOption.TRUNCATE_EXISTING); DurableFiles.forceFile(temporary); move(temporary, target); - ownerOnlyFile(target); - DurableFiles.forceDirectory(root); + try { + ownerOnlyFile(target); + DurableFiles.forceDirectory(root); + } catch (IOException postPublicationFailure) { + entries.put(id, State.RESERVED); + unconfirmedReservations.add(id); + throw new DurableFiles.PublishedException(postPublicationFailure); + } entries.put(id, State.RESERVED); } finally { Files.deleteIfExists(temporary); } } @@ -78,7 +92,10 @@ synchronized void reserve(String id) throws IOException { synchronized void recordCompleted(String id) throws IOException { requireWritable(); id = canonical(id); - if (entries.get(id) == State.COMPLETED) return; + if (entries.get(id) == State.COMPLETED) { + confirmCompleted(id); + return; + } if (entries.containsKey(id) || entries.size() >= MAX_ENTRIES) throw new IOException("HTTP acknowledgement confirmation fence is full"); requireRoot(); @@ -89,8 +106,15 @@ synchronized void recordCompleted(String id) throws IOException { Files.writeString(temporary, id, StandardCharsets.US_ASCII, StandardOpenOption.TRUNCATE_EXISTING); DurableFiles.forceFile(temporary); move(temporary, target); - ownerOnlyFile(target); - DurableFiles.forceDirectory(root); + try { + ownerOnlyFile(target); + DurableFiles.forceDirectory(root); + } + catch (IOException postPublicationFailure) { + entries.put(id, State.COMPLETED); + unconfirmedCompletions.add(id); + throw new DurableFiles.PublishedException(postPublicationFailure); + } entries.put(id, State.COMPLETED); } finally { Files.deleteIfExists(temporary); } } @@ -118,6 +142,8 @@ synchronized void remove(String id) throws IOException { requireRoot(); DurableFiles.deleteIfExists(file(id, state)); entries.remove(id); + unconfirmedReservations.remove(id); + unconfirmedCompletions.remove(id); } synchronized Map snapshot() { return Map.copyOf(entries); } @@ -125,6 +151,11 @@ synchronized void remove(String id) throws IOException { private void transition(String id, State expected, State replacement) throws IOException { requireWritable(); id = canonical(id); + if (expected == State.RESERVED) confirmReserved(id); + if (replacement == State.COMPLETED && entries.get(id) == replacement) { + confirmCompleted(id); + return; + } if (entries.get(id) != expected) throw new IOException("HTTP inbound delivery fence state is invalid"); requireRoot(); Path source = file(id, expected), target = file(id, replacement); @@ -132,10 +163,45 @@ private void transition(String id, State expected, State replacement) throws IOE || Files.exists(target, LinkOption.NOFOLLOW_LINKS)) throw new IOException("HTTP inbound delivery fence state is unsafe"); move(source, target); - DurableFiles.forceDirectory(root); + try { DurableFiles.forceDirectory(root); } + catch (IOException postPublicationFailure) { + entries.put(id, replacement); + if (replacement == State.COMPLETED) unconfirmedCompletions.add(id); + throw new DurableFiles.PublishedException(postPublicationFailure); + } entries.put(id, replacement); } + /** Retries the directory fsync required before exposing a reservation to a callback. */ + synchronized void confirmReserved(String id) throws IOException { + id = canonical(id); + if (entries.get(id) != State.RESERVED) throw new IOException("HTTP inbound delivery fence state is invalid"); + if (!unconfirmedReservations.contains(id)) return; + requireRoot(); + Path target = file(id, State.RESERVED); + if (Files.isSymbolicLink(target) || !Files.isRegularFile(target, LinkOption.NOFOLLOW_LINKS) + || Files.size(target) > 64L + || !Files.readString(target, StandardCharsets.US_ASCII).equals(id)) + throw new IOException("HTTP inbound delivery fence state is unsafe"); + DurableFiles.forceDirectory(root); + unconfirmedReservations.remove(id); + } + + /** Retries the directory fsync required before exposing a completed delivery for acknowledgement. */ + synchronized void confirmCompleted(String id) throws IOException { + id = canonical(id); + if (entries.get(id) != State.COMPLETED) throw new IOException("HTTP inbound delivery fence state is invalid"); + if (!unconfirmedCompletions.contains(id)) return; + requireRoot(); + Path target = file(id, State.COMPLETED); + if (Files.isSymbolicLink(target) || !Files.isRegularFile(target, LinkOption.NOFOLLOW_LINKS) + || Files.size(target) > 64L + || !Files.readString(target, StandardCharsets.US_ASCII).equals(id)) + throw new IOException("HTTP inbound delivery fence state is unsafe"); + DurableFiles.forceDirectory(root); + unconfirmedCompletions.remove(id); + } + private void load() throws IOException { try (DirectoryStream files = Files.newDirectoryStream(root)) { for (Path file : files) { @@ -167,6 +233,8 @@ private void load() throws IOException { DurableFiles.deleteIfExists(file(id, obsolete)); entries.put(id, retained); } + if (entries.get(id) == State.COMPLETED) unconfirmedCompletions.add(id); + if (entries.get(id) == State.RESERVED) unconfirmedReservations.add(id); if (entries.size() > MAX_ENTRIES) throw new IOException("HTTP inbound delivery fence exceeds its bound"); } } diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java index 9f12d28..9dffb3d 100644 --- a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java @@ -422,7 +422,10 @@ private BackendState(String serverId, DurableOutgoingQueue durableOutgoing, if (durableIncoming != null) for (Map.Entry entry : durableIncoming.snapshot().entrySet()) { if (entry.getValue() == HttpInboundDeliveryStore.State.COMPLETED) { - seen.add(entry.getKey()); queueAck(entry.getKey()); + try { + durableIncoming.confirmCompleted(entry.getKey()); + seen.add(entry.getKey()); queueAck(entry.getKey()); + } catch (IOException unconfirmed) { } } } } @@ -482,7 +485,12 @@ List acceptIncoming(List accepted = new java.util.ArrayList<>(); for (HttpTransportProtocol.Delivery delivery : received) { HttpInboundDeliveryStore.State persisted = durableIncoming == null ? null : durableIncoming.state(delivery.id()); - if (seen.contains(delivery.id()) || persisted == HttpInboundDeliveryStore.State.COMPLETED) { + if (persisted == HttpInboundDeliveryStore.State.COMPLETED) { + try { durableIncoming.confirmCompleted(delivery.id()); } + catch (IOException unconfirmed) { continue; } + seen.add(delivery.id()); queueAck(delivery.id()); continue; + } + if (seen.contains(delivery.id())) { seen.add(delivery.id()); queueAck(delivery.id()); continue; } if (persisted == HttpInboundDeliveryStore.State.RUNNING) continue; diff --git a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportRuntimeTest.java b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportRuntimeTest.java index 7a8ee0f..a213cc4 100644 --- a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportRuntimeTest.java +++ b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportRuntimeTest.java @@ -341,6 +341,86 @@ void proxyInboundCompletionSurvivesRestartBeforeAcknowledgement() throws Excepti "only an acknowledgement confirmation may retire the durable replay fence"); } + @Test + void completedInboundPublicationRetriesForceBeforeProxyAcknowledgement() throws Exception { + Path root = directory.resolve("proxy-incoming-force-retry"); + Files.createDirectory(root); + String deliveryId = java.util.UUID.randomUUID().toString(); + Path inboundDirectory = root.resolve("lobby-1"); + Path completed = inboundDirectory.resolve(deliveryId + ".completed"); + HttpTransportProtocol.Delivery delivery = new HttpTransportProtocol.Delivery(deliveryId, + JsonEnvelope.builder("backend-event").build()); + HttpInboundDeliveryStore store = HttpInboundDeliveryStore.open(root, "lobby-1"); + HttpProxyTransportServer.BackendState state = new HttpProxyTransportServer.BackendState( + "lobby-1", null, store, (server, id) -> { }); + try (var forces = org.mockito.Mockito.mockStatic(com.bencodez.simpleapi.file.DurableFiles.class, + org.mockito.Mockito.CALLS_REAL_METHODS)) { + forces.when(() -> com.bencodez.simpleapi.file.DurableFiles.forceDirectory(inboundDirectory)) + .thenAnswer(call -> { + if (Files.exists(completed)) throw new java.io.IOException("injected completed fsync failure"); + return call.callRealMethod(); + }); + assertEquals(java.util.List.of(delivery), state.acceptIncoming(java.util.List.of(delivery))); + state.beginIncoming(deliveryId); + assertThrows(com.bencodez.simpleapi.file.DurableFiles.PublishedException.class, + () -> state.completeIncomingDurably(deliveryId)); + state.completeIncoming(deliveryId, false); + assertEquals(HttpInboundDeliveryStore.State.COMPLETED, store.state(deliveryId)); + assertTrue(state.acceptIncoming(java.util.List.of(delivery)).isEmpty(), + "a visible completed target must never rerun its callback while fsync is unresolved"); + assertTrue(state.await("lobby-1", java.util.UUID.randomUUID().toString(), 0).acks().isEmpty(), + "a completed target must not be acknowledged before its fsync succeeds"); + } + assertTrue(state.acceptIncoming(java.util.List.of(delivery)).isEmpty(), + "recovered publication must be acknowledged rather than dispatched again"); + assertEquals(java.util.List.of(deliveryId), state.await("lobby-1", java.util.UUID.randomUUID().toString(), 0).acks()); + } + + @Test + void reservedInboundPublicationRetriesBeforeProxyCallback() throws Exception { + Path root = directory.resolve("reserved-force-proxy"); + Files.createDirectory(root); + String deliveryId = java.util.UUID.randomUUID().toString(); + Path inboundDirectory = root.resolve("lobby-1"); + Path reserved = inboundDirectory.resolve(deliveryId + ".reserved"); + CountDownLatch firstForce = new CountDownLatch(1), secondForce = new CountDownLatch(1); + java.util.concurrent.atomic.AtomicInteger forceCalls = new java.util.concurrent.atomic.AtomicInteger(); + java.util.concurrent.atomic.AtomicInteger callbacks = new java.util.concurrent.atomic.AtomicInteger(); + HttpTransportProtocol.Delivery delivery = new HttpTransportProtocol.Delivery(deliveryId, JsonEnvelope.builder("vote").build()); + HttpInboundDeliveryStore store = HttpInboundDeliveryStore.open(root, "lobby-1"); + HttpProxyTransportServer.BackendState state = new HttpProxyTransportServer.BackendState( + "lobby-1", null, store, (server, id) -> { }); + try (var forces = org.mockito.Mockito.mockStatic(com.bencodez.simpleapi.file.DurableFiles.class, + org.mockito.Mockito.CALLS_REAL_METHODS)) { + forces.when(() -> com.bencodez.simpleapi.file.DurableFiles.forceDirectory(inboundDirectory)) + .thenAnswer(call -> { + if (Files.exists(reserved)) { + if (forceCalls.incrementAndGet() == 1) firstForce.countDown(); else secondForce.countDown(); + throw new java.io.IOException("injected reservation fsync failure"); + } + return call.callRealMethod(); + }); + assertEquals(java.util.List.of(delivery), state.acceptIncoming(java.util.List.of(delivery))); + assertThrows(com.bencodez.simpleapi.file.DurableFiles.PublishedException.class, + () -> state.beginIncoming(deliveryId)); + state.completeIncoming(deliveryId, false); + assertTrue(firstForce.await(2, TimeUnit.SECONDS)); + assertEquals(java.util.List.of(delivery), state.acceptIncoming(java.util.List.of(delivery))); + assertThrows(java.io.IOException.class, () -> state.beginIncoming(deliveryId)); + state.completeIncoming(deliveryId, false); + assertTrue(secondForce.await(2, TimeUnit.SECONDS)); + assertEquals(0, callbacks.get(), "an unconfirmed reservation must not enter the callback"); + assertTrue(state.await("lobby-1", java.util.UUID.randomUUID().toString(), 0).acks().isEmpty()); + } + assertEquals(java.util.List.of(delivery), state.acceptIncoming(java.util.List.of(delivery))); + state.beginIncoming(deliveryId); + callbacks.incrementAndGet(); // Models the callback reached only after beginIncoming's durable RUNNING transition. + state.completeIncomingDurably(deliveryId); + state.completeIncoming(deliveryId, true); + assertEquals(java.util.List.of(deliveryId), state.await("lobby-1", java.util.UUID.randomUUID().toString(), 0).acks()); + assertEquals(1, callbacks.get()); + } + @Test void failedAcknowledgementCallbackRetainsProxyDelivery() throws Exception { HttpProxyTransportServer.BackendState state = new HttpProxyTransportServer.BackendState("lobby-1", null, diff --git a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportSecurityTest.java b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportSecurityTest.java index 8713661..157104b 100644 --- a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportSecurityTest.java +++ b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportSecurityTest.java @@ -416,6 +416,43 @@ void failedPendingCertificateWriteLeavesEnrollmentTokenRetryable() throws Except assertTrue(authority.authenticate("lobby-1", retried.certificate())); } + @Test + void enrollmentPublicationFailureDisablesAuthenticationUntilCodePersistenceRecovers() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("published-enrollment-proxy"), "localhost"); + Path stateDirectory = directory.resolve("published-enrollment-state"); + Files.createDirectories(stateDirectory); + java.util.concurrent.atomic.AtomicReference now = new java.util.concurrent.atomic.AtomicReference<>( + Instant.parse("2030-01-01T00:00:00Z")); + Clock clock = new Clock() { + @Override public ZoneOffset getZone() { return ZoneOffset.UTC; } + @Override public Clock withZone(java.time.ZoneId zone) { return this; } + @Override public Instant instant() { return now.get(); } + }; + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, clock, + stateDirectory.resolve("http-transport-clients.properties")); + URI endpoint = URI.create("https://localhost:8443/"); + HttpConnectionCode unrelatedCode = authority.createConnectionCode("unrelated", endpoint, Duration.ofMinutes(5)); + HttpTlsIdentity.IssuedClientCertificate unrelated = authority.enroll("unrelated", unrelatedCode.enrollmentToken()); + assertTrue(authority.authenticate("unrelated", unrelated.certificate())); + + HttpConnectionCode targetCode = authority.createConnectionCode("lobby-1", endpoint, Duration.ofMinutes(5)); + try (var forces = org.mockito.Mockito.mockStatic(com.bencodez.simpleapi.file.DurableFiles.class, + org.mockito.Mockito.CALLS_REAL_METHODS)) { + forces.when(() -> com.bencodez.simpleapi.file.DurableFiles.forceDirectory(stateDirectory)) + .thenThrow(new java.io.IOException("injected authority publication failure")); + assertThrows(com.bencodez.simpleapi.file.DurableFiles.PublishedException.class, + () -> authority.enroll("lobby-1", targetCode.enrollmentToken())); + } + + assertFalse(authority.authenticate("unrelated", unrelated.certificate()), + "authentication must fail closed while the published enrollment state is unresolved"); + now.set(Instant.parse("2030-01-01T00:06:00Z")); + HttpConnectionCode recoveryCode = authority.createConnectionCode("recovery", endpoint, Duration.ofMinutes(5)); + assertFalse(recoveryCode.enrollmentToken().isEmpty()); + assertTrue(authority.authenticate("unrelated", unrelated.certificate()), + "a successful full-state rewrite must restore authentication availability"); + } + @Test void authorityStatePrunesRevocationsAndBoundsActiveBindings() throws Exception { Path proxy = directory.resolve("bounded-proxy"); From 5bcce2dfbd808821382122f8baf098ec86299705 Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Sat, 5 Sep 2026 20:11:03 -0600 Subject: [PATCH 18/38] Retry pre-callback state writes and bound quarantined backends --- .../http/HttpBackendTransportConnector.java | 3 + .../http/HttpInboundDeliveryStore.java | 50 +++++++- .../http/HttpProxyTransportServer.java | 29 ++++- .../http/HttpOutgoingQueueCapacityTest.java | 76 ++++++++++++ .../http/HttpTransportRuntimeTest.java | 116 ++++++++++++++++++ 5 files changed, 271 insertions(+), 3 deletions(-) create mode 100644 SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpOutgoingQueueCapacityTest.java diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpBackendTransportConnector.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpBackendTransportConnector.java index dc72559..963c5c4 100644 --- a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpBackendTransportConnector.java +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpBackendTransportConnector.java @@ -309,6 +309,9 @@ List accept(List List accepted = new java.util.ArrayList<>(); for (HttpTransportProtocol.Delivery delivery : deliveries) { HttpInboundDeliveryStore.State persisted = inboundDeliveries == null ? null : inboundDeliveries.state(delivery.id()); + if (persisted == HttpInboundDeliveryStore.State.RUNNING) try { + if (inboundDeliveries.recoverKnownNotStartedRunning(delivery.id())) persisted = inboundDeliveries.state(delivery.id()); + } catch (IOException rollbackUnconfirmed) { continue; } if (persisted == HttpInboundDeliveryStore.State.COMPLETED) { try { inboundDeliveries.confirmCompleted(delivery.id()); } catch (IOException unconfirmed) { continue; } diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpInboundDeliveryStore.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpInboundDeliveryStore.java index 9fbec0a..e9444a3 100644 --- a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpInboundDeliveryStore.java +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpInboundDeliveryStore.java @@ -26,6 +26,9 @@ final class HttpInboundDeliveryStore { // A rename completed, but its directory entry still needs a successful fsync. private final Set unconfirmedReservations = new HashSet<>(); private final Set unconfirmedCompletions = new HashSet<>(); + // RUNNING was published before the callback, but restoring RESERVED was not yet confirmed. + // This is process-local evidence only: a RUNNING entry loaded after a restart remains ambiguous. + private final Set pendingRunningRollbacks = new HashSet<>(); private boolean sealed; HttpInboundDeliveryStore(Path credentialDirectory) throws IOException { @@ -119,7 +122,12 @@ synchronized void recordCompleted(String id) throws IOException { } finally { Files.deleteIfExists(temporary); } } - synchronized void markRunning(String id) throws IOException { transition(id, State.RESERVED, State.RUNNING); } + synchronized void markRunning(String id) throws IOException { + id = canonical(id); + if (entries.get(id) == State.RUNNING && pendingRunningRollbacks.contains(id)) + recoverKnownNotStartedRunning(id); + transition(id, State.RESERVED, State.RUNNING); + } synchronized void markCompleted(String id) throws IOException { transition(id, State.RUNNING, State.COMPLETED); } synchronized void seal() { sealed = true; } synchronized void sealAndDeleteIfEmpty() throws IOException { @@ -144,6 +152,7 @@ synchronized void remove(String id) throws IOException { entries.remove(id); unconfirmedReservations.remove(id); unconfirmedCompletions.remove(id); + pendingRunningRollbacks.remove(id); } synchronized Map snapshot() { return Map.copyOf(entries); } @@ -165,6 +174,15 @@ private void transition(String id, State expected, State replacement) throws IOE move(source, target); try { DurableFiles.forceDirectory(root); } catch (IOException postPublicationFailure) { + if (replacement == State.RUNNING) { + // The callback has not been exposed yet. Restore the durable reservation + // when possible, so a later redelivery can safely try the transition again. + entries.put(id, State.RUNNING); + pendingRunningRollbacks.add(id); + try { recoverKnownNotStartedRunning(id); } + catch (IOException rollbackFailure) { postPublicationFailure.addSuppressed(rollbackFailure); } + throw new DurableFiles.PublishedException(postPublicationFailure); + } entries.put(id, replacement); if (replacement == State.COMPLETED) unconfirmedCompletions.add(id); throw new DurableFiles.PublishedException(postPublicationFailure); @@ -172,6 +190,31 @@ private void transition(String id, State expected, State replacement) throws IOE entries.put(id, replacement); } + /** + * Retries a rollback known to have happened before this process could enter the callback. + * Entries loaded from disk are deliberately absent from this set: their callback is ambiguous. + */ + synchronized boolean recoverKnownNotStartedRunning(String id) throws IOException { + id = canonical(id); + if (!pendingRunningRollbacks.contains(id)) return false; + requireWritable(); + if (entries.get(id) != State.RUNNING) throw new IOException("HTTP inbound delivery fence state is invalid"); + requireRoot(); + Path reserved = file(id, State.RESERVED), running = file(id, State.RUNNING); + boolean hasReserved = Files.exists(reserved, LinkOption.NOFOLLOW_LINKS); + boolean hasRunning = Files.exists(running, LinkOption.NOFOLLOW_LINKS); + if (hasReserved == hasRunning) throw new IOException("HTTP inbound delivery fence rollback is unsafe"); + if (hasRunning) { + verifyStateFile(running, id); + move(running, reserved); + ownerOnlyFile(reserved); + } else verifyStateFile(reserved, id); + DurableFiles.forceDirectory(root); + entries.put(id, State.RESERVED); + pendingRunningRollbacks.remove(id); + return true; + } + /** Retries the directory fsync required before exposing a reservation to a callback. */ synchronized void confirmReserved(String id) throws IOException { id = canonical(id); @@ -241,6 +284,11 @@ private void load() throws IOException { } private Path file(String id, State state) { return root.resolve(id + state.suffix); } + private void verifyStateFile(Path file, String id) throws IOException { + if (Files.isSymbolicLink(file) || !Files.isRegularFile(file, LinkOption.NOFOLLOW_LINKS) + || Files.size(file) > 64L || !Files.readString(file, StandardCharsets.US_ASCII).equals(id)) + throw new IOException("HTTP inbound delivery fence state is unsafe"); + } private void requireWritable() throws IOException { if (sealed) throw new IOException("HTTP inbound delivery store ownership has ended"); } diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java index 9dffb3d..dac27f2 100644 --- a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java @@ -30,6 +30,7 @@ import java.util.LinkedHashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.UUID; import java.util.concurrent.ArrayBlockingQueue; @@ -80,7 +81,7 @@ public final class HttpProxyTransportServer implements AutoCloseable { /** In-memory constructor for tests; production callers must supply a durable state directory. */ HttpProxyTransportServer(InetSocketAddress bind, HttpTlsIdentity identity, HttpEnrollmentAuthority authority, Consumer onEnvelope) throws Exception { - this(bind, identity, authority, null, onEnvelope, (serverId, deliveryId) -> { }); + this(bind, identity, authority, null, onEnvelope, (serverId, deliveryId) -> { }, System::nanoTime); } public HttpProxyTransportServer(InetSocketAddress bind, HttpTlsIdentity identity, HttpEnrollmentAuthority authority, @@ -91,7 +92,8 @@ public HttpProxyTransportServer(InetSocketAddress bind, HttpTlsIdentity identity public HttpProxyTransportServer(InetSocketAddress bind, HttpTlsIdentity identity, HttpEnrollmentAuthority authority, Path outgoingDirectory, Consumer onEnvelope, DeliveryAcknowledgement onAcknowledged) throws Exception { - this(bind, identity, authority, outgoingDirectory, onEnvelope, onAcknowledged, System::nanoTime); + this(bind, identity, authority, Objects.requireNonNull(outgoingDirectory, "outgoingDirectory is required"), + onEnvelope, onAcknowledged, System::nanoTime); } HttpProxyTransportServer(InetSocketAddress bind, HttpTlsIdentity identity, HttpEnrollmentAuthority authority, @@ -485,6 +487,9 @@ List acceptIncoming(List accepted = new java.util.ArrayList<>(); for (HttpTransportProtocol.Delivery delivery : received) { HttpInboundDeliveryStore.State persisted = durableIncoming == null ? null : durableIncoming.state(delivery.id()); + if (persisted == HttpInboundDeliveryStore.State.RUNNING) try { + if (durableIncoming.recoverKnownNotStartedRunning(delivery.id())) persisted = durableIncoming.state(delivery.id()); + } catch (IOException rollbackUnconfirmed) { continue; } if (persisted == HttpInboundDeliveryStore.State.COMPLETED) { try { durableIncoming.confirmCompleted(delivery.id()); } catch (IOException unconfirmed) { continue; } @@ -672,6 +677,8 @@ synchronized Map> load() throws IOE private synchronized void persist(String serverId, HttpTransportProtocol.Delivery delivery) throws IOException { Path directory = root.resolve(serverId).normalize(); if (!directory.getParent().equals(root)) throw new IOException("HTTP outgoing queue server is invalid"); + if (!Files.exists(directory, LinkOption.NOFOLLOW_LINKS) && serverDirectoryCount() >= MAX_BACKENDS) + throw new IOException("HTTP outgoing queue exceeds its backend bound"); try { Files.createDirectory(directory); } catch (java.nio.file.FileAlreadyExistsException existing) { } if (Files.isSymbolicLink(directory) || !Files.isDirectory(directory, LinkOption.NOFOLLOW_LINKS)) @@ -732,6 +739,24 @@ private synchronized void persist(String serverId, HttpTransportProtocol.Deliver } finally { Files.deleteIfExists(temporary); } } + /** Counts durable backend directories, including quarantine-only queues omitted from load's deliverable map. */ + private int serverDirectoryCount() throws IOException { + int count = 0; + try (DirectoryStream directories = Files.newDirectoryStream(root)) { + for (Path directory : directories) { + if (Files.isSymbolicLink(directory) || !Files.isDirectory(directory, LinkOption.NOFOLLOW_LINKS)) + throw new IOException("HTTP outgoing queue contains an invalid entry"); + String serverId; + try { serverId = HttpTlsIdentity.canonicalServerId(directory.getFileName().toString()); } + catch (IllegalArgumentException invalid) { throw new IOException("HTTP outgoing queue server is invalid", invalid); } + if (!serverId.equals(directory.getFileName().toString())) + throw new IOException("HTTP outgoing queue server is not canonical"); + if (++count > MAX_BACKENDS) throw new IOException("HTTP outgoing queue exceeds its backend bound"); + } + } + return count; + } + private void quarantinePublished(Path directory, Path target, Path pending, Map serverFiles, Map quarantined, String id, IOException publicationFailure) throws IOException { try { diff --git a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpOutgoingQueueCapacityTest.java b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpOutgoingQueueCapacityTest.java new file mode 100644 index 0000000..b355d50 --- /dev/null +++ b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpOutgoingQueueCapacityTest.java @@ -0,0 +1,76 @@ +package com.bencodez.simpleapi.servercomm.http; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.bencodez.simpleapi.servercomm.codec.JsonEnvelope; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashSet; +import java.util.Set; +import java.util.UUID; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class HttpOutgoingQueueCapacityTest { + @TempDir Path directory; + + @Test + void quarantineDirectoriesCountTowardCapacityWithoutBlockingSameServerRetry() throws Exception { + Path queueRoot = directory.resolve("outgoing"); + Files.createDirectory(queueRoot); + HttpTransportProtocol.Delivery existingDelivery = null; + for (int index = 0; index < 128; index++) { + String serverId = "server-" + index; + String deliveryId = UUID.nameUUIDFromBytes(serverId.getBytes(StandardCharsets.UTF_8)).toString(); + HttpTransportProtocol.Delivery delivery = new HttpTransportProtocol.Delivery(deliveryId, + JsonEnvelope.builder("quarantined-" + index).build()); + Path serverDirectory = Files.createDirectory(queueRoot.resolve(serverId)); + Files.write(serverDirectory.resolve(".pending-" + deliveryId + ".json"), + HttpTransportProtocol.storedDelivery(delivery)); + if (index == 0) existingDelivery = delivery; + } + + HttpProxyTransportServer.DurableOutgoingQueue queue = new HttpProxyTransportServer.DurableOutgoingQueue( + queueRoot, com.bencodez.simpleapi.file.DurableFiles::forceDirectory); + queue.load(); + + String newServerId = "server-128"; + HttpTransportProtocol.Delivery newDelivery = new HttpTransportProtocol.Delivery(UUID.randomUUID().toString(), + JsonEnvelope.builder("new-server").build()); + HttpProxyTransportServer.BackendState newServer = new HttpProxyTransportServer.BackendState( + newServerId, queue, (server, id) -> { }); + assertFalse(newServer.enqueue(newDelivery), "a new backend must be rejected at the durable directory cap"); + assertFalse(Files.exists(queueRoot.resolve(newServerId)), "rejected enqueue must not publish a backend directory"); + + HttpProxyTransportServer.BackendState existingServer = new HttpProxyTransportServer.BackendState( + "server-0", queue, (server, id) -> { }); + assertTrue(existingServer.enqueue(existingDelivery), "same-server retry must recover its quarantine at capacity"); + + HttpProxyTransportServer.DurableOutgoingQueue restarted = new HttpProxyTransportServer.DurableOutgoingQueue( + queueRoot, com.bencodez.simpleapi.file.DurableFiles::forceDirectory); + restarted.load(); + + try (var servers = Files.list(queueRoot)) { + assertEquals(128L, servers.count(), "restart must not create a 129th backend directory"); + } + Set deliveryIds = new HashSet<>(); + long durableFiles = 0; + try (var servers = Files.list(queueRoot)) { + for (Path serverDirectory : servers.toList()) { + try (var messages = Files.list(serverDirectory)) { + for (Path message : messages.toList()) { + if (!message.getFileName().toString().endsWith(".json")) continue; + durableFiles++; + assertTrue(deliveryIds.add(HttpTransportProtocol.parseStoredDelivery(Files.readAllBytes(message)).id()), + "restart must not duplicate a durable delivery"); + } + } + } + } + assertEquals(128L, durableFiles, "each seeded delivery must remain represented exactly once"); + assertEquals(128, deliveryIds.size()); + } +} diff --git a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportRuntimeTest.java b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportRuntimeTest.java index a213cc4..70dc963 100644 --- a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportRuntimeTest.java +++ b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportRuntimeTest.java @@ -20,6 +20,7 @@ import java.time.Instant; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; import org.junit.jupiter.api.Test; @@ -28,6 +29,17 @@ class HttpTransportRuntimeTest { @TempDir Path directory; + @Test + void publicConstructorsRequireDurableOutgoingDirectory() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("guard-proxy"), "localhost"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, directory.resolve("guard-authority")); + InetSocketAddress bind = new InetSocketAddress("localhost", 0); + assertThrows(NullPointerException.class, () -> new HttpProxyTransportServer(bind, identity, authority, + null, ignored -> { })); + assertThrows(NullPointerException.class, () -> new HttpProxyTransportServer(bind, identity, authority, + null, ignored -> { }, (serverId, deliveryId) -> { })); + } + @Test void endpointHelperSupportsIpv6Literals() throws Exception { HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("ipv6-proxy"), "::1"); @@ -421,6 +433,110 @@ void reservedInboundPublicationRetriesBeforeProxyCallback() throws Exception { assertEquals(1, callbacks.get()); } + @Test + void runningPublicationRollbackLeavesAReservationSafeAfterRestart() throws Exception { + Path root = directory.resolve("running-rollback-restart"); + Files.createDirectory(root); + String deliveryId = java.util.UUID.randomUUID().toString(); + Path inboundDirectory = root.resolve("lobby-1"); + Path running = inboundDirectory.resolve(deliveryId + ".running"); + HttpInboundDeliveryStore store = HttpInboundDeliveryStore.open(root, "lobby-1"); + try (var forces = org.mockito.Mockito.mockStatic(com.bencodez.simpleapi.file.DurableFiles.class, + org.mockito.Mockito.CALLS_REAL_METHODS)) { + java.util.concurrent.atomic.AtomicBoolean failed = new java.util.concurrent.atomic.AtomicBoolean(); + forces.when(() -> com.bencodez.simpleapi.file.DurableFiles.forceDirectory(inboundDirectory)) + .thenAnswer(call -> { + if (Files.exists(running) && !failed.getAndSet(true)) + throw new java.io.IOException("injected running fsync failure"); + return call.callRealMethod(); + }); + store.reserve(deliveryId); + assertThrows(com.bencodez.simpleapi.file.DurableFiles.PublishedException.class, + () -> store.markRunning(deliveryId)); + assertEquals(HttpInboundDeliveryStore.State.RESERVED, store.state(deliveryId)); + } + store.seal(); + assertEquals(HttpInboundDeliveryStore.State.RESERVED, + HttpInboundDeliveryStore.open(root, "lobby-1").state(deliveryId), + "a callback never exposed must be recoverable after a one-shot RUNNING force failure"); + } + + @Test + void proxyRetriesKnownNotStartedRunningRollbackBeforeDispatch() throws Exception { + Path root = directory.resolve("running-rollback-proxy"); + Files.createDirectory(root); + String deliveryId = java.util.UUID.randomUUID().toString(); + Path inboundDirectory = root.resolve("lobby-1"); + Path running = inboundDirectory.resolve(deliveryId + ".running"); + AtomicInteger failures = new AtomicInteger(2), callbacks = new AtomicInteger(); + java.util.concurrent.atomic.AtomicBoolean rollbackStarted = new java.util.concurrent.atomic.AtomicBoolean(); + HttpTransportProtocol.Delivery delivery = new HttpTransportProtocol.Delivery(deliveryId, JsonEnvelope.builder("vote").build()); + HttpInboundDeliveryStore store = HttpInboundDeliveryStore.open(root, "lobby-1"); + HttpProxyTransportServer.BackendState state = new HttpProxyTransportServer.BackendState( + "lobby-1", null, store, (server, id) -> { }); + try (var forces = org.mockito.Mockito.mockStatic(com.bencodez.simpleapi.file.DurableFiles.class, + org.mockito.Mockito.CALLS_REAL_METHODS)) { + forces.when(() -> com.bencodez.simpleapi.file.DurableFiles.forceDirectory(inboundDirectory)) + .thenAnswer(call -> { + if (Files.exists(running)) rollbackStarted.set(true); + if (rollbackStarted.get() && failures.getAndDecrement() > 0) + throw new java.io.IOException("injected running or rollback fsync failure"); + return call.callRealMethod(); + }); + assertEquals(java.util.List.of(delivery), state.acceptIncoming(java.util.List.of(delivery))); + assertThrows(com.bencodez.simpleapi.file.DurableFiles.PublishedException.class, + () -> state.beginIncoming(deliveryId)); + state.completeIncoming(deliveryId, false); + assertEquals(java.util.List.of(delivery), state.acceptIncoming(java.util.List.of(delivery)), + "the in-process known-not-started RUNNING state must recover once its rollback force succeeds"); + state.beginIncoming(deliveryId); + callbacks.incrementAndGet(); + state.completeIncomingDurably(deliveryId); + state.completeIncoming(deliveryId, true); + } + assertEquals(1, callbacks.get(), "the callback becomes eligible exactly once after RUNNING is durable"); + } + + @Test + void backendRetriesKnownNotStartedRunningRollbackBeforeDispatch() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("running-backend-proxy"), "localhost"); + HttpTlsIdentity.IssuedClientCertificate issued = identity.issueClientCertificate("lobby-1"); + HttpConnectionCode code = new HttpConnectionCode("lobby-1", java.net.URI.create("https://localhost:8443/"), + identity.serverCertificatePin(), identity.caCertificatePin(), Instant.now().plusSeconds(300), "D".repeat(43)); + Path clientDirectory = directory.resolve("running-backend-client"); + HttpClientCredentialStore.saveEnrolled(clientDirectory, code, issued); + String deliveryId = java.util.UUID.randomUUID().toString(); + Path inboundDirectory = clientDirectory.resolve("http-transport-inbound-deliveries"); + Path running = inboundDirectory.resolve(deliveryId + ".running"); + AtomicInteger failures = new AtomicInteger(2), callbacks = new AtomicInteger(); + java.util.concurrent.atomic.AtomicBoolean rollbackStarted = new java.util.concurrent.atomic.AtomicBoolean(); + HttpTransportProtocol.Delivery delivery = new HttpTransportProtocol.Delivery(deliveryId, JsonEnvelope.builder("vote").build()); + try (var forces = org.mockito.Mockito.mockStatic(com.bencodez.simpleapi.file.DurableFiles.class, + org.mockito.Mockito.CALLS_REAL_METHODS); + HttpBackendTransportConnector connector = new HttpBackendTransportConnector(clientDirectory, + ignored -> callbacks.incrementAndGet())) { + forces.when(() -> com.bencodez.simpleapi.file.DurableFiles.forceDirectory(inboundDirectory)) + .thenAnswer(call -> { + if (Files.exists(running)) rollbackStarted.set(true); + if (rollbackStarted.get() && failures.getAndDecrement() > 0) + throw new java.io.IOException("injected running or rollback fsync failure"); + return call.callRealMethod(); + }); + var inboundField = HttpBackendTransportConnector.class.getDeclaredField("inboundDeliveries"); + inboundField.setAccessible(true); + HttpInboundDeliveryStore store = (HttpInboundDeliveryStore) inboundField.get(connector); + store.reserve(deliveryId); + assertThrows(com.bencodez.simpleapi.file.DurableFiles.PublishedException.class, + () -> store.markRunning(deliveryId)); + java.util.List retried = connector.accept(java.util.List.of(delivery)); + assertEquals(java.util.List.of(delivery), retried); + connector.dispatch(retried.get(0)); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(2); + while (callbacks.get() == 0 && System.nanoTime() < deadline) Thread.sleep(5); + assertEquals(1, callbacks.get()); + } + } + @Test void failedAcknowledgementCallbackRetainsProxyDelivery() throws Exception { HttpProxyTransportServer.BackendState state = new HttpProxyTransportServer.BackendState("lobby-1", null, From 11130ed742932cd0f5242177adbeb0250a9a1ff2 Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Sat, 5 Sep 2026 20:54:47 -0600 Subject: [PATCH 19/38] Enforce verified private permissions for HTTP transport state --- .../file/PrivateFilePermissions.java | 84 ++++++++ .../http/HttpClientCredentialStore.java | 27 +-- .../http/HttpEnrollmentAuthority.java | 21 +- .../http/HttpInboundDeliveryStore.java | 28 +-- .../http/HttpProxyTransportServer.java | 35 ++-- .../servercomm/http/HttpTlsIdentity.java | 16 +- .../file/PrivateFilePermissionsTest.java | 180 ++++++++++++++++++ 7 files changed, 307 insertions(+), 84 deletions(-) create mode 100644 SimpleAPI/src/main/java/com/bencodez/simpleapi/file/PrivateFilePermissions.java create mode 100644 SimpleAPI/src/test/java/com/bencodez/simpleapi/file/PrivateFilePermissionsTest.java diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/file/PrivateFilePermissions.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/file/PrivateFilePermissions.java new file mode 100644 index 0000000..f3d0f09 --- /dev/null +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/file/PrivateFilePermissions.java @@ -0,0 +1,84 @@ +package com.bencodez.simpleapi.file; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.nio.file.attribute.AclEntry; +import java.nio.file.attribute.AclEntryPermission; +import java.nio.file.attribute.AclEntryType; +import java.nio.file.attribute.AclFileAttributeView; +import java.nio.file.attribute.GroupPrincipal; +import java.nio.file.attribute.PosixFilePermission; +import java.nio.file.attribute.UserPrincipal; +import java.util.EnumSet; +import java.util.List; +import java.util.Set; + +/** Enforces and verifies owner-only access for persisted private material. */ +public final class PrivateFilePermissions { + private static final Set OWNER_FILE = EnumSet.of( + PosixFilePermission.OWNER_READ, PosixFilePermission.OWNER_WRITE); + private static final Set OWNER_DIRECTORY = EnumSet.of( + PosixFilePermission.OWNER_READ, PosixFilePermission.OWNER_WRITE, PosixFilePermission.OWNER_EXECUTE); + private static final Set OWNER_ACL = EnumSet.allOf(AclEntryPermission.class); + + private PrivateFilePermissions() { } + + /** Enforces 0600-equivalent access, or fails when the provider cannot prove it. */ + public static void ownerOnlyFile(Path path) throws IOException { + enforce(path, OWNER_FILE); + } + + /** Enforces 0700-equivalent access, or fails when the provider cannot prove it. */ + public static void ownerOnlyDirectory(Path path) throws IOException { + enforce(path, OWNER_DIRECTORY); + } + + private static void enforce(Path path, Set permissions) throws IOException { + if (path == null) throw new IllegalArgumentException("Private path is required"); + if (Files.isSymbolicLink(path)) throw new IOException("Refusing symbolic link for private storage: " + path); + try { + Files.setPosixFilePermissions(path, permissions); + if (!Files.getPosixFilePermissions(path, LinkOption.NOFOLLOW_LINKS).equals(permissions)) + throw new IOException("Could not verify owner-only POSIX permissions for " + path); + return; + } catch (UnsupportedOperationException unsupported) { + // A non-POSIX provider must expose an ACL that can be reduced and verified. + } + enforceWithAcl(path); + } + + private static void enforceWithAcl(Path path) throws IOException { + try { + AclFileAttributeView view = Files.getFileAttributeView(path, AclFileAttributeView.class, LinkOption.NOFOLLOW_LINKS); + if (view == null) throw new IOException("Owner-only private storage is unsupported for " + path); + enforceWithAcl(view, path); + } catch (UnsupportedOperationException unsupported) { + throw new IOException("Owner-only private storage is unsupported for " + path, unsupported); + } + } + + /** Package-visible for deterministic ACL-provider tests. */ + static void enforceWithAcl(AclFileAttributeView view, Path path) throws IOException { + try { + if (view == null) throw new IOException("Owner-only private storage is unsupported for " + path); + UserPrincipal owner = view.getOwner(); + if (owner == null || owner instanceof GroupPrincipal) + throw new IOException("Private storage owner is not an individual account for " + path); + AclEntry ownerEntry = AclEntry.newBuilder().setType(AclEntryType.ALLOW).setPrincipal(owner) + .setPermissions(OWNER_ACL).build(); + view.setAcl(List.of(ownerEntry)); + List acl = view.getAcl(); + if (acl.size() != 1 || !isVerifiedOwnerEntry(acl.get(0), owner)) + throw new IOException("Could not verify owner-only ACL permissions for " + path); + } catch (UnsupportedOperationException unsupported) { + throw new IOException("Owner-only private storage is unsupported for " + path, unsupported); + } + } + + private static boolean isVerifiedOwnerEntry(AclEntry entry, UserPrincipal owner) { + return entry.type() == AclEntryType.ALLOW && owner.equals(entry.principal()) && entry.flags().isEmpty() + && entry.permissions().equals(OWNER_ACL); + } +} diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpClientCredentialStore.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpClientCredentialStore.java index 2928461..3fddf65 100644 --- a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpClientCredentialStore.java +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpClientCredentialStore.java @@ -7,15 +7,14 @@ import java.nio.file.Path; import java.nio.file.StandardCopyOption; import java.nio.file.StandardOpenOption; -import java.nio.file.attribute.PosixFilePermission; import java.net.URI; import java.security.KeyStore; import java.security.PrivateKey; import java.security.cert.X509Certificate; import java.util.Base64; -import java.util.EnumSet; import java.util.Properties; import com.bencodez.simpleapi.file.DurableFiles; +import com.bencodez.simpleapi.file.PrivateFilePermissions; /** Owner-only persistence for the client certificate bundle returned by enrollment. */ public final class HttpClientCredentialStore { @@ -72,6 +71,8 @@ private static ClientCredential loadCredential(Path directory) throws Exception Path passwordFile = safe(directory.resolve(PASSWORD_FILE)); if (!Files.isRegularFile(bundle, LinkOption.NOFOLLOW_LINKS) || !Files.isRegularFile(passwordFile, LinkOption.NOFOLLOW_LINKS)) throw new IOException("HTTP client certificate has not been enrolled"); + PrivateFilePermissions.ownerOnlyFile(bundle); + PrivateFilePermissions.ownerOnlyFile(passwordFile); long passwordSize = Files.size(passwordFile); if (passwordSize < 40L || passwordSize > 128L) throw new IOException("HTTP client password is invalid"); byte[] passwordBytes; @@ -110,14 +111,14 @@ private static StagedCredential stage(Path directory, HttpTlsIdentity.IssuedClie Files.createDirectories(generations); if (Files.isSymbolicLink(generations) || !Files.isDirectory(generations, LinkOption.NOFOLLOW_LINKS)) throw new IOException("HTTP credential generation directory is unsafe"); - setOwnerOnlyDirectory(generations); + PrivateFilePermissions.ownerOnlyDirectory(generations); // Retry publication durability if an earlier staging attempt left this // directory behind after its parent fsync failed. DurableFiles.forceDirectory(credentialDirectory); String name = java.util.UUID.randomUUID().toString(); Path generation = generations.resolve(name); Files.createDirectory(generation); - setOwnerOnlyDirectory(generation); + PrivateFilePermissions.ownerOnlyDirectory(generation); try { save(generation, issued); ClientCredential replacement = loadCredential(generation); @@ -329,6 +330,7 @@ private static Path activeDirectory(Path directory) throws IOException { Path generation = generations.resolve(name).normalize(); if (!generation.getParent().equals(generations) || Files.isSymbolicLink(generation) || !Files.isDirectory(generation, LinkOption.NOFOLLOW_LINKS)) throw new IOException("HTTP client credential generation is invalid"); + PrivateFilePermissions.ownerOnlyDirectory(generation); return generation; } @@ -349,8 +351,8 @@ private static Path credentialRoot(Path directory, boolean create) throws IOExce if (create) Files.createDirectories(root); if (Files.isSymbolicLink(root) || !Files.isDirectory(root, LinkOption.NOFOLLOW_LINKS)) throw new IOException("HTTP credential directory is unsafe"); + PrivateFilePermissions.ownerOnlyDirectory(root); if (create) { - setOwnerOnlyDirectory(root); // Existing can mean create succeeded but publishing it durably did not. DurableFiles.forceDirectory(root.getParent()); } @@ -378,13 +380,13 @@ private static void restoreConnectionCodeDigest(Path directory, String digest) t private static void writePrivate(Path file, byte[] contents) throws IOException { Path temporary = Files.createTempFile(file.getParent(), file.getFileName().toString(), ".tmp"); try { - setOwnerOnly(temporary); + PrivateFilePermissions.ownerOnlyFile(temporary); Files.write(temporary, contents, StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING); DurableFiles.forceFile(temporary); try { Files.move(temporary, file, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); } catch (java.nio.file.AtomicMoveNotSupportedException unsupported) { Files.move(temporary, file, StandardCopyOption.REPLACE_EXISTING); } try { - setOwnerOnly(file); + PrivateFilePermissions.ownerOnlyFile(file); DurableFiles.forceDirectory(file.getParent()); } catch (IOException postPublicationFailure) { throw new DurableFiles.PublishedException(postPublicationFailure); @@ -392,17 +394,6 @@ private static void writePrivate(Path file, byte[] contents) throws IOException } finally { Files.deleteIfExists(temporary); } } - private static void setOwnerOnly(Path path) throws IOException { - try { Files.setPosixFilePermissions(path, EnumSet.of(PosixFilePermission.OWNER_READ, PosixFilePermission.OWNER_WRITE)); } - catch (UnsupportedOperationException ignored) { } - } - - private static void setOwnerOnlyDirectory(Path path) throws IOException { - try { Files.setPosixFilePermissions(path, EnumSet.of(PosixFilePermission.OWNER_READ, - PosixFilePermission.OWNER_WRITE, PosixFilePermission.OWNER_EXECUTE)); } - catch (UnsupportedOperationException ignored) { } - } - private static byte[] asciiBytes(char[] characters) { byte[] output = new byte[characters.length]; for (int index = 0; index < characters.length; index++) output[index] = (byte) characters[index]; diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentAuthority.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentAuthority.java index 912aafe..83e4b25 100644 --- a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentAuthority.java +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentAuthority.java @@ -7,7 +7,6 @@ import java.nio.file.Path; import java.nio.file.StandardCopyOption; import java.nio.file.StandardOpenOption; -import java.nio.file.attribute.PosixFilePermission; import java.time.Clock; import java.time.Duration; import java.time.Instant; @@ -15,8 +14,8 @@ import java.util.Map; import java.util.Properties; import java.util.Base64; -import java.util.EnumSet; import com.bencodez.simpleapi.file.DurableFiles; +import com.bencodez.simpleapi.file.PrivateFilePermissions; /** * Single-activation enrollment tokens and client-certificate binding. A token may retry certificate @@ -195,6 +194,7 @@ private synchronized void loadState() throws java.io.IOException { if (stateFile == null || !Files.exists(stateFile, LinkOption.NOFOLLOW_LINKS)) return; if (!Files.isRegularFile(stateFile, LinkOption.NOFOLLOW_LINKS) || Files.size(stateFile) > MAX_STATE_BYTES) throw new java.io.IOException("HTTP enrollment state is invalid"); + PrivateFilePermissions.ownerOnlyFile(stateFile); Properties properties = new Properties(); try (var input = Files.newInputStream(stateFile, LinkOption.NOFOLLOW_LINKS)) { properties.load(input); } String version = properties.getProperty("version"); @@ -271,13 +271,13 @@ private synchronized void persistState() throws java.io.IOException { if (bytes.size() > MAX_STATE_BYTES) throw new java.io.IOException("HTTP enrollment state exceeds its byte bound"); Path temporary = Files.createTempFile(stateFile.getParent(), stateFile.getFileName().toString(), ".tmp"); try { - setOwnerOnly(temporary); + PrivateFilePermissions.ownerOnlyFile(temporary); Files.write(temporary, bytes.toByteArray(), StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING); DurableFiles.forceFile(temporary); try { Files.move(temporary, stateFile, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); } catch (java.nio.file.AtomicMoveNotSupportedException unsupported) { Files.move(temporary, stateFile, StandardCopyOption.REPLACE_EXISTING); } try { - setOwnerOnly(stateFile); + PrivateFilePermissions.ownerOnlyFile(stateFile); DurableFiles.forceDirectory(stateFile.getParent()); } catch (java.io.IOException postPublicationFailure) { throw new DurableFiles.PublishedException(postPublicationFailure); @@ -291,7 +291,7 @@ private static Path stateFile(Path directory) throws java.io.IOException { Files.createDirectories(stateDirectory); if (Files.isSymbolicLink(stateDirectory) || !Files.isDirectory(stateDirectory, LinkOption.NOFOLLOW_LINKS)) throw new java.io.IOException("HTTP enrollment state directory is unsafe"); - setOwnerOnlyDirectory(stateDirectory); + PrivateFilePermissions.ownerOnlyDirectory(stateDirectory); // Existing can mean a previous create succeeded but its parent fsync did not. DurableFiles.forceDirectory(stateDirectory.getParent()); Path file = stateDirectory.resolve("http-transport-clients.properties"); @@ -299,17 +299,6 @@ private static Path stateFile(Path directory) throws java.io.IOException { return file; } - private static void setOwnerOnly(Path path) throws java.io.IOException { - try { Files.setPosixFilePermissions(path, EnumSet.of(PosixFilePermission.OWNER_READ, PosixFilePermission.OWNER_WRITE)); } - catch (UnsupportedOperationException ignored) { } - } - - private static void setOwnerOnlyDirectory(Path path) throws java.io.IOException { - try { Files.setPosixFilePermissions(path, EnumSet.of(PosixFilePermission.OWNER_READ, - PosixFilePermission.OWNER_WRITE, PosixFilePermission.OWNER_EXECUTE)); } - catch (UnsupportedOperationException ignored) { } - } - private void expireEnrollments() { Instant now = clock.instant(); enrollments.entrySet().removeIf(entry -> !entry.getValue().expiresAt().isAfter(now)); diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpInboundDeliveryStore.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpInboundDeliveryStore.java index e9444a3..f72d14a 100644 --- a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpInboundDeliveryStore.java +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpInboundDeliveryStore.java @@ -1,6 +1,7 @@ package com.bencodez.simpleapi.servercomm.http; import com.bencodez.simpleapi.file.DurableFiles; +import com.bencodez.simpleapi.file.PrivateFilePermissions; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.nio.file.DirectoryStream; @@ -9,8 +10,6 @@ import java.nio.file.Path; import java.nio.file.StandardCopyOption; import java.nio.file.StandardOpenOption; -import java.nio.file.attribute.PosixFilePermission; -import java.util.EnumSet; import java.util.HashSet; import java.util.LinkedHashMap; import java.util.Map; @@ -45,13 +44,13 @@ private HttpInboundDeliveryStore(Path parent, String directoryName) throws IOExc throw new IOException("HTTP credential directory is unsafe"); if (directoryName == null || !directoryName.matches("[A-Za-z0-9][A-Za-z0-9._-]{0,63}")) throw new IOException("HTTP inbound delivery directory name is invalid"); - ownerOnlyDirectory(credentials); + PrivateFilePermissions.ownerOnlyDirectory(credentials); root = credentials.resolve(directoryName).normalize(); if (!root.getParent().equals(credentials)) throw new IOException("HTTP inbound delivery directory is invalid"); try { Files.createDirectory(root); } catch (java.nio.file.FileAlreadyExistsException existing) { } requireRoot(); - ownerOnlyDirectory(root); + PrivateFilePermissions.ownerOnlyDirectory(root); // Retry a parent fsync that may have failed after creating this root. DurableFiles.forceDirectory(credentials); load(); @@ -75,12 +74,12 @@ synchronized void reserve(String id) throws IOException { throw new IOException("HTTP inbound delivery fence is inconsistent"); Path temporary = Files.createTempFile(root, ".pending-", ".tmp"); try { - ownerOnlyFile(temporary); + PrivateFilePermissions.ownerOnlyFile(temporary); Files.writeString(temporary, id, StandardCharsets.US_ASCII, StandardOpenOption.TRUNCATE_EXISTING); DurableFiles.forceFile(temporary); move(temporary, target); try { - ownerOnlyFile(target); + PrivateFilePermissions.ownerOnlyFile(target); DurableFiles.forceDirectory(root); } catch (IOException postPublicationFailure) { entries.put(id, State.RESERVED); @@ -105,12 +104,12 @@ synchronized void recordCompleted(String id) throws IOException { Path target = file(id, State.COMPLETED); Path temporary = Files.createTempFile(root, ".pending-", ".tmp"); try { - ownerOnlyFile(temporary); + PrivateFilePermissions.ownerOnlyFile(temporary); Files.writeString(temporary, id, StandardCharsets.US_ASCII, StandardOpenOption.TRUNCATE_EXISTING); DurableFiles.forceFile(temporary); move(temporary, target); try { - ownerOnlyFile(target); + PrivateFilePermissions.ownerOnlyFile(target); DurableFiles.forceDirectory(root); } catch (IOException postPublicationFailure) { @@ -207,7 +206,7 @@ synchronized boolean recoverKnownNotStartedRunning(String id) throws IOException if (hasRunning) { verifyStateFile(running, id); move(running, reserved); - ownerOnlyFile(reserved); + PrivateFilePermissions.ownerOnlyFile(reserved); } else verifyStateFile(reserved, id); DurableFiles.forceDirectory(root); entries.put(id, State.RESERVED); @@ -258,6 +257,7 @@ private void load() throws IOException { if (state == null || Files.isSymbolicLink(file) || !Files.isRegularFile(file, LinkOption.NOFOLLOW_LINKS) || Files.size(file) > 64L) throw new IOException("HTTP inbound delivery fence contains an invalid entry"); + PrivateFilePermissions.ownerOnlyFile(file); String id; try { id = canonical(name.substring(0, name.length() - state.suffix.length())); } catch (IllegalArgumentException invalid) { @@ -306,16 +306,6 @@ private void requireRoot() throws IOException { if (Files.isSymbolicLink(root) || !Files.isDirectory(root, LinkOption.NOFOLLOW_LINKS)) throw new IOException("HTTP inbound delivery directory is unsafe"); } - private static void ownerOnlyFile(Path path) throws IOException { - try { Files.setPosixFilePermissions(path, EnumSet.of(PosixFilePermission.OWNER_READ, PosixFilePermission.OWNER_WRITE)); } - catch (UnsupportedOperationException ignored) { } - } - private static void ownerOnlyDirectory(Path path) throws IOException { - try { Files.setPosixFilePermissions(path, EnumSet.of(PosixFilePermission.OWNER_READ, - PosixFilePermission.OWNER_WRITE, PosixFilePermission.OWNER_EXECUTE)); } - catch (UnsupportedOperationException ignored) { } - } - enum State { RESERVED(".reserved"), RUNNING(".running"), COMPLETED(".completed"); private final String suffix; diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java index dac27f2..fbcffef 100644 --- a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java @@ -2,6 +2,7 @@ import com.bencodez.simpleapi.servercomm.codec.JsonEnvelope; import com.bencodez.simpleapi.file.DurableFiles; +import com.bencodez.simpleapi.file.PrivateFilePermissions; import com.sun.net.httpserver.Headers; import com.sun.net.httpserver.HttpsConfigurator; import com.sun.net.httpserver.HttpsExchange; @@ -318,7 +319,7 @@ private static Path incomingRoot(Path outgoingDirectory) throws IOException { catch (java.nio.file.FileAlreadyExistsException existing) { } if (Files.isSymbolicLink(root) || !Files.isDirectory(root, LinkOption.NOFOLLOW_LINKS)) throw new IOException("HTTP incoming queue directory is invalid"); - DurableOutgoingQueue.ownerOnlyDirectory(root); + PrivateFilePermissions.ownerOnlyDirectory(root); // Retry publication durability even when an earlier attempt created the // directory but failed before its parent could be forced. DurableFiles.forceDirectory(parent); @@ -605,7 +606,7 @@ private DurableOutgoingQueue(Path root) throws IOException { catch (java.nio.file.FileAlreadyExistsException existing) { } if (Files.isSymbolicLink(this.root) || !Files.isDirectory(this.root, LinkOption.NOFOLLOW_LINKS)) throw new IOException("HTTP outgoing queue directory is invalid"); - ownerOnlyDirectory(this.root); + PrivateFilePermissions.ownerOnlyDirectory(this.root); // A failed parent fsync can leave the directory present but not durable. // Reopening must retry it before the queue can accept work. directoryForcer.force(this.root.getParent()); @@ -619,6 +620,7 @@ synchronized Map> load() throws IOE if (++serverDirectories > MAX_BACKENDS) throw new IOException("HTTP outgoing queue exceeds its backend bound"); if (Files.isSymbolicLink(directory) || !Files.isDirectory(directory, LinkOption.NOFOLLOW_LINKS)) throw new IOException("HTTP outgoing queue contains an invalid entry"); + PrivateFilePermissions.ownerOnlyDirectory(directory); String serverId; try { serverId = HttpTlsIdentity.canonicalServerId(directory.getFileName().toString()); } catch (IllegalArgumentException invalid) { throw new IOException("HTTP outgoing queue server is invalid", invalid); } @@ -644,6 +646,7 @@ synchronized Map> load() throws IOE if (Files.isSymbolicLink(message) || !Files.isRegularFile(message, LinkOption.NOFOLLOW_LINKS) || Files.size(message) > HttpTransportProtocol.MAX_ENVELOPE_BYTES * 2L) throw new IOException("HTTP outgoing queue quarantine is invalid"); + PrivateFilePermissions.ownerOnlyFile(message); HttpTransportProtocol.Delivery delivery; try { delivery = HttpTransportProtocol.parseStoredDelivery(Files.readAllBytes(message)); } catch (IllegalArgumentException invalid) { throw new IOException("HTTP outgoing queue quarantine is invalid", invalid); } @@ -657,6 +660,7 @@ synchronized Map> load() throws IOE if (Files.isSymbolicLink(message) || !Files.isRegularFile(message, LinkOption.NOFOLLOW_LINKS) || !name.matches(FILE_PATTERN) || Files.size(message) > HttpTransportProtocol.MAX_ENVELOPE_BYTES * 2L) throw new IOException("HTTP outgoing queue message is invalid"); + PrivateFilePermissions.ownerOnlyFile(message); HttpTransportProtocol.Delivery delivery; try { delivery = HttpTransportProtocol.parseStoredDelivery(Files.readAllBytes(message)); } catch (IllegalArgumentException invalid) { throw new IOException("HTTP outgoing queue message is invalid", invalid); } @@ -683,7 +687,7 @@ private synchronized void persist(String serverId, HttpTransportProtocol.Deliver catch (java.nio.file.FileAlreadyExistsException existing) { } if (Files.isSymbolicLink(directory) || !Files.isDirectory(directory, LinkOption.NOFOLLOW_LINKS)) throw new IOException("HTTP outgoing queue server directory is invalid"); - ownerOnlyDirectory(directory); + PrivateFilePermissions.ownerOnlyDirectory(directory); // The child fsync below cannot make this published name durable in its // parent. Repeat it so a prior failed attempt is recoverable. directoryForcer.force(root); @@ -695,7 +699,7 @@ private synchronized void persist(String serverId, HttpTransportProtocol.Deliver || Files.size(existing) > HttpTransportProtocol.MAX_ENVELOPE_BYTES * 2L || !Arrays.equals(Files.readAllBytes(existing), HttpTransportProtocol.storedDelivery(delivery))) throw new IOException("HTTP outgoing queue delivery id conflicts with persisted data"); - ownerOnlyFile(existing); + PrivateFilePermissions.ownerOnlyFile(existing); directoryForcer.force(directory); return; } @@ -710,7 +714,7 @@ private synchronized void persist(String serverId, HttpTransportProtocol.Deliver Path target = directory.resolve(name); try { Files.move(pending, target, StandardCopyOption.ATOMIC_MOVE); } catch (java.nio.file.AtomicMoveNotSupportedException unsupported) { Files.move(pending, target); } - try { ownerOnlyFile(target); directoryForcer.force(directory); } + try { PrivateFilePermissions.ownerOnlyFile(target); directoryForcer.force(directory); } catch (IOException postPublicationFailure) { quarantinePublished(directory, target, pending, serverFiles, quarantined, delivery.id(), postPublicationFailure); } @@ -725,12 +729,12 @@ private synchronized void persist(String serverId, HttpTransportProtocol.Deliver Path target = directory.resolve(name); Path temporary = Files.createTempFile(directory, ".pending-", ".tmp"); try { - ownerOnlyFile(temporary); + PrivateFilePermissions.ownerOnlyFile(temporary); Files.write(temporary, HttpTransportProtocol.storedDelivery(delivery), StandardOpenOption.TRUNCATE_EXISTING); DurableFiles.forceFile(temporary); try { Files.move(temporary, target, StandardCopyOption.ATOMIC_MOVE); } catch (java.nio.file.AtomicMoveNotSupportedException unsupported) { Files.move(temporary, target); } - try { ownerOnlyFile(target); directoryForcer.force(directory); } + try { PrivateFilePermissions.ownerOnlyFile(target); directoryForcer.force(directory); } catch (IOException postPublicationFailure) { pending = directory.resolve(".pending-" + delivery.id() + ".json"); quarantinePublished(directory, target, pending, serverFiles, quarantined, delivery.id(), postPublicationFailure); @@ -765,7 +769,7 @@ private void quarantinePublished(Path directory, Path target, Path pending, Map< // Record the observed rename before metadata writeback. If the force is // indeterminate, a same-process retry must still find this quarantine. quarantined.put(id, pending); - ownerOnlyFile(pending); + PrivateFilePermissions.ownerOnlyFile(pending); directoryForcer.force(directory); } catch (IOException quarantineFailure) { // A failed quarantine rename leaves the original published name in place on @@ -789,7 +793,7 @@ private synchronized void confirm(String serverId, String id) throws IOException Path file = serverFiles == null ? null : serverFiles.get(id); if (file == null || Files.isSymbolicLink(file) || !Files.isRegularFile(file, LinkOption.NOFOLLOW_LINKS)) throw new IOException("HTTP outgoing queue delivery is unavailable"); - ownerOnlyFile(file); + PrivateFilePermissions.ownerOnlyFile(file); directoryForcer.force(file.getParent()); } @@ -818,18 +822,5 @@ private synchronized void remove(String serverId, String id) throws IOException } } - private static void ownerOnlyFile(Path path) throws IOException { - try { Files.setPosixFilePermissions(path, java.util.EnumSet.of( - java.nio.file.attribute.PosixFilePermission.OWNER_READ, - java.nio.file.attribute.PosixFilePermission.OWNER_WRITE)); } - catch (UnsupportedOperationException ignored) { } - } - private static void ownerOnlyDirectory(Path path) throws IOException { - try { Files.setPosixFilePermissions(path, java.util.EnumSet.of( - java.nio.file.attribute.PosixFilePermission.OWNER_READ, - java.nio.file.attribute.PosixFilePermission.OWNER_WRITE, - java.nio.file.attribute.PosixFilePermission.OWNER_EXECUTE)); } - catch (UnsupportedOperationException ignored) { } - } } } diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpTlsIdentity.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpTlsIdentity.java index ba2babc..42771fc 100644 --- a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpTlsIdentity.java +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpTlsIdentity.java @@ -8,7 +8,6 @@ import java.nio.file.Path; import java.nio.file.StandardCopyOption; import java.nio.file.StandardOpenOption; -import java.nio.file.attribute.PosixFilePermission; import java.security.KeyPair; import java.security.KeyPairGenerator; import java.security.KeyStore; @@ -21,7 +20,6 @@ import java.time.Clock; import java.time.Duration; import java.util.Date; -import java.util.EnumSet; import java.util.Arrays; import java.util.Collection; import java.util.List; @@ -48,6 +46,7 @@ import org.bouncycastle.operator.ContentSigner; import org.bouncycastle.operator.jcajce.JcaContentSignerBuilder; import com.bencodez.simpleapi.file.DurableFiles; +import com.bencodez.simpleapi.file.PrivateFilePermissions; /** Durable private CA plus server identity used by the proxy HTTP listener. */ public final class HttpTlsIdentity { @@ -95,6 +94,7 @@ static HttpTlsIdentity loadOrCreate(Path directory, String advertisedHost, Clock Files.createDirectories(identityDirectory); if (Files.isSymbolicLink(identityDirectory) || !Files.isDirectory(identityDirectory, LinkOption.NOFOLLOW_LINKS)) throw new IOException("HTTP TLS identity directory is unsafe"); + PrivateFilePermissions.ownerOnlyDirectory(identityDirectory); // The identity files cannot make the newly created directory entry durable. // Persist its parent before the TLS identity is returned for listener use. DurableFiles.forceDirectory(identityDirectory.getParent()); @@ -123,6 +123,9 @@ static HttpTlsIdentity loadOrCreate(Path directory, String advertisedHost, Clock } if (caExists || serverExists || passwordExists) { if (!completeIdentity) throw new IOException("HTTP TLS identity files are incomplete"); + PrivateFilePermissions.ownerOnlyFile(caFile); + PrivateFilePermissions.ownerOnlyFile(serverFile); + PrivateFilePermissions.ownerOnlyFile(passwordFile); char[] password = readPassword(passwordFile); try { KeyStore ca = load(caFile, password); @@ -416,12 +419,12 @@ private static void writeStore(Path file, KeyStore store, char[] password) throw private static void writePrivate(Path file, byte[] contents) throws IOException { Path temporary = Files.createTempFile(file.getParent(), file.getFileName().toString(), ".tmp"); try { - setOwnerOnly(temporary); + PrivateFilePermissions.ownerOnlyFile(temporary); Files.write(temporary, contents, StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING); DurableFiles.forceFile(temporary); try { Files.move(temporary, file, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); } catch (java.nio.file.AtomicMoveNotSupportedException unsupported) { Files.move(temporary, file, StandardCopyOption.REPLACE_EXISTING); } - setOwnerOnly(file); + PrivateFilePermissions.ownerOnlyFile(file); DurableFiles.forceDirectory(file.getParent()); } finally { Files.deleteIfExists(temporary); } } @@ -447,11 +450,6 @@ private static byte[] asciiBytes(char[] characters) { return output; } - private static void setOwnerOnly(Path path) throws IOException { - try { Files.setPosixFilePermissions(path, EnumSet.of(PosixFilePermission.OWNER_READ, PosixFilePermission.OWNER_WRITE)); } - catch (UnsupportedOperationException ignored) { /* Windows ACLs are inherited; never make the file world-readable. */ } - } - private final class RotatingServerKeyManager extends X509ExtendedKeyManager { private static final String ALIAS = "server"; private void refresh() { diff --git a/SimpleAPI/src/test/java/com/bencodez/simpleapi/file/PrivateFilePermissionsTest.java b/SimpleAPI/src/test/java/com/bencodez/simpleapi/file/PrivateFilePermissionsTest.java new file mode 100644 index 0000000..dcad178 --- /dev/null +++ b/SimpleAPI/src/test/java/com/bencodez/simpleapi/file/PrivateFilePermissionsTest.java @@ -0,0 +1,180 @@ +package com.bencodez.simpleapi.file; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.CALLS_REAL_METHODS; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.AclEntry; +import java.nio.file.attribute.AclEntryPermission; +import java.nio.file.attribute.AclEntryType; +import java.nio.file.attribute.AclFileAttributeView; +import java.nio.file.attribute.GroupPrincipal; +import java.nio.file.attribute.PosixFileAttributeView; +import java.nio.file.attribute.PosixFilePermission; +import java.nio.file.attribute.UserPrincipal; +import java.util.EnumSet; +import java.util.List; +import java.util.Set; + +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class PrivateFilePermissionsTest { + + @TempDir + Path tempDir; + + @Test + void fileIsOwnerReadWriteOnlyOnPosixProvider() throws Exception { + Path file = Files.createFile(tempDir.resolve("secret")); + Assumptions.assumeTrue(Files.getFileAttributeView(file, PosixFileAttributeView.class) != null); + + PrivateFilePermissions.ownerOnlyFile(file); + + assertEquals(Set.of(PosixFilePermission.OWNER_READ, PosixFilePermission.OWNER_WRITE), + Files.getPosixFilePermissions(file)); + } + + @Test + void directoryIsOwnerReadWriteExecuteOnlyOnPosixProvider() throws Exception { + Path directory = Files.createDirectory(tempDir.resolve("private")); + Assumptions.assumeTrue(Files.getFileAttributeView(directory, PosixFileAttributeView.class) != null); + + PrivateFilePermissions.ownerOnlyDirectory(directory); + + assertEquals(Set.of(PosixFilePermission.OWNER_READ, PosixFilePermission.OWNER_WRITE, + PosixFilePermission.OWNER_EXECUTE), Files.getPosixFilePermissions(directory)); + } + + @Test + void aclReplacementLeavesOnlyOwnerAllowEntry() throws Exception { + Path file = tempDir.resolve("acl-secret"); + UserPrincipal owner = mock(UserPrincipal.class); + AclFileAttributeView acl = mock(AclFileAttributeView.class); + when(acl.getOwner()).thenReturn(owner); + when(acl.getAcl()).thenReturn(List.of(allow(owner), allow(mock(UserPrincipal.class)))); + try (var files = mockStatic(Files.class, CALLS_REAL_METHODS)) { + stubAcl(files, file, acl); + doAnswer(invocation -> { + when(acl.getAcl()).thenReturn(invocation.getArgument(0)); + return null; + }).when(acl).setAcl(org.mockito.ArgumentMatchers.anyList()); + + PrivateFilePermissions.ownerOnlyFile(file); + + var replacement = org.mockito.ArgumentCaptor.forClass(List.class); + verify(acl).setAcl(replacement.capture()); + assertEquals(List.of(allow(owner)), replacement.getValue()); + } + } + + @Test + void ignoredSetAclOrRemainingNonOwnerAccessIsRejected() throws Exception { + Path file = tempDir.resolve("bad-acl"); + UserPrincipal owner = mock(UserPrincipal.class); + AclFileAttributeView acl = mock(AclFileAttributeView.class); + when(acl.getOwner()).thenReturn(owner); + when(acl.getAcl()).thenReturn(List.of(allow(owner), allow(mock(UserPrincipal.class)))); + doAnswer(invocation -> null).when(acl).setAcl(org.mockito.ArgumentMatchers.anyList()); + try (var files = mockStatic(Files.class, CALLS_REAL_METHODS)) { + stubAcl(files, file, acl); + + assertThrows(IOException.class, () -> PrivateFilePermissions.ownerOnlyFile(file)); + } + } + + @Test + void missingAclAndPosixSupportIsRejected() throws Exception { + Path file = tempDir.resolve("unsupported"); + try (var files = mockStatic(Files.class, CALLS_REAL_METHODS)) { + files.when(() -> Files.setPosixFilePermissions(eq(file), org.mockito.ArgumentMatchers.anySet())) + .thenThrow(new UnsupportedOperationException()); + files.when(() -> Files.getFileAttributeView(file, PosixFileAttributeView.class)).thenReturn(null); + files.when(() -> Files.getFileAttributeView(file, AclFileAttributeView.class)).thenReturn(null); + files.when(() -> Files.getFileAttributeView(file, AclFileAttributeView.class, + java.nio.file.LinkOption.NOFOLLOW_LINKS)).thenReturn(null); + assertThrows(IOException.class, () -> PrivateFilePermissions.ownerOnlyFile(file)); + } + } + + @Test + void groupOwnerIsRejected() throws Exception { + Path file = tempDir.resolve("group-owner"); + AclFileAttributeView acl = mock(AclFileAttributeView.class); + when(acl.getOwner()).thenReturn(mock(GroupPrincipal.class)); + try (var files = mockStatic(Files.class, CALLS_REAL_METHODS)) { + stubAcl(files, file, acl); + assertThrows(IOException.class, () -> PrivateFilePermissions.ownerOnlyFile(file)); + } + } + + @Test + void aclReadbackFailureIsRejected() throws Exception { + Path file = tempDir.resolve("readback-failure"); + UserPrincipal owner = mock(UserPrincipal.class); + AclFileAttributeView acl = mock(AclFileAttributeView.class); + when(acl.getOwner()).thenReturn(owner); + when(acl.getAcl()).thenThrow(new IOException("readback")); + try (var files = mockStatic(Files.class, CALLS_REAL_METHODS)) { + stubAcl(files, file, acl); + assertThrows(IOException.class, () -> PrivateFilePermissions.ownerOnlyFile(file)); + } + } + + @Test + void unsupportedAclMutationIsRejectedAsIoException() throws Exception { + Path file = tempDir.resolve("acl-unsupported-mutation"); + UserPrincipal owner = mock(UserPrincipal.class); + AclFileAttributeView acl = mock(AclFileAttributeView.class); + when(acl.getOwner()).thenReturn(owner); + doAnswer(invocation -> { throw new UnsupportedOperationException("ACL mutation"); }) + .when(acl).setAcl(org.mockito.ArgumentMatchers.anyList()); + try (var files = mockStatic(Files.class, CALLS_REAL_METHODS)) { + stubAcl(files, file, acl); + assertThrows(IOException.class, () -> PrivateFilePermissions.ownerOnlyFile(file)); + } + } + + @Test + void symlinkIsRejectedWithoutChangingReferentPermissions() throws Exception { + Path target = Files.createFile(tempDir.resolve("referent")); + Assumptions.assumeTrue(Files.getFileAttributeView(target, PosixFileAttributeView.class) != null); + Set original = Set.of(PosixFilePermission.OWNER_READ, + PosixFilePermission.OWNER_WRITE, PosixFilePermission.GROUP_READ); + Files.setPosixFilePermissions(target, original); + Path link = tempDir.resolve("referent-link"); + try { + Files.createSymbolicLink(link, target.getFileName()); + } catch (UnsupportedOperationException | IOException unavailable) { + Assumptions.assumeTrue(false, "symbolic links unavailable: " + unavailable); + return; + } + + assertThrows(IOException.class, () -> PrivateFilePermissions.ownerOnlyFile(link)); + assertEquals(original, Files.getPosixFilePermissions(target)); + } + + private static AclEntry allow(UserPrincipal principal) { + return AclEntry.newBuilder().setType(AclEntryType.ALLOW).setPrincipal(principal) + .setPermissions(EnumSet.allOf(AclEntryPermission.class)).setFlags(Set.of()).build(); + } + + private static void stubAcl(org.mockito.MockedStatic files, Path path, AclFileAttributeView acl) { + files.when(() -> Files.setPosixFilePermissions(eq(path), org.mockito.ArgumentMatchers.anySet())) + .thenThrow(new UnsupportedOperationException()); + files.when(() -> Files.getFileAttributeView(path, PosixFileAttributeView.class)).thenReturn(null); + files.when(() -> Files.getFileAttributeView(path, AclFileAttributeView.class)).thenReturn(acl); + files.when(() -> Files.getFileAttributeView(path, AclFileAttributeView.class, + java.nio.file.LinkOption.NOFOLLOW_LINKS)).thenReturn(acl); + } +} From 99ebcc64cd6376a45583326e687f2440cd4f97a7 Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Sat, 5 Sep 2026 21:19:20 -0600 Subject: [PATCH 20/38] Confirm credential activation and safely finish callback shutdown --- .../http/HttpBackendTransportConnector.java | 97 ++++++++++--- .../http/HttpProxyTransportServer.java | 117 +++++++++++++-- .../HttpBackendLifecycleRecoveryTest.java | 51 +++++++ .../http/HttpProxyLifecycleRecoveryTest.java | 136 ++++++++++++++++++ .../http/HttpTransportRuntimeTest.java | 17 ++- 5 files changed, 384 insertions(+), 34 deletions(-) create mode 100644 SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpBackendLifecycleRecoveryTest.java create mode 100644 SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpProxyLifecycleRecoveryTest.java diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpBackendTransportConnector.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpBackendTransportConnector.java index 963c5c4..76ea61f 100644 --- a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpBackendTransportConnector.java +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpBackendTransportConnector.java @@ -58,7 +58,10 @@ public final class HttpBackendTransportConnector implements AutoCloseable { private final URI transportEndpoint; private final ThreadPoolExecutor callbackExecutor; private final AtomicBoolean running = new AtomicBoolean(); + private final AtomicBoolean closing = new AtomicBoolean(); + private final CountDownLatch closed = new CountDownLatch(1); private final CountDownLatch firstResponse = new CountDownLatch(1); + private final Object lifecycle = new Object(); private final Object state = new Object(); private final Object renewal = new Object(); private final LinkedHashMap outgoing = new LinkedHashMap<>(); @@ -67,11 +70,13 @@ public final class HttpBackendTransportConnector implements AutoCloseable { private final ArrayDeque acknowledgementConfirmations = new ArrayDeque<>(); private final String session = UUID.randomUUID().toString(); private volatile Thread poller; + // Set by the executor's worker wrapper. Thread names are not an ownership boundary. + private volatile Thread callbackWorker; private long sequence; private volatile long nextRenewalCheckNanos; // Guarded by renewal: do not supersede a visibly published credential until // the pointer selecting it has been confirmed durable. - private HttpClientCredentialStore.StagedCredential pendingActivation; + private PendingActivation pendingActivation; /** In-memory test constructor; production transport must use a directory-backed constructor. */ HttpBackendTransportConnector(HttpConnectionCode code, String serverId, @@ -123,7 +128,7 @@ private HttpBackendTransportConnector(HttpClientCredentialStore.HttpClientProfil // GlobalMessageHandler routes mutate backend vote state and must observe the // wire order. One bounded lane preserves batch ordering without running work on // the long-poll thread; bounded admission below backpressures this poller. - callbackExecutor = executor("SimpleAPI-HTTP-callback", 1, CALLBACK_QUEUE_CAPACITY); + callbackExecutor = callbackExecutor(); } /** Convenience constructor for the owner-only credential directory produced by {@link #enroll}. */ @@ -156,8 +161,12 @@ public static HttpClientCredentialStore.ClientCredential enroll(HttpConnectionCo } public void start() { - if (!running.compareAndSet(false, true)) return; - poller = new Thread(this::pollLoop, "SimpleAPI-HTTP-poll"); poller.setDaemon(true); poller.start(); + synchronized (lifecycle) { + if (closing.get() || !running.compareAndSet(false, true)) return; + poller = new Thread(this::pollLoop, "SimpleAPI-HTTP-poll"); + poller.setDaemon(true); + poller.start(); + } } /** Waits for one authenticated, protocol-valid transport response. */ public boolean awaitFirstResponse(long deadlineNanos) throws InterruptedException { @@ -246,19 +255,58 @@ public boolean flushOutgoing(long deadlineNanos) { } return true; } + /** + * Stops polling and drains callbacks before sealing their journals. When called by the + * callback worker itself, terminal cleanup is deferred until that callback returns so + * its COMPLETED transition cannot be sealed out from under it. + */ @Override public void close() { - running.getAndSet(false); - firstResponse.countDown(); - Thread current = poller; if (current != null) current.interrupt(); + boolean calledByCallbackWorker = Thread.currentThread() == callbackWorker; + Thread current; + boolean alreadyClosing; + synchronized (lifecycle) { + alreadyClosing = !closing.compareAndSet(false, true); + if (alreadyClosing) current = null; + else { + running.set(false); + firstResponse.countDown(); + current = poller; + if (current != null) current.interrupt(); + } + } + if (alreadyClosing) { + if (!calledByCallbackWorker) awaitClosed(); + return; + } + // Closing admission before joining the poller releases an executeOrdered call + // that is backpressured behind this callback. + callbackExecutor.shutdown(); // The owning transport may release the credential-directory semaphore as soon // as close returns. Wait for the interrupted poller so an in-flight renewal // cannot activate an old credential generation after that ownership handoff. joinPoller(current); - shutdownCallbacks(); + if (calledByCallbackWorker) { + Thread cleanup = new Thread(this::finishClose, "SimpleAPI-HTTP-callback-close"); + cleanup.setDaemon(true); + cleanup.start(); + return; + } + finishClose(); + } + private void finishClose() { + try { + shutdownCallbacks(); // No poller can enqueue more callbacks and every running journal transition has // finished, so ownership can now be revoked without stranding completed work. - if (inboundDeliveries != null) inboundDeliveries.seal(); - if (acknowledgementConfirmationStore != null) acknowledgementConfirmationStore.seal(); + if (inboundDeliveries != null) inboundDeliveries.seal(); + if (acknowledgementConfirmationStore != null) acknowledgementConfirmationStore.seal(); + } finally { closed.countDown(); } + } + private void awaitClosed() { + boolean interrupted = false; + while (closed.getCount() != 0L) try { closed.await(); } + catch (InterruptedException stopRequested) { interrupted = true; } + if (interrupted) Thread.currentThread().interrupt(); } private void shutdownCallbacks() { callbackExecutor.shutdown(); @@ -397,7 +445,18 @@ private boolean confirmSentAcknowledgementConfirmations(Collection ids) List drainAcknowledgements() { synchronized (state) { return drain(acknowledgements); } } private static List first(Collection values) { List output = new java.util.ArrayList<>(); for (T value : values) { output.add(value); if (output.size() == HttpTransportProtocol.MAX_BATCH) break; } return output; } private static List drain(ArrayDeque values) { List output = new java.util.ArrayList<>(); while (!values.isEmpty() && output.size() < HttpTransportProtocol.MAX_BATCH) output.add(values.remove()); return output; } - private static ThreadPoolExecutor executor(String name, int threads, int queue) { ThreadFactory factory = task -> { Thread thread = new Thread(task, name); thread.setDaemon(true); return thread; }; return new ThreadPoolExecutor(threads, threads, 0L, TimeUnit.MILLISECONDS, new ArrayBlockingQueue<>(queue), factory, new ThreadPoolExecutor.AbortPolicy()); } + private ThreadPoolExecutor callbackExecutor() { + ThreadFactory factory = task -> { + Thread thread = new Thread(() -> { + callbackWorker = Thread.currentThread(); + task.run(); + }, "SimpleAPI-HTTP-callback"); + thread.setDaemon(true); + return thread; + }; + return new ThreadPoolExecutor(1, 1, 0L, TimeUnit.MILLISECONDS, + new ArrayBlockingQueue<>(CALLBACK_QUEUE_CAPACITY), factory, new ThreadPoolExecutor.AbortPolicy()); + } static boolean executeOrdered(ThreadPoolExecutor executor, Runnable task) { try { executor.execute(task); return true; } catch (RejectedExecutionException fullOrClosed) { @@ -433,7 +492,10 @@ private void maybeRenewCredential() { if (directory == null) return; if (pendingActivation != null) { try { - HttpClientCredentialStore.activateReplacement(directory, pendingActivation); + HttpClientCredentialStore.activateReplacement(directory, pendingActivation.staged()); + profile = pendingActivation.staged().profile(); + client = pendingActivation.client(); + credential = pendingActivation.staged().credential(); pendingActivation = null; } catch (IOException unconfirmed) { return; } } @@ -459,10 +521,12 @@ private void maybeRenewCredential() { if (!matchesCredential(replacementProfile, replacement)) throw new IllegalArgumentException("Renewed HTTP certificate is invalid"); HttpClient replacementClient = client(replacementProfile, replacement); try { HttpClientCredentialStore.activateReplacement(directory, staged); } - catch (com.bencodez.simpleapi.file.DurableFiles.PublishedException published) { - // CURRENT already selects this generation. Adopt it in memory as well, - // and retry this publication before requesting any newer certificate. - pendingActivation = staged; + catch (IOException unconfirmed) { + // A rename may already be visible while its parent fsync is unresolved. + // Keep using the restart-safe old identity and retry precisely this + // generation; requesting another renewal could revoke both identities. + pendingActivation = new PendingActivation(staged, replacementClient); + return; } profile = replacementProfile; client = replacementClient; @@ -474,6 +538,7 @@ private void maybeRenewCredential() { } } } + private record PendingActivation(HttpClientCredentialStore.StagedCredential staged, HttpClient client) { } static Duration renewalRetryDelay(Duration remainingValidity) { if (remainingValidity == null || remainingValidity.isNegative() || remainingValidity.isZero()) return Duration.ofSeconds(1); diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java index fbcffef..e3c20fa 100644 --- a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java @@ -40,6 +40,7 @@ import java.util.concurrent.ThreadFactory; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; import java.util.function.LongSupplier; import javax.net.ssl.SSLParameters; @@ -70,6 +71,8 @@ public final class HttpProxyTransportServer implements AutoCloseable { private final HttpsServer server; private final ThreadPoolExecutor listenerExecutor; private final ThreadPoolExecutor handlerExecutor; + private final AtomicReference handlerWorker = new AtomicReference<>(); + private final Object closeMonitor = new Object(); private final Semaphore admission = new Semaphore(64); private final Map backends = new HashMap<>(); private final DurableOutgoingQueue durableOutgoing; @@ -78,6 +81,7 @@ public final class HttpProxyTransportServer implements AutoCloseable { private final DeliveryAcknowledgement onAcknowledged; private final LongSupplier nanoTime; private volatile boolean closed; + private boolean closeFinalizing, closeFinalized; /** In-memory constructor for tests; production callers must supply a durable state directory. */ HttpProxyTransportServer(InetSocketAddress bind, HttpTlsIdentity identity, HttpEnrollmentAuthority authority, @@ -124,7 +128,8 @@ public HttpProxyTransportServer(InetSocketAddress bind, HttpTlsIdentity identity listenerExecutor = executor("SimpleAPI-HTTP-listener", 72, 72); // The proxy router mutates shared presence, vote, and reward state. A separate // bounded FIFO lane keeps wire order without blocking long-poll workers. - handlerExecutor = executor("SimpleAPI-HTTP-handler", 1, HttpBackendTransportConnector.CALLBACK_QUEUE_CAPACITY); + handlerExecutor = executor("SimpleAPI-HTTP-handler", 1, + HttpBackendTransportConnector.CALLBACK_QUEUE_CAPACITY, handlerWorker); server.setExecutor(listenerExecutor); server.createContext("/v1/enroll", exchange -> enroll((HttpsExchange) exchange)); server.createContext("/v1/renew", exchange -> renew((HttpsExchange) exchange)); @@ -212,9 +217,47 @@ private boolean send(String serverId, String deliveryId, JsonEnvelope envelope, } @Override public void close() { - if (closed) return; closed = true; server.stop(1); - shutdown(handlerExecutor); shutdown(listenerExecutor); - synchronized (backends) { for (BackendState backend : backends.values()) { backend.seal(); backend.signal(); } backends.clear(); } + boolean callbackWorker = Thread.currentThread() == handlerWorker.get(); + boolean finalizeHere = false; + synchronized (closeMonitor) { + if (!closed) { + closed = true; + server.stop(1); + closeFinalizing = true; + if (callbackWorker) { + Thread finalizer = new Thread(this::finishClose, "SimpleAPI-HTTP-proxy-close"); + finalizer.setDaemon(true); + finalizer.start(); + return; + } + finalizeHere = true; + } else if (callbackWorker || closeFinalized) { + return; + } + } + if (finalizeHere) finishClose(); else awaitClose(); + } + + private void finishClose() { + try { + shutdown(handlerExecutor); + shutdown(listenerExecutor); + synchronized (backends) { + for (BackendState backend : backends.values()) { backend.seal(); backend.signal(); } + backends.clear(); + } + } finally { + synchronized (closeMonitor) { closeFinalizing = false; closeFinalized = true; closeMonitor.notifyAll(); } + } + } + + private void awaitClose() { + boolean interrupted = false; + synchronized (closeMonitor) { + while (closeFinalizing && !closeFinalized) try { closeMonitor.wait(); } + catch (InterruptedException stopRequested) { interrupted = true; } + } + if (interrupted) Thread.currentThread().interrupt(); } private void enroll(HttpsExchange exchange) throws IOException { @@ -358,7 +401,17 @@ static int fixedBodyErrorStatus(Headers headers, int maximum) { } catch (NumberFormatException invalid) { return 400; } } private static ThreadPoolExecutor executor(String name, int threads, int queue) { - ThreadFactory factory = task -> { Thread thread = new Thread(task, name); thread.setDaemon(true); return thread; }; + return executor(name, threads, queue, null); + } + private static ThreadPoolExecutor executor(String name, int threads, int queue, AtomicReference worker) { + ThreadFactory factory = task -> { + Thread thread = new Thread(() -> { + if (worker != null) worker.set(Thread.currentThread()); + task.run(); + }, name); + thread.setDaemon(true); + return thread; + }; return new ThreadPoolExecutor(threads, threads, 0L, TimeUnit.MILLISECONDS, new ArrayBlockingQueue<>(queue), factory, new ThreadPoolExecutor.AbortPolicy()); } private static void setDefault(String name, String value) { if (System.getProperty(name) == null) System.setProperty(name, value); } @@ -617,7 +670,6 @@ synchronized Map> load() throws IOE int serverDirectories = 0; try (DirectoryStream servers = Files.newDirectoryStream(root)) { for (Path directory : servers) { - if (++serverDirectories > MAX_BACKENDS) throw new IOException("HTTP outgoing queue exceeds its backend bound"); if (Files.isSymbolicLink(directory) || !Files.isDirectory(directory, LinkOption.NOFOLLOW_LINKS)) throw new IOException("HTTP outgoing queue contains an invalid entry"); PrivateFilePermissions.ownerOnlyDirectory(directory); @@ -672,6 +724,14 @@ synchronized Map> load() throws IOE throw new IOException("HTTP outgoing queue exceeds its bound"); sequence = Math.max(sequence, Long.parseLong(name.substring(0, 20))); } + if (durableEntries == 0 && hasNoIndexedDeliveries(serverId)) { + deleteVerifiedEmptyDirectory(directory); + files.remove(serverId); + quarantinedFiles.remove(serverId); + continue; + } + if (++serverDirectories > MAX_BACKENDS) + throw new IOException("HTTP outgoing queue exceeds its backend bound"); if (!deliveries.isEmpty()) loaded.put(serverId, deliveries); } } @@ -683,14 +743,21 @@ private synchronized void persist(String serverId, HttpTransportProtocol.Deliver if (!directory.getParent().equals(root)) throw new IOException("HTTP outgoing queue server is invalid"); if (!Files.exists(directory, LinkOption.NOFOLLOW_LINKS) && serverDirectoryCount() >= MAX_BACKENDS) throw new IOException("HTTP outgoing queue exceeds its backend bound"); - try { Files.createDirectory(directory); } + boolean created = false; + try { Files.createDirectory(directory); created = true; } catch (java.nio.file.FileAlreadyExistsException existing) { } - if (Files.isSymbolicLink(directory) || !Files.isDirectory(directory, LinkOption.NOFOLLOW_LINKS)) - throw new IOException("HTTP outgoing queue server directory is invalid"); - PrivateFilePermissions.ownerOnlyDirectory(directory); - // The child fsync below cannot make this published name durable in its - // parent. Repeat it so a prior failed attempt is recoverable. - directoryForcer.force(root); + try { + if (Files.isSymbolicLink(directory) || !Files.isDirectory(directory, LinkOption.NOFOLLOW_LINKS)) + throw new IOException("HTTP outgoing queue server directory is invalid"); + PrivateFilePermissions.ownerOnlyDirectory(directory); + // The child fsync below cannot make this published name durable in its + // parent. Repeat it so a prior failed attempt is recoverable. + directoryForcer.force(root); + } catch (IOException setupFailure) { + if (created) try { deleteVerifiedEmptyDirectory(directory); } + catch (IOException cleanupFailure) { setupFailure.addSuppressed(cleanupFailure); } + throw setupFailure; + } Map serverFiles = files.computeIfAbsent(serverId, ignored -> new HashMap<>()); Map quarantined = quarantinedFiles.computeIfAbsent(serverId, ignored -> new HashMap<>()); Path existing = serverFiles.get(delivery.id()); @@ -755,12 +822,36 @@ private int serverDirectoryCount() throws IOException { catch (IllegalArgumentException invalid) { throw new IOException("HTTP outgoing queue server is invalid", invalid); } if (!serverId.equals(directory.getFileName().toString())) throw new IOException("HTTP outgoing queue server is not canonical"); + PrivateFilePermissions.ownerOnlyDirectory(directory); + if (isEmptyDirectory(directory) && hasNoIndexedDeliveries(serverId)) { + deleteVerifiedEmptyDirectory(directory); + continue; + } if (++count > MAX_BACKENDS) throw new IOException("HTTP outgoing queue exceeds its backend bound"); } } return count; } + private boolean isEmptyDirectory(Path directory) throws IOException { + try (DirectoryStream entries = Files.newDirectoryStream(directory)) { return !entries.iterator().hasNext(); } + } + + private boolean hasNoIndexedDeliveries(String serverId) { + Map serverFiles = files.get(serverId); + Map quarantined = quarantinedFiles.get(serverId); + return (serverFiles == null || serverFiles.isEmpty()) && (quarantined == null || quarantined.isEmpty()); + } + + /** Deletes only a validated, observed-empty backend directory and makes its removal durable. */ + private void deleteVerifiedEmptyDirectory(Path directory) throws IOException { + if (Files.isSymbolicLink(directory) || !Files.isDirectory(directory, LinkOption.NOFOLLOW_LINKS) + || !isEmptyDirectory(directory)) + throw new IOException("HTTP outgoing queue server directory is no longer empty"); + Files.delete(directory); + directoryForcer.force(root); + } + private void quarantinePublished(Path directory, Path target, Path pending, Map serverFiles, Map quarantined, String id, IOException publicationFailure) throws IOException { try { diff --git a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpBackendLifecycleRecoveryTest.java b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpBackendLifecycleRecoveryTest.java new file mode 100644 index 0000000..e8c6c44 --- /dev/null +++ b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpBackendLifecycleRecoveryTest.java @@ -0,0 +1,51 @@ +package com.bencodez.simpleapi.servercomm.http; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.bencodez.simpleapi.servercomm.codec.JsonEnvelope; +import java.net.URI; +import java.nio.file.Path; +import java.time.Instant; +import java.util.UUID; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class HttpBackendLifecycleRecoveryTest { + @TempDir Path directory; + + @Test + void callbackOwnedCloseCompletesItsJournalBeforeAsynchronousSealing() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); + HttpTlsIdentity.IssuedClientCertificate issued = identity.issueClientCertificate("lobby-1"); + HttpConnectionCode code = new HttpConnectionCode("lobby-1", URI.create("https://localhost:8443/"), + identity.serverCertificatePin(), identity.caCertificatePin(), Instant.now().plusSeconds(300), "A".repeat(43)); + Path clientDirectory = directory.resolve("client"); + HttpClientCredentialStore.saveEnrolled(clientDirectory, code, issued); + AtomicReference connector = new AtomicReference<>(); + CountDownLatch callbackReturned = new CountDownLatch(1); + connector.set(new HttpBackendTransportConnector(clientDirectory, ignored -> { + connector.get().close(); + callbackReturned.countDown(); + })); + String id = UUID.randomUUID().toString(); + connector.get().dispatch(new HttpTransportProtocol.Delivery(id, JsonEnvelope.builder("self-close").build())); + assertTrue(callbackReturned.await(2, TimeUnit.SECONDS), "callback-owned close must not drain its own worker"); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(2); + HttpInboundDeliveryStore.State state = null; + while (System.nanoTime() < deadline) { + state = new HttpInboundDeliveryStore(clientDirectory).state(id); + if (state == HttpInboundDeliveryStore.State.COMPLETED) break; + Thread.sleep(5); + } + assertEquals(HttpInboundDeliveryStore.State.COMPLETED, state, + "the callback wrapper must mark completed before close seals the journal"); + connector.get().start(); + assertFalse(connector.get().pollerAlive(), "a closed connector must not restart its poller"); + connector.get().close(); + } +} diff --git a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpProxyLifecycleRecoveryTest.java b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpProxyLifecycleRecoveryTest.java new file mode 100644 index 0000000..3d0cd5f --- /dev/null +++ b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpProxyLifecycleRecoveryTest.java @@ -0,0 +1,136 @@ +package com.bencodez.simpleapi.servercomm.http; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.bencodez.simpleapi.servercomm.codec.JsonEnvelope; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.util.UUID; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class HttpProxyLifecycleRecoveryTest { + @TempDir Path directory; + + @Test + void callbackOwnedCloseCompletesItsInboundJournalBeforeItIsSealed() throws Exception { + Path outgoing = directory.resolve("outgoing"); + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, directory.resolve("authority")); + AtomicReference proxy = new AtomicReference<>(); + AtomicReference deliveryId = new AtomicReference<>(); + CountDownLatch callbackStarted = new CountDownLatch(1), callbackReturned = new CountDownLatch(1); + HttpProxyTransportServer server = new HttpProxyTransportServer(new InetSocketAddress("localhost", 0), identity, + authority, outgoing, received -> { + deliveryId.set(received.messageId()); + callbackStarted.countDown(); + proxy.get().close(); + callbackReturned.countDown(); + }); + proxy.set(server); + try { + server.start(); + HttpConnectionCode code = authority.createConnectionCode("lobby-1", server.endpoint("localhost"), + Duration.ofMinutes(5)); + HttpClientCredentialStore.ClientCredential credential = HttpBackendTransportConnector.enroll(code, "lobby-1", + directory.resolve("client")); + try (HttpBackendTransportConnector connector = new HttpBackendTransportConnector(code, "lobby-1", credential, + ignored -> { })) { + connector.start(); + assertTrue(connector.awaitFirstResponse(System.nanoTime() + TimeUnit.SECONDS.toNanos(8))); + assertTrue(connector.send(JsonEnvelope.builder("callback-close").build())); + assertTrue(callbackStarted.await(8, TimeUnit.SECONDS)); + assertTrue(callbackReturned.await(3, TimeUnit.SECONDS), + "a callback must not wait for shutdown of its own handler worker"); + } + server.close(); + assertNotNull(deliveryId.get()); + assertEquals(HttpInboundDeliveryStore.State.COMPLETED, + HttpInboundDeliveryStore.open(directory.resolve("outgoing-incoming"), "lobby-1").state(deliveryId.get()), + "the callback completion must be durable before close seals the journal"); + } finally { + server.close(); + } + } + + @Test + void failedNewBackendSetupCleansItsEmptyDirectoryBeforeRetry() throws Exception { + Path root = directory.resolve("outgoing"); + AtomicBoolean failFirstBackendPublication = new AtomicBoolean(true); + HttpProxyTransportServer.DurableOutgoingQueue queue = new HttpProxyTransportServer.DurableOutgoingQueue(root, + forced -> { + if (forced.equals(root) && failFirstBackendPublication.getAndSet(false)) + throw new IOException("injected backend directory publication failure"); + }); + HttpProxyTransportServer.BackendState state = new HttpProxyTransportServer.BackendState("lobby-1", queue, + (server, id) -> { }); + HttpTransportProtocol.Delivery delivery = new HttpTransportProtocol.Delivery(UUID.randomUUID().toString(), + JsonEnvelope.builder("retry-directory").build()); + + assertFalse(state.enqueue(delivery)); + assertFalse(Files.exists(root.resolve("lobby-1")), + "a failed setup must not leave an empty backend directory consuming the global bound"); + assertTrue(state.enqueue(delivery)); + } + + @Test + void restartPrunesAccumulatedEmptyBackendDirectoriesBeforeApplyingTheCap() throws Exception { + Path root = directory.resolve("outgoing"); + new HttpProxyTransportServer.DurableOutgoingQueue(root, ignored -> { }); + for (int index = 0; index < 128; index++) Files.createDirectory(root.resolve("server-" + index)); + + HttpProxyTransportServer.DurableOutgoingQueue restarted = new HttpProxyTransportServer.DurableOutgoingQueue(root, + ignored -> { }); + assertTrue(restarted.load().isEmpty()); + HttpProxyTransportServer.BackendState state = new HttpProxyTransportServer.BackendState("replacement", restarted, + (server, id) -> { }); + assertTrue(state.enqueue(new HttpTransportProtocol.Delivery(UUID.randomUUID().toString(), + JsonEnvelope.builder("recovered-capacity").build()))); + assertTrue(Files.isDirectory(root.resolve("replacement"))); + } + + @Test + void capPruningPreservesAnEmptyDirectoryUntilItsUnconfirmedAckCanRetry() throws Exception { + Path root = directory.resolve("outgoing"); + Path lobbyDirectory = root.resolve("lobby-1"); + HttpProxyTransportServer.DurableOutgoingQueue queue = new HttpProxyTransportServer.DurableOutgoingQueue(root, + ignored -> { }); + HttpProxyTransportServer.BackendState lobby = new HttpProxyTransportServer.BackendState("lobby-1", queue, + (server, id) -> { }); + HttpTransportProtocol.Delivery delivery = new HttpTransportProtocol.Delivery(UUID.randomUUID().toString(), + JsonEnvelope.builder("ack-retry").build()); + assertTrue(lobby.enqueue(delivery)); + + AtomicBoolean failFinalAckForce = new AtomicBoolean(true); + try (var forces = org.mockito.Mockito.mockStatic(com.bencodez.simpleapi.file.DurableFiles.class, + org.mockito.Mockito.CALLS_REAL_METHODS)) { + forces.when(() -> com.bencodez.simpleapi.file.DurableFiles.forceDirectory(lobbyDirectory)).thenAnswer(call -> { + if (failFinalAckForce.getAndSet(false)) throw new IOException("injected acknowledgement force failure"); + return call.callRealMethod(); + }); + assertThrows(IOException.class, () -> lobby.acknowledge(List.of(delivery.id()))); + assertTrue(Files.isDirectory(lobbyDirectory), "the unlink has happened but its directory fsync is unresolved"); + + HttpProxyTransportServer.BackendState replacement = new HttpProxyTransportServer.BackendState("replacement", + queue, (server, id) -> { }); + assertTrue(replacement.enqueue(new HttpTransportProtocol.Delivery(UUID.randomUUID().toString(), + JsonEnvelope.builder("trigger-cap-count").build()))); + assertTrue(Files.isDirectory(lobbyDirectory), + "capacity pruning must retain the directory indexed by the failed acknowledgement"); + lobby.acknowledge(List.of(delivery.id())); + } + assertFalse(Files.exists(lobbyDirectory), "the same acknowledgement retry must be able to finish cleanup"); + } +} diff --git a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportRuntimeTest.java b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportRuntimeTest.java index 70dc963..555d859 100644 --- a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportRuntimeTest.java +++ b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportRuntimeTest.java @@ -174,8 +174,9 @@ void outgoingQueueRetriesDirectoryPublicationAfterForceFailure() throws Exceptio HttpTransportProtocol.Delivery delivery = new HttpTransportProtocol.Delivery( java.util.UUID.randomUUID().toString(), JsonEnvelope.builder("retry-directory-force").build()); assertFalse(state.enqueue(delivery)); + assertEquals(2L, serverRootForces.get(), "failed publication and empty-directory cleanup must both force the parent"); assertTrue(state.enqueue(delivery)); - assertEquals(2L, serverRootForces.get(), "retrying an existing backend directory must force its parent again"); + assertEquals(3L, serverRootForces.get(), "retrying backend directory creation must force its parent again"); } @Test @@ -994,7 +995,7 @@ void persistedBackendConnectsAfterAutomaticServerLeafRotation() throws Exception } @Test - void renewalAdoptsPublishedPointerAndRetriesItBeforeAnotherRenewal() throws Exception { + void renewalRetainsOldCredentialUntilPublishedPointerIsConfirmed() throws Exception { HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("pointer-proxy"), "localhost"); HttpTlsIdentity.IssuedClientCertificate expiring = identity.issueClientCertificate("lobby-1", Instant.now().minus(Duration.ofDays(340))); @@ -1038,11 +1039,11 @@ void renewalAdoptsPublishedPointerAndRetriesItBeforeAnotherRenewal() throws Exce var selected = HttpClientCredentialStore.load(clientDirectory); String selectedPin = HttpTransportSecrets.certificatePin(selected.certificate()); assertFalse(originalPin.equals(selectedPin)); - assertEquals(selectedPin, HttpTransportSecrets.certificatePin( + assertEquals(originalPin, HttpTransportSecrets.certificatePin( ((HttpClientCredentialStore.ClientCredential) credential.get(connector)).certificate())); assertTrue(pending.get(connector) != null); - assertTrue(authority.authenticate("lobby-1", selected.certificate())); - assertFalse(authority.authenticate("lobby-1", expiring.certificate())); + assertTrue(authority.authenticate("lobby-1", expiring.certificate()), + "uncertain pointer publication must not invalidate the restart-safe old credential"); renew.invoke(connector); assertEquals(generation, Files.readString(pointer)); assertTrue(pending.get(connector) != null, "failed retry must retain the same pending activation"); @@ -1050,6 +1051,12 @@ void renewalAdoptsPublishedPointerAndRetriesItBeforeAnotherRenewal() throws Exce renew.invoke(connector); assertTrue(pending.get(connector) == null); assertEquals(generation, Files.readString(pointer)); + var confirmed = HttpClientCredentialStore.load(clientDirectory); + assertEquals(HttpTransportSecrets.certificatePin(confirmed.certificate()), + HttpTransportSecrets.certificatePin(((HttpClientCredentialStore.ClientCredential) + credential.get(connector)).certificate())); + assertTrue(authority.authenticate("lobby-1", confirmed.certificate())); + assertFalse(authority.authenticate("lobby-1", expiring.certificate())); connector.start(); assertTrue(connector.send(JsonEnvelope.builder("after-pointer-recovery").build())); assertTrue(received.await(8, TimeUnit.SECONDS), "transport must use the adopted TLS client"); From 4c7887ff9b49ef82e8e2174bf01f85869ee8ba4f Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Sat, 5 Sep 2026 21:39:23 -0600 Subject: [PATCH 21/38] Preserve enrollment CA trust and validate codes before reservation --- .../http/HttpClientCredentialStore.java | 12 +- .../http/HttpEnrollmentAuthority.java | 9 +- .../http/HttpEnrollmentPinTest.java | 107 ++++++++++++++++++ 3 files changed, 120 insertions(+), 8 deletions(-) create mode 100644 SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentPinTest.java diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpClientCredentialStore.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpClientCredentialStore.java index 3fddf65..f62e4d3 100644 --- a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpClientCredentialStore.java +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpClientCredentialStore.java @@ -44,7 +44,7 @@ public static void saveEnrolled(Path directory, HttpConnectionCode code, HttpTls HttpClientProfile profile = new HttpClientProfile(HttpTlsIdentity.canonicalServerId(issued.serverId()), code.endpoint(), code.serverCertificatePin(), code.caCertificatePin()); try { - StagedCredential staged = stage(directory, issued, profile, connectionCodeDigest(code)); + StagedCredential staged = stage(directory, issued, profile, connectionCodeDigest(code), false); activateReplacement(directory, staged); } catch (IOException failure) { throw failure; } catch (Exception failure) { throw new IOException("Could not persist HTTP client credential", failure); } @@ -99,11 +99,11 @@ private static ClientCredential loadCredential(Path directory) throws Exception /** Writes and validates a replacement generation without touching the active credential. */ static StagedCredential stageReplacement(Path directory, HttpTlsIdentity.IssuedClientCertificate issued) throws Exception { Path active = activeDirectory(directory); - return stage(directory, issued, loadProfileFile(active), readConnectionCodeDigest(active)); + return stage(directory, issued, loadProfileFile(active), readConnectionCodeDigest(active), true); } private static StagedCredential stage(Path directory, HttpTlsIdentity.IssuedClientCertificate issued, - HttpClientProfile profile, String connectionCodeDigest) throws Exception { + HttpClientProfile profile, String connectionCodeDigest, boolean allowCaRotation) throws Exception { if (directory == null || issued == null) throw new IllegalArgumentException("Credential replacement is required"); Path credentialDirectory = credentialRoot(directory, true); Path generations = credentialDirectory.resolve(GENERATIONS_DIRECTORY); @@ -122,8 +122,10 @@ private static StagedCredential stage(Path directory, HttpTlsIdentity.IssuedClie try { save(generation, issued); ClientCredential replacement = loadCredential(generation); - profile = new HttpClientProfile(profile.serverId(), profile.endpoint(), profile.serverCertificatePin(), - HttpTransportSecrets.certificatePin(replacement.caCertificate())); + // Initial enrollment must retain the connection code's trust anchor. Only + // renewal over an already authenticated connection may introduce a new CA. + if (allowCaRotation) profile = new HttpClientProfile(profile.serverId(), profile.endpoint(), + profile.serverCertificatePin(), HttpTransportSecrets.certificatePin(replacement.caCertificate())); writeProfile(generation, profile); if (connectionCodeDigest != null) writePrivate(safe(generation.resolve(CONNECTION_CODE_DIGEST_FILE)), connectionCodeDigest.getBytes(StandardCharsets.US_ASCII)); diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentAuthority.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentAuthority.java index 83e4b25..cc6a4ce 100644 --- a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentAuthority.java +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentAuthority.java @@ -58,11 +58,14 @@ public synchronized HttpConnectionCode createConnectionCode(String serverId, URI serverId = HttpTlsIdentity.canonicalServerId(serverId); if (lifetime == null || lifetime.isNegative() || lifetime.isZero() || lifetime.compareTo(MAX_ENROLLMENT_LIFETIME) > 0) throw new IllegalArgumentException("Enrollment lifetime must be between one second and fifteen minutes"); + Instant expiresAt = clock.instant().plus(lifetime); + String token = HttpTransportSecrets.randomToken(); + // Validate the complete code before reserving or persisting a pending slot. + HttpConnectionCode code = new HttpConnectionCode(serverId, endpoint, identity.serverCertificatePin(), + identity.caCertificatePin(), expiresAt, token); expireEnrollments(); if (enrollments.size() >= MAX_PENDING_ENROLLMENTS) throw new IllegalStateException("Too many pending HTTP enrollments"); - Instant expiresAt = clock.instant().plus(lifetime); - String token = HttpTransportSecrets.randomToken(); byte[] tokenHash = HttpTransportSecrets.sha256(token.getBytes(StandardCharsets.US_ASCII)); String lookup = java.util.Base64.getUrlEncoder().withoutPadding().encodeToString(tokenHash); enrollments.put(lookup, new Enrollment(tokenHash, expiresAt, serverId, null)); @@ -71,7 +74,7 @@ public synchronized HttpConnectionCode createConnectionCode(String serverId, URI enrollments.remove(lookup); throw new IllegalStateException("Could not persist HTTP enrollment", failure); } - return new HttpConnectionCode(serverId, endpoint, identity.serverCertificatePin(), identity.caCertificatePin(), expiresAt, token); + return code; } public synchronized HttpTlsIdentity.IssuedClientCertificate enroll(String serverId, String enrollmentToken) throws Exception { diff --git a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentPinTest.java b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentPinTest.java new file mode 100644 index 0000000..952ab71 --- /dev/null +++ b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentPinTest.java @@ -0,0 +1,107 @@ +package com.bencodez.simpleapi.servercomm.http; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.net.URI; +import java.nio.file.Path; +import java.time.Duration; +import java.time.Instant; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class HttpEnrollmentPinTest { + @TempDir Path directory; + + @Test + void rejectsDifferentCaBundleDuringInitialEnrollment() throws Exception { + HttpTlsIdentity proxy = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); + HttpTlsIdentity foreign = HttpTlsIdentity.loadOrCreate(directory.resolve("foreign"), "localhost"); + HttpConnectionCode code = code(proxy, "lobby-1"); + + assertThrows(java.io.IOException.class, () -> HttpClientCredentialStore.saveEnrolled( + directory.resolve("client"), code, foreign.issueClientCertificate("lobby-1"))); + assertFalse(HttpClientCredentialStore.hasEnrolledProfile(directory.resolve("client"))); + } + + @Test + void rejectsDifferentCaBundleWithoutReplacingExistingEnrollment() throws Exception { + HttpTlsIdentity proxy = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); + HttpTlsIdentity foreign = HttpTlsIdentity.loadOrCreate(directory.resolve("foreign"), "localhost"); + Path client = directory.resolve("client"); + HttpConnectionCode code = code(proxy, "lobby-1"); + HttpTlsIdentity.IssuedClientCertificate original = proxy.issueClientCertificate("lobby-1"); + HttpClientCredentialStore.saveEnrolled(client, code, original); + HttpClientCredentialStore.EnrolledClient before = HttpClientCredentialStore.loadEnrolled(client); + + assertThrows(java.io.IOException.class, () -> HttpClientCredentialStore.saveEnrolled( + client, code, foreign.issueClientCertificate("lobby-1"))); + HttpClientCredentialStore.EnrolledClient after = HttpClientCredentialStore.loadEnrolled(client); + assertEquals(before.profile(), after.profile()); + assertArrayEquals(before.credential().certificate().getEncoded(), after.credential().certificate().getEncoded()); + } + + @Test + void acceptsBundleSignedByConnectionCodeCa() throws Exception { + HttpTlsIdentity proxy = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); + Path client = directory.resolve("client"); + HttpConnectionCode code = code(proxy, "lobby-1"); + + HttpClientCredentialStore.saveEnrolled(client, code, proxy.issueClientCertificate("lobby-1")); + HttpClientCredentialStore.EnrolledClient enrolled = HttpClientCredentialStore.loadEnrolled(client); + assertEquals(code.caCertificatePin(), enrolled.profile().caCertificatePin()); + assertEquals(code.caCertificatePin(), HttpTransportSecrets.certificatePin(enrolled.credential().caCertificate())); + } + + @Test + void stageReplacementCanRotateCaAndActivatesValidatedCredential() throws Exception { + HttpTlsIdentity proxy = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); + HttpTlsIdentity foreign = HttpTlsIdentity.loadOrCreate(directory.resolve("foreign"), "localhost"); + Path client = directory.resolve("client"); + HttpConnectionCode code = code(proxy, "lobby-1"); + HttpClientCredentialStore.saveEnrolled(client, code, proxy.issueClientCertificate("lobby-1")); + String originalCaPin = HttpClientCredentialStore.loadProfile(client).caCertificatePin(); + + HttpClientCredentialStore.StagedCredential staged = HttpClientCredentialStore.stageReplacement( + client, foreign.issueClientCertificate("lobby-1")); + String replacementCaPin = foreign.caCertificatePin(); + assertNotEquals(originalCaPin, replacementCaPin); + assertEquals(replacementCaPin, staged.profile().caCertificatePin()); + assertEquals(replacementCaPin, HttpTransportSecrets.certificatePin(staged.credential().caCertificate())); + + HttpClientCredentialStore.activateReplacement(client, staged); + HttpClientCredentialStore.EnrolledClient active = HttpClientCredentialStore.loadEnrolled(client); + assertEquals(replacementCaPin, active.profile().caCertificatePin()); + assertEquals(replacementCaPin, HttpTransportSecrets.certificatePin(active.credential().caCertificate())); + } + + @Test + void invalidConnectionCodeEndpointsDoNotConsumePendingCapacityOrBreakReload() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); + Path state = directory.resolve("state"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, state); + for (int i = 0; i < 128; i++) { + final int index = i; + URI endpoint = (i & 1) == 0 ? null : URI.create("http://localhost:8443/"); + assertThrows(IllegalArgumentException.class, () -> authority.createConnectionCode( + "lobby-" + index, endpoint, Duration.ofMinutes(5))); + } + + HttpConnectionCode first = authority.createConnectionCode("lobby-1", URI.create("https://localhost:8443/"), + Duration.ofMinutes(5)); + assertEquals("lobby-1", first.serverId()); + HttpEnrollmentAuthority reloaded = new HttpEnrollmentAuthority(identity, state); + HttpConnectionCode second = reloaded.createConnectionCode("lobby-2", URI.create("https://localhost:8443/"), + Duration.ofMinutes(5)); + assertEquals("lobby-2", second.serverId()); + } + + private HttpConnectionCode code(HttpTlsIdentity identity, String serverId) { + return new HttpConnectionCode(serverId, URI.create("https://localhost:8443/"), + identity.serverCertificatePin(), identity.caCertificatePin(), Instant.now().plusSeconds(300), + HttpTransportSecrets.randomToken()); + } +} From c8befc40ffc5e36a9a79a34c0aae4b0d03a78218 Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Sat, 5 Sep 2026 21:53:25 -0600 Subject: [PATCH 22/38] Require trusted CA key continuity for credential renewal --- .../http/HttpClientCredentialStore.java | 21 ++++++--- .../http/HttpEnrollmentPinTest.java | 45 ++++++++++++++----- 2 files changed, 47 insertions(+), 19 deletions(-) diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpClientCredentialStore.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpClientCredentialStore.java index f62e4d3..d3052da 100644 --- a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpClientCredentialStore.java +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpClientCredentialStore.java @@ -44,7 +44,7 @@ public static void saveEnrolled(Path directory, HttpConnectionCode code, HttpTls HttpClientProfile profile = new HttpClientProfile(HttpTlsIdentity.canonicalServerId(issued.serverId()), code.endpoint(), code.serverCertificatePin(), code.caCertificatePin()); try { - StagedCredential staged = stage(directory, issued, profile, connectionCodeDigest(code), false); + StagedCredential staged = stage(directory, issued, profile, connectionCodeDigest(code), null); activateReplacement(directory, staged); } catch (IOException failure) { throw failure; } catch (Exception failure) { throw new IOException("Could not persist HTTP client credential", failure); } @@ -99,11 +99,13 @@ private static ClientCredential loadCredential(Path directory) throws Exception /** Writes and validates a replacement generation without touching the active credential. */ static StagedCredential stageReplacement(Path directory, HttpTlsIdentity.IssuedClientCertificate issued) throws Exception { Path active = activeDirectory(directory); - return stage(directory, issued, loadProfileFile(active), readConnectionCodeDigest(active), true); + EnrolledClient enrolled = loadEnrolledDirectory(active); + return stage(directory, issued, enrolled.profile(), readConnectionCodeDigest(active), + enrolled.credential().caCertificate().getPublicKey()); } private static StagedCredential stage(Path directory, HttpTlsIdentity.IssuedClientCertificate issued, - HttpClientProfile profile, String connectionCodeDigest, boolean allowCaRotation) throws Exception { + HttpClientProfile profile, String connectionCodeDigest, java.security.PublicKey trustedCaKey) throws Exception { if (directory == null || issued == null) throw new IllegalArgumentException("Credential replacement is required"); Path credentialDirectory = credentialRoot(directory, true); Path generations = credentialDirectory.resolve(GENERATIONS_DIRECTORY); @@ -122,10 +124,15 @@ private static StagedCredential stage(Path directory, HttpTlsIdentity.IssuedClie try { save(generation, issued); ClientCredential replacement = loadCredential(generation); - // Initial enrollment must retain the connection code's trust anchor. Only - // renewal over an already authenticated connection may introduce a new CA. - if (allowCaRotation) profile = new HttpClientProfile(profile.serverId(), profile.endpoint(), - profile.serverCertificatePin(), HttpTransportSecrets.certificatePin(replacement.caCertificate())); + // Initial enrollment retains the code's CA pin. Renewal may refresh the CA + // certificate, but must preserve the previously trusted CA public key. + if (trustedCaKey != null) { + if (!java.security.MessageDigest.isEqual(trustedCaKey.getEncoded(), + replacement.caCertificate().getPublicKey().getEncoded())) + throw new IOException("Renewed HTTP credential changes the trusted CA key"); + profile = new HttpClientProfile(profile.serverId(), profile.endpoint(), + profile.serverCertificatePin(), HttpTransportSecrets.certificatePin(replacement.caCertificate())); + } writeProfile(generation, profile); if (connectionCodeDigest != null) writePrivate(safe(generation.resolve(CONNECTION_CODE_DIGEST_FILE)), connectionCodeDigest.getBytes(StandardCharsets.US_ASCII)); diff --git a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentPinTest.java b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentPinTest.java index 952ab71..108042c 100644 --- a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentPinTest.java +++ b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentPinTest.java @@ -57,25 +57,46 @@ void acceptsBundleSignedByConnectionCodeCa() throws Exception { } @Test - void stageReplacementCanRotateCaAndActivatesValidatedCredential() throws Exception { + void stageReplacementRejectsForeignCaAndPreservesActiveEnrollment() throws Exception { HttpTlsIdentity proxy = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); HttpTlsIdentity foreign = HttpTlsIdentity.loadOrCreate(directory.resolve("foreign"), "localhost"); Path client = directory.resolve("client"); HttpConnectionCode code = code(proxy, "lobby-1"); - HttpClientCredentialStore.saveEnrolled(client, code, proxy.issueClientCertificate("lobby-1")); - String originalCaPin = HttpClientCredentialStore.loadProfile(client).caCertificatePin(); + HttpTlsIdentity.IssuedClientCertificate originalCredential = proxy.issueClientCertificate("lobby-1"); + HttpClientCredentialStore.saveEnrolled(client, code, originalCredential); + HttpClientCredentialStore.EnrolledClient before = HttpClientCredentialStore.loadEnrolled(client); - HttpClientCredentialStore.StagedCredential staged = HttpClientCredentialStore.stageReplacement( - client, foreign.issueClientCertificate("lobby-1")); - String replacementCaPin = foreign.caCertificatePin(); - assertNotEquals(originalCaPin, replacementCaPin); - assertEquals(replacementCaPin, staged.profile().caCertificatePin()); - assertEquals(replacementCaPin, HttpTransportSecrets.certificatePin(staged.credential().caCertificate())); + assertThrows(java.io.IOException.class, () -> HttpClientCredentialStore.stageReplacement( + client, foreign.issueClientCertificate("lobby-1"))); + HttpClientCredentialStore.EnrolledClient after = HttpClientCredentialStore.loadEnrolled(client); + assertEquals(before.profile(), after.profile()); + assertArrayEquals(before.credential().certificate().getEncoded(), after.credential().certificate().getEncoded()); + } + + @Test + void stageReplacementAcceptsCaRenewalWithSameAuthorityKey() throws Exception { + Instant now = Instant.now(); + java.time.Clock originalClock = java.time.Clock.fixed( + now.minus(Duration.ofDays(9 * 365L + 30L)), java.time.ZoneOffset.UTC); + Path proxyDirectory = directory.resolve("proxy"); + HttpTlsIdentity original = HttpTlsIdentity.loadOrCreate(proxyDirectory, "localhost", originalClock); + HttpTlsIdentity.IssuedClientCertificate originalCredential = original.issueClientCertificate("lobby-1", now); + Path client = directory.resolve("client"); + HttpConnectionCode code = new HttpConnectionCode("lobby-1", URI.create("https://localhost:8443/"), + HttpTransportSecrets.certificatePin(original.serverCertificate()), + HttpTransportSecrets.certificatePin(original.caCertificate()), now.plusSeconds(60), "A".repeat(43)); + HttpClientCredentialStore.saveEnrolled(client, code, originalCredential); + + HttpTlsIdentity renewed = HttpTlsIdentity.loadOrCreate(proxyDirectory, "localhost", + java.time.Clock.fixed(now, java.time.ZoneOffset.UTC)); + assertNotEquals(original.caCertificatePin(), renewed.caCertificatePin()); + assertEquals(original.caCertificate().getPublicKey(), renewed.caCertificate().getPublicKey()); + HttpClientCredentialStore.StagedCredential staged = HttpClientCredentialStore.stageReplacement( + client, renewed.issueClientCertificate("lobby-1", now)); + assertEquals(renewed.caCertificatePin(), staged.profile().caCertificatePin()); HttpClientCredentialStore.activateReplacement(client, staged); - HttpClientCredentialStore.EnrolledClient active = HttpClientCredentialStore.loadEnrolled(client); - assertEquals(replacementCaPin, active.profile().caCertificatePin()); - assertEquals(replacementCaPin, HttpTransportSecrets.certificatePin(active.credential().caCertificate())); + assertEquals(renewed.caCertificatePin(), HttpClientCredentialStore.loadProfile(client).caCertificatePin()); } @Test From 2ab7908454f7613d8325d25241ee2bcc9aa343a5 Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Sat, 5 Sep 2026 22:01:22 -0600 Subject: [PATCH 23/38] Enforce one-second minimum enrollment lifetime --- .../servercomm/http/HttpEnrollmentAuthority.java | 2 +- .../servercomm/http/HttpEnrollmentPinTest.java | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentAuthority.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentAuthority.java index cc6a4ce..afc66a2 100644 --- a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentAuthority.java +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentAuthority.java @@ -56,7 +56,7 @@ public synchronized HttpConnectionCode createConnectionCode(String serverId, URI if (revocationRetryRequired) throw new IllegalStateException("HTTP certificate revocation durability must be retried"); serverId = HttpTlsIdentity.canonicalServerId(serverId); - if (lifetime == null || lifetime.isNegative() || lifetime.isZero() || lifetime.compareTo(MAX_ENROLLMENT_LIFETIME) > 0) + if (lifetime == null || lifetime.compareTo(Duration.ofSeconds(1)) < 0 || lifetime.compareTo(MAX_ENROLLMENT_LIFETIME) > 0) throw new IllegalArgumentException("Enrollment lifetime must be between one second and fifteen minutes"); Instant expiresAt = clock.instant().plus(lifetime); String token = HttpTransportSecrets.randomToken(); diff --git a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentPinTest.java b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentPinTest.java index 108042c..000f71a 100644 --- a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentPinTest.java +++ b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentPinTest.java @@ -16,6 +16,22 @@ class HttpEnrollmentPinTest { @TempDir Path directory; + @Test + void enrollmentLifetimeRequiresAtLeastOneSecond() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); + java.time.Clock clock = java.time.Clock.fixed(Instant.parse("2026-09-05T12:00:00.999Z"), + java.time.ZoneOffset.UTC); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, clock); + URI endpoint = URI.create("https://localhost:8443/"); + for (Duration lifetime : new Duration[] { Duration.ofNanos(1), Duration.ofMillis(999), + Duration.ofSeconds(1).minusNanos(1), Duration.ZERO, Duration.ofSeconds(-1) }) { + assertThrows(IllegalArgumentException.class, + () -> authority.createConnectionCode("lobby-1", endpoint, lifetime)); + } + HttpConnectionCode minimum = authority.createConnectionCode("lobby-1", endpoint, Duration.ofSeconds(1)); + org.junit.jupiter.api.Assertions.assertTrue(HttpConnectionCode.parse(minimum.encode()).expiresAt().isAfter(clock.instant())); + } + @Test void rejectsDifferentCaBundleDuringInitialEnrollment() throws Exception { HttpTlsIdentity proxy = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); From d6a202ce399a22b90be57214d31d2a590f8b233a Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Sat, 5 Sep 2026 22:30:12 -0600 Subject: [PATCH 24/38] Lock inbound journal ownership and validate client key pairs --- .../http/HttpBackendTransportConnector.java | 55 +++++---- .../http/HttpClientCredentialStore.java | 21 ++++ .../http/HttpInboundDeliveryStore.java | 106 +++++++++++++++--- .../http/HttpProxyTransportServer.java | 77 ++++++++----- .../HttpBackendLifecycleRecoveryTest.java | 2 +- .../http/HttpEnrollmentPinTest.java | 40 +++++++ .../http/HttpProxyLifecycleRecoveryTest.java | 2 +- .../http/HttpTransportRuntimeTest.java | 46 +++++++- 8 files changed, 277 insertions(+), 72 deletions(-) diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpBackendTransportConnector.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpBackendTransportConnector.java index 76ea61f..69ed075 100644 --- a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpBackendTransportConnector.java +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpBackendTransportConnector.java @@ -107,28 +107,43 @@ private HttpBackendTransportConnector(HttpClientCredentialStore.HttpClientProfil this.profile = profile; this.serverId = profile.serverId(); this.onEnvelope = onEnvelope; this.credential = credential; this.credentialDirectory = credentialDirectory; - inboundDeliveries = credentialDirectory == null ? null : new HttpInboundDeliveryStore(credentialDirectory); - acknowledgementConfirmationStore = credentialDirectory == null ? null - : HttpInboundDeliveryStore.open(credentialDirectory, "http-transport-ack-confirmations"); - if (inboundDeliveries != null) for (var entry : inboundDeliveries.snapshot().entrySet()) { - if (entry.getValue() == HttpInboundDeliveryStore.State.COMPLETED) { - inboundDeliveries.confirmCompleted(entry.getKey()); - received.add(entry.getKey()); - queueAck(entry.getKey()); + HttpInboundDeliveryStore loadedInbound = null, loadedAcknowledgements = null; + HttpClient initializedClient; + URI initializedEndpoint; + ThreadPoolExecutor initializedCallbacks; + try { + loadedInbound = credentialDirectory == null ? null : new HttpInboundDeliveryStore(credentialDirectory); + loadedAcknowledgements = credentialDirectory == null ? null + : HttpInboundDeliveryStore.open(credentialDirectory, "http-transport-ack-confirmations"); + if (loadedInbound != null) for (var entry : loadedInbound.snapshot().entrySet()) { + if (entry.getValue() == HttpInboundDeliveryStore.State.COMPLETED) { + loadedInbound.confirmCompleted(entry.getKey()); + received.add(entry.getKey()); + queueAck(entry.getKey()); + } } + if (loadedAcknowledgements != null) for (var entry : loadedAcknowledgements.snapshot().entrySet()) { + if (entry.getValue() != HttpInboundDeliveryStore.State.COMPLETED) + throw new IOException("HTTP acknowledgement confirmation state is invalid"); + loadedAcknowledgements.confirmCompleted(entry.getKey()); + queueAcknowledgementConfirmation(entry.getKey()); + } + initializedClient = client(profile, credential); + initializedEndpoint = profile.endpoint().resolve("v1/transport"); + // GlobalMessageHandler routes mutate backend vote state and must observe the + // wire order. One bounded lane preserves batch ordering without running work on + // the long-poll thread; bounded admission below backpressures this poller. + initializedCallbacks = callbackExecutor(); + } catch (Exception | Error setupFailure) { + if (loadedAcknowledgements != null) loadedAcknowledgements.seal(); + if (loadedInbound != null) loadedInbound.seal(); + throw setupFailure; } - if (acknowledgementConfirmationStore != null) for (var entry : acknowledgementConfirmationStore.snapshot().entrySet()) { - if (entry.getValue() != HttpInboundDeliveryStore.State.COMPLETED) - throw new IOException("HTTP acknowledgement confirmation state is invalid"); - acknowledgementConfirmationStore.confirmCompleted(entry.getKey()); - queueAcknowledgementConfirmation(entry.getKey()); - } - client = client(profile, credential); - transportEndpoint = profile.endpoint().resolve("v1/transport"); - // GlobalMessageHandler routes mutate backend vote state and must observe the - // wire order. One bounded lane preserves batch ordering without running work on - // the long-poll thread; bounded admission below backpressures this poller. - callbackExecutor = callbackExecutor(); + inboundDeliveries = loadedInbound; + acknowledgementConfirmationStore = loadedAcknowledgements; + client = initializedClient; + transportEndpoint = initializedEndpoint; + callbackExecutor = initializedCallbacks; } /** Convenience constructor for the owner-only credential directory produced by {@link #enroll}. */ diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpClientCredentialStore.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpClientCredentialStore.java index d3052da..cc431df 100644 --- a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpClientCredentialStore.java +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpClientCredentialStore.java @@ -92,10 +92,31 @@ private static ClientCredential loadCredential(Path directory) throws Exception if (privateKey == null || chain == null || chain.length != 2 || !(chain[0] instanceof X509Certificate client) || !(chain[1] instanceof X509Certificate authority)) throw new IOException("HTTP client certificate bundle is invalid"); + verifyKeyPair(privateKey, client); return new ClientCredential(privateKey, client, authority, password); } finally { java.util.Arrays.fill(password, '\0'); } } + private static void verifyKeyPair(PrivateKey privateKey, X509Certificate certificate) throws Exception { + String algorithm = switch (certificate.getPublicKey().getAlgorithm()) { + case "EC" -> "SHA256withECDSA"; + case "RSA" -> "SHA256withRSA"; + case "DSA" -> "SHA256withDSA"; + case "Ed25519", "Ed448" -> certificate.getPublicKey().getAlgorithm(); + case "EdDSA" -> ((java.security.interfaces.EdECKey) certificate.getPublicKey()).getParams().getName(); + default -> throw new IOException("HTTP client key algorithm is unsupported"); + }; + byte[] challenge = new byte[32]; + new java.security.SecureRandom().nextBytes(challenge); + java.security.Signature signature = java.security.Signature.getInstance(algorithm); + signature.initSign(privateKey); + signature.update(challenge); + byte[] proof = signature.sign(); + signature.initVerify(certificate.getPublicKey()); + signature.update(challenge); + if (!signature.verify(proof)) throw new IOException("HTTP client private key does not match its certificate"); + } + /** Writes and validates a replacement generation without touching the active credential. */ static StagedCredential stageReplacement(Path directory, HttpTlsIdentity.IssuedClientCertificate issued) throws Exception { Path active = activeDirectory(directory); diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpInboundDeliveryStore.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpInboundDeliveryStore.java index f72d14a..9ad230f 100644 --- a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpInboundDeliveryStore.java +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpInboundDeliveryStore.java @@ -3,6 +3,9 @@ import com.bencodez.simpleapi.file.DurableFiles; import com.bencodez.simpleapi.file.PrivateFilePermissions; import java.io.IOException; +import java.nio.channels.FileChannel; +import java.nio.channels.FileLock; +import java.nio.channels.OverlappingFileLockException; import java.nio.charset.StandardCharsets; import java.nio.file.DirectoryStream; import java.nio.file.Files; @@ -19,8 +22,13 @@ /** Crash-durable state for proxy deliveries around a non-transactional application callback. */ final class HttpInboundDeliveryStore { private static final String DIRECTORY = "http-transport-inbound-deliveries"; + // This sidecar is deliberately outside the deletable journal directory. Never remove it: + // deleting and recreating a locked file could split ownership across two inodes. + private static final String OWNER_LOCK_PREFIX = ".http-inbound-owner-"; + private static final String OWNER_LOCK_SUFFIX = ".lock"; private static final int MAX_ENTRIES = HttpTransportProtocol.MAX_QUEUE; private final Path root; + private final boolean readOnly; private final Map entries = new LinkedHashMap<>(); // A rename completed, but its directory entry still needs a successful fsync. private final Set unconfirmedReservations = new HashSet<>(); @@ -28,32 +36,57 @@ final class HttpInboundDeliveryStore { // RUNNING was published before the callback, but restoring RESERVED was not yet confirmed. // This is process-local evidence only: a RUNNING entry loaded after a restart remains ambiguous. private final Set pendingRunningRollbacks = new HashSet<>(); + private FileChannel ownershipChannel; + private FileLock ownershipLock; private boolean sealed; HttpInboundDeliveryStore(Path credentialDirectory) throws IOException { - this(credentialDirectory, DIRECTORY); + this(credentialDirectory, DIRECTORY, false); } static HttpInboundDeliveryStore open(Path parent, String directoryName) throws IOException { - return new HttpInboundDeliveryStore(parent, directoryName); + return new HttpInboundDeliveryStore(parent, directoryName, false); } - private HttpInboundDeliveryStore(Path parent, String directoryName) throws IOException { + /** Opens an existing journal for state inspection without claiming its writer ownership. */ + static HttpInboundDeliveryStore inspect(Path parent, String directoryName) throws IOException { + return new HttpInboundDeliveryStore(parent, directoryName, true); + } + + static HttpInboundDeliveryStore inspect(Path credentialDirectory) throws IOException { + return new HttpInboundDeliveryStore(credentialDirectory, DIRECTORY, true); + } + + private HttpInboundDeliveryStore(Path parent, String directoryName, boolean readOnly) throws IOException { Path credentials = parent.toAbsolutePath().normalize(); if (Files.isSymbolicLink(credentials) || !Files.isDirectory(credentials, LinkOption.NOFOLLOW_LINKS)) throw new IOException("HTTP credential directory is unsafe"); if (directoryName == null || !directoryName.matches("[A-Za-z0-9][A-Za-z0-9._-]{0,63}")) throw new IOException("HTTP inbound delivery directory name is invalid"); - PrivateFilePermissions.ownerOnlyDirectory(credentials); + this.readOnly = readOnly; + if (!readOnly) PrivateFilePermissions.ownerOnlyDirectory(credentials); root = credentials.resolve(directoryName).normalize(); if (!root.getParent().equals(credentials)) throw new IOException("HTTP inbound delivery directory is invalid"); - try { Files.createDirectory(root); } - catch (java.nio.file.FileAlreadyExistsException existing) { } - requireRoot(); - PrivateFilePermissions.ownerOnlyDirectory(root); - // Retry a parent fsync that may have failed after creating this root. - DurableFiles.forceDirectory(credentials); - load(); + if (!readOnly) { + try { + claimOwnership(credentials.resolve(OWNER_LOCK_PREFIX + directoryName + OWNER_LOCK_SUFFIX)); + // Claim before creating or checking the journal root: a retiring owner may be + // between its empty check and deletion, and only one owner may cross that boundary. + try { Files.createDirectory(root); } + catch (java.nio.file.FileAlreadyExistsException existing) { } + requireRoot(); + PrivateFilePermissions.ownerOnlyDirectory(root); + // Retry a parent fsync that may have failed after creating this root. + DurableFiles.forceDirectory(credentials); + load(); + } catch (IOException | RuntimeException failure) { + releaseOwnership(); + throw failure; + } + } else { + requireRoot(); + load(); + } } synchronized State state(String id) { return entries.get(canonical(id)); } @@ -128,8 +161,13 @@ synchronized void markRunning(String id) throws IOException { transition(id, State.RESERVED, State.RUNNING); } synchronized void markCompleted(String id) throws IOException { transition(id, State.RUNNING, State.COMPLETED); } - synchronized void seal() { sealed = true; } + synchronized void seal() { + if (sealed) return; + sealed = true; + releaseOwnership(); + } synchronized void sealAndDeleteIfEmpty() throws IOException { + requireWritable(); if (!entries.isEmpty()) throw new IOException("HTTP inbound delivery store is not empty"); requireRoot(); try (DirectoryStream files = Files.newDirectoryStream(root)) { @@ -137,8 +175,10 @@ synchronized void sealAndDeleteIfEmpty() throws IOException { } sealed = true; Path parent = root.getParent(); - Files.delete(root); - DurableFiles.forceDirectory(parent); + try { + Files.delete(root); + DurableFiles.forceDirectory(parent); + } finally { releaseOwnership(); } } synchronized void remove(String id) throws IOException { @@ -216,6 +256,7 @@ synchronized boolean recoverKnownNotStartedRunning(String id) throws IOException /** Retries the directory fsync required before exposing a reservation to a callback. */ synchronized void confirmReserved(String id) throws IOException { + requireWritable(); id = canonical(id); if (entries.get(id) != State.RESERVED) throw new IOException("HTTP inbound delivery fence state is invalid"); if (!unconfirmedReservations.contains(id)) return; @@ -231,6 +272,7 @@ synchronized void confirmReserved(String id) throws IOException { /** Retries the directory fsync required before exposing a completed delivery for acknowledgement. */ synchronized void confirmCompleted(String id) throws IOException { + requireWritable(); id = canonical(id); if (entries.get(id) != State.COMPLETED) throw new IOException("HTTP inbound delivery fence state is invalid"); if (!unconfirmedCompletions.contains(id)) return; @@ -250,14 +292,14 @@ private void load() throws IOException { String name = file.getFileName().toString(); if (name.startsWith(".pending-") && name.endsWith(".tmp") && !Files.isSymbolicLink(file) && Files.isRegularFile(file, LinkOption.NOFOLLOW_LINKS)) { - DurableFiles.deleteIfExists(file); + if (!readOnly) DurableFiles.deleteIfExists(file); continue; } State state = State.fromFileName(name); if (state == null || Files.isSymbolicLink(file) || !Files.isRegularFile(file, LinkOption.NOFOLLOW_LINKS) || Files.size(file) > 64L) throw new IOException("HTTP inbound delivery fence contains an invalid entry"); - PrivateFilePermissions.ownerOnlyFile(file); + if (!readOnly) PrivateFilePermissions.ownerOnlyFile(file); String id; try { id = canonical(name.substring(0, name.length() - state.suffix.length())); } catch (IllegalArgumentException invalid) { @@ -273,7 +315,7 @@ private void load() throws IOException { // RUNNING never replays, and COMPLETED alone may be acknowledged. State retained = existing.ordinal() >= state.ordinal() ? existing : state; State obsolete = retained == existing ? state : existing; - DurableFiles.deleteIfExists(file(id, obsolete)); + if (!readOnly) DurableFiles.deleteIfExists(file(id, obsolete)); entries.put(id, retained); } if (entries.get(id) == State.COMPLETED) unconfirmedCompletions.add(id); @@ -290,7 +332,35 @@ private void verifyStateFile(Path file, String id) throws IOException { throw new IOException("HTTP inbound delivery fence state is unsafe"); } private void requireWritable() throws IOException { - if (sealed) throw new IOException("HTTP inbound delivery store ownership has ended"); + if (readOnly || sealed || ownershipLock == null || !ownershipLock.isValid()) + throw new IOException("HTTP inbound delivery store ownership has ended"); + } + private void claimOwnership(Path sidecar) throws IOException { + if (Files.isSymbolicLink(sidecar) || Files.exists(sidecar, LinkOption.NOFOLLOW_LINKS) + && !Files.isRegularFile(sidecar, LinkOption.NOFOLLOW_LINKS)) + throw new IOException("HTTP inbound delivery ownership lock is unsafe"); + FileChannel channel = FileChannel.open(sidecar, StandardOpenOption.CREATE, StandardOpenOption.WRITE, + LinkOption.NOFOLLOW_LINKS); + try { + PrivateFilePermissions.ownerOnlyFile(sidecar); + FileLock lock; + try { lock = channel.tryLock(); } + catch (OverlappingFileLockException alreadyOwned) { throw new IOException("HTTP inbound delivery store is already owned", alreadyOwned); } + if (lock == null) throw new IOException("HTTP inbound delivery store is already owned"); + ownershipChannel = channel; + ownershipLock = lock; + } catch (IOException | RuntimeException failure) { + try { channel.close(); } catch (IOException closeFailure) { failure.addSuppressed(closeFailure); } + throw failure; + } + } + private void releaseOwnership() { + FileLock lock = ownershipLock; + FileChannel channel = ownershipChannel; + ownershipLock = null; + ownershipChannel = null; + if (lock != null) try { lock.release(); } catch (IOException ignored) { } + if (channel != null) try { channel.close(); } catch (IOException ignored) { } } private static void move(Path source, Path target) throws IOException { try { Files.move(source, target, StandardCopyOption.ATOMIC_MOVE); } diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java index e3c20fa..586b99b 100644 --- a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java @@ -111,29 +111,46 @@ public HttpProxyTransportServer(InetSocketAddress bind, HttpTlsIdentity identity this.onAcknowledged = onAcknowledged; this.nanoTime = nanoTime; durableOutgoing = outgoingDirectory == null ? null : new DurableOutgoingQueue(outgoingDirectory); durableIncomingRoot = outgoingDirectory == null ? null : incomingRoot(outgoingDirectory); - if (durableOutgoing != null) for (Map.Entry> pending - : durableOutgoing.load().entrySet()) { - BackendState state = backendState(pending.getKey()); - state.restore(pending.getValue()); - } - server = HttpsServer.create(bind, 32); - server.setHttpsConfigurator(new HttpsConfigurator(identity.serverContext()) { - @Override public void configure(HttpsParameters parameters) { - SSLParameters ssl = HttpPinnedTls.secureParameters(getSSLContext()); - ssl.setWantClientAuth(true); parameters.setSSLParameters(ssl); + try { + if (durableOutgoing != null) for (Map.Entry> pending + : durableOutgoing.load().entrySet()) { + BackendState state = backendState(pending.getKey()); + state.restore(pending.getValue()); } - }); - // Long polls are blocking by design. Capacity is bounded by admission, while enough workers - // remain available for all admitted polls plus setup requests. - listenerExecutor = executor("SimpleAPI-HTTP-listener", 72, 72); - // The proxy router mutates shared presence, vote, and reward state. A separate - // bounded FIFO lane keeps wire order without blocking long-poll workers. - handlerExecutor = executor("SimpleAPI-HTTP-handler", 1, - HttpBackendTransportConnector.CALLBACK_QUEUE_CAPACITY, handlerWorker); - server.setExecutor(listenerExecutor); - server.createContext("/v1/enroll", exchange -> enroll((HttpsExchange) exchange)); - server.createContext("/v1/renew", exchange -> renew((HttpsExchange) exchange)); - server.createContext("/v1/transport", exchange -> transport((HttpsExchange) exchange)); + } catch (Exception | Error setupFailure) { + releaseBackendOwnership(); + throw setupFailure; + } + try { + server = HttpsServer.create(bind, 32); + server.setHttpsConfigurator(new HttpsConfigurator(identity.serverContext()) { + @Override public void configure(HttpsParameters parameters) { + SSLParameters ssl = HttpPinnedTls.secureParameters(getSSLContext()); + ssl.setWantClientAuth(true); parameters.setSSLParameters(ssl); + } + }); + // Long polls are blocking by design. Capacity is bounded by admission, while enough workers + // remain available for all admitted polls plus setup requests. + listenerExecutor = executor("SimpleAPI-HTTP-listener", 72, 72); + // The proxy router mutates shared presence, vote, and reward state. A separate + // bounded FIFO lane keeps wire order without blocking long-poll workers. + handlerExecutor = executor("SimpleAPI-HTTP-handler", 1, + HttpBackendTransportConnector.CALLBACK_QUEUE_CAPACITY, handlerWorker); + server.setExecutor(listenerExecutor); + server.createContext("/v1/enroll", exchange -> enroll((HttpsExchange) exchange)); + server.createContext("/v1/renew", exchange -> renew((HttpsExchange) exchange)); + server.createContext("/v1/transport", exchange -> transport((HttpsExchange) exchange)); + } catch (Exception | Error setupFailure) { + releaseBackendOwnership(); + throw setupFailure; + } + } + + private void releaseBackendOwnership() { + synchronized (backends) { + for (BackendState backend : backends.values()) backend.seal(); + backends.clear(); + } } private void renew(HttpsExchange exchange) throws IOException { @@ -335,11 +352,17 @@ private BackendState backendState(String serverId) throws IOException { if (existing != null) return existing; if (backends.size() >= MAX_BACKENDS) reclaimInactiveBackend(); if (backends.size() >= MAX_BACKENDS) throw new IOException("HTTP backend state exceeds its bound"); - HttpInboundDeliveryStore inbound = durableIncomingRoot == null ? null - : HttpInboundDeliveryStore.open(durableIncomingRoot, serverId); - BackendState created = new BackendState(serverId, durableOutgoing, inbound, onAcknowledged, nanoTime); - backends.put(serverId, created); - return created; + HttpInboundDeliveryStore inbound = null; + try { + inbound = durableIncomingRoot == null ? null + : HttpInboundDeliveryStore.open(durableIncomingRoot, serverId); + BackendState created = new BackendState(serverId, durableOutgoing, inbound, onAcknowledged, nanoTime); + backends.put(serverId, created); + return created; + } catch (Exception | Error setupFailure) { + if (inbound != null) inbound.seal(); + throw setupFailure; + } } } private void reclaimInactiveBackend() throws IOException { diff --git a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpBackendLifecycleRecoveryTest.java b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpBackendLifecycleRecoveryTest.java index e8c6c44..c6e90e9 100644 --- a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpBackendLifecycleRecoveryTest.java +++ b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpBackendLifecycleRecoveryTest.java @@ -38,7 +38,7 @@ void callbackOwnedCloseCompletesItsJournalBeforeAsynchronousSealing() throws Exc long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(2); HttpInboundDeliveryStore.State state = null; while (System.nanoTime() < deadline) { - state = new HttpInboundDeliveryStore(clientDirectory).state(id); + state = HttpInboundDeliveryStore.inspect(clientDirectory).state(id); if (state == HttpInboundDeliveryStore.State.COMPLETED) break; Thread.sleep(5); } diff --git a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentPinTest.java b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentPinTest.java index 000f71a..93a3763 100644 --- a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentPinTest.java +++ b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentPinTest.java @@ -7,7 +7,12 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import java.net.URI; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; import java.nio.file.Path; +import java.security.KeyPair; +import java.security.KeyPairGenerator; +import java.security.KeyStore; import java.time.Duration; import java.time.Instant; import org.junit.jupiter.api.Test; @@ -72,6 +77,25 @@ void acceptsBundleSignedByConnectionCodeCa() throws Exception { assertEquals(code.caCertificatePin(), HttpTransportSecrets.certificatePin(enrolled.credential().caCertificate())); } + @Test + void rejectsPkcs12WithUnrelatedPrivateKeyDuringEnrollmentAndReplacement() throws Exception { + HttpTlsIdentity proxy = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); + HttpConnectionCode code = code(proxy, "lobby-1"); + HttpTlsIdentity.IssuedClientCertificate original = proxy.issueClientCertificate("lobby-1"); + HttpTlsIdentity.IssuedClientCertificate mismatched = withUnrelatedPrivateKey(original); + Path client = directory.resolve("client"); + + assertThrows(java.io.IOException.class, () -> HttpClientCredentialStore.saveEnrolled(client, code, mismatched)); + assertFalse(HttpClientCredentialStore.hasEnrolledProfile(client)); + + HttpClientCredentialStore.saveEnrolled(client, code, original); + HttpClientCredentialStore.EnrolledClient before = HttpClientCredentialStore.loadEnrolled(client); + assertThrows(java.io.IOException.class, () -> HttpClientCredentialStore.stageReplacement(client, mismatched)); + HttpClientCredentialStore.EnrolledClient after = HttpClientCredentialStore.loadEnrolled(client); + assertEquals(before.profile(), after.profile()); + assertArrayEquals(before.credential().certificate().getEncoded(), after.credential().certificate().getEncoded()); + } + @Test void stageReplacementRejectsForeignCaAndPreservesActiveEnrollment() throws Exception { HttpTlsIdentity proxy = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); @@ -141,4 +165,20 @@ private HttpConnectionCode code(HttpTlsIdentity identity, String serverId) { identity.serverCertificatePin(), identity.caCertificatePin(), Instant.now().plusSeconds(300), HttpTransportSecrets.randomToken()); } + + private HttpTlsIdentity.IssuedClientCertificate withUnrelatedPrivateKey( + HttpTlsIdentity.IssuedClientCertificate issued) throws Exception { + char[] password = issued.password(); + KeyStore source = KeyStore.getInstance("PKCS12"); + source.load(new ByteArrayInputStream(issued.pkcs12()), password); + KeyPairGenerator generator = KeyPairGenerator.getInstance("EC"); + generator.initialize(new java.security.spec.ECGenParameterSpec("secp256r1")); + KeyPair unrelated = generator.generateKeyPair(); + KeyStore replacement = KeyStore.getInstance("PKCS12"); + replacement.load(null, password); + replacement.setKeyEntry("client", unrelated.getPrivate(), password, source.getCertificateChain("client")); + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + replacement.store(bytes, password); + return new HttpTlsIdentity.IssuedClientCertificate(issued.serverId(), issued.certificate(), bytes.toByteArray(), password); + } } diff --git a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpProxyLifecycleRecoveryTest.java b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpProxyLifecycleRecoveryTest.java index 3d0cd5f..625580c 100644 --- a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpProxyLifecycleRecoveryTest.java +++ b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpProxyLifecycleRecoveryTest.java @@ -58,7 +58,7 @@ void callbackOwnedCloseCompletesItsInboundJournalBeforeItIsSealed() throws Excep server.close(); assertNotNull(deliveryId.get()); assertEquals(HttpInboundDeliveryStore.State.COMPLETED, - HttpInboundDeliveryStore.open(directory.resolve("outgoing-incoming"), "lobby-1").state(deliveryId.get()), + HttpInboundDeliveryStore.inspect(directory.resolve("outgoing-incoming"), "lobby-1").state(deliveryId.get()), "the callback completion must be durable before close seals the journal"); } finally { server.close(); diff --git a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportRuntimeTest.java b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportRuntimeTest.java index 555d859..2519f30 100644 --- a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportRuntimeTest.java +++ b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportRuntimeTest.java @@ -79,7 +79,7 @@ void enrollsThenDeliversBothDirectionsWithAuthenticatedIdentity() throws Excepti long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(3); while (connector.queuedOutgoing() != 0 && System.nanoTime() < deadline) Thread.sleep(10); assertEquals(0, connector.queuedOutgoing(), "proxy ACK must remove the exact outbound delivery ID"); - Path proxyInboundFence = directory.resolve("proxy-outgoing-incoming"); + Path proxyInboundFence = directory.resolve("proxy-outgoing-incoming").resolve("lobby-1"); long confirmationDeadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(3); while (countRegularFiles(proxyInboundFence) != 0L && System.nanoTime() < confirmationDeadline) Thread.sleep(10); assertEquals(0L, countRegularFiles(proxyInboundFence), @@ -354,6 +354,40 @@ void proxyInboundCompletionSurvivesRestartBeforeAcknowledgement() throws Excepti "only an acknowledgement confirmation may retire the durable replay fence"); } + @Test + void inboundJournalHasOneWriterWhileReadOnlyInspectionRemainsAvailable() throws Exception { + Path root = directory.resolve("exclusive-incoming"); + Files.createDirectory(root); + String deliveryId = java.util.UUID.randomUUID().toString(); + HttpInboundDeliveryStore owner = HttpInboundDeliveryStore.open(root, "lobby-1"); + owner.reserve(deliveryId); + assertThrows(java.io.IOException.class, () -> HttpInboundDeliveryStore.open(root, "lobby-1"), + "a live journal owner must exclude a stale in-memory writer"); + assertEquals(HttpInboundDeliveryStore.State.RESERVED, + HttpInboundDeliveryStore.inspect(root, "lobby-1").state(deliveryId)); + owner.seal(); + HttpInboundDeliveryStore successor = HttpInboundDeliveryStore.open(root, "lobby-1"); + successor.markRunning(deliveryId); + successor.seal(); + } + + @Test + void backendConnectorReleasesJournalOwnershipOnlyAfterClose() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("owner-proxy"), "localhost"); + HttpTlsIdentity.IssuedClientCertificate issued = identity.issueClientCertificate("lobby-1"); + HttpConnectionCode code = new HttpConnectionCode("lobby-1", URI.create("https://localhost:8443/"), + identity.serverCertificatePin(), identity.caCertificatePin(), Instant.now().plusSeconds(300), "A".repeat(43)); + Path credentials = directory.resolve("owner-client"); + HttpClientCredentialStore.saveEnrolled(credentials, code, issued); + try (HttpBackendTransportConnector owner = new HttpBackendTransportConnector(credentials, ignored -> { })) { + assertThrows(java.io.IOException.class, () -> new HttpBackendTransportConnector(credentials, ignored -> { }), + "two live connectors must not own the same inbound delivery journal"); + } + try (HttpBackendTransportConnector successor = new HttpBackendTransportConnector(credentials, ignored -> { })) { + assertFalse(successor.pollerAlive()); + } + } + @Test void completedInboundPublicationRetriesForceBeforeProxyAcknowledgement() throws Exception { Path root = directory.resolve("proxy-incoming-force-retry"); @@ -458,7 +492,7 @@ void runningPublicationRollbackLeavesAReservationSafeAfterRestart() throws Excep } store.seal(); assertEquals(HttpInboundDeliveryStore.State.RESERVED, - HttpInboundDeliveryStore.open(root, "lobby-1").state(deliveryId), + HttpInboundDeliveryStore.inspect(root, "lobby-1").state(deliveryId), "a callback never exposed must be recoverable after a one-shot RUNNING force failure"); } @@ -624,7 +658,7 @@ void closeDrainsRunningCallbacksBeforeSealingTheirJournal() throws Exception { closer.join(3000); assertFalse(closer.isAlive()); assertEquals(HttpInboundDeliveryStore.State.COMPLETED, - new HttpInboundDeliveryStore(clientDirectory).state(id)); + HttpInboundDeliveryStore.inspect(clientDirectory).state(id)); } @Test @@ -655,7 +689,7 @@ void interruptedCloseRemainsBoundedWhenACallbackIgnoresInterruption() throws Exc assertTrue(stopped.await(2, TimeUnit.SECONDS)); } finally { release.countDown(); } assertEquals(HttpInboundDeliveryStore.State.RUNNING, - new HttpInboundDeliveryStore(clientDirectory).state(id), + HttpInboundDeliveryStore.inspect(clientDirectory).state(id), "an ambiguous callback must remain fail-closed after bounded shutdown"); } @@ -1356,7 +1390,9 @@ void reservedButNotStartedDeliveryResumesAfterRestart() throws Exception { Path clientDirectory = directory.resolve("reserved-client"); HttpClientCredentialStore.saveEnrolled(clientDirectory, code, issued); String id = java.util.UUID.randomUUID().toString(); - new HttpInboundDeliveryStore(clientDirectory).reserve(id); + HttpInboundDeliveryStore reservedStore = new HttpInboundDeliveryStore(clientDirectory); + reservedStore.reserve(id); + reservedStore.seal(); CountDownLatch completed = new CountDownLatch(1); HttpTransportProtocol.Delivery delivery = new HttpTransportProtocol.Delivery(id, JsonEnvelope.builder("vote").build()); try (HttpBackendTransportConnector restarted = new HttpBackendTransportConnector(clientDirectory, ignored -> completed.countDown())) { From c9b9e6f2e4e1b2b1b36e6fe758588064f203feef Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Sat, 5 Sep 2026 22:48:22 -0600 Subject: [PATCH 25/38] Recover journal retirement and lock outgoing proxy queues --- .../http/HttpInboundDeliveryStore.java | 20 ++- .../http/HttpProxyTransportServer.java | 118 +++++++++++++----- .../http/HttpInboundRetirementTest.java | 60 +++++++++ .../http/HttpOutgoingOwnershipTest.java | 56 +++++++++ .../http/HttpOutgoingQueueCapacityTest.java | 2 + .../http/HttpProxyLifecycleRecoveryTest.java | 7 +- .../http/HttpTransportRuntimeTest.java | 13 +- 7 files changed, 240 insertions(+), 36 deletions(-) create mode 100644 SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpInboundRetirementTest.java create mode 100644 SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpOutgoingOwnershipTest.java diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpInboundDeliveryStore.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpInboundDeliveryStore.java index 9ad230f..6c0bcd5 100644 --- a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpInboundDeliveryStore.java +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpInboundDeliveryStore.java @@ -39,6 +39,7 @@ final class HttpInboundDeliveryStore { private FileChannel ownershipChannel; private FileLock ownershipLock; private boolean sealed; + private boolean retirementRecoveryRequired; HttpInboundDeliveryStore(Path credentialDirectory) throws IOException { this(credentialDirectory, DIRECTORY, false); @@ -173,12 +174,13 @@ synchronized void sealAndDeleteIfEmpty() throws IOException { try (DirectoryStream files = Files.newDirectoryStream(root)) { if (files.iterator().hasNext()) throw new IOException("HTTP inbound delivery directory is not empty"); } - sealed = true; Path parent = root.getParent(); - try { - Files.delete(root); - DurableFiles.forceDirectory(parent); - } finally { releaseOwnership(); } + // Keep exclusive ownership if deletion or its publication cannot be confirmed. + // The live backend may retry retirement or reconnect and resume journal writes. + retirementRecoveryRequired = true; + Files.delete(root); + DurableFiles.forceDirectory(parent); + seal(); } synchronized void remove(String id) throws IOException { @@ -334,6 +336,14 @@ private void verifyStateFile(Path file, String id) throws IOException { private void requireWritable() throws IOException { if (readOnly || sealed || ownershipLock == null || !ownershipLock.isValid()) throw new IOException("HTTP inbound delivery store ownership has ended"); + if (retirementRecoveryRequired) { + try { Files.createDirectory(root); } + catch (java.nio.file.FileAlreadyExistsException existing) { } + requireRoot(); + PrivateFilePermissions.ownerOnlyDirectory(root); + DurableFiles.forceDirectory(root.getParent()); + retirementRecoveryRequired = false; + } } private void claimOwnership(Path sidecar) throws IOException { if (Files.isSymbolicLink(sidecar) || Files.exists(sidecar, LinkOption.NOFOLLOW_LINKS) diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java index 586b99b..ad97cca 100644 --- a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java @@ -13,6 +13,9 @@ import java.io.InputStream; import java.net.InetSocketAddress; import java.net.URI; +import java.nio.channels.FileChannel; +import java.nio.channels.FileLock; +import java.nio.channels.OverlappingFileLockException; import java.nio.file.DirectoryStream; import java.nio.file.Files; import java.nio.file.LinkOption; @@ -110,20 +113,17 @@ public HttpProxyTransportServer(InetSocketAddress bind, HttpTlsIdentity identity this.identity = identity; this.authority = authority; this.onEnvelope = onEnvelope; this.onAcknowledged = onAcknowledged; this.nanoTime = nanoTime; durableOutgoing = outgoingDirectory == null ? null : new DurableOutgoingQueue(outgoingDirectory); - durableIncomingRoot = outgoingDirectory == null ? null : incomingRoot(outgoingDirectory); + HttpsServer createdServer = null; + ThreadPoolExecutor createdListener = null, createdHandler = null; try { + durableIncomingRoot = outgoingDirectory == null ? null : incomingRoot(outgoingDirectory); if (durableOutgoing != null) for (Map.Entry> pending : durableOutgoing.load().entrySet()) { BackendState state = backendState(pending.getKey()); state.restore(pending.getValue()); } - } catch (Exception | Error setupFailure) { - releaseBackendOwnership(); - throw setupFailure; - } - try { - server = HttpsServer.create(bind, 32); - server.setHttpsConfigurator(new HttpsConfigurator(identity.serverContext()) { + createdServer = HttpsServer.create(bind, 32); + createdServer.setHttpsConfigurator(new HttpsConfigurator(identity.serverContext()) { @Override public void configure(HttpsParameters parameters) { SSLParameters ssl = HttpPinnedTls.secureParameters(getSSLContext()); ssl.setWantClientAuth(true); parameters.setSSLParameters(ssl); @@ -131,17 +131,24 @@ public HttpProxyTransportServer(InetSocketAddress bind, HttpTlsIdentity identity }); // Long polls are blocking by design. Capacity is bounded by admission, while enough workers // remain available for all admitted polls plus setup requests. - listenerExecutor = executor("SimpleAPI-HTTP-listener", 72, 72); + createdListener = executor("SimpleAPI-HTTP-listener", 72, 72); // The proxy router mutates shared presence, vote, and reward state. A separate // bounded FIFO lane keeps wire order without blocking long-poll workers. - handlerExecutor = executor("SimpleAPI-HTTP-handler", 1, + createdHandler = executor("SimpleAPI-HTTP-handler", 1, HttpBackendTransportConnector.CALLBACK_QUEUE_CAPACITY, handlerWorker); - server.setExecutor(listenerExecutor); - server.createContext("/v1/enroll", exchange -> enroll((HttpsExchange) exchange)); - server.createContext("/v1/renew", exchange -> renew((HttpsExchange) exchange)); - server.createContext("/v1/transport", exchange -> transport((HttpsExchange) exchange)); + createdServer.setExecutor(createdListener); + createdServer.createContext("/v1/enroll", exchange -> enroll((HttpsExchange) exchange)); + createdServer.createContext("/v1/renew", exchange -> renew((HttpsExchange) exchange)); + createdServer.createContext("/v1/transport", exchange -> transport((HttpsExchange) exchange)); + server = createdServer; + listenerExecutor = createdListener; + handlerExecutor = createdHandler; } catch (Exception | Error setupFailure) { + if (createdServer != null) createdServer.stop(0); + if (createdHandler != null) shutdown(createdHandler); + if (createdListener != null) shutdown(createdListener); releaseBackendOwnership(); + if (durableOutgoing != null) durableOutgoing.close(); throw setupFailure; } } @@ -257,14 +264,18 @@ private boolean send(String serverId, String deliveryId, JsonEnvelope envelope, private void finishClose() { try { - shutdown(handlerExecutor); - shutdown(listenerExecutor); - synchronized (backends) { - for (BackendState backend : backends.values()) { backend.seal(); backend.signal(); } - backends.clear(); - } + try { shutdown(handlerExecutor); } + finally { shutdown(listenerExecutor); } } finally { - synchronized (closeMonitor) { closeFinalizing = false; closeFinalized = true; closeMonitor.notifyAll(); } + try { + synchronized (backends) { + for (BackendState backend : backends.values()) { backend.seal(); backend.signal(); } + backends.clear(); + } + } finally { try { if (durableOutgoing != null) durableOutgoing.close(); } + finally { + synchronized (closeMonitor) { closeFinalizing = false; closeFinalized = true; closeMonitor.notifyAll(); } + } } } } @@ -659,7 +670,7 @@ private long nanosUntilRedelivery(long now) { private synchronized void signal() { notifyAll(); } } - static final class DurableOutgoingQueue { + static final class DurableOutgoingQueue implements AutoCloseable { @FunctionalInterface interface DirectoryForcer { void force(Path directory) throws IOException; } private static final String FILE_PATTERN = "[0-9]{20}-[0-9a-f-]{36}\\.json"; @@ -667,6 +678,8 @@ interface DirectoryForcer { void force(Path directory) throws IOException; } private final DirectoryForcer directoryForcer; private final Map> files = new HashMap<>(); private final Map> quarantinedFiles = new HashMap<>(); + private FileChannel ownershipChannel; + private FileLock ownershipLock; private long sequence; private DurableOutgoingQueue(Path root) throws IOException { @@ -678,17 +691,27 @@ private DurableOutgoingQueue(Path root) throws IOException { throw new IllegalArgumentException("HTTP outgoing queue configuration is required"); this.directoryForcer = directoryForcer; this.root = root.toAbsolutePath().normalize(); - try { Files.createDirectory(this.root); } - catch (java.nio.file.FileAlreadyExistsException existing) { } - if (Files.isSymbolicLink(this.root) || !Files.isDirectory(this.root, LinkOption.NOFOLLOW_LINKS)) + Path parent = this.root.getParent(); + if (parent == null || this.root.getFileName() == null) throw new IOException("HTTP outgoing queue directory is invalid"); - PrivateFilePermissions.ownerOnlyDirectory(this.root); - // A failed parent fsync can leave the directory present but not durable. - // Reopening must retry it before the queue can accept work. - directoryForcer.force(this.root.getParent()); + try { + claimOwnership(parent.resolve("." + this.root.getFileName() + ".http-outgoing-owner.lock")); + try { Files.createDirectory(this.root); } + catch (java.nio.file.FileAlreadyExistsException existing) { } + if (Files.isSymbolicLink(this.root) || !Files.isDirectory(this.root, LinkOption.NOFOLLOW_LINKS)) + throw new IOException("HTTP outgoing queue directory is invalid"); + PrivateFilePermissions.ownerOnlyDirectory(this.root); + // A failed parent fsync can leave the directory present but not durable. + // Reopening must retry it before the queue can accept work. + directoryForcer.force(parent); + } catch (IOException | RuntimeException setupFailure) { + releaseOwnership(); + throw setupFailure; + } } synchronized Map> load() throws IOException { + requireOwnership(); Map> loaded = new LinkedHashMap<>(); int serverDirectories = 0; try (DirectoryStream servers = Files.newDirectoryStream(root)) { @@ -762,6 +785,7 @@ synchronized Map> load() throws IOE } private synchronized void persist(String serverId, HttpTransportProtocol.Delivery delivery) throws IOException { + requireOwnership(); Path directory = root.resolve(serverId).normalize(); if (!directory.getParent().equals(root)) throw new IOException("HTTP outgoing queue server is invalid"); if (!Files.exists(directory, LinkOption.NOFOLLOW_LINKS) && serverDirectoryCount() >= MAX_BACKENDS) @@ -903,6 +927,7 @@ private void quarantinePublished(Path directory, Path target, Path pending, Map< } private synchronized void confirm(String serverId, String id) throws IOException { + requireOwnership(); Map serverFiles = files.get(serverId); Path file = serverFiles == null ? null : serverFiles.get(id); if (file == null || Files.isSymbolicLink(file) || !Files.isRegularFile(file, LinkOption.NOFOLLOW_LINKS)) @@ -912,6 +937,7 @@ private synchronized void confirm(String serverId, String id) throws IOException } private synchronized void remove(String serverId, String id) throws IOException { + requireOwnership(); Map serverFiles = files.get(serverId); if (serverFiles == null) throw new IOException("HTTP outgoing queue acknowledgement is unknown"); Path file = serverFiles.get(id); @@ -936,5 +962,39 @@ private synchronized void remove(String serverId, String id) throws IOException } } + @Override public synchronized void close() { releaseOwnership(); } + + private void requireOwnership() throws IOException { + if (ownershipLock == null || !ownershipLock.isValid()) + throw new IOException("HTTP outgoing queue ownership has ended"); + } + private void claimOwnership(Path sidecar) throws IOException { + if (Files.isSymbolicLink(sidecar) || Files.exists(sidecar, LinkOption.NOFOLLOW_LINKS) + && !Files.isRegularFile(sidecar, LinkOption.NOFOLLOW_LINKS)) + throw new IOException("HTTP outgoing queue ownership lock is unsafe"); + FileChannel channel = FileChannel.open(sidecar, StandardOpenOption.CREATE, StandardOpenOption.WRITE, + LinkOption.NOFOLLOW_LINKS); + try { + PrivateFilePermissions.ownerOnlyFile(sidecar); + FileLock lock; + try { lock = channel.tryLock(); } + catch (OverlappingFileLockException alreadyOwned) { throw new IOException("HTTP outgoing queue is already owned", alreadyOwned); } + if (lock == null) throw new IOException("HTTP outgoing queue is already owned"); + ownershipChannel = channel; + ownershipLock = lock; + } catch (IOException | RuntimeException failure) { + try { channel.close(); } catch (IOException closeFailure) { failure.addSuppressed(closeFailure); } + throw failure; + } + } + private void releaseOwnership() { + FileLock lock = ownershipLock; + FileChannel channel = ownershipChannel; + ownershipLock = null; + ownershipChannel = null; + if (lock != null) try { lock.release(); } catch (IOException ignored) { } + if (channel != null) try { channel.close(); } catch (IOException ignored) { } + } + } } diff --git a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpInboundRetirementTest.java b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpInboundRetirementTest.java new file mode 100644 index 0000000..4941b09 --- /dev/null +++ b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpInboundRetirementTest.java @@ -0,0 +1,60 @@ +package com.bencodez.simpleapi.servercomm.http; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicBoolean; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class HttpInboundRetirementTest { + @TempDir Path directory; + + @Test + void failedRetirementKeepsOwnershipAndRestoresRootOnNextWrite() throws Exception { + Path parent = directory.resolve("incoming"); + Files.createDirectory(parent); + HttpInboundDeliveryStore store = HttpInboundDeliveryStore.open(parent, "lobby-1"); + AtomicBoolean fail = new AtomicBoolean(true); + try (var forces = org.mockito.Mockito.mockStatic(com.bencodez.simpleapi.file.DurableFiles.class, + org.mockito.Mockito.CALLS_REAL_METHODS)) { + forces.when(() -> com.bencodez.simpleapi.file.DurableFiles.forceDirectory(parent)).thenAnswer(call -> { + if (fail.getAndSet(false)) throw new IOException("injected retirement force failure"); + return call.callRealMethod(); + }); + assertThrows(IOException.class, store::sealAndDeleteIfEmpty); + assertThrows(IOException.class, () -> HttpInboundDeliveryStore.open(parent, "lobby-1")); + String id = UUID.randomUUID().toString(); + store.reserve(id); + store.markRunning(id); + assertEquals(HttpInboundDeliveryStore.State.RUNNING, store.state(id)); + } + store.seal(); + } + + @Test + void failedRetirementCanRetryAfterDeleteFailure() throws Exception { + Path parent = directory.resolve("incoming"); + Files.createDirectory(parent); + HttpInboundDeliveryStore store = HttpInboundDeliveryStore.open(parent, "lobby-1"); + Path root = parent.resolve("lobby-1"); + AtomicBoolean fail = new AtomicBoolean(true); + try (var files = org.mockito.Mockito.mockStatic(Files.class, org.mockito.Mockito.CALLS_REAL_METHODS)) { + files.when(() -> Files.delete(root)).thenAnswer(call -> { + if (fail.getAndSet(false)) throw new IOException("injected retirement delete failure"); + return call.callRealMethod(); + }); + // CALLS_REAL_METHODS executes the unstubbed delete while registering it. + Files.createDirectory(root); + assertThrows(IOException.class, store::sealAndDeleteIfEmpty); + assertThrows(IOException.class, () -> HttpInboundDeliveryStore.open(parent, "lobby-1")); + store.sealAndDeleteIfEmpty(); + } + HttpInboundDeliveryStore successor = HttpInboundDeliveryStore.open(parent, "lobby-1"); + successor.seal(); + } +} diff --git a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpOutgoingOwnershipTest.java b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpOutgoingOwnershipTest.java new file mode 100644 index 0000000..df2ab34 --- /dev/null +++ b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpOutgoingOwnershipTest.java @@ -0,0 +1,56 @@ +package com.bencodez.simpleapi.servercomm.http; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.bencodez.simpleapi.servercomm.codec.JsonEnvelope; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.nio.file.Path; +import java.util.List; +import java.util.UUID; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class HttpOutgoingOwnershipTest { + @TempDir Path directory; + + @Test + void outgoingQueueAllowsOneOwnerAndFailsClosedAfterClose() throws Exception { + Path root = directory.resolve("outgoing"); + HttpProxyTransportServer.DurableOutgoingQueue owner = new HttpProxyTransportServer.DurableOutgoingQueue(root, + com.bencodez.simpleapi.file.DurableFiles::forceDirectory); + HttpProxyTransportServer.BackendState state = new HttpProxyTransportServer.BackendState("lobby-1", owner, + (server, id) -> { }); + HttpTransportProtocol.Delivery delivery = new HttpTransportProtocol.Delivery(UUID.randomUUID().toString(), + JsonEnvelope.builder("owner").build()); + assertTrue(state.enqueue(delivery)); + assertThrows(IOException.class, () -> new HttpProxyTransportServer.DurableOutgoingQueue(root, + com.bencodez.simpleapi.file.DurableFiles::forceDirectory)); + owner.close(); + assertThrows(IOException.class, owner::load); + assertFalse(state.enqueue(new HttpTransportProtocol.Delivery(UUID.randomUUID().toString(), + JsonEnvelope.builder("closed").build()))); + assertThrows(IOException.class, () -> state.acknowledge(List.of(delivery.id()))); + HttpProxyTransportServer.DurableOutgoingQueue successor = new HttpProxyTransportServer.DurableOutgoingQueue(root, + com.bencodez.simpleapi.file.DurableFiles::forceDirectory); + successor.close(); + } + + @Test + void proxyClaimsOutgoingOwnershipBeforeAnyBackendStateExists() throws Exception { + Path root = directory.resolve("outgoing"); + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, directory.resolve("authority")); + HttpProxyTransportServer owner = new HttpProxyTransportServer(new InetSocketAddress("localhost", 0), identity, + authority, root, ignored -> { }); + try { + assertThrows(IOException.class, () -> new HttpProxyTransportServer(new InetSocketAddress("localhost", 0), identity, + authority, root, ignored -> { }), + "even an idle proxy must own the durable outgoing queue exclusively"); + } finally { owner.close(); } + try (HttpProxyTransportServer successor = new HttpProxyTransportServer(new InetSocketAddress("localhost", 0), identity, + authority, root, ignored -> { })) { } + } +} diff --git a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpOutgoingQueueCapacityTest.java b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpOutgoingQueueCapacityTest.java index b355d50..422c3e4 100644 --- a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpOutgoingQueueCapacityTest.java +++ b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpOutgoingQueueCapacityTest.java @@ -48,6 +48,7 @@ void quarantineDirectoriesCountTowardCapacityWithoutBlockingSameServerRetry() th HttpProxyTransportServer.BackendState existingServer = new HttpProxyTransportServer.BackendState( "server-0", queue, (server, id) -> { }); assertTrue(existingServer.enqueue(existingDelivery), "same-server retry must recover its quarantine at capacity"); + queue.close(); HttpProxyTransportServer.DurableOutgoingQueue restarted = new HttpProxyTransportServer.DurableOutgoingQueue( queueRoot, com.bencodez.simpleapi.file.DurableFiles::forceDirectory); @@ -72,5 +73,6 @@ void quarantineDirectoriesCountTowardCapacityWithoutBlockingSameServerRetry() th } assertEquals(128L, durableFiles, "each seeded delivery must remain represented exactly once"); assertEquals(128, deliveryIds.size()); + restarted.close(); } } diff --git a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpProxyLifecycleRecoveryTest.java b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpProxyLifecycleRecoveryTest.java index 625580c..bdf8218 100644 --- a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpProxyLifecycleRecoveryTest.java +++ b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpProxyLifecycleRecoveryTest.java @@ -83,12 +83,15 @@ void failedNewBackendSetupCleansItsEmptyDirectoryBeforeRetry() throws Exception assertFalse(Files.exists(root.resolve("lobby-1")), "a failed setup must not leave an empty backend directory consuming the global bound"); assertTrue(state.enqueue(delivery)); + queue.close(); } @Test void restartPrunesAccumulatedEmptyBackendDirectoriesBeforeApplyingTheCap() throws Exception { Path root = directory.resolve("outgoing"); - new HttpProxyTransportServer.DurableOutgoingQueue(root, ignored -> { }); + HttpProxyTransportServer.DurableOutgoingQueue initial = new HttpProxyTransportServer.DurableOutgoingQueue(root, + ignored -> { }); + initial.close(); for (int index = 0; index < 128; index++) Files.createDirectory(root.resolve("server-" + index)); HttpProxyTransportServer.DurableOutgoingQueue restarted = new HttpProxyTransportServer.DurableOutgoingQueue(root, @@ -99,6 +102,7 @@ void restartPrunesAccumulatedEmptyBackendDirectoriesBeforeApplyingTheCap() throw assertTrue(state.enqueue(new HttpTransportProtocol.Delivery(UUID.randomUUID().toString(), JsonEnvelope.builder("recovered-capacity").build()))); assertTrue(Files.isDirectory(root.resolve("replacement"))); + restarted.close(); } @Test @@ -132,5 +136,6 @@ void capPruningPreservesAnEmptyDirectoryUntilItsUnconfirmedAckCanRetry() throws lobby.acknowledge(List.of(delivery.id())); } assertFalse(Files.exists(lobbyDirectory), "the same acknowledgement retry must be able to finish cleanup"); + queue.close(); } } diff --git a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportRuntimeTest.java b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportRuntimeTest.java index 2519f30..d1d0fa1 100644 --- a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportRuntimeTest.java +++ b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportRuntimeTest.java @@ -177,6 +177,8 @@ void outgoingQueueRetriesDirectoryPublicationAfterForceFailure() throws Exceptio assertEquals(2L, serverRootForces.get(), "failed publication and empty-directory cleanup must both force the parent"); assertTrue(state.enqueue(delivery)); assertEquals(3L, serverRootForces.get(), "retrying backend directory creation must force its parent again"); + queue.close(); + serverQueue.close(); } @Test @@ -255,6 +257,7 @@ void outgoingQueueRecoversWhenPromotionQuarantineRenameFails() throws Exception "target fallback must clear the stale quarantine index"); state.acknowledge(java.util.List.of(deliveryId)); assertFalse(Files.exists(serverDirectory)); + queue.close(); } @Test @@ -275,6 +278,7 @@ void publishedOutgoingDeliveryRemainsTrackedUntilDurabilityCanBeConfirmed() thro assertTrue(state.await("lobby-1", java.util.UUID.randomUUID().toString(), 0).messages().isEmpty(), "an uncertain publication must remain hidden until its durability retry succeeds"); assertEquals(1L, countRegularFiles(queueRoot)); + queue.close(); HttpProxyTransportServer.DurableOutgoingQueue restartedQueue = new HttpProxyTransportServer.DurableOutgoingQueue( queueRoot, com.bencodez.simpleapi.file.DurableFiles::forceDirectory); assertTrue(restartedQueue.load().isEmpty(), @@ -286,6 +290,7 @@ void publishedOutgoingDeliveryRemainsTrackedUntilDurabilityCanBeConfirmed() thro assertEquals(1L, countRegularFiles(queueRoot), "durability retry must not create a duplicate file"); assertEquals(java.util.List.of(delivery), restarted.await("lobby-1", java.util.UUID.randomUUID().toString(), 0).messages()); + restartedQueue.close(); HttpProxyTransportServer.DurableOutgoingQueue confirmedQueue = new HttpProxyTransportServer.DurableOutgoingQueue( queueRoot, com.bencodez.simpleapi.file.DurableFiles::forceDirectory); java.util.List confirmed = confirmedQueue.load().get("lobby-1"); @@ -293,8 +298,12 @@ void publishedOutgoingDeliveryRemainsTrackedUntilDurabilityCanBeConfirmed() thro assertTrue(java.util.Arrays.equals(HttpTransportProtocol.storedDelivery(delivery), HttpTransportProtocol.storedDelivery(confirmed.get(0))), "a confirmed same-ID retry must become deliverable after restart"); - restarted.acknowledge(java.util.List.of(deliveryId)); + HttpProxyTransportServer.BackendState confirmedState = new HttpProxyTransportServer.BackendState( + "lobby-1", confirmedQueue, (server, id) -> { }); + assertTrue(confirmedState.enqueue(delivery)); + confirmedState.acknowledge(java.util.List.of(deliveryId)); assertEquals(0L, countRegularFiles(queueRoot), "the tracked published file must be removable by ACK"); + confirmedQueue.close(); } @Test @@ -316,9 +325,11 @@ void unresolvedOutgoingPublicationDoesNotReportAFalseRejection() throws Exceptio failForces.set(false); assertTrue(state.enqueue(delivery), "a same-ID retry must recover the observed quarantine"); assertEquals(1L, countRegularFiles(queueRoot), "recovery must not leave duplicate queue entries"); + queue.close(); HttpProxyTransportServer.DurableOutgoingQueue restarted = new HttpProxyTransportServer.DurableOutgoingQueue( queueRoot, com.bencodez.simpleapi.file.DurableFiles::forceDirectory); assertEquals(1, restarted.load().get("lobby-1").size()); + restarted.close(); } @Test From d5e84743c4ce67f1210bfadfda0c474b69931670 Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Sat, 5 Sep 2026 23:01:13 -0600 Subject: [PATCH 26/38] Prevent backend state creation after proxy shutdown --- .../http/HttpProxyTransportServer.java | 1 + .../http/HttpOutgoingOwnershipTest.java | 32 +++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java index ad97cca..e3bff5c 100644 --- a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java @@ -359,6 +359,7 @@ private void dispatch(String serverId, BackendState backend, HttpTransportProtoc } private BackendState backendState(String serverId) throws IOException { synchronized (backends) { + if (closed) throw new IOException("HTTP proxy transport is closed"); BackendState existing = backends.get(serverId); if (existing != null) return existing; if (backends.size() >= MAX_BACKENDS) reclaimInactiveBackend(); diff --git a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpOutgoingOwnershipTest.java b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpOutgoingOwnershipTest.java index df2ab34..1fe22a8 100644 --- a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpOutgoingOwnershipTest.java +++ b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpOutgoingOwnershipTest.java @@ -16,6 +16,38 @@ class HttpOutgoingOwnershipTest { @TempDir Path directory; + @Test + void senderWaitingForBackendMonitorCannotReopenJournalAfterClose() throws Exception { + Path root = directory.resolve("outgoing"); + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, directory.resolve("authority")); + HttpProxyTransportServer server = new HttpProxyTransportServer(new InetSocketAddress("localhost", 0), + identity, authority, root, ignored -> { }); + var field = HttpProxyTransportServer.class.getDeclaredField("backends"); + field.setAccessible(true); + java.util.concurrent.atomic.AtomicBoolean sent = new java.util.concurrent.atomic.AtomicBoolean(true); + Thread sender = new Thread(() -> sent.set(server.send("lobby-1", JsonEnvelope.builder("close-race").build()))); + try { + synchronized (field.get(server)) { + sender.start(); + long deadline = System.nanoTime() + java.util.concurrent.TimeUnit.SECONDS.toNanos(3); + while (sender.getState() != Thread.State.BLOCKED && System.nanoTime() < deadline) Thread.sleep(5); + org.junit.jupiter.api.Assertions.assertEquals(Thread.State.BLOCKED, sender.getState()); + server.close(); + } + sender.join(3000); + assertFalse(sender.isAlive()); + assertFalse(sent.get()); + try (HttpProxyTransportServer successor = new HttpProxyTransportServer(new InetSocketAddress("localhost", 0), + identity, authority, root, ignored -> { })) { + org.junit.jupiter.api.Assertions.assertNotNull(successor.backendStateForTest("lobby-1")); + } + } finally { + server.close(); + sender.join(3000); + } + } + @Test void outgoingQueueAllowsOneOwnerAndFailsClosedAfterClose() throws Exception { Path root = directory.resolve("outgoing"); From 2fe4343d9551763dfe3a5c25d815b7126e4733d8 Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Sat, 5 Sep 2026 23:16:30 -0600 Subject: [PATCH 27/38] Close backend send admission and enforce pending enrollment expiry --- .../http/HttpBackendTransportConnector.java | 63 ++++++---- .../http/HttpEnrollmentAuthority.java | 3 +- .../http/HttpBackendSendLifecycleTest.java | 113 ++++++++++++++++++ .../http/HttpPendingEnrollmentExpiryTest.java | 42 +++++++ 4 files changed, 198 insertions(+), 23 deletions(-) create mode 100644 SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpBackendSendLifecycleTest.java create mode 100644 SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpPendingEnrollmentExpiryTest.java diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpBackendTransportConnector.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpBackendTransportConnector.java index 69ed075..4bd5028 100644 --- a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpBackendTransportConnector.java +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpBackendTransportConnector.java @@ -72,6 +72,10 @@ public final class HttpBackendTransportConnector implements AutoCloseable { private volatile Thread poller; // Set by the executor's worker wrapper. Thread names are not an ownership boundary. private volatile Thread callbackWorker; + // Guarded by lifecycle. A flush drains the accepted queue but must not admit a new send. + private boolean flushingOutgoing; + // Guarded by state. Keep admission and queue insertion in the same critical section. + private boolean sendAdmissionOpen; private long sequence; private volatile long nextRenewalCheckNanos; // Guarded by renewal: do not supersede a visibly published credential until @@ -177,11 +181,16 @@ public static HttpClientCredentialStore.ClientCredential enroll(HttpConnectionCo public void start() { synchronized (lifecycle) { - if (closing.get() || !running.compareAndSet(false, true)) return; + if (closing.get() || flushingOutgoing || !running.compareAndSet(false, true)) return; poller = new Thread(this::pollLoop, "SimpleAPI-HTTP-poll"); poller.setDaemon(true); poller.start(); } + // Do not nest lifecycle and state: close/flush transition running before taking + // state, while send observes both values under state before enqueuing. + synchronized (state) { + if (running.get() && !closing.get()) sendAdmissionOpen = true; + } } /** Waits for one authenticated, protocol-valid transport response. */ public boolean awaitFirstResponse(long deadlineNanos) throws InterruptedException { @@ -193,10 +202,11 @@ public boolean awaitFirstResponse(long deadlineNanos) throws InterruptedExceptio * callers needing restart durability must retain the application operation independently. */ public boolean send(JsonEnvelope envelope) { - if (envelope == null || !running.get()) return false; + if (envelope == null) return false; try { HttpTransportProtocol.validateEnvelope(envelope); } catch (IllegalArgumentException invalid) { return false; } synchronized (state) { + if (!sendAdmissionOpen || !running.get()) return false; if (outgoing.size() >= HttpTransportProtocol.MAX_QUEUE) return false; String id = UUID.randomUUID().toString(); outgoing.put(id, new HttpTransportProtocol.Delivery(id, envelope)); return true; } @@ -248,27 +258,35 @@ private boolean pollOnce(Duration timeout, boolean requireRunning, boolean accep } /** Stops normal polling and gives already-queued outbound messages a bounded final delivery attempt. */ public boolean flushOutgoing(long deadlineNanos) { - running.set(false); - firstResponse.countDown(); - Thread current = poller; - if (current != null) current.interrupt(); - if (!joinPoller(current, deadlineNanos)) return false; - while (queuedOutgoing() != 0 || queuedAcknowledgements() != 0) { - long remaining = deadlineNanos - System.nanoTime(); - if (remaining <= 0L) return false; - Duration timeout = Duration.ofNanos(Math.min(CLIENT_TIMEOUT.toNanos(), remaining)); - synchronized (this) { - if (pollOnce(timeout, false, false)) continue; - } - // The proxy may retain its one-active-poll guard briefly after the old - // client request is interrupted. Retry that transient 409 without busy - // spinning, but never extend the caller's shutdown deadline. - remaining = deadlineNanos - System.nanoTime(); - if (remaining <= 0L) return false; - try { TimeUnit.NANOSECONDS.sleep(Math.min(TimeUnit.MILLISECONDS.toNanos(50), remaining)); } - catch (InterruptedException interrupted) { Thread.currentThread().interrupt(); return false; } + Thread current; + synchronized (lifecycle) { + if (closing.get() || flushingOutgoing) return false; + flushingOutgoing = true; + running.set(false); + firstResponse.countDown(); + current = poller; + if (current != null) current.interrupt(); } - return true; + synchronized (state) { sendAdmissionOpen = false; } + try { + if (!joinPoller(current, deadlineNanos)) return false; + while (queuedOutgoing() != 0 || queuedAcknowledgements() != 0) { + long remaining = deadlineNanos - System.nanoTime(); + if (remaining <= 0L) return false; + Duration timeout = Duration.ofNanos(Math.min(CLIENT_TIMEOUT.toNanos(), remaining)); + synchronized (this) { + if (pollOnce(timeout, false, false)) continue; + } + // The proxy may retain its one-active-poll guard briefly after the old + // client request is interrupted. Retry that transient 409 without busy + // spinning, but never extend the caller's shutdown deadline. + remaining = deadlineNanos - System.nanoTime(); + if (remaining <= 0L) return false; + try { TimeUnit.NANOSECONDS.sleep(Math.min(TimeUnit.MILLISECONDS.toNanos(50), remaining)); } + catch (InterruptedException interrupted) { Thread.currentThread().interrupt(); return false; } + } + return true; + } finally { synchronized (lifecycle) { flushingOutgoing = false; } } } /** * Stops polling and drains callbacks before sealing their journals. When called by the @@ -289,6 +307,7 @@ public boolean flushOutgoing(long deadlineNanos) { if (current != null) current.interrupt(); } } + if (!alreadyClosing) synchronized (state) { sendAdmissionOpen = false; } if (alreadyClosing) { if (!calledByCallbackWorker) awaitClosed(); return; diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentAuthority.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentAuthority.java index afc66a2..84bff7b 100644 --- a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentAuthority.java +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentAuthority.java @@ -127,7 +127,8 @@ public synchronized boolean authenticate(String serverId, java.security.cert.X50 catch (java.io.IOException failure) { bindings.put(serverId, binding); return false; } } Map.Entry pending = pendingCertificate(serverId, pin); - if (pending == null || bindings.size() >= MAX_BINDINGS) return false; + if (pending == null || !pending.getValue().expiresAt().isAfter(clock.instant()) + || bindings.size() >= MAX_BINDINGS) return false; enrollments.remove(pending.getKey()); bindings.put(serverId, new ClientBinding(pin, null, false)); try { persistState(); return true; } diff --git a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpBackendSendLifecycleTest.java b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpBackendSendLifecycleTest.java new file mode 100644 index 0000000..3537ac8 --- /dev/null +++ b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpBackendSendLifecycleTest.java @@ -0,0 +1,113 @@ +package com.bencodez.simpleapi.servercomm.http; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.bencodez.simpleapi.servercomm.codec.JsonEnvelope; +import java.lang.reflect.Field; +import java.net.InetSocketAddress; +import java.nio.file.Path; +import java.time.Duration; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class HttpBackendSendLifecycleTest { + @TempDir Path directory; + + @Test + void pausedSendCannotEnterAfterCloseAndFlushCutsOffNewSends() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, directory.resolve("authority")); + try (HttpProxyTransportServer server = new HttpProxyTransportServer(new InetSocketAddress("localhost", 0), identity, + authority, ignored -> { })) { + server.start(); + HttpConnectionCode code = authority.createConnectionCode("lobby-1", server.endpoint("localhost"), Duration.ofMinutes(5)); + Path firstDirectory = directory.resolve("close-client"); + HttpBackendTransportConnector.enroll(code, "lobby-1", firstDirectory); + HttpBackendTransportConnector first = new HttpBackendTransportConnector(firstDirectory, ignored -> { }); + try { + first.start(); + assertTrue(first.awaitFirstResponse(System.nanoTime() + TimeUnit.SECONDS.toNanos(8))); + assertPausedSendIsRejectedByClose(first); + } finally { first.close(); } + + Path secondDirectory = directory.resolve("flush-client"); + HttpConnectionCode secondCode = authority.createConnectionCode("lobby-2", server.endpoint("localhost"), Duration.ofMinutes(5)); + HttpBackendTransportConnector.enroll(secondCode, "lobby-2", secondDirectory); + try (HttpBackendTransportConnector second = new HttpBackendTransportConnector(secondDirectory, ignored -> { })) { + second.start(); + assertTrue(second.awaitFirstResponse(System.nanoTime() + TimeUnit.SECONDS.toNanos(8))); + assertTrue(second.send(JsonEnvelope.builder("accepted-before-flush").build())); + assertPausedSendIsRejectedByFlush(second); + assertFalse(second.send(JsonEnvelope.builder("rejected-after-flush").build())); + assertEquals(0, second.queuedOutgoing()); + second.start(); + assertTrue(second.send(JsonEnvelope.builder("accepted-after-restart").build())); + } + } + } + + private static void assertPausedSendIsRejectedByClose(HttpBackendTransportConnector connector) throws Exception { + Object state = field("state").get(connector); + AtomicBoolean closing = (AtomicBoolean) field("closing").get(connector); + AtomicBoolean accepted = new AtomicBoolean(true); + CountDownLatch senderStarted = new CountDownLatch(1); + Thread sender = new Thread(() -> { + senderStarted.countDown(); + accepted.set(connector.send(JsonEnvelope.builder("paused-send").build())); + }, "paused-backend-send"); + Thread closer = new Thread(connector::close, "backend-close"); + synchronized (state) { + sender.start(); + assertTrue(senderStarted.await(1, TimeUnit.SECONDS)); + closer.start(); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(2); + while (!closing.get() && System.nanoTime() < deadline) Thread.onSpinWait(); + assertTrue(closing.get(), "close must transition running before it waits for state"); + } + sender.join(3000); + closer.join(3000); + assertFalse(sender.isAlive()); + assertFalse(closer.isAlive()); + assertFalse(accepted.get()); + assertEquals(0, connector.queuedOutgoing()); + } + + private static void assertPausedSendIsRejectedByFlush(HttpBackendTransportConnector connector) throws Exception { + Object state = field("state").get(connector); + AtomicBoolean running = (AtomicBoolean) field("running").get(connector); + AtomicBoolean accepted = new AtomicBoolean(true), flushed = new AtomicBoolean(); + CountDownLatch senderStarted = new CountDownLatch(1); + Thread sender = new Thread(() -> { + senderStarted.countDown(); + accepted.set(connector.send(JsonEnvelope.builder("paused-flush-send").build())); + }, "paused-flush-send"); + Thread flusher = new Thread(() -> flushed.set(connector.flushOutgoing( + System.nanoTime() + TimeUnit.SECONDS.toNanos(5))), "backend-flush"); + synchronized (state) { + sender.start(); + assertTrue(senderStarted.await(1, TimeUnit.SECONDS)); + flusher.start(); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(2); + while (running.get() && System.nanoTime() < deadline) Thread.onSpinWait(); + assertFalse(running.get(), "flush must cut off admission before it waits for state"); + } + sender.join(3000); + flusher.join(6000); + assertFalse(sender.isAlive()); + assertFalse(flusher.isAlive()); + assertFalse(accepted.get()); + assertTrue(flushed.get()); + assertEquals(0, connector.queuedOutgoing()); + } + + private static Field field(String name) throws Exception { + Field field = HttpBackendTransportConnector.class.getDeclaredField(name); + field.setAccessible(true); + return field; + } +} diff --git a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpPendingEnrollmentExpiryTest.java b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpPendingEnrollmentExpiryTest.java new file mode 100644 index 0000000..9d6cc1c --- /dev/null +++ b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpPendingEnrollmentExpiryTest.java @@ -0,0 +1,42 @@ +package com.bencodez.simpleapi.servercomm.http; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.net.URI; +import java.nio.file.Path; +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.util.concurrent.atomic.AtomicReference; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class HttpPendingEnrollmentExpiryTest { + @TempDir Path directory; + + @Test + void expiredPendingCertificateCannotActivateButActiveBindingRemainsValid() throws Exception { + AtomicReference now = new AtomicReference<>(Instant.now()); + Clock clock = new Clock() { + @Override public ZoneId getZone() { return ZoneOffset.UTC; } + @Override public Clock withZone(ZoneId zone) { return this; } + @Override public Instant instant() { return now.get(); } + }; + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("identity"), "localhost"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, clock); + URI endpoint = URI.create("https://localhost:8443/"); + HttpConnectionCode pendingCode = authority.createConnectionCode("pending", endpoint, Duration.ofSeconds(1)); + var pending = authority.enroll("pending", pendingCode.enrollmentToken()); + HttpConnectionCode activeCode = authority.createConnectionCode("active", endpoint, Duration.ofSeconds(1)); + var active = authority.enroll("active", activeCode.enrollmentToken()); + assertTrue(authority.authenticate("active", active.certificate())); + now.set(pendingCode.expiresAt()); + assertFalse(authority.authenticate("pending", pending.certificate())); + assertTrue(authority.authenticate("active", active.certificate())); + now.set(now.get().plusSeconds(1)); + assertFalse(authority.authenticate("pending", pending.certificate())); + } +} From d2754e96709d528a46aee8f0ef6892eb70fd4d19 Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Sat, 5 Sep 2026 23:29:48 -0600 Subject: [PATCH 28/38] Rate limit certificate renewal and serialize TLS identity writes --- .../http/HttpEnrollmentAuthority.java | 15 +++++- .../http/HttpProxyTransportServer.java | 3 ++ .../servercomm/http/HttpTlsIdentity.java | 41 ++++++++++++++++ .../http/HttpRenewalRateLimitTest.java | 47 ++++++++++++++++++ .../http/HttpTlsIdentityOwnershipTest.java | 48 +++++++++++++++++++ .../http/HttpTransportSecurityTest.java | 11 ++++- 6 files changed, 163 insertions(+), 2 deletions(-) create mode 100644 SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpRenewalRateLimitTest.java create mode 100644 SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTlsIdentityOwnershipTest.java diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentAuthority.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentAuthority.java index 84bff7b..636b2bf 100644 --- a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentAuthority.java +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentAuthority.java @@ -24,6 +24,7 @@ */ public final class HttpEnrollmentAuthority { private static final Duration MAX_ENROLLMENT_LIFETIME = Duration.ofMinutes(15); + private static final Duration MIN_RENEWAL_INTERVAL = Duration.ofMinutes(1); private static final int MAX_PENDING_ENROLLMENTS = 128; private static final int MAX_BINDINGS = 128; private static final long MAX_STATE_BYTES = 65536; @@ -32,6 +33,7 @@ public final class HttpEnrollmentAuthority { private final Path stateFile; private final Map enrollments = new HashMap<>(); private final Map bindings = new HashMap<>(); + private final Map renewalNotBefore = new HashMap<>(); private boolean persistenceFailure; private boolean revocationRetryRequired; @@ -148,8 +150,14 @@ public synchronized boolean authenticate(String serverId, java.security.cert.X50 * until the replacement successfully authenticates, making a lost renewal response safe to retry. */ public synchronized HttpTlsIdentity.IssuedClientCertificate renew(String serverId, java.security.cert.X509Certificate currentCertificate) throws Exception { - if (!authenticate(serverId, currentCertificate)) throw new IllegalArgumentException("Certificate renewal was rejected"); serverId = HttpTlsIdentity.canonicalServerId(serverId); + Instant now = clock.instant(); + Instant nextAllowed = renewalNotBefore.get(serverId); + if (nextAllowed != null && now.isBefore(nextAllowed)) throw new RenewalRateLimitException(); + if (!authenticate(serverId, currentCertificate)) throw new IllegalArgumentException("Certificate renewal was rejected"); + // Bound the limiter by active bindings; failed issuance still consumes the window. + renewalNotBefore.keySet().retainAll(bindings.keySet()); + renewalNotBefore.put(serverId, now.plus(MIN_RENEWAL_INTERVAL)); ClientBinding binding = bindings.get(serverId); HttpTlsIdentity.IssuedClientCertificate issued = identity.issueClientCertificate(serverId); bindings.put(serverId, new ClientBinding(binding.certificatePin(), @@ -164,6 +172,11 @@ public synchronized HttpTlsIdentity.IssuedClientCertificate renew(String serverI return issued; } + static final class RenewalRateLimitException extends IllegalStateException { + private static final long serialVersionUID = 1L; + RenewalRateLimitException() { super("HTTP certificate renewal is rate limited"); } + } + public synchronized void revoke(String serverId) { try { serverId = HttpTlsIdentity.canonicalServerId(serverId); } catch (IllegalArgumentException invalid) { return; } diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java index e3bff5c..9d16bf0 100644 --- a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java @@ -173,6 +173,9 @@ private void renew(HttpsExchange exchange) throws IOException { if (certificate == null || !authority.authenticate(serverId, certificate)) { reply(exchange, 401, new byte[0]); return; } HttpTlsIdentity.IssuedClientCertificate issued = authority.renew(serverId, certificate); reply(exchange, 201, HttpTransportProtocol.enrollmentResponse(issued)); + } catch (HttpEnrollmentAuthority.RenewalRateLimitException limited) { + exchange.getResponseHeaders().set("Retry-After", "60"); + reply(exchange, 429, new byte[0]); } catch (IllegalArgumentException rejected) { reply(exchange, 403, new byte[0]); } catch (Exception failure) { reply(exchange, 503, new byte[0]); } finally { admission.release(); } diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpTlsIdentity.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpTlsIdentity.java index 42771fc..449a8c0 100644 --- a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpTlsIdentity.java +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpTlsIdentity.java @@ -3,6 +3,9 @@ import java.io.IOException; import java.io.OutputStream; import java.math.BigInteger; +import java.nio.channels.FileChannel; +import java.nio.channels.FileLock; +import java.nio.channels.OverlappingFileLockException; import java.nio.file.Files; import java.nio.file.LinkOption; import java.nio.file.Path; @@ -54,9 +57,11 @@ public final class HttpTlsIdentity { private static final String SERVER_FILE = "http-transport-server.p12"; private static final String PASSWORD_FILE = "http-transport-password"; private static final String INITIALIZING_FILE = "http-transport-initializing"; + private static final String OWNERSHIP_LOCK_FILE = ".http-transport-identity.lock"; private static final String ENROLLMENT_STATE_FILE = "http-transport-clients.properties"; private static final String OUTGOING_DIRECTORY = "outgoing-v1"; private static final char[] EMPTY_PASSWORD = new char[0]; + private static final Object IDENTITY_LOCK_MONITOR = new Object(); static final Duration RENEW_BEFORE = Duration.ofDays(30); static final Duration CA_RENEW_BEFORE = Duration.ofDays(365); private final PrivateKey caKey; @@ -98,6 +103,8 @@ static HttpTlsIdentity loadOrCreate(Path directory, String advertisedHost, Clock // The identity files cannot make the newly created directory entry durable. // Persist its parent before the TLS identity is returned for listener use. DurableFiles.forceDirectory(identityDirectory.getParent()); + synchronized (IDENTITY_LOCK_MONITOR) { + try (IdentityLock ignored = claimIdentityLock(identityDirectory)) { directory = identityDirectory; Path caFile = safe(directory.resolve(CA_FILE)); Path serverFile = safe(directory.resolve(SERVER_FILE)); @@ -189,6 +196,8 @@ static HttpTlsIdentity loadOrCreate(Path directory, String advertisedHost, Clock return new HttpTlsIdentity(caPair.getPrivate(), caCertificate, serverPair.getPrivate(), serverCertificate, password, caFile, serverFile, advertisedHost); } finally { Arrays.fill(password, '\0'); } + } + } } public String serverCertificatePin() { @@ -229,6 +238,8 @@ private synchronized void renewIdentityIfNeeded() throws Exception { Clock clock = Clock.systemUTC(); boolean renewCa = needsCaRenewal(caCertificate, clock); if (!renewCa && !needsRenewal(serverCertificate, clock)) return; + synchronized (IDENTITY_LOCK_MONITOR) { + try (IdentityLock ignored = claimIdentityLock(caFile.getParent())) { ensureBouncyCastle(); X509Certificate replacementCa = caCertificate; if (renewCa) { @@ -250,6 +261,8 @@ private synchronized void renewIdentityIfNeeded() throws Exception { caCertificate = replacementCa; serverKey = pair.getPrivate(); serverCertificate = replacement; + } + } } public IssuedClientCertificate issueClientCertificate(String serverId) throws Exception { @@ -385,6 +398,34 @@ private static Path safe(Path file) throws IOException { if (parent == null || Files.isSymbolicLink(file)) throw new IOException("Refusing unsafe HTTP TLS identity path"); return file.toAbsolutePath().normalize(); } + private static IdentityLock claimIdentityLock(Path directory) throws IOException { + Path sidecar = safe(directory.resolve(OWNERSHIP_LOCK_FILE)); + if (Files.exists(sidecar, LinkOption.NOFOLLOW_LINKS) + && !Files.isRegularFile(sidecar, LinkOption.NOFOLLOW_LINKS)) + throw new IOException("HTTP TLS identity ownership lock is unsafe"); + FileChannel channel = FileChannel.open(sidecar, StandardOpenOption.CREATE, StandardOpenOption.WRITE, + LinkOption.NOFOLLOW_LINKS); + try { + PrivateFilePermissions.ownerOnlyFile(sidecar); + FileLock lock; + try { lock = channel.tryLock(); } + catch (OverlappingFileLockException alreadyOwned) { throw new IOException("HTTP TLS identity is already initializing", alreadyOwned); } + if (lock == null) throw new IOException("HTTP TLS identity is already initializing"); + return new IdentityLock(channel, lock); + } catch (IOException | RuntimeException failure) { + try { channel.close(); } catch (IOException closeFailure) { failure.addSuppressed(closeFailure); } + throw failure; + } + } + private static final class IdentityLock implements AutoCloseable { + private final FileChannel channel; + private final FileLock lock; + private IdentityLock(FileChannel channel, FileLock lock) { this.channel = channel; this.lock = lock; } + @Override public void close() { + try { lock.release(); } catch (IOException ignored) { } + try { channel.close(); } catch (IOException ignored) { } + } + } private static KeyStore load(Path path, char[] password) throws Exception { KeyStore store = KeyStore.getInstance("PKCS12"); diff --git a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpRenewalRateLimitTest.java b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpRenewalRateLimitTest.java new file mode 100644 index 0000000..5977f49 --- /dev/null +++ b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpRenewalRateLimitTest.java @@ -0,0 +1,47 @@ +package com.bencodez.simpleapi.servercomm.http; + +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.net.URI; +import java.nio.file.Path; +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.util.concurrent.atomic.AtomicReference; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class HttpRenewalRateLimitTest { + @TempDir Path directory; + + @Test + void repeatedRenewalIsLimitedPerBackendBeforeCertificateIssuance() throws Exception { + AtomicReference now = new AtomicReference<>(Instant.now()); + Clock clock = new Clock() { + @Override public ZoneId getZone() { return ZoneOffset.UTC; } + @Override public Clock withZone(ZoneId zone) { return this; } + @Override public Instant instant() { return now.get(); } + }; + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("identity"), "localhost"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, clock); + URI endpoint = URI.create("https://localhost:8443/"); + var code = authority.createConnectionCode("lobby-1", endpoint, Duration.ofMinutes(5)); + var original = authority.enroll("lobby-1", code.enrollmentToken()); + assertTrue(authority.authenticate("lobby-1", original.certificate())); + var replacement = authority.renew("lobby-1", original.certificate()); + for (int index = 0; index < 10; index++) assertThrows(HttpEnrollmentAuthority.RenewalRateLimitException.class, + () -> authority.renew("lobby-1", original.certificate())); + assertTrue(authority.authenticate("lobby-1", replacement.certificate())); + assertThrows(HttpEnrollmentAuthority.RenewalRateLimitException.class, + () -> authority.renew("lobby-1", replacement.certificate())); + var otherCode = authority.createConnectionCode("lobby-2", endpoint, Duration.ofMinutes(5)); + var other = authority.enroll("lobby-2", otherCode.enrollmentToken()); + assertNotNull(authority.renew("lobby-2", other.certificate())); + now.set(now.get().plusSeconds(60)); + assertNotNull(authority.renew("lobby-1", replacement.certificate())); + } +} diff --git a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTlsIdentityOwnershipTest.java b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTlsIdentityOwnershipTest.java new file mode 100644 index 0000000..966b87a --- /dev/null +++ b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTlsIdentityOwnershipTest.java @@ -0,0 +1,48 @@ +package com.bencodez.simpleapi.servercomm.http; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class HttpTlsIdentityOwnershipTest { + @TempDir Path directory; + + @Test + void concurrentFirstOpenPublishesOneReloadableIdentity() throws Exception { + Path identityDirectory = directory.resolve("identity"); + CountDownLatch ready = new CountDownLatch(2), start = new CountDownLatch(1); + ExecutorService workers = Executors.newFixedThreadPool(2); + try { + Future first = workers.submit(() -> openTogether(identityDirectory, ready, start)); + Future second = workers.submit(() -> openTogether(identityDirectory, ready, start)); + assertTrue(ready.await(2, TimeUnit.SECONDS)); + start.countDown(); + HttpTlsIdentity firstIdentity = first.get(10, TimeUnit.SECONDS); + HttpTlsIdentity secondIdentity = second.get(10, TimeUnit.SECONDS); + assertEquals(firstIdentity.caCertificatePin(), secondIdentity.caCertificatePin()); + assertEquals(firstIdentity.serverCertificatePin(), secondIdentity.serverCertificatePin()); + HttpTlsIdentity reloaded = HttpTlsIdentity.loadOrCreate(identityDirectory, "localhost"); + assertEquals(firstIdentity.caCertificatePin(), reloaded.caCertificatePin()); + assertEquals(firstIdentity.serverCertificatePin(), reloaded.serverCertificatePin()); + assertTrue(Files.isRegularFile(identityDirectory.resolve(".http-transport-identity.lock"))); + } finally { + workers.shutdownNow(); + workers.awaitTermination(2, TimeUnit.SECONDS); + } + } + + private static HttpTlsIdentity openTogether(Path directory, CountDownLatch ready, CountDownLatch start) throws Exception { + ready.countDown(); + if (!start.await(5, TimeUnit.SECONDS)) throw new IllegalStateException("concurrent identity start timed out"); + return HttpTlsIdentity.loadOrCreate(directory, "localhost"); + } +} diff --git a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportSecurityTest.java b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportSecurityTest.java index 157104b..ba88748 100644 --- a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportSecurityTest.java +++ b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportSecurityTest.java @@ -515,7 +515,15 @@ void renewalKeepsOldCredentialUntilReplacementAuthenticates() throws Exception { void failedRenewalPersistenceRestoresTheActiveBindingAndCanRetry() throws Exception { HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("retry-renewal-proxy"), "localhost"); Path stateDirectory = directory.resolve("retry-renewal-state"); - HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, stateDirectory); + Files.createDirectory(stateDirectory); + java.util.concurrent.atomic.AtomicReference now = new java.util.concurrent.atomic.AtomicReference<>(Instant.now()); + Clock clock = new Clock() { + @Override public java.time.ZoneId getZone() { return ZoneOffset.UTC; } + @Override public Clock withZone(java.time.ZoneId zone) { return this; } + @Override public Instant instant() { return now.get(); } + }; + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, clock, + stateDirectory.resolve("http-transport-clients.properties")); HttpConnectionCode code = authority.createConnectionCode("lobby-1", URI.create("https://localhost:8443/"), Duration.ofMinutes(5)); HttpTlsIdentity.IssuedClientCertificate original = authority.enroll("lobby-1", code.enrollmentToken()); @@ -528,6 +536,7 @@ void failedRenewalPersistenceRestoresTheActiveBindingAndCanRetry() throws Except assertTrue(authority.authenticate("lobby-1", original.certificate()), "a pre-publication renewal failure must leave the active credential usable"); Files.delete(stateFile); + now.set(now.get().plusSeconds(60)); HttpTlsIdentity.IssuedClientCertificate retried = authority.renew("lobby-1", original.certificate()); assertTrue(authority.authenticate("lobby-1", retried.certificate()), "renewal must remain retryable after persistence recovers"); From 8a994f09238c3ff54ae4d557da839e8a73d9e5e8 Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Sun, 6 Sep 2026 06:28:56 -0600 Subject: [PATCH 29/38] Reload persisted TLS identity before renewal --- .../servercomm/http/HttpTlsIdentity.java | 20 ++++++++++++++++ .../http/HttpEnrollmentPinTest.java | 6 +++-- .../http/HttpTlsIdentityOwnershipTest.java | 23 +++++++++++++++++++ 3 files changed, 47 insertions(+), 2 deletions(-) diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpTlsIdentity.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpTlsIdentity.java index 449a8c0..3b42748 100644 --- a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpTlsIdentity.java +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpTlsIdentity.java @@ -240,6 +240,26 @@ private synchronized void renewIdentityIfNeeded() throws Exception { if (!renewCa && !needsRenewal(serverCertificate, clock)) return; synchronized (IDENTITY_LOCK_MONITOR) { try (IdentityLock ignored = claimIdentityLock(caFile.getParent())) { + // Another process or long-lived instance may have renewed while this object + // waited for the identity lock. Adopt the durable state before deciding whether + // another renewal is still necessary. + KeyStore persistedCaStore = load(caFile, password); + KeyStore persistedServerStore = load(serverFile, password); + PrivateKey persistedCaKey = (PrivateKey) persistedCaStore.getKey("ca", password); + X509Certificate persistedCa = (X509Certificate) persistedCaStore.getCertificate("ca"); + PrivateKey persistedServerKey = (PrivateKey) persistedServerStore.getKey("server", password); + X509Certificate persistedServer = (X509Certificate) persistedServerStore.getCertificate("server"); + if (persistedCaKey == null || persistedCa == null || persistedServerKey == null || persistedServer == null + || !java.security.MessageDigest.isEqual(caKey.getEncoded(), persistedCaKey.getEncoded())) + throw new IOException("HTTP TLS identity files are invalid"); + persistedServer.verify(persistedCa.getPublicKey()); + if (!hasServerName(persistedServer, advertisedHost)) + throw new IOException("HTTP TLS server identity does not match its advertised host"); + caCertificate = persistedCa; + serverKey = persistedServerKey; + serverCertificate = persistedServer; + renewCa = needsCaRenewal(caCertificate, clock); + if (!renewCa && !needsRenewal(serverCertificate, clock)) return; ensureBouncyCastle(); X509Certificate replacementCa = caCertificate; if (renewCa) { diff --git a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentPinTest.java b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentPinTest.java index 93a3763..6df6ae4 100644 --- a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentPinTest.java +++ b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentPinTest.java @@ -121,6 +121,8 @@ void stageReplacementAcceptsCaRenewalWithSameAuthorityKey() throws Exception { Path proxyDirectory = directory.resolve("proxy"); HttpTlsIdentity original = HttpTlsIdentity.loadOrCreate(proxyDirectory, "localhost", originalClock); HttpTlsIdentity.IssuedClientCertificate originalCredential = original.issueClientCertificate("lobby-1", now); + String originalCaPin = HttpTransportSecrets.certificatePin(original.caCertificate()); + java.security.PublicKey originalCaKey = original.caCertificate().getPublicKey(); Path client = directory.resolve("client"); HttpConnectionCode code = new HttpConnectionCode("lobby-1", URI.create("https://localhost:8443/"), HttpTransportSecrets.certificatePin(original.serverCertificate()), @@ -129,8 +131,8 @@ void stageReplacementAcceptsCaRenewalWithSameAuthorityKey() throws Exception { HttpTlsIdentity renewed = HttpTlsIdentity.loadOrCreate(proxyDirectory, "localhost", java.time.Clock.fixed(now, java.time.ZoneOffset.UTC)); - assertNotEquals(original.caCertificatePin(), renewed.caCertificatePin()); - assertEquals(original.caCertificate().getPublicKey(), renewed.caCertificate().getPublicKey()); + assertNotEquals(originalCaPin, renewed.caCertificatePin()); + assertEquals(originalCaKey, renewed.caCertificate().getPublicKey()); HttpClientCredentialStore.StagedCredential staged = HttpClientCredentialStore.stageReplacement( client, renewed.issueClientCertificate("lobby-1", now)); diff --git a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTlsIdentityOwnershipTest.java b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTlsIdentityOwnershipTest.java index 966b87a..138821b 100644 --- a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTlsIdentityOwnershipTest.java +++ b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTlsIdentityOwnershipTest.java @@ -1,10 +1,15 @@ package com.bencodez.simpleapi.servercomm.http; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertTrue; import java.nio.file.Files; import java.nio.file.Path; +import java.time.Clock; +import java.time.Instant; +import java.time.Duration; +import java.time.ZoneOffset; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -40,6 +45,24 @@ void concurrentFirstOpenPublishesOneReloadableIdentity() throws Exception { } } + @Test + void staleInstanceAdoptsAnotherInstancesRenewedServerCertificate() throws Exception { + Instant now = Instant.now(); + Path identityDirectory = directory.resolve("stale-renewal"); + Clock creationClock = Clock.fixed(now.minus(Duration.ofDays(340)), ZoneOffset.UTC); + HttpTlsIdentity first = HttpTlsIdentity.loadOrCreate(identityDirectory, "localhost", creationClock); + HttpTlsIdentity second = HttpTlsIdentity.loadOrCreate(identityDirectory, "localhost", creationClock); + String originalPin = HttpTransportSecrets.certificatePin(first.serverCertificate()); + + String renewedByFirst = first.serverCertificatePin(); + assertNotEquals(originalPin, renewedByFirst); + String adoptedBySecond = second.serverCertificatePin(); + assertEquals(renewedByFirst, adoptedBySecond, + "a stale instance must reload the persisted renewal instead of generating another server key"); + HttpTlsIdentity reloaded = HttpTlsIdentity.loadOrCreate(identityDirectory, "localhost"); + assertEquals(renewedByFirst, reloaded.serverCertificatePin()); + } + private static HttpTlsIdentity openTogether(Path directory, CountDownLatch ready, CountDownLatch start) throws Exception { ready.countDown(); if (!start.await(5, TimeUnit.SECONDS)) throw new IllegalStateException("concurrent identity start timed out"); From 7bf06f981ee18ec093158ee11bd72b6515fb4b64 Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Sun, 6 Sep 2026 06:40:28 -0600 Subject: [PATCH 30/38] Restore inbound journals within the backend capacity bound --- .../http/HttpInboundDeliveryStore.java | 30 +++++++ .../http/HttpProxyTransportServer.java | 17 ++++ .../http/HttpInboundJournalCapacityTest.java | 81 +++++++++++++++++++ 3 files changed, 128 insertions(+) create mode 100644 SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpInboundJournalCapacityTest.java diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpInboundDeliveryStore.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpInboundDeliveryStore.java index 6c0bcd5..3fbb140 100644 --- a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpInboundDeliveryStore.java +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpInboundDeliveryStore.java @@ -17,6 +17,7 @@ import java.util.LinkedHashMap; import java.util.Map; import java.util.Set; +import java.util.TreeSet; import java.util.UUID; /** Crash-durable state for proxy deliveries around a non-transactional application callback. */ @@ -58,6 +59,35 @@ static HttpInboundDeliveryStore inspect(Path credentialDirectory) throws IOExcep return new HttpInboundDeliveryStore(credentialDirectory, DIRECTORY, true); } + /** Returns validated journal directory names while tolerating their persistent ownership sidecars. */ + static Set discover(Path parent) throws IOException { + Path root = parent.toAbsolutePath().normalize(); + if (Files.isSymbolicLink(root) || !Files.isDirectory(root, LinkOption.NOFOLLOW_LINKS)) + throw new IOException("HTTP inbound delivery parent is unsafe"); + PrivateFilePermissions.ownerOnlyDirectory(root); + Set directories = new TreeSet<>(); + try (DirectoryStream entries = Files.newDirectoryStream(root)) { + for (Path entry : entries) { + String name = entry.getFileName().toString(); + if (!Files.isSymbolicLink(entry) && Files.isDirectory(entry, LinkOption.NOFOLLOW_LINKS)) { + if (!name.matches("[A-Za-z0-9][A-Za-z0-9._-]{0,63}")) + throw new IOException("HTTP inbound delivery directory name is invalid"); + PrivateFilePermissions.ownerOnlyDirectory(entry); + directories.add(name); + continue; + } + if (!name.startsWith(OWNER_LOCK_PREFIX) || !name.endsWith(OWNER_LOCK_SUFFIX) + || Files.isSymbolicLink(entry) || !Files.isRegularFile(entry, LinkOption.NOFOLLOW_LINKS)) + throw new IOException("HTTP inbound delivery parent contains an invalid entry"); + String journal = name.substring(OWNER_LOCK_PREFIX.length(), name.length() - OWNER_LOCK_SUFFIX.length()); + if (!journal.matches("[A-Za-z0-9][A-Za-z0-9._-]{0,63}")) + throw new IOException("HTTP inbound delivery ownership lock name is invalid"); + PrivateFilePermissions.ownerOnlyFile(entry); + } + } + return directories; + } + private HttpInboundDeliveryStore(Path parent, String directoryName, boolean readOnly) throws IOException { Path credentials = parent.toAbsolutePath().normalize(); if (Files.isSymbolicLink(credentials) || !Files.isDirectory(credentials, LinkOption.NOFOLLOW_LINKS)) diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java index 9d16bf0..30b4e88 100644 --- a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java @@ -117,6 +117,23 @@ public HttpProxyTransportServer(InetSocketAddress bind, HttpTlsIdentity identity ThreadPoolExecutor createdListener = null, createdHandler = null; try { durableIncomingRoot = outgoingDirectory == null ? null : incomingRoot(outgoingDirectory); + if (durableIncomingRoot != null) for (String serverId : HttpInboundDeliveryStore.discover(durableIncomingRoot)) { + String canonical; + try { canonical = HttpTlsIdentity.canonicalServerId(serverId); } + catch (IllegalArgumentException invalid) { throw new IOException("HTTP inbound backend id is invalid", invalid); } + if (!canonical.equals(serverId)) throw new IOException("HTTP inbound backend id is not canonical"); + HttpInboundDeliveryStore inbound = HttpInboundDeliveryStore.open(durableIncomingRoot, serverId); + if (inbound.snapshot().isEmpty()) { + try { inbound.sealAndDeleteIfEmpty(); } + catch (IOException cleanupFailure) { inbound.seal(); throw cleanupFailure; } + continue; + } + if (backends.size() >= MAX_BACKENDS) { + inbound.seal(); + throw new IOException("HTTP backend state exceeds its bound"); + } + backends.put(serverId, new BackendState(serverId, durableOutgoing, inbound, onAcknowledged, nanoTime)); + } if (durableOutgoing != null) for (Map.Entry> pending : durableOutgoing.load().entrySet()) { BackendState state = backendState(pending.getKey()); diff --git a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpInboundJournalCapacityTest.java b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpInboundJournalCapacityTest.java new file mode 100644 index 0000000..5455e73 --- /dev/null +++ b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpInboundJournalCapacityTest.java @@ -0,0 +1,81 @@ +package com.bencodez.simpleapi.servercomm.http; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.UUID; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class HttpInboundJournalCapacityTest { + @TempDir Path directory; + + @Test + void restoresRunningInboundJournalsAndEnforcesCapacityAcrossRestart() throws Exception { + Path outgoing = directory.resolve("outgoing"); + Files.createDirectory(outgoing); + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("identity"), "localhost"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, directory.resolve("authority")); + seedRunningJournals(outgoing, 128); + + try (HttpProxyTransportServer proxy = new HttpProxyTransportServer(new InetSocketAddress("localhost", 0), + identity, authority, outgoing, ignored -> { })) { + assertEquals(128, proxy.backendCountForTest(), "validated journals must restore backend state"); + assertNotNull(proxy.backendStateForTest("server-0")); + assertThrows(java.io.IOException.class, () -> proxy.backendStateForTest("server-128"), + "a 129th backend must be rejected while durable journals occupy the cap"); + } + + try (HttpProxyTransportServer restarted = new HttpProxyTransportServer(new InetSocketAddress("localhost", 0), + identity, authority, outgoing, ignored -> { })) { + assertEquals(128, restarted.backendCountForTest(), "restart must retain the restored backend count"); + assertThrows(java.io.IOException.class, () -> restarted.backendStateForTest("server-128"), + "the cap must remain enforced after restart"); + } + } + + @Test + void prunesEmptySafeJournalsAndAllowsAReplacementBackend() throws Exception { + Path outgoing = directory.resolve("outgoing"); + Files.createDirectory(outgoing); + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("identity"), "localhost"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, directory.resolve("authority")); + seedEmptyJournals(outgoing, 128); + + try (HttpProxyTransportServer proxy = new HttpProxyTransportServer(new InetSocketAddress("localhost", 0), + identity, authority, outgoing, ignored -> { })) { + assertEquals(0, proxy.backendCountForTest(), "empty journals are safe to prune at startup"); + assertNotNull(proxy.backendStateForTest("replacement")); + assertEquals(1, proxy.backendCountForTest()); + assertFalse(Files.exists(outgoing.getParent().resolve("outgoing-incoming").resolve("server-0"))); + } + } + + private static void seedRunningJournals(Path outgoing, int count) throws Exception { + Path incoming = outgoing.getParent().resolve("outgoing-incoming"); + Files.createDirectory(incoming); + for (int index = 0; index < count; index++) { + String serverId = "server-" + index; + HttpInboundDeliveryStore store = HttpInboundDeliveryStore.open(incoming, serverId); + String deliveryId = UUID.nameUUIDFromBytes(serverId.getBytes(StandardCharsets.UTF_8)).toString(); + store.reserve(deliveryId); + store.markRunning(deliveryId); + store.seal(); + } + } + + private static void seedEmptyJournals(Path outgoing, int count) throws Exception { + Path incoming = outgoing.getParent().resolve("outgoing-incoming"); + Files.createDirectory(incoming); + for (int index = 0; index < count; index++) { + HttpInboundDeliveryStore store = HttpInboundDeliveryStore.open(incoming, "server-" + index); + store.seal(); + } + } +} From c390c0f1d0c080638efec04791461b120a47ed76 Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Sun, 6 Sep 2026 06:48:43 -0600 Subject: [PATCH 31/38] test: isolate failed HTTP revocation retries --- .../HttpRevocationRetryIsolationTest.java | 58 +++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpRevocationRetryIsolationTest.java diff --git a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpRevocationRetryIsolationTest.java b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpRevocationRetryIsolationTest.java new file mode 100644 index 0000000..ad4b783 --- /dev/null +++ b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpRevocationRetryIsolationTest.java @@ -0,0 +1,58 @@ +package com.bencodez.simpleapi.servercomm.http; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.net.URI; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class HttpRevocationRetryIsolationTest { + @TempDir Path directory; + + @Test + void failedRevocationRetryCannotClearOrReplaceAnotherServerBinding() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); + Path state = directory.resolve("authority"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, state); + URI endpoint = URI.create("https://localhost:8443/"); + + HttpConnectionCode codeA = authority.createConnectionCode("backend-a", endpoint, Duration.ofMinutes(5)); + HttpTlsIdentity.IssuedClientCertificate certificateA = authority.enroll("backend-a", codeA.enrollmentToken()); + HttpConnectionCode codeB = authority.createConnectionCode("backend-b", endpoint, Duration.ofMinutes(5)); + HttpTlsIdentity.IssuedClientCertificate certificateB = authority.enroll("backend-b", codeB.enrollmentToken()); + assertTrue(authority.authenticate("backend-a", certificateA.certificate())); + assertTrue(authority.authenticate("backend-b", certificateB.certificate())); + + Path stateFile = state.resolve("http-transport-clients.properties"); + Files.delete(stateFile); + Files.createDirectory(stateFile); + + assertThrows(IllegalStateException.class, () -> authority.revoke("backend-a")); + assertFalse(authority.authenticate("backend-a", certificateA.certificate()), + "authentication must fail closed while the revocation state is unresolved"); + assertFalse(authority.authenticate("backend-b", certificateB.certificate()), + "the global retry guard must fail closed for every backend"); + + assertThrows(IllegalStateException.class, () -> authority.revoke("nonexistent")); + assertThrows(IllegalStateException.class, () -> authority.revoke("backend-b")); + assertFalse(authority.authenticate("backend-a", certificateA.certificate())); + assertFalse(authority.authenticate("backend-b", certificateB.certificate()), + "unrelated revocation attempts must not clear the retry guard or restore state"); + + Files.delete(stateFile); + authority.revoke("backend-a"); + assertFalse(authority.authenticate("backend-a", certificateA.certificate())); + assertTrue(authority.authenticate("backend-b", certificateB.certificate()), + "retrying the original target must revoke only that target"); + + HttpEnrollmentAuthority restarted = new HttpEnrollmentAuthority(identity, state); + assertFalse(restarted.authenticate("backend-a", certificateA.certificate())); + assertTrue(restarted.authenticate("backend-b", certificateB.certificate()), + "the isolated revocation result must survive restart"); + } +} From 7beae58df9e30cf6d703d89cfa737d787e37b579 Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Sun, 6 Sep 2026 06:50:46 -0600 Subject: [PATCH 32/38] Preserve failed revocation target across retries --- .../simpleapi/servercomm/http/HttpEnrollmentAuthority.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentAuthority.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentAuthority.java index 636b2bf..3ea9823 100644 --- a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentAuthority.java +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentAuthority.java @@ -36,6 +36,7 @@ public final class HttpEnrollmentAuthority { private final Map renewalNotBefore = new HashMap<>(); private boolean persistenceFailure; private boolean revocationRetryRequired; + private String revocationRetryServerId; /** Creates a restart-safe authority. State contains public certificate pins plus bounded hashes of pending tokens. */ public HttpEnrollmentAuthority(HttpTlsIdentity identity, Path stateDirectory) throws java.io.IOException { @@ -180,6 +181,8 @@ static final class RenewalRateLimitException extends IllegalStateException { public synchronized void revoke(String serverId) { try { serverId = HttpTlsIdentity.canonicalServerId(serverId); } catch (IllegalArgumentException invalid) { return; } + if (revocationRetryRequired && !serverId.equals(revocationRetryServerId)) + throw new IllegalStateException("A different HTTP certificate revocation must be retried first"); final String revokedServer = serverId; Map removedEnrollments = new HashMap<>(); enrollments.entrySet().removeIf(entry -> { @@ -193,6 +196,7 @@ public synchronized void revoke(String serverId) { persistState(); persistenceFailure = false; revocationRetryRequired = false; + revocationRetryServerId = null; } catch (java.io.IOException failure) { // Before publication, restore the exact disk-backed state so a retry still has work to persist. @@ -203,6 +207,7 @@ public synchronized void revoke(String serverId) { } persistenceFailure = true; revocationRetryRequired = true; + revocationRetryServerId = serverId; throw new IllegalStateException("Could not persist HTTP certificate revocation", failure); } } From 826e680754502d14ed888e0e43ac960b3587fa36 Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Sun, 6 Sep 2026 10:08:41 -0600 Subject: [PATCH 33/38] Wait for concurrent JSON saves in tests --- .../tests/file/VelocityJsonFileTest.java | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/SimpleAPI/src/test/java/com/bencodez/simpleapi/tests/file/VelocityJsonFileTest.java b/SimpleAPI/src/test/java/com/bencodez/simpleapi/tests/file/VelocityJsonFileTest.java index 4826268..0b94f6c 100644 --- a/SimpleAPI/src/test/java/com/bencodez/simpleapi/tests/file/VelocityJsonFileTest.java +++ b/SimpleAPI/src/test/java/com/bencodez/simpleapi/tests/file/VelocityJsonFileTest.java @@ -8,10 +8,13 @@ import java.nio.file.Files; import java.nio.file.Path; +import java.util.ArrayList; import java.util.Comparator; import java.util.List; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; @@ -118,18 +121,25 @@ void concurrentSavesAreSafeEnough() throws Exception { Path file = tmpRoot.resolve("conc/config.json"); VelocityJSONFile v = new VelocityJSONFile(file); - ExecutorService pool = Executors.newFixedThreadPool(4); + ExecutorService pool = Executors.newFixedThreadPool(4); + List> saves = new ArrayList<>(); try { for (int i = 0; i < 20; i++) { final int n = i; - pool.submit(() -> { + saves.add(pool.submit(() -> { v.set(new Object[] { "counter" }, n); v.save(); - }); + })); + } + for (Future save : saves) { + save.get(); } } finally { pool.shutdown(); - Thread.sleep(200); // small settle time + if (!pool.awaitTermination(10, TimeUnit.SECONDS)) { + pool.shutdownNow(); + assertTrue(pool.awaitTermination(10, TimeUnit.SECONDS), "save workers did not terminate"); + } } VelocityJSONFile re = new VelocityJSONFile(file); From af984b80ba13a958ebb95177214bab6e1b6bec36 Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Sun, 6 Sep 2026 10:32:17 -0600 Subject: [PATCH 34/38] Recover durable HTTP setup failure paths --- .../http/HttpBackendTransportConnector.java | 19 +++++-- .../http/HttpClientCredentialStore.java | 29 ++++++++++- .../http/HttpProxyTransportServer.java | 5 +- ...kendTransportConnectorConstructorTest.java | 34 +++++++++++++ ...HttpEnrollmentPublicationRecoveryTest.java | 51 +++++++++++++++++++ .../http/HttpInboundJournalCapacityTest.java | 33 +++++++++++- .../http/HttpTransportRuntimeTest.java | 4 +- 7 files changed, 164 insertions(+), 11 deletions(-) create mode 100644 SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpBackendTransportConnectorConstructorTest.java create mode 100644 SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentPublicationRecoveryTest.java diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpBackendTransportConnector.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpBackendTransportConnector.java index 4bd5028..38455bd 100644 --- a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpBackendTransportConnector.java +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpBackendTransportConnector.java @@ -153,8 +153,7 @@ private HttpBackendTransportConnector(HttpClientCredentialStore.HttpClientProfil /** Convenience constructor for the owner-only credential directory produced by {@link #enroll}. */ public HttpBackendTransportConnector(HttpConnectionCode code, String serverId, Path credentials, Consumer onEnvelope) throws Exception { - this(HttpClientCredentialStore.loadEnrolled(credentials), onEnvelope, credentials); - if (code == null || !profile(code, serverId).equals(this.profile)) throw new IllegalArgumentException("HTTP transport profile does not match connection code"); + this(validatedEnrollment(code, serverId, credentials), onEnvelope, credentials); } /** Starts normal transport using only the persisted certificate and non-secret profile. */ @@ -164,9 +163,11 @@ public HttpBackendTransportConnector(Path credentials, Consumer on /** Performs enrollment network I/O; call this from a connector/setup worker, never a platform main thread. */ public static HttpClientCredentialStore.ClientCredential enroll(HttpConnectionCode code, String serverId, Path credentials) throws Exception { - if (code == null || credentials == null || serverId == null || !serverId.matches("[A-Za-z0-9][A-Za-z0-9._-]{0,63}")) throw new IllegalArgumentException("Enrollment configuration is invalid"); - if (!code.serverId().equals(HttpTlsIdentity.canonicalServerId(serverId))) - throw new IllegalArgumentException("HTTP connection code belongs to a different backend"); + if (credentials == null) throw new IllegalArgumentException("Enrollment configuration is invalid"); + HttpClientCredentialStore.HttpClientProfile expectedProfile = profile(code, serverId); + HttpClientCredentialStore.ClientCredential published = HttpClientCredentialStore.recoverPublishedEnrollment( + credentials, code, expectedProfile); + if (published != null) return published; code.requireActive(Clock.systemUTC()); byte[] payload = ("{\"server\":\"" + serverId + "\",\"token\":\"" + code.enrollmentToken() + "\"}").getBytes(StandardCharsets.UTF_8); HttpClient client = HttpClient.newBuilder().version(HttpClient.Version.HTTP_1_1).followRedirects(HttpClient.Redirect.NEVER) @@ -510,6 +511,14 @@ private static HttpClientCredentialStore.HttpClientProfile profile(HttpConnectio if (!code.serverId().equals(HttpTlsIdentity.canonicalServerId(serverId))) throw new IllegalArgumentException("HTTP connection code belongs to a different backend"); return new HttpClientCredentialStore.HttpClientProfile(serverId, code.endpoint(), code.serverCertificatePin(), code.caCertificatePin()); } + private static HttpClientCredentialStore.EnrolledClient validatedEnrollment(HttpConnectionCode code, + String serverId, Path credentials) throws Exception { + HttpClientCredentialStore.HttpClientProfile expected = profile(code, serverId); + HttpClientCredentialStore.EnrolledClient enrolled = HttpClientCredentialStore.loadEnrolled(credentials); + if (!expected.equals(enrolled.profile())) + throw new IllegalArgumentException("HTTP transport profile does not match connection code"); + return enrolled; + } private static boolean matchesCredential(HttpClientCredentialStore.HttpClientProfile profile, HttpClientCredentialStore.ClientCredential credential) { try { diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpClientCredentialStore.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpClientCredentialStore.java index cc431df..7040e6c 100644 --- a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpClientCredentialStore.java +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpClientCredentialStore.java @@ -217,7 +217,34 @@ public static boolean hasEnrolledProfile(Path directory) { public static boolean matchesEnrollmentCode(Path directory, HttpConnectionCode code) throws IOException { if (code == null) throw new IllegalArgumentException("Connection code is required"); String stored = readConnectionCodeDigest(activeDirectory(directory)); - if (stored == null) return false; + return stored != null && matchesConnectionCodeDigest(stored, code); + } + + /** Confirms and returns an already-published initial enrollment before its one-time code is sent again. */ + static ClientCredential recoverPublishedEnrollment(Path directory, HttpConnectionCode code, + HttpClientProfile expectedProfile) throws Exception { + if (directory == null || code == null || expectedProfile == null) + throw new IllegalArgumentException("Enrollment recovery is required"); + Path candidate = directory.toAbsolutePath().normalize(); + if (!Files.exists(candidate, LinkOption.NOFOLLOW_LINKS)) return null; + Path root = credentialRoot(candidate, false); + Path active = activeDirectory(root); + if (!Files.isRegularFile(active.resolve(BUNDLE_FILE), LinkOption.NOFOLLOW_LINKS) + || !Files.isRegularFile(active.resolve(PASSWORD_FILE), LinkOption.NOFOLLOW_LINKS) + || !Files.isRegularFile(active.resolve(PROFILE_FILE), LinkOption.NOFOLLOW_LINKS) + || !Files.isRegularFile(active.resolve(CONNECTION_CODE_DIGEST_FILE), LinkOption.NOFOLLOW_LINKS)) + return null; + EnrolledClient enrolled = loadEnrolledDirectory(active); + String stored = readConnectionCodeDigest(active); + if (!expectedProfile.equals(enrolled.profile()) || stored == null || !matchesConnectionCodeDigest(stored, code)) + return null; + // CURRENT and every generation file were already forced before publication. + // Re-forcing the root confirms a pointer whose post-rename directory force failed. + DurableFiles.forceDirectory(root); + return enrolled.credential(); + } + + private static boolean matchesConnectionCodeDigest(String stored, HttpConnectionCode code) { byte[] storedBytes = stored.getBytes(StandardCharsets.US_ASCII); return HttpTransportSecrets.constantTimeEquals(storedBytes, connectionCodeDigest(code.encode()).getBytes(StandardCharsets.US_ASCII)) diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java index 30b4e88..5f8bbc2 100644 --- a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java @@ -799,7 +799,10 @@ synchronized Map> load() throws IOE } if (++serverDirectories > MAX_BACKENDS) throw new IOException("HTTP outgoing queue exceeds its backend bound"); - if (!deliveries.isEmpty()) loaded.put(serverId, deliveries); + // Quarantine-only queues still reserve this backend's runtime state. Omitting + // them can consume the cap with unrelated inbound journals and make the + // identical retry unable to reach persist()'s quarantine recovery path. + loaded.put(serverId, deliveries); } } return loaded; diff --git a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpBackendTransportConnectorConstructorTest.java b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpBackendTransportConnectorConstructorTest.java new file mode 100644 index 0000000..5e61ad8 --- /dev/null +++ b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpBackendTransportConnectorConstructorTest.java @@ -0,0 +1,34 @@ +package com.bencodez.simpleapi.servercomm.http; + +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.net.InetSocketAddress; +import java.nio.file.Path; +import java.time.Duration; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class HttpBackendTransportConnectorConstructorTest { + @TempDir Path directory; + + @Test + void invalidConnectionCodeDoesNotPreventImmediateValidRetry() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, directory.resolve("authority")); + try (HttpProxyTransportServer server = new HttpProxyTransportServer(new InetSocketAddress("localhost", 0), + identity, authority, ignored -> { })) { + server.start(); + HttpConnectionCode code = authority.createConnectionCode("lobby-1", server.endpoint("localhost"), + Duration.ofMinutes(5)); + Path credentials = directory.resolve("client"); + HttpBackendTransportConnector.enroll(code, "lobby-1", credentials); + + assertThrows(IllegalArgumentException.class, + () -> new HttpBackendTransportConnector(null, "lobby-1", credentials, ignored -> { })); + try (HttpBackendTransportConnector ignored = new HttpBackendTransportConnector(code, "lobby-1", + credentials, ignoredMessage -> { })) { + // The valid retry must be able to claim both durable journal locks. + } + } + } +} diff --git a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentPublicationRecoveryTest.java b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentPublicationRecoveryTest.java new file mode 100644 index 0000000..a050fdd --- /dev/null +++ b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentPublicationRecoveryTest.java @@ -0,0 +1,51 @@ +package com.bencodez.simpleapi.servercomm.http; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.net.InetSocketAddress; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.time.Duration; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import com.bencodez.simpleapi.file.DurableFiles; + +class HttpEnrollmentPublicationRecoveryTest { + @TempDir Path directory; + + @Test + void publishedInitialCredentialIsConfirmedWithoutReusingTheEnrollmentToken() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, directory.resolve("authority")); + Path credentials = directory.resolve("client").toAbsolutePath().normalize(); + HttpConnectionCode code; + HttpClientCredentialStore.ClientCredential published; + try (HttpProxyTransportServer server = new HttpProxyTransportServer(new InetSocketAddress("localhost", 0), + identity, authority, ignored -> { })) { + server.start(); + code = authority.createConnectionCode("lobby-1", server.endpoint("localhost"), Duration.ofMinutes(5)); + Path current = credentials.resolve("http-transport-client-current"); + try (var forces = org.mockito.Mockito.mockStatic(DurableFiles.class, org.mockito.Mockito.CALLS_REAL_METHODS)) { + forces.when(() -> DurableFiles.forceDirectory(credentials)).thenAnswer(call -> { + if (Files.isRegularFile(current, LinkOption.NOFOLLOW_LINKS)) + throw new java.io.IOException("injected CURRENT publication failure"); + return call.callRealMethod(); + }); + assertThrows(DurableFiles.PublishedException.class, + () -> HttpBackendTransportConnector.enroll(code, "lobby-1", credentials)); + } + published = HttpClientCredentialStore.load(credentials); + } + + // The endpoint is now closed. Recovery can succeed only by confirming the + // already-published CURRENT pointer rather than sending the token again. + HttpClientCredentialStore.ClientCredential recovered = HttpBackendTransportConnector.enroll( + code, "lobby-1", credentials); + assertArrayEquals(published.certificate().getEncoded(), recovered.certificate().getEncoded()); + assertTrue(authority.authenticate("lobby-1", recovered.certificate())); + } +} diff --git a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpInboundJournalCapacityTest.java b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpInboundJournalCapacityTest.java index 5455e73..47f7a2e 100644 --- a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpInboundJournalCapacityTest.java +++ b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpInboundJournalCapacityTest.java @@ -4,7 +4,9 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import com.bencodez.simpleapi.servercomm.codec.JsonEnvelope; import java.net.InetSocketAddress; import java.nio.charset.StandardCharsets; import java.nio.file.Files; @@ -57,8 +59,35 @@ void prunesEmptySafeJournalsAndAllowsAReplacementBackend() throws Exception { } } + @Test + void quarantineOnlyBackendIsIncludedInTheCombinedStartupCapacity() throws Exception { + Path outgoing = directory.resolve("quarantine-outgoing"); + Files.createDirectory(outgoing); + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("quarantine-identity"), "localhost"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, directory.resolve("quarantine-authority")); + seedRunningJournals(outgoing, 128); + String serverId = "server-128"; + String deliveryId = UUID.nameUUIDFromBytes(serverId.getBytes(StandardCharsets.UTF_8)).toString(); + HttpTransportProtocol.Delivery delivery = new HttpTransportProtocol.Delivery(deliveryId, + JsonEnvelope.builder("quarantined-retry").build()); + Path serverDirectory = Files.createDirectory(outgoing.resolve(serverId)); + Files.write(serverDirectory.resolve(".pending-" + deliveryId + ".json"), + HttpTransportProtocol.storedDelivery(delivery)); + + assertThrows(java.io.IOException.class, () -> new HttpProxyTransportServer( + new InetSocketAddress("localhost", 0), identity, authority, outgoing, ignored -> { }), + "129 distinct durable backend states must be rejected during startup rather than blocking a later retry"); + + try (HttpProxyTransportServer.DurableOutgoingQueue queue = new HttpProxyTransportServer.DurableOutgoingQueue( + outgoing, com.bencodez.simpleapi.file.DurableFiles::forceDirectory)) { + var loaded = queue.load(); + assertTrue(loaded.containsKey(serverId), "quarantine-only state must reserve its backend identity"); + assertTrue(loaded.get(serverId).isEmpty(), "an unconfirmed quarantine must remain hidden from delivery"); + } + } + private static void seedRunningJournals(Path outgoing, int count) throws Exception { - Path incoming = outgoing.getParent().resolve("outgoing-incoming"); + Path incoming = outgoing.getParent().resolve(outgoing.getFileName() + "-incoming"); Files.createDirectory(incoming); for (int index = 0; index < count; index++) { String serverId = "server-" + index; @@ -71,7 +100,7 @@ private static void seedRunningJournals(Path outgoing, int count) throws Excepti } private static void seedEmptyJournals(Path outgoing, int count) throws Exception { - Path incoming = outgoing.getParent().resolve("outgoing-incoming"); + Path incoming = outgoing.getParent().resolve(outgoing.getFileName() + "-incoming"); Files.createDirectory(incoming); for (int index = 0; index < count; index++) { HttpInboundDeliveryStore store = HttpInboundDeliveryStore.open(incoming, "server-" + index); diff --git a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportRuntimeTest.java b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportRuntimeTest.java index d1d0fa1..9a67aa4 100644 --- a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportRuntimeTest.java +++ b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportRuntimeTest.java @@ -281,8 +281,8 @@ void publishedOutgoingDeliveryRemainsTrackedUntilDurabilityCanBeConfirmed() thro queue.close(); HttpProxyTransportServer.DurableOutgoingQueue restartedQueue = new HttpProxyTransportServer.DurableOutgoingQueue( queueRoot, com.bencodez.simpleapi.file.DurableFiles::forceDirectory); - assertTrue(restartedQueue.load().isEmpty(), - "a restart must not expose an operation whose sender observed rejection"); + assertEquals(java.util.List.of(), restartedQueue.load().get("lobby-1"), + "a quarantine must reserve its backend without exposing an operation whose sender observed rejection"); HttpProxyTransportServer.BackendState restarted = new HttpProxyTransportServer.BackendState( "lobby-1", restartedQueue, (server, id) -> { }); assertTrue(restarted.await("lobby-1", java.util.UUID.randomUUID().toString(), 0).messages().isEmpty()); From cf61b6f2e3f2724c94379c25aed8d770888a40c1 Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Sun, 6 Sep 2026 11:10:21 -0600 Subject: [PATCH 35/38] Fix HTTP transport restart and persistence recovery --- .../http/HttpBackendTransportConnector.java | 32 ++++- .../servercomm/http/HttpConnectionCode.java | 13 +- .../http/HttpEnrollmentAuthority.java | 123 ++++++++++++++++-- .../http/HttpProxyTransportServer.java | 9 +- .../http/HttpBackendSendLifecycleTest.java | 23 ++++ .../HttpEnrollmentAuthorityOwnershipTest.java | 36 +++++ .../http/HttpTransportRuntimeTest.java | 36 +++++ .../http/HttpTransportSecurityTest.java | 7 + 8 files changed, 260 insertions(+), 19 deletions(-) create mode 100644 SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentAuthorityOwnershipTest.java diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpBackendTransportConnector.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpBackendTransportConnector.java index 38455bd..d162821 100644 --- a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpBackendTransportConnector.java +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpBackendTransportConnector.java @@ -60,7 +60,8 @@ public final class HttpBackendTransportConnector implements AutoCloseable { private final AtomicBoolean running = new AtomicBoolean(); private final AtomicBoolean closing = new AtomicBoolean(); private final CountDownLatch closed = new CountDownLatch(1); - private final CountDownLatch firstResponse = new CountDownLatch(1); + // Replaced for each start. A response from a stopped run must not satisfy a later run. + private volatile ResponseState responseState = new ResponseState(); private final Object lifecycle = new Object(); private final Object state = new Object(); private final Object renewal = new Object(); @@ -183,6 +184,8 @@ public static HttpClientCredentialStore.ClientCredential enroll(HttpConnectionCo public void start() { synchronized (lifecycle) { if (closing.get() || flushingOutgoing || !running.compareAndSet(false, true)) return; + responseState.cancel(); + responseState = new ResponseState(); poller = new Thread(this::pollLoop, "SimpleAPI-HTTP-poll"); poller.setDaemon(true); poller.start(); @@ -195,9 +198,13 @@ public void start() { } /** Waits for one authenticated, protocol-valid transport response. */ public boolean awaitFirstResponse(long deadlineNanos) throws InterruptedException { + ResponseState expected = responseState; long remaining = deadlineNanos - System.nanoTime(); - return remaining > 0L && firstResponse.await(remaining, TimeUnit.NANOSECONDS) && running.get(); + return remaining > 0L && expected.await(remaining) && responseStateIsActive(expected); } + private boolean responseStateIsActive(ResponseState expected) { synchronized (lifecycle) { + return responseState == expected && running.get(); + } } /** * Inserts an in-memory at-least-once delivery. It survives retry/lost responses while this process remains alive; * callers needing restart durability must retain the application operation independently. @@ -217,7 +224,11 @@ public synchronized boolean pollOnce() { return pollOnce(CLIENT_TIMEOUT, true, true); } private boolean pollOnce(Duration timeout, boolean requireRunning, boolean acceptIncoming) { - if (requireRunning && !running.get()) return false; + ResponseState runResponse; + synchronized (lifecycle) { + if (requireRunning && !running.get()) return false; + runResponse = responseState; + } List acks = List.of(), ackConfirmations = List.of(); boolean acknowledgementsConfirmed = false, confirmationsConfirmed = false; try { @@ -249,7 +260,7 @@ private boolean pollOnce(Duration timeout, boolean requireRunning, boolean accep outgoing.remove(ack); queueAcknowledgementConfirmation(ack); } } if (acceptIncoming) for (HttpTransportProtocol.Delivery delivery : accept(packet.messages())) dispatch(delivery); - firstResponse.countDown(); + if (requireRunning) runResponse.received(); return true; } catch (Exception failure) { return false; } finally { @@ -264,7 +275,7 @@ public boolean flushOutgoing(long deadlineNanos) { if (closing.get() || flushingOutgoing) return false; flushingOutgoing = true; running.set(false); - firstResponse.countDown(); + responseState.cancel(); current = poller; if (current != null) current.interrupt(); } @@ -303,7 +314,7 @@ public boolean flushOutgoing(long deadlineNanos) { if (alreadyClosing) current = null; else { running.set(false); - firstResponse.countDown(); + responseState.cancel(); current = poller; if (current != null) current.interrupt(); } @@ -382,6 +393,15 @@ private static void joinPoller(Thread poller) { catch (InterruptedException stopRequested) { interrupted = true; poller.interrupt(); } if (interrupted) Thread.currentThread().interrupt(); } + private static final class ResponseState { + private final CountDownLatch complete = new CountDownLatch(1); + private final AtomicBoolean received = new AtomicBoolean(); + void received() { received.set(true); complete.countDown(); } + void cancel() { complete.countDown(); } + boolean await(long timeoutNanos) throws InterruptedException { + return complete.await(timeoutNanos, TimeUnit.NANOSECONDS) && received.get(); + } + } private void pollLoop() { long retry = 1000L; diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpConnectionCode.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpConnectionCode.java index dfad475..463729d 100644 --- a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpConnectionCode.java +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpConnectionCode.java @@ -60,8 +60,19 @@ public static HttpConnectionCode parse(String code) { if (code == null || code.length() > MAX_CODE_LENGTH || code.indexOf('\n') >= 0 || code.indexOf('\r') >= 0) throw new IllegalArgumentException("Connection code is invalid"); String[] parts = code.split("\\.", -1); - if (parts.length != 8 || (!VERSION.equals(parts[0]) && !LEGACY_VERSION.equals(parts[0]))) + if (parts.length < 8 || (!VERSION.equals(parts[0]) && !LEGACY_VERSION.equals(parts[0]))) throw new IllegalArgumentException("Connection code is invalid"); + if (LEGACY_VERSION.equals(parts[0]) && parts.length > 8) { + int endpointIndex = parts.length - 6; + StringBuilder serverId = new StringBuilder(parts[1]); + for (int index = 2; index < endpointIndex; index++) serverId.append('.').append(parts[index]); + String[] normalized = new String[8]; + normalized[0] = parts[0]; + normalized[1] = serverId.toString(); + System.arraycopy(parts, endpointIndex, normalized, 2, 6); + parts = normalized; + } + if (parts.length != 8) throw new IllegalArgumentException("Connection code is invalid"); try { String serverId = LEGACY_VERSION.equals(parts[0]) ? parts[1] : new String(Base64.getUrlDecoder().decode(parts[1]), StandardCharsets.UTF_8); diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentAuthority.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentAuthority.java index 3ea9823..79abd78 100644 --- a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentAuthority.java +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentAuthority.java @@ -1,6 +1,9 @@ package com.bencodez.simpleapi.servercomm.http; import java.net.URI; +import java.nio.channels.FileChannel; +import java.nio.channels.FileLock; +import java.nio.channels.OverlappingFileLockException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.LinkOption; @@ -23,6 +26,8 @@ * retained; only SHA-256 hashes are kept until expiry. This type is thread-safe. */ public final class HttpEnrollmentAuthority { + private static final Object PROCESS_STATE_LOCK = new Object(); + private static final String STATE_LOCK_FILE = ".http-transport-authority.lock"; private static final Duration MAX_ENROLLMENT_LIFETIME = Duration.ofMinutes(15); private static final Duration MIN_RENEWAL_INTERVAL = Duration.ofMinutes(1); private static final int MAX_PENDING_ENROLLMENTS = 128; @@ -35,13 +40,14 @@ public final class HttpEnrollmentAuthority { private final Map bindings = new HashMap<>(); private final Map renewalNotBefore = new HashMap<>(); private boolean persistenceFailure; + private boolean rollbackStateAvailable; private boolean revocationRetryRequired; private String revocationRetryServerId; /** Creates a restart-safe authority. State contains public certificate pins plus bounded hashes of pending tokens. */ public HttpEnrollmentAuthority(HttpTlsIdentity identity, Path stateDirectory) throws java.io.IOException { this(identity, Clock.systemUTC(), stateFile(stateDirectory)); - loadState(); + withStateLock(() -> null); } HttpEnrollmentAuthority(HttpTlsIdentity identity, Clock clock) { @@ -56,6 +62,11 @@ public HttpEnrollmentAuthority(HttpTlsIdentity identity, Path stateDirectory) th } public synchronized HttpConnectionCode createConnectionCode(String serverId, URI endpoint, Duration lifetime) { + try { return withMutationLock(() -> createConnectionCodeLocked(serverId, endpoint, lifetime)); } + catch (java.io.IOException failure) { throw new IllegalStateException("Could not read HTTP enrollment state", failure); } + } + + private HttpConnectionCode createConnectionCodeLocked(String serverId, URI endpoint, Duration lifetime) { if (revocationRetryRequired) throw new IllegalStateException("HTTP certificate revocation durability must be retried"); serverId = HttpTlsIdentity.canonicalServerId(serverId); @@ -72,15 +83,20 @@ public synchronized HttpConnectionCode createConnectionCode(String serverId, URI byte[] tokenHash = HttpTransportSecrets.sha256(token.getBytes(StandardCharsets.US_ASCII)); String lookup = java.util.Base64.getUrlEncoder().withoutPadding().encodeToString(tokenHash); enrollments.put(lookup, new Enrollment(tokenHash, expiresAt, serverId, null)); - try { persistState(); persistenceFailure = false; } + try { persistState(); persistenceFailure = false; rollbackStateAvailable = false; } catch (java.io.IOException failure) { enrollments.remove(lookup); + rollbackStateAvailable = !(failure instanceof DurableFiles.PublishedException); throw new IllegalStateException("Could not persist HTTP enrollment", failure); } return code; } public synchronized HttpTlsIdentity.IssuedClientCertificate enroll(String serverId, String enrollmentToken) throws Exception { + return withMutationLock(() -> enrollLocked(serverId, enrollmentToken)); + } + + private HttpTlsIdentity.IssuedClientCertificate enrollLocked(String serverId, String enrollmentToken) throws Exception { // A failed revoke may have restored a still-valid pending token in memory. // Do not let enrollment persist that stale state before the revoke is retried. if (revocationRetryRequired) @@ -101,16 +117,24 @@ public synchronized HttpTlsIdentity.IssuedClientCertificate enroll(String server HttpTlsIdentity.IssuedClientCertificate issued = identity.issueClientCertificate(serverId); enrollments.put(lookup, new Enrollment(enrollment.tokenHash(), enrollment.expiresAt(), serverId, HttpTransportSecrets.certificatePin(issued.certificate()))); - try { persistState(); persistenceFailure = false; } + try { persistState(); persistenceFailure = false; rollbackStateAvailable = false; } catch (java.io.IOException failure) { - if (!(failure instanceof DurableFiles.PublishedException)) enrollments.put(lookup, enrollment); - else persistenceFailure = true; + if (!(failure instanceof DurableFiles.PublishedException)) { + enrollments.put(lookup, enrollment); + rollbackStateAvailable = true; + } + else { persistenceFailure = true; rollbackStateAvailable = false; } throw failure; } return issued; } public synchronized boolean authenticate(String serverId, java.security.cert.X509Certificate certificate) { + try { return withStateLock(() -> authenticateLocked(serverId, certificate), rollbackStateAvailable); } + catch (java.io.IOException failure) { return false; } + } + + private boolean authenticateLocked(String serverId, java.security.cert.X509Certificate certificate) { if (persistenceFailure || serverId == null || certificate == null) return false; try { serverId = HttpTlsIdentity.canonicalServerId(serverId); } catch (IllegalArgumentException invalid) { return false; } @@ -121,21 +145,23 @@ public synchronized boolean authenticate(String serverId, java.security.cert.X50 if (samePin(binding.certificatePin(), pin)) return true; if (!samePin(binding.pendingCertificatePin(), pin)) return false; bindings.put(serverId, new ClientBinding(pin, null, false)); - try { persistState(); return true; } + try { persistState(); rollbackStateAvailable = false; return true; } catch (DurableFiles.PublishedException published) { + rollbackStateAvailable = false; // The replacement is visible. If publication is lost on a crash, the // previous durable pending binding can promote this certificate again. return true; } - catch (java.io.IOException failure) { bindings.put(serverId, binding); return false; } + catch (java.io.IOException failure) { bindings.put(serverId, binding); rollbackStateAvailable = true; return false; } } Map.Entry pending = pendingCertificate(serverId, pin); if (pending == null || !pending.getValue().expiresAt().isAfter(clock.instant()) || bindings.size() >= MAX_BINDINGS) return false; enrollments.remove(pending.getKey()); bindings.put(serverId, new ClientBinding(pin, null, false)); - try { persistState(); return true; } + try { persistState(); rollbackStateAvailable = false; return true; } catch (DurableFiles.PublishedException published) { + rollbackStateAvailable = false; // The visible state is active. If its directory entry is lost on a crash, the // prior pending state can promote this same certificate again after restart. return true; @@ -143,6 +169,7 @@ public synchronized boolean authenticate(String serverId, java.security.cert.X50 catch (java.io.IOException failure) { bindings.remove(serverId); enrollments.put(pending.getKey(), pending.getValue()); + rollbackStateAvailable = true; return false; } } @@ -151,11 +178,16 @@ public synchronized boolean authenticate(String serverId, java.security.cert.X50 * until the replacement successfully authenticates, making a lost renewal response safe to retry. */ public synchronized HttpTlsIdentity.IssuedClientCertificate renew(String serverId, java.security.cert.X509Certificate currentCertificate) throws Exception { + return withMutationLock(() -> renewLocked(serverId, currentCertificate)); + } + + private HttpTlsIdentity.IssuedClientCertificate renewLocked(String serverId, + java.security.cert.X509Certificate currentCertificate) throws Exception { serverId = HttpTlsIdentity.canonicalServerId(serverId); Instant now = clock.instant(); Instant nextAllowed = renewalNotBefore.get(serverId); if (nextAllowed != null && now.isBefore(nextAllowed)) throw new RenewalRateLimitException(); - if (!authenticate(serverId, currentCertificate)) throw new IllegalArgumentException("Certificate renewal was rejected"); + if (!authenticateLocked(serverId, currentCertificate)) throw new IllegalArgumentException("Certificate renewal was rejected"); // Bound the limiter by active bindings; failed issuance still consumes the window. renewalNotBefore.keySet().retainAll(bindings.keySet()); renewalNotBefore.put(serverId, now.plus(MIN_RENEWAL_INTERVAL)); @@ -163,13 +195,14 @@ public synchronized HttpTlsIdentity.IssuedClientCertificate renew(String serverI HttpTlsIdentity.IssuedClientCertificate issued = identity.issueClientCertificate(serverId); bindings.put(serverId, new ClientBinding(binding.certificatePin(), HttpTransportSecrets.certificatePin(issued.certificate()), false)); - try { persistState(); } + try { persistState(); rollbackStateAvailable = false; } catch (DurableFiles.PublishedException published) { + rollbackStateAvailable = false; // The old certificate remains active in both the old and newly visible state, // so a lost response can safely retry and republish the complete state. throw published; } - catch (java.io.IOException failure) { bindings.put(serverId, binding); throw failure; } + catch (java.io.IOException failure) { bindings.put(serverId, binding); rollbackStateAvailable = true; throw failure; } return issued; } @@ -179,6 +212,11 @@ static final class RenewalRateLimitException extends IllegalStateException { } public synchronized void revoke(String serverId) { + try { withMutationLock(() -> { revokeLocked(serverId); return null; }); } + catch (java.io.IOException failure) { throw new IllegalStateException("Could not read HTTP enrollment state", failure); } + } + + private void revokeLocked(String serverId) { try { serverId = HttpTlsIdentity.canonicalServerId(serverId); } catch (IllegalArgumentException invalid) { return; } if (revocationRetryRequired && !serverId.equals(revocationRetryServerId)) @@ -195,6 +233,7 @@ public synchronized void revoke(String serverId) { if (removedBinding != null || !removedEnrollments.isEmpty() || revocationRetryRequired) try { persistState(); persistenceFailure = false; + rollbackStateAvailable = false; revocationRetryRequired = false; revocationRetryServerId = null; } @@ -204,14 +243,76 @@ public synchronized void revoke(String serverId) { if (!(failure instanceof DurableFiles.PublishedException)) { if (removedBinding != null) bindings.put(serverId, removedBinding); enrollments.putAll(removedEnrollments); + rollbackStateAvailable = true; } persistenceFailure = true; + if (failure instanceof DurableFiles.PublishedException) rollbackStateAvailable = false; revocationRetryRequired = true; revocationRetryServerId = serverId; throw new IllegalStateException("Could not persist HTTP certificate revocation", failure); } } + @FunctionalInterface + private interface StateOperation { T run() throws E; } + + private T withMutationLock(StateOperation operation) throws java.io.IOException, E { + return withStateLock(operation, true); + } + + /** Serializes every durable-state operation across both authority instances and processes, then + * adopts the latest complete file before making a decision or rewriting it. */ + private T withStateLock(StateOperation operation) throws java.io.IOException, E { + return withStateLock(operation, false); + } + + private T withStateLock(StateOperation operation, + boolean allowDirectoryFailure) throws java.io.IOException, E { + if (stateFile == null) return operation.run(); + synchronized (PROCESS_STATE_LOCK) { + Path sidecar = stateFile.resolveSibling(STATE_LOCK_FILE); + if (Files.isSymbolicLink(sidecar) || Files.exists(sidecar, LinkOption.NOFOLLOW_LINKS) + && !Files.isRegularFile(sidecar, LinkOption.NOFOLLOW_LINKS)) + throw new java.io.IOException("HTTP enrollment state lock is unsafe"); + try (FileChannel channel = FileChannel.open(sidecar, StandardOpenOption.CREATE, + StandardOpenOption.WRITE, LinkOption.NOFOLLOW_LINKS)) { + PrivateFilePermissions.ownerOnlyFile(sidecar); + try (FileLock ignored = lock(channel)) { + refreshState(allowDirectoryFailure); + return operation.run(); + } + } + } + } + + private static FileLock lock(FileChannel channel) throws java.io.IOException { + try { return channel.lock(); } + catch (OverlappingFileLockException alreadyOwned) { + throw new java.io.IOException("HTTP enrollment state is already being updated", alreadyOwned); + } + } + + private void refreshState(boolean allowDirectoryFailure) throws java.io.IOException { + // A pre-publication failure may temporarily leave no state file while this instance + // retains the exact rollback state needed for a retry. Any successful peer mutation + // recreates the file under this same lock and will therefore be adopted here. + if (!Files.exists(stateFile, LinkOption.NOFOLLOW_LINKS)) return; + // Tests and recovery callers may expose a failed replacement as a directory at + // the target path. Mutations must reach persistState() so its existing rollback + // and fail-closed retry semantics run; reads still reject the invalid state. + if (allowDirectoryFailure && Files.isDirectory(stateFile, LinkOption.NOFOLLOW_LINKS)) return; + Map previousEnrollments = new HashMap<>(enrollments); + Map previousBindings = new HashMap<>(bindings); + enrollments.clear(); + bindings.clear(); + try { loadState(); } + catch (java.io.IOException failure) { + enrollments.putAll(previousEnrollments); + bindings.putAll(previousBindings); + throw failure; + } + } + private synchronized void loadState() throws java.io.IOException { if (stateFile == null || !Files.exists(stateFile, LinkOption.NOFOLLOW_LINKS)) return; if (!Files.isRegularFile(stateFile, LinkOption.NOFOLLOW_LINKS) || Files.size(stateFile) > MAX_STATE_BYTES) diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java index 5f8bbc2..caaa2c7 100644 --- a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java @@ -635,7 +635,8 @@ synchronized void confirmIncoming(Collection ids) throws IOException { private synchronized boolean retireIfQuiescent(long now, long retentionNanos) throws IOException { if (retired || activePoll || !outgoing.isEmpty() || !deliveredAtNanos.isEmpty() || !seen.isEmpty() || !processing.isEmpty() || !acknowledgements.isEmpty() || now - lastActivityNanos < retentionNanos - || durableIncoming != null && !durableIncoming.snapshot().isEmpty()) return false; + || durableIncoming != null && !durableIncoming.snapshot().isEmpty() + || durableOutgoing != null && durableOutgoing.hasQuarantined(serverId)) return false; if (durableIncoming != null) durableIncoming.sealAndDeleteIfEmpty(); retired = true; return true; @@ -914,6 +915,12 @@ private boolean hasNoIndexedDeliveries(String serverId) { return (serverFiles == null || serverFiles.isEmpty()) && (quarantined == null || quarantined.isEmpty()); } + private synchronized boolean hasQuarantined(String serverId) throws IOException { + requireOwnership(); + Map quarantined = quarantinedFiles.get(serverId); + return quarantined != null && !quarantined.isEmpty(); + } + /** Deletes only a validated, observed-empty backend directory and makes its removal durable. */ private void deleteVerifiedEmptyDirectory(Path directory) throws IOException { if (Files.isSymbolicLink(directory) || !Files.isDirectory(directory, LinkOption.NOFOLLOW_LINKS) diff --git a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpBackendSendLifecycleTest.java b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpBackendSendLifecycleTest.java index 3537ac8..ca7d64a 100644 --- a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpBackendSendLifecycleTest.java +++ b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpBackendSendLifecycleTest.java @@ -51,6 +51,29 @@ void pausedSendCannotEnterAfterCloseAndFlushCutsOffNewSends() throws Exception { } } + @Test + void restartedConnectorRequiresAResponseFromItsNewRun() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("restart-proxy"), "localhost"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, directory.resolve("restart-authority")); + try (HttpProxyTransportServer server = new HttpProxyTransportServer(new InetSocketAddress("localhost", 0), identity, + authority, ignored -> { })) { + server.start(); + HttpConnectionCode code = authority.createConnectionCode("lobby-1", server.endpoint("localhost"), Duration.ofMinutes(5)); + Path clientDirectory = directory.resolve("restart-client"); + HttpBackendTransportConnector.enroll(code, "lobby-1", clientDirectory); + try (HttpBackendTransportConnector connector = new HttpBackendTransportConnector(clientDirectory, ignored -> { })) { + connector.start(); + assertTrue(connector.awaitFirstResponse(System.nanoTime() + TimeUnit.SECONDS.toNanos(8))); + assertTrue(connector.flushOutgoing(System.nanoTime() + TimeUnit.SECONDS.toNanos(5))); + server.close(); + + connector.start(); + assertFalse(connector.awaitFirstResponse(System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(250)), + "a completed flush must not satisfy the restarted run's first-response wait"); + } + } + } + private static void assertPausedSendIsRejectedByClose(HttpBackendTransportConnector connector) throws Exception { Object state = field("state").get(connector); AtomicBoolean closing = (AtomicBoolean) field("closing").get(connector); diff --git a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentAuthorityOwnershipTest.java b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentAuthorityOwnershipTest.java new file mode 100644 index 0000000..11e8eaf --- /dev/null +++ b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentAuthorityOwnershipTest.java @@ -0,0 +1,36 @@ +package com.bencodez.simpleapi.servercomm.http; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +import java.net.URI; +import java.nio.file.Path; +import java.time.Duration; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class HttpEnrollmentAuthorityOwnershipTest { + @TempDir Path directory; + + @Test + void staleAuthorityCannotEraseAnotherEnrollmentOrRestoreARevokedBinding() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("identity"), "localhost"); + Path state = directory.resolve("authority"); + HttpEnrollmentAuthority first = new HttpEnrollmentAuthority(identity, state); + HttpEnrollmentAuthority stale = new HttpEnrollmentAuthority(identity, state); + URI endpoint = URI.create("https://localhost:8443/"); + + HttpConnectionCode firstCode = first.createConnectionCode("lobby-1", endpoint, Duration.ofMinutes(5)); + HttpConnectionCode secondCode = stale.createConnectionCode("lobby-2", endpoint, Duration.ofMinutes(5)); + assertNotNull(first.enroll("lobby-1", firstCode.enrollmentToken()), + "a stale authority rewrite must retain another instance's pending enrollment"); + var secondCertificate = stale.enroll("lobby-2", secondCode.enrollmentToken()); + assertNotNull(secondCertificate); + + first.revoke("lobby-2"); + stale.createConnectionCode("lobby-3", endpoint, Duration.ofMinutes(5)); + HttpEnrollmentAuthority reloaded = new HttpEnrollmentAuthority(identity, state); + assertFalse(reloaded.authenticate("lobby-2", secondCertificate.certificate()), + "a stale authority rewrite must not restore a revoked certificate"); + } +} diff --git a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportRuntimeTest.java b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportRuntimeTest.java index 9a67aa4..05bed61 100644 --- a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportRuntimeTest.java +++ b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportRuntimeTest.java @@ -859,6 +859,42 @@ void proxyReclaimsOnlyQuiescentBackendStateAfterReplayWindow() throws Exception } } + @Test + void proxyDoesNotReclaimAQuarantineOnlyBackendAfterReplayWindow() throws Exception { + Path queueRoot = directory.resolve("quarantine-reclamation-outgoing"); + AtomicLong forceCalls = new AtomicLong(); + String serverId = "quarantined"; + HttpTransportProtocol.Delivery retry = new HttpTransportProtocol.Delivery( + java.util.UUID.randomUUID().toString(), JsonEnvelope.builder("retry").build()); + try (HttpProxyTransportServer.DurableOutgoingQueue queue = new HttpProxyTransportServer.DurableOutgoingQueue( + queueRoot, ignored -> { + if (forceCalls.incrementAndGet() == 3L) throw new java.io.IOException("injected directory force failure"); + })) { + HttpProxyTransportServer.BackendState state = new HttpProxyTransportServer.BackendState( + serverId, queue, (server, id) -> { }); + assertFalse(state.enqueue(retry)); + } + + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("quarantine-reclamation-proxy"), "localhost"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, + directory.resolve("quarantine-reclamation-authority")); + AtomicLong nanoTime = new AtomicLong(); + try (HttpProxyTransportServer server = new HttpProxyTransportServer(new InetSocketAddress("localhost", 0), + identity, authority, queueRoot, ignored -> { }, (backend, delivery) -> { }, nanoTime::get)) { + for (int index = 0; index < 127; index++) { + HttpProxyTransportServer.BackendState state = server.backendStateForTest("idle-" + index); + assertTrue(state.beginPollForTest()); + state.endPollForTest(); + } + nanoTime.addAndGet(TimeUnit.MILLISECONDS.toNanos(HttpTransportProtocol.MAX_CLOCK_SKEW_MILLIS) + 1L); + for (int index = 0; index < 127; index++) + assertTrue(server.send("replacement-" + index, JsonEnvelope.builder("replacement").build())); + assertFalse(server.send("overflow", JsonEnvelope.builder("overflow").build())); + assertTrue(server.backendStateForTest(serverId).enqueue(retry), + "the identical retry must still reach and promote its quarantined delivery at capacity"); + } + } + @Test void proxyOutgoingQueueSurvivesRestartUntilBackendAcknowledges() throws Exception { Path proxyDirectory = directory.resolve("proxy"); diff --git a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportSecurityTest.java b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportSecurityTest.java index ba88748..0c0352d 100644 --- a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportSecurityTest.java +++ b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportSecurityTest.java @@ -93,6 +93,13 @@ void legacyConnectionCodesAndConsumedMarkersRemainCompatible() throws Exception HttpConnectionCode.parse(legacy.encodeLegacy()))); } + @Test + void legacyConnectionCodesSupportDottedServerIds() { + HttpConnectionCode original = new HttpConnectionCode("lobby.eu", URI.create("https://proxy.example.test:8443/"), + pin('a'), pin('b'), Instant.parse("2030-01-01T00:00:00Z"), HttpTransportSecrets.randomToken()); + assertEquals(original, HttpConnectionCode.parse(original.encodeLegacy())); + } + @Test void expiredCodesAreNotActive() { HttpConnectionCode code = new HttpConnectionCode("lobby", URI.create("https://proxy.example.test/"), pin('a'), pin('b'), From b2e5f7bfb44b4aef4f192d7c07e85d492317bb0c Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Sun, 6 Sep 2026 19:09:22 -0600 Subject: [PATCH 36/38] Fix HTTP enrollment review findings --- .../http/HttpEnrollmentAuthority.java | 74 ++++++++++++++++--- .../http/HttpProxyTransportServer.java | 6 +- .../http/HttpEnrollmentPinTest.java | 6 +- .../http/HttpRenewalRateLimitTest.java | 49 ++++++++++++ 4 files changed, 123 insertions(+), 12 deletions(-) diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentAuthority.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentAuthority.java index 79abd78..fabb51a 100644 --- a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentAuthority.java +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentAuthority.java @@ -13,6 +13,7 @@ import java.time.Clock; import java.time.Duration; import java.time.Instant; +import java.time.temporal.ChronoUnit; import java.util.HashMap; import java.util.Map; import java.util.Properties; @@ -72,7 +73,13 @@ private HttpConnectionCode createConnectionCodeLocked(String serverId, URI endpo serverId = HttpTlsIdentity.canonicalServerId(serverId); if (lifetime == null || lifetime.compareTo(Duration.ofSeconds(1)) < 0 || lifetime.compareTo(MAX_ENROLLMENT_LIFETIME) > 0) throw new IllegalArgumentException("Enrollment lifetime must be between one second and fifteen minutes"); - Instant expiresAt = clock.instant().plus(lifetime); + Instant now = clock.instant(); + Instant expiresAt = now.plus(lifetime); + if (expiresAt.getNano() != 0) { + expiresAt = expiresAt.plusSeconds(1).truncatedTo(ChronoUnit.SECONDS); + Instant maximumExpiry = now.plus(MAX_ENROLLMENT_LIFETIME).truncatedTo(ChronoUnit.SECONDS); + if (expiresAt.isAfter(maximumExpiry)) expiresAt = maximumExpiry; + } String token = HttpTransportSecrets.randomToken(); // Validate the complete code before reserving or persisting a pending slot. HttpConnectionCode code = new HttpConnectionCode(serverId, endpoint, identity.serverCertificatePin(), @@ -188,9 +195,18 @@ private HttpTlsIdentity.IssuedClientCertificate renewLocked(String serverId, Instant nextAllowed = renewalNotBefore.get(serverId); if (nextAllowed != null && now.isBefore(nextAllowed)) throw new RenewalRateLimitException(); if (!authenticateLocked(serverId, currentCertificate)) throw new IllegalArgumentException("Certificate renewal was rejected"); - // Bound the limiter by active bindings; failed issuance still consumes the window. + // Persist the limiter before certificate generation so failed issuance still consumes + // the same window across every authority sharing this state. renewalNotBefore.keySet().retainAll(bindings.keySet()); - renewalNotBefore.put(serverId, now.plus(MIN_RENEWAL_INTERVAL)); + Instant previousRenewalNotBefore = renewalNotBefore.put(serverId, now.plus(MIN_RENEWAL_INTERVAL)); + try { persistState(); rollbackStateAvailable = false; } + catch (DurableFiles.PublishedException published) { rollbackStateAvailable = false; throw published; } + catch (java.io.IOException failure) { + if (previousRenewalNotBefore == null) renewalNotBefore.remove(serverId); + else renewalNotBefore.put(serverId, previousRenewalNotBefore); + rollbackStateAvailable = true; + throw failure; + } ClientBinding binding = bindings.get(serverId); HttpTlsIdentity.IssuedClientCertificate issued = identity.issueClientCertificate(serverId); bindings.put(serverId, new ClientBinding(binding.certificatePin(), @@ -230,7 +246,9 @@ private void revokeLocked(String serverId) { }); // Absence is the durable revocation fence: authentication always requires an exact active binding. ClientBinding removedBinding = bindings.remove(serverId); - if (removedBinding != null || !removedEnrollments.isEmpty() || revocationRetryRequired) try { + Instant removedRenewalNotBefore = renewalNotBefore.remove(serverId); + if (removedBinding != null || !removedEnrollments.isEmpty() || removedRenewalNotBefore != null + || revocationRetryRequired) try { persistState(); persistenceFailure = false; rollbackStateAvailable = false; @@ -243,6 +261,7 @@ private void revokeLocked(String serverId) { if (!(failure instanceof DurableFiles.PublishedException)) { if (removedBinding != null) bindings.put(serverId, removedBinding); enrollments.putAll(removedEnrollments); + if (removedRenewalNotBefore != null) renewalNotBefore.put(serverId, removedRenewalNotBefore); rollbackStateAvailable = true; } persistenceFailure = true; @@ -303,12 +322,15 @@ private void refreshState(boolean allowDirectoryFailure) throws java.io.IOExcept if (allowDirectoryFailure && Files.isDirectory(stateFile, LinkOption.NOFOLLOW_LINKS)) return; Map previousEnrollments = new HashMap<>(enrollments); Map previousBindings = new HashMap<>(bindings); + Map previousRenewalNotBefore = new HashMap<>(renewalNotBefore); enrollments.clear(); bindings.clear(); + renewalNotBefore.clear(); try { loadState(); } catch (java.io.IOException failure) { enrollments.putAll(previousEnrollments); bindings.putAll(previousBindings); + renewalNotBefore.putAll(previousRenewalNotBefore); throw failure; } } @@ -321,7 +343,8 @@ private synchronized void loadState() throws java.io.IOException { Properties properties = new Properties(); try (var input = Files.newInputStream(stateFile, LinkOption.NOFOLLOW_LINKS)) { properties.load(input); } String version = properties.getProperty("version"); - if (!("1".equals(version) || "2".equals(version) || "3".equals(version) || "4".equals(version))) + if (!("1".equals(version) || "2".equals(version) || "3".equals(version) || "4".equals(version) + || "5".equals(version))) throw new java.io.IOException("HTTP enrollment state is invalid"); for (String key : properties.stringPropertyNames()) { if (key.startsWith("binding.")) { @@ -329,7 +352,8 @@ private synchronized void loadState() throws java.io.IOException { serverId = HttpTlsIdentity.canonicalServerId(serverId); String[] value = properties.getProperty(key, "").split(":", -1); if (!((value.length == 2 && "1".equals(version)) - || (value.length == 3 && ("2".equals(version) || "3".equals(version) || "4".equals(version)))) + || (value.length == 3 && ("2".equals(version) || "3".equals(version) + || "4".equals(version) || "5".equals(version)))) || !value[0].matches("[0-9a-f]{64}")) throw new java.io.IOException("HTTP enrollment state is invalid"); String pending = value.length == 3 && !"-".equals(value[1]) ? value[1] : null; @@ -340,7 +364,8 @@ private synchronized void loadState() throws java.io.IOException { if (bindings.size() >= MAX_BINDINGS) throw new java.io.IOException("HTTP enrollment state exceeds its bound"); bindings.put(serverId, new ClientBinding(value[0], pending, false)); } - } else if (key.startsWith("enrollment.") && ("3".equals(version) || "4".equals(version))) { + } else if (key.startsWith("enrollment.") + && ("3".equals(version) || "4".equals(version) || "5".equals(version))) { String lookup = key.substring("enrollment.".length()); if (!lookup.matches("[A-Za-z0-9_-]{43}")) throw new java.io.IOException("HTTP enrollment state is invalid"); byte[] tokenHash; @@ -348,7 +373,8 @@ private synchronized void loadState() throws java.io.IOException { catch (IllegalArgumentException invalid) { throw new java.io.IOException("HTTP enrollment state is invalid", invalid); } if (tokenHash.length != 32) throw new java.io.IOException("HTTP enrollment state is invalid"); String[] value = properties.getProperty(key, "").split(":", -1); - if (!(value.length == 2 && "3".equals(version)) && !(value.length == 3 && "4".equals(version))) + if (!(value.length == 2 && "3".equals(version)) + && !(value.length == 3 && ("4".equals(version) || "5".equals(version)))) throw new java.io.IOException("HTTP enrollment state is invalid"); Instant expiresAt; String serverId; @@ -364,18 +390,43 @@ private synchronized void loadState() throws java.io.IOException { throw new java.io.IOException("HTTP enrollment state exceeds its bound"); enrollments.put(lookup, new Enrollment(tokenHash, expiresAt, serverId, pendingPin)); } + } else if (key.startsWith("renewal.") && "5".equals(version)) { + String encodedServer = key.substring("renewal.".length()); + String serverId; + Instant notBefore; + try { + serverId = HttpTlsIdentity.canonicalServerId(new String( + Base64.getUrlDecoder().decode(encodedServer), StandardCharsets.UTF_8)); + String canonicalEncoding = Base64.getUrlEncoder().withoutPadding().encodeToString( + serverId.getBytes(StandardCharsets.UTF_8)); + if (!canonicalEncoding.equals(encodedServer)) throw new IllegalArgumentException(); + String value = properties.getProperty(key, ""); + if (!value.matches("[0-9]{1,19}")) throw new IllegalArgumentException(); + notBefore = Instant.ofEpochMilli(Long.parseLong(value)); + } catch (RuntimeException invalid) { + throw new java.io.IOException("HTTP enrollment state is invalid", invalid); + } + if (renewalNotBefore.size() >= MAX_BINDINGS + || renewalNotBefore.putIfAbsent(serverId, notBefore) != null) + throw new java.io.IOException("HTTP enrollment state exceeds its bound"); } else if (!"version".equals(key)) throw new java.io.IOException("HTTP enrollment state is invalid"); } + Instant now = clock.instant(); + renewalNotBefore.entrySet().removeIf(entry -> !bindings.containsKey(entry.getKey()) + || !entry.getValue().isAfter(now)); if (reservedBindingCount() > MAX_BINDINGS) throw new java.io.IOException("HTTP enrollment state exceeds its bound"); } private synchronized void persistState() throws java.io.IOException { if (stateFile == null) return; + Instant now = clock.instant(); + renewalNotBefore.entrySet().removeIf(entry -> !bindings.containsKey(entry.getKey()) + || !entry.getValue().isAfter(now)); if (reservedBindingCount() > MAX_BINDINGS || enrollments.size() > MAX_PENDING_ENROLLMENTS) throw new java.io.IOException("HTTP enrollment state exceeds its bound"); Properties properties = new Properties(); - properties.setProperty("version", "4"); + properties.setProperty("version", "5"); for (Map.Entry entry : bindings.entrySet()) { String key = Base64.getUrlEncoder().withoutPadding().encodeToString(entry.getKey().getBytes(StandardCharsets.UTF_8)); properties.setProperty("binding." + key, entry.getValue().certificatePin() + ":" @@ -389,6 +440,11 @@ private synchronized void persistState() throws java.io.IOException { entry.getValue().expiresAt().toEpochMilli() + ":" + server + ":" + (entry.getValue().pendingCertificatePin() == null ? "-" : entry.getValue().pendingCertificatePin())); } + for (Map.Entry entry : renewalNotBefore.entrySet()) { + String server = Base64.getUrlEncoder().withoutPadding().encodeToString( + entry.getKey().getBytes(StandardCharsets.UTF_8)); + properties.setProperty("renewal." + server, Long.toString(entry.getValue().toEpochMilli())); + } java.io.ByteArrayOutputStream bytes = new java.io.ByteArrayOutputStream(); properties.store(bytes, "VotingPlugin HTTP transport authority state"); if (bytes.size() > MAX_STATE_BYTES) throw new java.io.IOException("HTTP enrollment state exceeds its byte bound"); diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java index caaa2c7..b055576 100644 --- a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java @@ -55,6 +55,8 @@ */ public final class HttpProxyTransportServer implements AutoCloseable { private static final int MAX_BACKENDS = 128; + private static final int SETUP_REQUEST_HEADROOM = 8; + private static final int MAX_ADMITTED_REQUESTS = MAX_BACKENDS + SETUP_REQUEST_HEADROOM; private static final long BACKEND_REPLAY_RETENTION_NANOS = TimeUnit.MILLISECONDS.toNanos(HttpTransportProtocol.MAX_CLOCK_SKEW_MILLIS) + 1L; static { @@ -76,7 +78,7 @@ public final class HttpProxyTransportServer implements AutoCloseable { private final ThreadPoolExecutor handlerExecutor; private final AtomicReference handlerWorker = new AtomicReference<>(); private final Object closeMonitor = new Object(); - private final Semaphore admission = new Semaphore(64); + private final Semaphore admission = new Semaphore(MAX_ADMITTED_REQUESTS); private final Map backends = new HashMap<>(); private final DurableOutgoingQueue durableOutgoing; private final Path durableIncomingRoot; @@ -148,7 +150,7 @@ public HttpProxyTransportServer(InetSocketAddress bind, HttpTlsIdentity identity }); // Long polls are blocking by design. Capacity is bounded by admission, while enough workers // remain available for all admitted polls plus setup requests. - createdListener = executor("SimpleAPI-HTTP-listener", 72, 72); + createdListener = executor("SimpleAPI-HTTP-listener", MAX_ADMITTED_REQUESTS, MAX_ADMITTED_REQUESTS); // The proxy router mutates shared presence, vote, and reward state. A separate // bounded FIFO lane keeps wire order without blocking long-poll workers. createdHandler = executor("SimpleAPI-HTTP-handler", 1, diff --git a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentPinTest.java b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentPinTest.java index 6df6ae4..db27218 100644 --- a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentPinTest.java +++ b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentPinTest.java @@ -34,7 +34,11 @@ void enrollmentLifetimeRequiresAtLeastOneSecond() throws Exception { () -> authority.createConnectionCode("lobby-1", endpoint, lifetime)); } HttpConnectionCode minimum = authority.createConnectionCode("lobby-1", endpoint, Duration.ofSeconds(1)); - org.junit.jupiter.api.Assertions.assertTrue(HttpConnectionCode.parse(minimum.encode()).expiresAt().isAfter(clock.instant())); + assertEquals(Instant.parse("2026-09-05T12:00:02Z"), + HttpConnectionCode.parse(minimum.encode()).expiresAt()); + HttpConnectionCode maximum = authority.createConnectionCode("lobby-2", endpoint, Duration.ofMinutes(15)); + assertEquals(Instant.parse("2026-09-05T12:15:00Z"), + HttpConnectionCode.parse(maximum.encode()).expiresAt()); } @Test diff --git a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpRenewalRateLimitTest.java b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpRenewalRateLimitTest.java index 5977f49..039d710 100644 --- a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpRenewalRateLimitTest.java +++ b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpRenewalRateLimitTest.java @@ -44,4 +44,53 @@ void repeatedRenewalIsLimitedPerBackendBeforeCertificateIssuance() throws Except now.set(now.get().plusSeconds(60)); assertNotNull(authority.renew("lobby-1", replacement.certificate())); } + + @Test + void renewalWindowIsSharedAcrossAuthorities() throws Exception { + AtomicReference now = new AtomicReference<>(Instant.now()); + Clock clock = new Clock() { + @Override public ZoneId getZone() { return ZoneOffset.UTC; } + @Override public Clock withZone(ZoneId zone) { return this; } + @Override public Instant instant() { return now.get(); } + }; + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("shared-identity"), "localhost"); + Path stateDirectory = directory.resolve("shared-authority"); + java.nio.file.Files.createDirectory(stateDirectory); + Path stateFile = stateDirectory.resolve("http-transport-clients.properties"); + HttpEnrollmentAuthority first = new HttpEnrollmentAuthority(identity, clock, stateFile); + HttpEnrollmentAuthority second = new HttpEnrollmentAuthority(identity, clock, stateFile); + URI endpoint = URI.create("https://localhost:8443/"); + var code = first.createConnectionCode("lobby-1", endpoint, Duration.ofMinutes(5)); + var original = first.enroll("lobby-1", code.enrollmentToken()); + assertTrue(first.authenticate("lobby-1", original.certificate())); + + first.renew("lobby-1", original.certificate()); + assertThrows(HttpEnrollmentAuthority.RenewalRateLimitException.class, + () -> second.renew("lobby-1", original.certificate())); + now.set(now.get().plusSeconds(60)); + assertNotNull(second.renew("lobby-1", original.certificate())); + } + + @Test + void revocationRemovesTheDurableRenewalWindow() throws Exception { + Instant now = Instant.now(); + Clock clock = Clock.fixed(now, ZoneOffset.UTC); + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("revoked-identity"), "localhost"); + Path stateDirectory = directory.resolve("revoked-authority"); + java.nio.file.Files.createDirectory(stateDirectory); + Path stateFile = stateDirectory.resolve("http-transport-clients.properties"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, clock, stateFile); + URI endpoint = URI.create("https://localhost:8443/"); + var firstCode = authority.createConnectionCode("lobby-1", endpoint, Duration.ofMinutes(5)); + var firstCertificate = authority.enroll("lobby-1", firstCode.enrollmentToken()); + assertTrue(authority.authenticate("lobby-1", firstCertificate.certificate())); + authority.renew("lobby-1", firstCertificate.certificate()); + authority.revoke("lobby-1"); + + var secondCode = authority.createConnectionCode("lobby-1", endpoint, Duration.ofMinutes(5)); + var secondCertificate = authority.enroll("lobby-1", secondCode.enrollmentToken()); + assertTrue(authority.authenticate("lobby-1", secondCertificate.certificate())); + assertNotNull(new HttpEnrollmentAuthority(identity, clock, stateFile) + .renew("lobby-1", secondCertificate.certificate())); + } } From f451f789441d7bc710b3d30bccba58c64e71a811 Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Sun, 6 Sep 2026 20:00:07 -0600 Subject: [PATCH 37/38] Reclaim retired inbound ownership locks --- .../http/HttpInboundDeliveryStore.java | 97 +++++++++++++++---- .../http/HttpInboundRetirementTest.java | 17 ++++ 2 files changed, 93 insertions(+), 21 deletions(-) diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpInboundDeliveryStore.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpInboundDeliveryStore.java index 3fbb140..afa35d2 100644 --- a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpInboundDeliveryStore.java +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpInboundDeliveryStore.java @@ -23,10 +23,12 @@ /** Crash-durable state for proxy deliveries around a non-transactional application callback. */ final class HttpInboundDeliveryStore { private static final String DIRECTORY = "http-transport-inbound-deliveries"; - // This sidecar is deliberately outside the deletable journal directory. Never remove it: - // deleting and recreating a locked file could split ownership across two inodes. + // Per-journal sidecars are reclaimed only while holding the stable owners guard, so an + // opener cannot race the unlink and acquire a different inode for the same journal. private static final String OWNER_LOCK_PREFIX = ".http-inbound-owner-"; private static final String OWNER_LOCK_SUFFIX = ".lock"; + private static final String OWNER_GUARD_LOCK = ".http-inbound-owners.lock"; + private static final Object OWNER_GUARD_MONITOR = new Object(); private static final int MAX_ENTRIES = HttpTransportProtocol.MAX_QUEUE; private final Path root; private final boolean readOnly; @@ -39,6 +41,7 @@ final class HttpInboundDeliveryStore { private final Set pendingRunningRollbacks = new HashSet<>(); private FileChannel ownershipChannel; private FileLock ownershipLock; + private Path ownershipSidecar; private boolean sealed; private boolean retirementRecoveryRequired; @@ -76,6 +79,11 @@ static Set discover(Path parent) throws IOException { directories.add(name); continue; } + if (OWNER_GUARD_LOCK.equals(name) && !Files.isSymbolicLink(entry) + && Files.isRegularFile(entry, LinkOption.NOFOLLOW_LINKS)) { + PrivateFilePermissions.ownerOnlyFile(entry); + continue; + } if (!name.startsWith(OWNER_LOCK_PREFIX) || !name.endsWith(OWNER_LOCK_SUFFIX) || Files.isSymbolicLink(entry) || !Files.isRegularFile(entry, LinkOption.NOFOLLOW_LINKS)) throw new IOException("HTTP inbound delivery parent contains an invalid entry"); @@ -100,7 +108,7 @@ private HttpInboundDeliveryStore(Path parent, String directoryName, boolean read if (!root.getParent().equals(credentials)) throw new IOException("HTTP inbound delivery directory is invalid"); if (!readOnly) { try { - claimOwnership(credentials.resolve(OWNER_LOCK_PREFIX + directoryName + OWNER_LOCK_SUFFIX)); + claimOwnership(credentials, credentials.resolve(OWNER_LOCK_PREFIX + directoryName + OWNER_LOCK_SUFFIX)); // Claim before creating or checking the journal root: a retiring owner may be // between its empty check and deletion, and only one owner may cross that boundary. try { Files.createDirectory(root); } @@ -210,7 +218,8 @@ synchronized void sealAndDeleteIfEmpty() throws IOException { retirementRecoveryRequired = true; Files.delete(root); DurableFiles.forceDirectory(parent); - seal(); + retireOwnership(); + sealed = true; } synchronized void remove(String id) throws IOException { @@ -375,33 +384,79 @@ private void requireWritable() throws IOException { retirementRecoveryRequired = false; } } - private void claimOwnership(Path sidecar) throws IOException { - if (Files.isSymbolicLink(sidecar) || Files.exists(sidecar, LinkOption.NOFOLLOW_LINKS) - && !Files.isRegularFile(sidecar, LinkOption.NOFOLLOW_LINKS)) - throw new IOException("HTTP inbound delivery ownership lock is unsafe"); - FileChannel channel = FileChannel.open(sidecar, StandardOpenOption.CREATE, StandardOpenOption.WRITE, - LinkOption.NOFOLLOW_LINKS); - try { - PrivateFilePermissions.ownerOnlyFile(sidecar); - FileLock lock; - try { lock = channel.tryLock(); } - catch (OverlappingFileLockException alreadyOwned) { throw new IOException("HTTP inbound delivery store is already owned", alreadyOwned); } - if (lock == null) throw new IOException("HTTP inbound delivery store is already owned"); - ownershipChannel = channel; - ownershipLock = lock; - } catch (IOException | RuntimeException failure) { - try { channel.close(); } catch (IOException closeFailure) { failure.addSuppressed(closeFailure); } - throw failure; + private void claimOwnership(Path parent, Path sidecar) throws IOException { + withOwnerGuard(parent, () -> { + if (Files.isSymbolicLink(sidecar) || Files.exists(sidecar, LinkOption.NOFOLLOW_LINKS) + && !Files.isRegularFile(sidecar, LinkOption.NOFOLLOW_LINKS)) + throw new IOException("HTTP inbound delivery ownership lock is unsafe"); + FileChannel channel = FileChannel.open(sidecar, StandardOpenOption.CREATE, StandardOpenOption.WRITE, + LinkOption.NOFOLLOW_LINKS); + try { + PrivateFilePermissions.ownerOnlyFile(sidecar); + FileLock lock; + try { lock = channel.tryLock(); } + catch (OverlappingFileLockException alreadyOwned) { + throw new IOException("HTTP inbound delivery store is already owned", alreadyOwned); + } + if (lock == null) throw new IOException("HTTP inbound delivery store is already owned"); + ownershipChannel = channel; + ownershipLock = lock; + ownershipSidecar = sidecar; + } catch (IOException | RuntimeException failure) { + try { channel.close(); } catch (IOException closeFailure) { failure.addSuppressed(closeFailure); } + throw failure; + } + }); + } + private void retireOwnership() throws IOException { + Path sidecar = ownershipSidecar; + if (sidecar == null) throw new IOException("HTTP inbound delivery store ownership has ended"); + withOwnerGuard(root.getParent(), () -> { + releaseOwnershipForRetirement(); + DurableFiles.deleteIfExists(sidecar); + }); + } + private void releaseOwnershipForRetirement() throws IOException { + FileLock lock = ownershipLock; + FileChannel channel = ownershipChannel; + ownershipLock = null; + ownershipChannel = null; + ownershipSidecar = null; + IOException failure = null; + if (lock != null) try { lock.release(); } catch (IOException releaseFailure) { failure = releaseFailure; } + if (channel != null) try { channel.close(); } catch (IOException closeFailure) { + if (failure == null) failure = closeFailure; + else failure.addSuppressed(closeFailure); } + if (failure != null) throw failure; } private void releaseOwnership() { FileLock lock = ownershipLock; FileChannel channel = ownershipChannel; ownershipLock = null; ownershipChannel = null; + ownershipSidecar = null; if (lock != null) try { lock.release(); } catch (IOException ignored) { } if (channel != null) try { channel.close(); } catch (IOException ignored) { } } + private static void withOwnerGuard(Path parent, IoAction action) throws IOException { + synchronized (OWNER_GUARD_MONITOR) { + Path guard = parent.resolve(OWNER_GUARD_LOCK); + if (Files.isSymbolicLink(guard) || Files.exists(guard, LinkOption.NOFOLLOW_LINKS) + && !Files.isRegularFile(guard, LinkOption.NOFOLLOW_LINKS)) + throw new IOException("HTTP inbound delivery owners guard is unsafe"); + try (FileChannel channel = FileChannel.open(guard, StandardOpenOption.CREATE, StandardOpenOption.WRITE, + LinkOption.NOFOLLOW_LINKS)) { + PrivateFilePermissions.ownerOnlyFile(guard); + try (FileLock ignored = channel.lock()) { action.run(); } + catch (OverlappingFileLockException overlapping) { + throw new IOException("HTTP inbound delivery owners guard is already held", overlapping); + } + } + } + } + @FunctionalInterface + private interface IoAction { void run() throws IOException; } private static void move(Path source, Path target) throws IOException { try { Files.move(source, target, StandardCopyOption.ATOMIC_MOVE); } catch (java.nio.file.AtomicMoveNotSupportedException unsupported) { Files.move(source, target); } diff --git a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpInboundRetirementTest.java b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpInboundRetirementTest.java index 4941b09..2845fc7 100644 --- a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpInboundRetirementTest.java +++ b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpInboundRetirementTest.java @@ -2,6 +2,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; import java.nio.file.Files; @@ -57,4 +58,20 @@ void failedRetirementCanRetryAfterDeleteFailure() throws Exception { HttpInboundDeliveryStore successor = HttpInboundDeliveryStore.open(parent, "lobby-1"); successor.seal(); } + + @Test + void successfulRetirementReclaimsPerJournalOwnershipSidecars() throws Exception { + Path parent = directory.resolve("incoming"); + Files.createDirectory(parent); + for (int index = 0; index < 200; index++) { + HttpInboundDeliveryStore store = HttpInboundDeliveryStore.open(parent, "backend-" + index); + store.sealAndDeleteIfEmpty(); + } + + try (var files = Files.list(parent)) { + assertEquals(java.util.List.of(".http-inbound-owners.lock"), + files.map(path -> path.getFileName().toString()).sorted().toList()); + } + assertTrue(HttpInboundDeliveryStore.discover(parent).isEmpty()); + } } From 5e6a1490df17a06acd4711461e557c154801783b Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Sun, 6 Sep 2026 20:25:21 -0600 Subject: [PATCH 38/38] Retry and bound orphaned ownership locks --- .../http/HttpInboundDeliveryStore.java | 84 ++++++++++++++----- .../http/HttpInboundRetirementTest.java | 17 ++++ 2 files changed, 79 insertions(+), 22 deletions(-) diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpInboundDeliveryStore.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpInboundDeliveryStore.java index afa35d2..9e82dd1 100644 --- a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpInboundDeliveryStore.java +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpInboundDeliveryStore.java @@ -29,6 +29,7 @@ final class HttpInboundDeliveryStore { private static final String OWNER_LOCK_SUFFIX = ".lock"; private static final String OWNER_GUARD_LOCK = ".http-inbound-owners.lock"; private static final Object OWNER_GUARD_MONITOR = new Object(); + private static final int MAX_OWNER_SIDECARS = 128; private static final int MAX_ENTRIES = HttpTransportProtocol.MAX_QUEUE; private final Path root; private final boolean readOnly; @@ -69,30 +70,33 @@ static Set discover(Path parent) throws IOException { throw new IOException("HTTP inbound delivery parent is unsafe"); PrivateFilePermissions.ownerOnlyDirectory(root); Set directories = new TreeSet<>(); - try (DirectoryStream entries = Files.newDirectoryStream(root)) { - for (Path entry : entries) { - String name = entry.getFileName().toString(); - if (!Files.isSymbolicLink(entry) && Files.isDirectory(entry, LinkOption.NOFOLLOW_LINKS)) { - if (!name.matches("[A-Za-z0-9][A-Za-z0-9._-]{0,63}")) - throw new IOException("HTTP inbound delivery directory name is invalid"); - PrivateFilePermissions.ownerOnlyDirectory(entry); - directories.add(name); - continue; - } - if (OWNER_GUARD_LOCK.equals(name) && !Files.isSymbolicLink(entry) - && Files.isRegularFile(entry, LinkOption.NOFOLLOW_LINKS)) { + withOwnerGuard(root, () -> { + reclaimOrphanSidecars(root); + try (DirectoryStream entries = Files.newDirectoryStream(root)) { + for (Path entry : entries) { + String name = entry.getFileName().toString(); + if (!Files.isSymbolicLink(entry) && Files.isDirectory(entry, LinkOption.NOFOLLOW_LINKS)) { + if (!name.matches("[A-Za-z0-9][A-Za-z0-9._-]{0,63}")) + throw new IOException("HTTP inbound delivery directory name is invalid"); + PrivateFilePermissions.ownerOnlyDirectory(entry); + directories.add(name); + continue; + } + if (OWNER_GUARD_LOCK.equals(name) && !Files.isSymbolicLink(entry) + && Files.isRegularFile(entry, LinkOption.NOFOLLOW_LINKS)) { + PrivateFilePermissions.ownerOnlyFile(entry); + continue; + } + if (!name.startsWith(OWNER_LOCK_PREFIX) || !name.endsWith(OWNER_LOCK_SUFFIX) + || Files.isSymbolicLink(entry) || !Files.isRegularFile(entry, LinkOption.NOFOLLOW_LINKS)) + throw new IOException("HTTP inbound delivery parent contains an invalid entry"); + String journal = name.substring(OWNER_LOCK_PREFIX.length(), name.length() - OWNER_LOCK_SUFFIX.length()); + if (!journal.matches("[A-Za-z0-9][A-Za-z0-9._-]{0,63}")) + throw new IOException("HTTP inbound delivery ownership lock name is invalid"); PrivateFilePermissions.ownerOnlyFile(entry); - continue; } - if (!name.startsWith(OWNER_LOCK_PREFIX) || !name.endsWith(OWNER_LOCK_SUFFIX) - || Files.isSymbolicLink(entry) || !Files.isRegularFile(entry, LinkOption.NOFOLLOW_LINKS)) - throw new IOException("HTTP inbound delivery parent contains an invalid entry"); - String journal = name.substring(OWNER_LOCK_PREFIX.length(), name.length() - OWNER_LOCK_SUFFIX.length()); - if (!journal.matches("[A-Za-z0-9][A-Za-z0-9._-]{0,63}")) - throw new IOException("HTTP inbound delivery ownership lock name is invalid"); - PrivateFilePermissions.ownerOnlyFile(entry); } - } + }); return directories; } @@ -386,6 +390,10 @@ private void requireWritable() throws IOException { } private void claimOwnership(Path parent, Path sidecar) throws IOException { withOwnerGuard(parent, () -> { + reclaimOrphanSidecars(parent); + if (!Files.exists(sidecar, LinkOption.NOFOLLOW_LINKS) + && ownershipSidecarCount(parent) >= MAX_OWNER_SIDECARS) + throw new IOException("HTTP inbound delivery ownership locks exceed their bound"); if (Files.isSymbolicLink(sidecar) || Files.exists(sidecar, LinkOption.NOFOLLOW_LINKS) && !Files.isRegularFile(sidecar, LinkOption.NOFOLLOW_LINKS)) throw new IOException("HTTP inbound delivery ownership lock is unsafe"); @@ -413,7 +421,10 @@ private void retireOwnership() throws IOException { if (sidecar == null) throw new IOException("HTTP inbound delivery store ownership has ended"); withOwnerGuard(root.getParent(), () -> { releaseOwnershipForRetirement(); - DurableFiles.deleteIfExists(sidecar); + // Ownership has ended and the journal root is durably gone. A cleanup + // failure may leave one safe, reusable sidecar, but must not leave the + // caller retaining an unusable BackendState that can never retire. + try { DurableFiles.deleteIfExists(sidecar); } catch (IOException ignored) { } }); } private void releaseOwnershipForRetirement() throws IOException { @@ -455,6 +466,35 @@ private static void withOwnerGuard(Path parent, IoAction action) throws IOExcept } } } + private static void reclaimOrphanSidecars(Path parent) throws IOException { + try (DirectoryStream entries = Files.newDirectoryStream(parent, + OWNER_LOCK_PREFIX + "*" + OWNER_LOCK_SUFFIX)) { + for (Path sidecar : entries) { + String name = sidecar.getFileName().toString(); + String journal = name.substring(OWNER_LOCK_PREFIX.length(), name.length() - OWNER_LOCK_SUFFIX.length()); + if (!journal.matches("[A-Za-z0-9][A-Za-z0-9._-]{0,63}") || Files.isSymbolicLink(sidecar) + || !Files.isRegularFile(sidecar, LinkOption.NOFOLLOW_LINKS)) continue; + if (Files.exists(parent.resolve(journal), LinkOption.NOFOLLOW_LINKS)) continue; + try (FileChannel channel = FileChannel.open(sidecar, StandardOpenOption.WRITE, LinkOption.NOFOLLOW_LINKS)) { + FileLock lock; + try { lock = channel.tryLock(); } catch (OverlappingFileLockException owned) { continue; } + if (lock == null) continue; + try (lock) { } + } catch (java.nio.file.NoSuchFileException alreadyRemoved) { continue; } + try { DurableFiles.deleteIfExists(sidecar); } catch (IOException cleanupFailure) { } + } + } + } + private static int ownershipSidecarCount(Path parent) throws IOException { + int count = 0; + try (DirectoryStream entries = Files.newDirectoryStream(parent, + OWNER_LOCK_PREFIX + "*" + OWNER_LOCK_SUFFIX)) { + for (Path ignored : entries) { + if (++count > MAX_OWNER_SIDECARS) break; + } + } + return count; + } @FunctionalInterface private interface IoAction { void run() throws IOException; } private static void move(Path source, Path target) throws IOException { diff --git a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpInboundRetirementTest.java b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpInboundRetirementTest.java index 2845fc7..8f02649 100644 --- a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpInboundRetirementTest.java +++ b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpInboundRetirementTest.java @@ -74,4 +74,21 @@ void successfulRetirementReclaimsPerJournalOwnershipSidecars() throws Exception } assertTrue(HttpInboundDeliveryStore.discover(parent).isEmpty()); } + + @Test + void sidecarCleanupFailureStillCompletesRetirement() throws Exception { + Path parent = directory.resolve("incoming"); + Files.createDirectory(parent); + HttpInboundDeliveryStore store = HttpInboundDeliveryStore.open(parent, "lobby-1"); + Path sidecar = parent.resolve(".http-inbound-owner-lobby-1.lock"); + try (var files = org.mockito.Mockito.mockStatic(com.bencodez.simpleapi.file.DurableFiles.class, + org.mockito.Mockito.CALLS_REAL_METHODS)) { + files.when(() -> com.bencodez.simpleapi.file.DurableFiles.deleteIfExists(sidecar)) + .thenThrow(new IOException("injected sidecar cleanup failure")); + store.sealAndDeleteIfEmpty(); + } + + HttpInboundDeliveryStore successor = HttpInboundDeliveryStore.open(parent, "lobby-1"); + successor.sealAndDeleteIfEmpty(); + } }