diff --git a/SimpleAPI/pom.xml b/SimpleAPI/pom.xml index c02f0ba..f9559f7 100644 --- a/SimpleAPI/pom.xml +++ b/SimpleAPI/pom.xml @@ -16,6 +16,7 @@ 21 21 21 + 1.85 src/main/java @@ -71,6 +72,11 @@ ${project.name} + + org.apache.maven.plugins + maven-surefire-plugin + 3.5.4 + org.apache.maven.plugins maven-shade-plugin @@ -155,6 +161,16 @@ + + org.bouncycastle + bcpkix-jdk18on + ${bouncycastle.version} + + + org.bouncycastle + bcprov-jdk18on + ${bouncycastle.version} + org.spigotmc spigot-api @@ -407,4 +423,4 @@ - \ No newline at end of file + diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/file/DurableFiles.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/file/DurableFiles.java new file mode 100644 index 0000000..3969f64 --- /dev/null +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/file/DurableFiles.java @@ -0,0 +1,65 @@ +package com.bencodez.simpleapi.file; + +import java.io.IOException; +import java.nio.channels.FileChannel; +import java.nio.file.AccessDeniedException; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.Locale; + +/** Cross-platform helpers for forcing file contents and published directory entries. */ +public final class DurableFiles { + private DurableFiles() { } + + public static void forceFile(Path file) throws IOException { + try (FileChannel channel = FileChannel.open(file, StandardOpenOption.WRITE, LinkOption.NOFOLLOW_LINKS)) { + channel.force(true); + } + } + + public static void forceDirectory(Path directory) throws IOException { + if (directory == null) return; + try { + try (FileChannel channel = FileChannel.open(directory, StandardOpenOption.READ)) { + channel.force(true); + } + } catch (AccessDeniedException unsupportedDirectoryHandle) { + // The Windows NIO provider cannot open directory handles. File contents are + // still forced before atomic publication; do not make persistence unusable. + if (!isWindowsName(System.getProperty("os.name", ""))) throw unsupportedDirectoryHandle; + } catch (UnsupportedOperationException unsupportedDirectoryForce) { + // Some providers support atomic moves but expose no directory-force operation. + } + } + + public static boolean deleteIfExists(Path target) throws IOException { + boolean deleted = Files.deleteIfExists(target); + if (deleted) forceDirectory(target.toAbsolutePath().normalize().getParent()); + return deleted; + } + + public static void forceMoveDirectories(Path source, Path target) throws IOException { + try { + Path sourceParent = source.toAbsolutePath().normalize().getParent(); + Path targetParent = target.toAbsolutePath().normalize().getParent(); + forceDirectory(targetParent); + if (sourceParent != null && !sourceParent.equals(targetParent)) forceDirectory(sourceParent); + } catch (IOException failure) { + throw new PublishedException(failure); + } + } + + public static boolean isWindowsName(String name) { + return name != null && name.trim().toLowerCase(Locale.ROOT).startsWith("windows"); + } + + /** Indicates that an atomic rename completed before metadata writeback failed. */ + @SuppressWarnings("serial") + public static final class PublishedException extends IOException { + public PublishedException(IOException cause) { + super("File was published but its directory metadata could not be forced", cause); + } + } +} diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/file/PrivateFilePermissions.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/file/PrivateFilePermissions.java new file mode 100644 index 0000000..f3d0f09 --- /dev/null +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/file/PrivateFilePermissions.java @@ -0,0 +1,84 @@ +package com.bencodez.simpleapi.file; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.nio.file.attribute.AclEntry; +import java.nio.file.attribute.AclEntryPermission; +import java.nio.file.attribute.AclEntryType; +import java.nio.file.attribute.AclFileAttributeView; +import java.nio.file.attribute.GroupPrincipal; +import java.nio.file.attribute.PosixFilePermission; +import java.nio.file.attribute.UserPrincipal; +import java.util.EnumSet; +import java.util.List; +import java.util.Set; + +/** Enforces and verifies owner-only access for persisted private material. */ +public final class PrivateFilePermissions { + private static final Set OWNER_FILE = EnumSet.of( + PosixFilePermission.OWNER_READ, PosixFilePermission.OWNER_WRITE); + private static final Set OWNER_DIRECTORY = EnumSet.of( + PosixFilePermission.OWNER_READ, PosixFilePermission.OWNER_WRITE, PosixFilePermission.OWNER_EXECUTE); + private static final Set OWNER_ACL = EnumSet.allOf(AclEntryPermission.class); + + private PrivateFilePermissions() { } + + /** Enforces 0600-equivalent access, or fails when the provider cannot prove it. */ + public static void ownerOnlyFile(Path path) throws IOException { + enforce(path, OWNER_FILE); + } + + /** Enforces 0700-equivalent access, or fails when the provider cannot prove it. */ + public static void ownerOnlyDirectory(Path path) throws IOException { + enforce(path, OWNER_DIRECTORY); + } + + private static void enforce(Path path, Set permissions) throws IOException { + if (path == null) throw new IllegalArgumentException("Private path is required"); + if (Files.isSymbolicLink(path)) throw new IOException("Refusing symbolic link for private storage: " + path); + try { + Files.setPosixFilePermissions(path, permissions); + if (!Files.getPosixFilePermissions(path, LinkOption.NOFOLLOW_LINKS).equals(permissions)) + throw new IOException("Could not verify owner-only POSIX permissions for " + path); + return; + } catch (UnsupportedOperationException unsupported) { + // A non-POSIX provider must expose an ACL that can be reduced and verified. + } + enforceWithAcl(path); + } + + private static void enforceWithAcl(Path path) throws IOException { + try { + AclFileAttributeView view = Files.getFileAttributeView(path, AclFileAttributeView.class, LinkOption.NOFOLLOW_LINKS); + if (view == null) throw new IOException("Owner-only private storage is unsupported for " + path); + enforceWithAcl(view, path); + } catch (UnsupportedOperationException unsupported) { + throw new IOException("Owner-only private storage is unsupported for " + path, unsupported); + } + } + + /** Package-visible for deterministic ACL-provider tests. */ + static void enforceWithAcl(AclFileAttributeView view, Path path) throws IOException { + try { + if (view == null) throw new IOException("Owner-only private storage is unsupported for " + path); + UserPrincipal owner = view.getOwner(); + if (owner == null || owner instanceof GroupPrincipal) + throw new IOException("Private storage owner is not an individual account for " + path); + AclEntry ownerEntry = AclEntry.newBuilder().setType(AclEntryType.ALLOW).setPrincipal(owner) + .setPermissions(OWNER_ACL).build(); + view.setAcl(List.of(ownerEntry)); + List acl = view.getAcl(); + if (acl.size() != 1 || !isVerifiedOwnerEntry(acl.get(0), owner)) + throw new IOException("Could not verify owner-only ACL permissions for " + path); + } catch (UnsupportedOperationException unsupported) { + throw new IOException("Owner-only private storage is unsupported for " + path, unsupported); + } + } + + private static boolean isVerifiedOwnerEntry(AclEntry entry, UserPrincipal owner) { + return entry.type() == AclEntryType.ALLOW && owner.equals(entry.principal()) && entry.flags().isEmpty() + && entry.permissions().equals(OWNER_ACL); + } +} diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpBackendTransportConnector.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpBackendTransportConnector.java new file mode 100644 index 0000000..d162821 --- /dev/null +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpBackendTransportConnector.java @@ -0,0 +1,727 @@ +package com.bencodez.simpleapi.servercomm.http; + +import com.bencodez.simpleapi.servercomm.codec.JsonEnvelope; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.net.http.HttpTimeoutException; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.security.KeyStore; +import java.security.cert.X509Certificate; +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayDeque; +import java.util.Collection; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Flow; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Consumer; +import javax.net.ssl.KeyManagerFactory; +import javax.net.ssl.SSLContext; + +/** Backend-side, persistent HTTP/1.1 long-poll connector. */ +public final class HttpBackendTransportConnector implements AutoCloseable { + public static final Duration CLIENT_TIMEOUT = Duration.ofSeconds(35); + private static final Duration RENEWAL_SUCCESS_CHECK = Duration.ofHours(6); + private static final Duration RENEWAL_FAILURE_RETRY = Duration.ofMinutes(5); + static final int CALLBACK_QUEUE_CAPACITY = 128; + private volatile HttpClientCredentialStore.HttpClientProfile profile; + private final String serverId; + private final Consumer onEnvelope; + private volatile HttpClient client; + private volatile HttpClientCredentialStore.ClientCredential credential; + private final Path credentialDirectory; + private final HttpInboundDeliveryStore inboundDeliveries; + private final HttpInboundDeliveryStore acknowledgementConfirmationStore; + private final URI transportEndpoint; + private final ThreadPoolExecutor callbackExecutor; + private final AtomicBoolean running = new AtomicBoolean(); + private final AtomicBoolean closing = new AtomicBoolean(); + private final CountDownLatch closed = new CountDownLatch(1); + // Replaced for each start. A response from a stopped run must not satisfy a later run. + private volatile ResponseState responseState = new ResponseState(); + private final Object lifecycle = new Object(); + private final Object state = new Object(); + private final Object renewal = new Object(); + private final LinkedHashMap outgoing = new LinkedHashMap<>(); + private final Set received = new LinkedHashSet<>(), processing = new LinkedHashSet<>(); + private final ArrayDeque acknowledgements = new ArrayDeque<>(); + private final ArrayDeque acknowledgementConfirmations = new ArrayDeque<>(); + private final String session = UUID.randomUUID().toString(); + private volatile Thread poller; + // Set by the executor's worker wrapper. Thread names are not an ownership boundary. + private volatile Thread callbackWorker; + // Guarded by lifecycle. A flush drains the accepted queue but must not admit a new send. + private boolean flushingOutgoing; + // Guarded by state. Keep admission and queue insertion in the same critical section. + private boolean sendAdmissionOpen; + private long sequence; + private volatile long nextRenewalCheckNanos; + // Guarded by renewal: do not supersede a visibly published credential until + // the pointer selecting it has been confirmed durable. + private PendingActivation pendingActivation; + + /** In-memory test constructor; production transport must use a directory-backed constructor. */ + HttpBackendTransportConnector(HttpConnectionCode code, String serverId, + HttpClientCredentialStore.ClientCredential credential, Consumer onEnvelope) throws Exception { + this(profile(code, serverId), credential, onEnvelope); + } + + /** In-memory test constructor; production transport must use a directory-backed constructor. */ + HttpBackendTransportConnector(HttpClientCredentialStore.EnrolledClient enrolled, Consumer onEnvelope) throws Exception { + this(enrolled, onEnvelope, null); + } + + /** In-memory test constructor; production transport must use a directory-backed constructor. */ + HttpBackendTransportConnector(HttpClientCredentialStore.HttpClientProfile profile, + HttpClientCredentialStore.ClientCredential credential, Consumer onEnvelope) throws Exception { + this(profile, credential, onEnvelope, null); + } + + private HttpBackendTransportConnector(HttpClientCredentialStore.EnrolledClient enrolled, Consumer onEnvelope, + Path credentialDirectory) throws Exception { + this(enrolled == null ? null : enrolled.profile(), enrolled == null ? null : enrolled.credential(), onEnvelope, credentialDirectory); + } + + private HttpBackendTransportConnector(HttpClientCredentialStore.HttpClientProfile profile, + HttpClientCredentialStore.ClientCredential credential, Consumer onEnvelope, Path credentialDirectory) throws Exception { + if (profile == null || credential == null || onEnvelope == null) throw new IllegalArgumentException("HTTP backend transport configuration is invalid"); + if (!matchesCredential(profile, credential)) throw new IllegalArgumentException("HTTP client certificate does not match transport profile"); + this.profile = profile; this.serverId = profile.serverId(); this.onEnvelope = onEnvelope; + this.credential = credential; + this.credentialDirectory = credentialDirectory; + HttpInboundDeliveryStore loadedInbound = null, loadedAcknowledgements = null; + HttpClient initializedClient; + URI initializedEndpoint; + ThreadPoolExecutor initializedCallbacks; + try { + loadedInbound = credentialDirectory == null ? null : new HttpInboundDeliveryStore(credentialDirectory); + loadedAcknowledgements = credentialDirectory == null ? null + : HttpInboundDeliveryStore.open(credentialDirectory, "http-transport-ack-confirmations"); + if (loadedInbound != null) for (var entry : loadedInbound.snapshot().entrySet()) { + if (entry.getValue() == HttpInboundDeliveryStore.State.COMPLETED) { + loadedInbound.confirmCompleted(entry.getKey()); + received.add(entry.getKey()); + queueAck(entry.getKey()); + } + } + if (loadedAcknowledgements != null) for (var entry : loadedAcknowledgements.snapshot().entrySet()) { + if (entry.getValue() != HttpInboundDeliveryStore.State.COMPLETED) + throw new IOException("HTTP acknowledgement confirmation state is invalid"); + loadedAcknowledgements.confirmCompleted(entry.getKey()); + queueAcknowledgementConfirmation(entry.getKey()); + } + initializedClient = client(profile, credential); + initializedEndpoint = profile.endpoint().resolve("v1/transport"); + // GlobalMessageHandler routes mutate backend vote state and must observe the + // wire order. One bounded lane preserves batch ordering without running work on + // the long-poll thread; bounded admission below backpressures this poller. + initializedCallbacks = callbackExecutor(); + } catch (Exception | Error setupFailure) { + if (loadedAcknowledgements != null) loadedAcknowledgements.seal(); + if (loadedInbound != null) loadedInbound.seal(); + throw setupFailure; + } + inboundDeliveries = loadedInbound; + acknowledgementConfirmationStore = loadedAcknowledgements; + client = initializedClient; + transportEndpoint = initializedEndpoint; + callbackExecutor = initializedCallbacks; + } + + /** Convenience constructor for the owner-only credential directory produced by {@link #enroll}. */ + public HttpBackendTransportConnector(HttpConnectionCode code, String serverId, Path credentials, + Consumer onEnvelope) throws Exception { + this(validatedEnrollment(code, serverId, credentials), onEnvelope, credentials); + } + + /** Starts normal transport using only the persisted certificate and non-secret profile. */ + public HttpBackendTransportConnector(Path credentials, Consumer onEnvelope) throws Exception { + this(HttpClientCredentialStore.loadEnrolled(credentials), onEnvelope, credentials); + } + + /** Performs enrollment network I/O; call this from a connector/setup worker, never a platform main thread. */ + public static HttpClientCredentialStore.ClientCredential enroll(HttpConnectionCode code, String serverId, Path credentials) throws Exception { + if (credentials == null) throw new IllegalArgumentException("Enrollment configuration is invalid"); + HttpClientCredentialStore.HttpClientProfile expectedProfile = profile(code, serverId); + HttpClientCredentialStore.ClientCredential published = HttpClientCredentialStore.recoverPublishedEnrollment( + credentials, code, expectedProfile); + if (published != null) return published; + code.requireActive(Clock.systemUTC()); + byte[] payload = ("{\"server\":\"" + serverId + "\",\"token\":\"" + code.enrollmentToken() + "\"}").getBytes(StandardCharsets.UTF_8); + HttpClient client = HttpClient.newBuilder().version(HttpClient.Version.HTTP_1_1).followRedirects(HttpClient.Redirect.NEVER) + .connectTimeout(Duration.ofSeconds(5)).sslContext(HttpPinnedTls.clientContext(code)).build(); + HttpRequest request = HttpRequest.newBuilder(code.endpoint().resolve("v1/enroll")).timeout(CLIENT_TIMEOUT) + .header("Content-Type", "application/json").header("Cache-Control", "no-store").POST(HttpRequest.BodyPublishers.ofByteArray(payload)).build(); + LimitedResponse response = sendLimited(client, request); + if (response.statusCode() != 201) throw new IllegalArgumentException("Enrollment was rejected"); + HttpTlsIdentity.IssuedClientCertificate issued = HttpTransportProtocol.parseEnrollmentResponse(serverId, response.body()); + HttpClientCredentialStore.saveEnrolled(credentials, code, issued); return HttpClientCredentialStore.load(credentials); + } + + public void start() { + synchronized (lifecycle) { + if (closing.get() || flushingOutgoing || !running.compareAndSet(false, true)) return; + responseState.cancel(); + responseState = new ResponseState(); + poller = new Thread(this::pollLoop, "SimpleAPI-HTTP-poll"); + poller.setDaemon(true); + poller.start(); + } + // Do not nest lifecycle and state: close/flush transition running before taking + // state, while send observes both values under state before enqueuing. + synchronized (state) { + if (running.get() && !closing.get()) sendAdmissionOpen = true; + } + } + /** Waits for one authenticated, protocol-valid transport response. */ + public boolean awaitFirstResponse(long deadlineNanos) throws InterruptedException { + ResponseState expected = responseState; + long remaining = deadlineNanos - System.nanoTime(); + return remaining > 0L && expected.await(remaining) && responseStateIsActive(expected); + } + private boolean responseStateIsActive(ResponseState expected) { synchronized (lifecycle) { + return responseState == expected && running.get(); + } } + /** + * Inserts an in-memory at-least-once delivery. It survives retry/lost responses while this process remains alive; + * callers needing restart durability must retain the application operation independently. + */ + public boolean send(JsonEnvelope envelope) { + if (envelope == null) return false; + try { HttpTransportProtocol.validateEnvelope(envelope); } + catch (IllegalArgumentException invalid) { return false; } + synchronized (state) { + if (!sendAdmissionOpen || !running.get()) return false; + if (outgoing.size() >= HttpTransportProtocol.MAX_QUEUE) return false; + String id = UUID.randomUUID().toString(); outgoing.put(id, new HttpTransportProtocol.Delivery(id, envelope)); return true; + } + } + /** A synchronous single poll, useful for lifecycle-controlled integrations and tests. */ + public synchronized boolean pollOnce() { + return pollOnce(CLIENT_TIMEOUT, true, true); + } + private boolean pollOnce(Duration timeout, boolean requireRunning, boolean acceptIncoming) { + ResponseState runResponse; + synchronized (lifecycle) { + if (requireRunning && !running.get()) return false; + runResponse = responseState; + } + List acks = List.of(), ackConfirmations = List.of(); + boolean acknowledgementsConfirmed = false, confirmationsConfirmed = false; + try { + if (requireRunning) maybeRenewCredential(); + List messages; long requestSequence; + synchronized (state) { + acks = first(acknowledgements); + ackConfirmations = first(acknowledgementConfirmations); + requestSequence = sequence++; + messages = HttpTransportProtocol.fittingMessages(serverId, session, requestSequence, acks, + ackConfirmations, outgoing.values()); + for (int index = 0; index < acks.size(); index++) acknowledgements.removeFirst(); + for (int index = 0; index < ackConfirmations.size(); index++) acknowledgementConfirmations.removeFirst(); + } + HttpRequest request = HttpRequest.newBuilder(transportEndpoint).timeout(timeout).header("Content-Type", "application/json") + .header("Cache-Control", "no-store").POST(HttpRequest.BodyPublishers.ofByteArray(HttpTransportProtocol.request( + serverId, session, requestSequence, acks, ackConfirmations, messages))).build(); + LimitedResponse response = sendLimited(client, request); + if (response.statusCode() != 200) return false; + HttpTransportProtocol.Packet packet = HttpTransportProtocol.parsePacket(response.body()); + if (!serverId.equals(packet.server()) || !session.equals(packet.session()) || packet.sequence() != requestSequence) return false; + if (!acks.equals(packet.ackConfirmations())) return false; + confirmAcknowledgements(packet.ackConfirmations()); + acknowledgementsConfirmed = true; + if (!confirmSentAcknowledgementConfirmations(ackConfirmations)) return false; + confirmationsConfirmed = true; + if (!recordAcknowledgementConfirmations(packet.acks())) return false; + synchronized (state) { for (String ack : packet.acks()) { + outgoing.remove(ack); queueAcknowledgementConfirmation(ack); + } } + if (acceptIncoming) for (HttpTransportProtocol.Delivery delivery : accept(packet.messages())) dispatch(delivery); + if (requireRunning) runResponse.received(); + return true; + } catch (Exception failure) { return false; + } finally { + if (!acknowledgementsConfirmed) requeue(acknowledgements, acks); + if (!confirmationsConfirmed) requeue(acknowledgementConfirmations, ackConfirmations); + } + } + /** Stops normal polling and gives already-queued outbound messages a bounded final delivery attempt. */ + public boolean flushOutgoing(long deadlineNanos) { + Thread current; + synchronized (lifecycle) { + if (closing.get() || flushingOutgoing) return false; + flushingOutgoing = true; + running.set(false); + responseState.cancel(); + current = poller; + if (current != null) current.interrupt(); + } + synchronized (state) { sendAdmissionOpen = false; } + try { + if (!joinPoller(current, deadlineNanos)) return false; + while (queuedOutgoing() != 0 || queuedAcknowledgements() != 0) { + long remaining = deadlineNanos - System.nanoTime(); + if (remaining <= 0L) return false; + Duration timeout = Duration.ofNanos(Math.min(CLIENT_TIMEOUT.toNanos(), remaining)); + synchronized (this) { + if (pollOnce(timeout, false, false)) continue; + } + // The proxy may retain its one-active-poll guard briefly after the old + // client request is interrupted. Retry that transient 409 without busy + // spinning, but never extend the caller's shutdown deadline. + remaining = deadlineNanos - System.nanoTime(); + if (remaining <= 0L) return false; + try { TimeUnit.NANOSECONDS.sleep(Math.min(TimeUnit.MILLISECONDS.toNanos(50), remaining)); } + catch (InterruptedException interrupted) { Thread.currentThread().interrupt(); return false; } + } + return true; + } finally { synchronized (lifecycle) { flushingOutgoing = false; } } + } + /** + * Stops polling and drains callbacks before sealing their journals. When called by the + * callback worker itself, terminal cleanup is deferred until that callback returns so + * its COMPLETED transition cannot be sealed out from under it. + */ + @Override public void close() { + boolean calledByCallbackWorker = Thread.currentThread() == callbackWorker; + Thread current; + boolean alreadyClosing; + synchronized (lifecycle) { + alreadyClosing = !closing.compareAndSet(false, true); + if (alreadyClosing) current = null; + else { + running.set(false); + responseState.cancel(); + current = poller; + if (current != null) current.interrupt(); + } + } + if (!alreadyClosing) synchronized (state) { sendAdmissionOpen = false; } + if (alreadyClosing) { + if (!calledByCallbackWorker) awaitClosed(); + return; + } + // Closing admission before joining the poller releases an executeOrdered call + // that is backpressured behind this callback. + callbackExecutor.shutdown(); + // The owning transport may release the credential-directory semaphore as soon + // as close returns. Wait for the interrupted poller so an in-flight renewal + // cannot activate an old credential generation after that ownership handoff. + joinPoller(current); + if (calledByCallbackWorker) { + Thread cleanup = new Thread(this::finishClose, "SimpleAPI-HTTP-callback-close"); + cleanup.setDaemon(true); + cleanup.start(); + return; + } + finishClose(); + } + private void finishClose() { + try { + shutdownCallbacks(); + // No poller can enqueue more callbacks and every running journal transition has + // finished, so ownership can now be revoked without stranding completed work. + if (inboundDeliveries != null) inboundDeliveries.seal(); + if (acknowledgementConfirmationStore != null) acknowledgementConfirmationStore.seal(); + } finally { closed.countDown(); } + } + private void awaitClosed() { + boolean interrupted = false; + while (closed.getCount() != 0L) try { closed.await(); } + catch (InterruptedException stopRequested) { interrupted = true; } + if (interrupted) Thread.currentThread().interrupt(); + } + private void shutdownCallbacks() { + callbackExecutor.shutdown(); + boolean interrupted = false; + try { + if (!callbackExecutor.awaitTermination(5, TimeUnit.SECONDS)) { + callbackExecutor.shutdownNow(); + callbackExecutor.awaitTermination(1, TimeUnit.SECONDS); + } + } catch (InterruptedException stopRequested) { + interrupted = true; + callbackExecutor.shutdownNow(); + try { callbackExecutor.awaitTermination(1, TimeUnit.SECONDS); } + catch (InterruptedException repeated) { interrupted = true; } + } + if (interrupted) Thread.currentThread().interrupt(); + } + boolean pollerAlive() { Thread current = poller; return current != null && current.isAlive(); } + private static boolean joinPoller(Thread poller, long deadlineNanos) { + if (poller == null || poller == Thread.currentThread()) return true; + boolean interrupted = false; + while (poller.isAlive()) { + long remaining = deadlineNanos - System.nanoTime(); + if (remaining <= 0L) { + if (interrupted) Thread.currentThread().interrupt(); + return false; + } + try { TimeUnit.NANOSECONDS.timedJoin(poller, remaining); } + catch (InterruptedException stopRequested) { interrupted = true; poller.interrupt(); } + } + if (interrupted) Thread.currentThread().interrupt(); + return true; + } + private static void joinPoller(Thread poller) { + if (poller == null || poller == Thread.currentThread()) return; + boolean interrupted = false; + while (poller.isAlive()) try { poller.join(); } + catch (InterruptedException stopRequested) { interrupted = true; poller.interrupt(); } + if (interrupted) Thread.currentThread().interrupt(); + } + private static final class ResponseState { + private final CountDownLatch complete = new CountDownLatch(1); + private final AtomicBoolean received = new AtomicBoolean(); + void received() { received.set(true); complete.countDown(); } + void cancel() { complete.countDown(); } + boolean await(long timeoutNanos) throws InterruptedException { + return complete.await(timeoutNanos, TimeUnit.NANOSECONDS) && received.get(); + } + } + + private void pollLoop() { + long retry = 1000L; + while (running.get()) { if (pollOnce()) { retry = 1000L; continue; } try { Thread.sleep(retry); } catch (InterruptedException interrupted) { Thread.currentThread().interrupt(); break; } retry = Math.min(30_000L, retry * 2); } + } + List accept(List deliveries) { + synchronized (state) { + List accepted = new java.util.ArrayList<>(); + for (HttpTransportProtocol.Delivery delivery : deliveries) { + HttpInboundDeliveryStore.State persisted = inboundDeliveries == null ? null : inboundDeliveries.state(delivery.id()); + if (persisted == HttpInboundDeliveryStore.State.RUNNING) try { + if (inboundDeliveries.recoverKnownNotStartedRunning(delivery.id())) persisted = inboundDeliveries.state(delivery.id()); + } catch (IOException rollbackUnconfirmed) { continue; } + if (persisted == HttpInboundDeliveryStore.State.COMPLETED) { + try { inboundDeliveries.confirmCompleted(delivery.id()); } + catch (IOException unconfirmed) { continue; } + received.add(delivery.id()); queueAck(delivery.id()); continue; + } + if (received.contains(delivery.id())) { + received.add(delivery.id()); queueAck(delivery.id()); continue; + } + // A callback that was running when the process stopped may already have + // produced external side effects. Keep the proxy copy without replaying or + // acknowledging it; arbitrary plugin callbacks cannot share this journal. + if (persisted == HttpInboundDeliveryStore.State.RUNNING) continue; + if (!processing.contains(delivery.id())) { + processing.add(delivery.id()); accepted.add(delivery); + } + } + return accepted; + } + } + void dispatch(HttpTransportProtocol.Delivery delivery) { + Runnable callback = () -> { + boolean success = false; + try { + if (inboundDeliveries != null) { + if (inboundDeliveries.state(delivery.id()) == null) inboundDeliveries.reserve(delivery.id()); + inboundDeliveries.markRunning(delivery.id()); + } + onEnvelope.accept(delivery.envelope()); + if (inboundDeliveries != null) inboundDeliveries.markCompleted(delivery.id()); + success = true; + } catch (IOException persistenceFailure) { + // Never run before RUNNING is durable and never acknowledge until + // COMPLETED is durable. An uncertain transition stays fail-closed. + } catch (RuntimeException callbackFailure) { + // The callback may have failed after partial external effects. Leave RUNNING + // unacknowledged so a restart cannot silently lose or duplicate the delivery. + } + completeIncoming(delivery.id(), success); + }; + if (!executeOrdered(callbackExecutor, callback)) completeIncoming(delivery.id(), false); + } + void completeIncoming(String id, boolean success) { synchronized (state) { + processing.remove(id); + if (success) { received.add(id); while (received.size() > HttpTransportProtocol.MAX_QUEUE) received.remove(received.iterator().next()); queueAck(id); } + } } + private void queueAck(String id) { if (acknowledgements.size() < HttpTransportProtocol.MAX_QUEUE && !acknowledgements.contains(id)) acknowledgements.add(id); } + private void queueAcknowledgementConfirmation(String id) { + if (acknowledgementConfirmations.size() < HttpTransportProtocol.MAX_QUEUE + && !acknowledgementConfirmations.contains(id)) acknowledgementConfirmations.add(id); + } + private void requeue(ArrayDeque queue, List ids) { synchronized (state) { + for (int index = ids.size() - 1; index >= 0; index--) { + String id = ids.get(index); + if (!queue.contains(id)) { + while (queue.size() >= HttpTransportProtocol.MAX_QUEUE) queue.removeLast(); + queue.addFirst(id); + } + } + } } + private void confirmAcknowledgements(Collection ids) { + for (String id : ids) { + boolean removed = true; + if (inboundDeliveries != null) try { inboundDeliveries.remove(id); } + catch (IOException cleanupFailure) { removed = false; } + synchronized (state) { + if (removed) received.remove(id); + else queueAck(id); + } + } + } + private boolean recordAcknowledgementConfirmations(Collection ids) { + if (acknowledgementConfirmationStore == null) return true; + try { for (String id : ids) acknowledgementConfirmationStore.recordCompleted(id); return true; } + catch (IOException persistenceFailure) { return false; } + } + private boolean confirmSentAcknowledgementConfirmations(Collection ids) { + if (acknowledgementConfirmationStore == null) return true; + try { for (String id : ids) acknowledgementConfirmationStore.remove(id); return true; } + catch (IOException cleanupFailure) { return false; } + } + int queuedOutgoing() { synchronized (state) { return outgoing.size(); } } + int queuedAcknowledgements() { synchronized (state) { return acknowledgements.size() + acknowledgementConfirmations.size(); } } + List drainAcknowledgements() { synchronized (state) { return drain(acknowledgements); } } + private static List first(Collection values) { List output = new java.util.ArrayList<>(); for (T value : values) { output.add(value); if (output.size() == HttpTransportProtocol.MAX_BATCH) break; } return output; } + private static List drain(ArrayDeque values) { List output = new java.util.ArrayList<>(); while (!values.isEmpty() && output.size() < HttpTransportProtocol.MAX_BATCH) output.add(values.remove()); return output; } + private ThreadPoolExecutor callbackExecutor() { + ThreadFactory factory = task -> { + Thread thread = new Thread(() -> { + callbackWorker = Thread.currentThread(); + task.run(); + }, "SimpleAPI-HTTP-callback"); + thread.setDaemon(true); + return thread; + }; + return new ThreadPoolExecutor(1, 1, 0L, TimeUnit.MILLISECONDS, + new ArrayBlockingQueue<>(CALLBACK_QUEUE_CAPACITY), factory, new ThreadPoolExecutor.AbortPolicy()); + } + static boolean executeOrdered(ThreadPoolExecutor executor, Runnable task) { + try { executor.execute(task); return true; } + catch (RejectedExecutionException fullOrClosed) { + if (executor.isShutdown()) return false; + try { + while (!executor.isShutdown()) { + if (!executor.getQueue().offer(task, 100L, TimeUnit.MILLISECONDS)) continue; + if (executor.isShutdown() && executor.remove(task)) return false; + return true; + } + } catch (InterruptedException interrupted) { Thread.currentThread().interrupt(); } + return false; + } + } + private static HttpClientCredentialStore.HttpClientProfile profile(HttpConnectionCode code, String serverId) { + if (code == null || serverId == null) throw new IllegalArgumentException("HTTP backend transport configuration is invalid"); + if (!code.serverId().equals(HttpTlsIdentity.canonicalServerId(serverId))) throw new IllegalArgumentException("HTTP connection code belongs to a different backend"); + return new HttpClientCredentialStore.HttpClientProfile(serverId, code.endpoint(), code.serverCertificatePin(), code.caCertificatePin()); + } + private static HttpClientCredentialStore.EnrolledClient validatedEnrollment(HttpConnectionCode code, + String serverId, Path credentials) throws Exception { + HttpClientCredentialStore.HttpClientProfile expected = profile(code, serverId); + HttpClientCredentialStore.EnrolledClient enrolled = HttpClientCredentialStore.loadEnrolled(credentials); + if (!expected.equals(enrolled.profile())) + throw new IllegalArgumentException("HTTP transport profile does not match connection code"); + return enrolled; + } + private static boolean matchesCredential(HttpClientCredentialStore.HttpClientProfile profile, + HttpClientCredentialStore.ClientCredential credential) { + try { + credential.certificate().checkValidity(); credential.certificate().verify(credential.caCertificate().getPublicKey()); + String expected = "urn:votingplugin:http-backend:" + profile.serverId(); + var names = credential.certificate().getSubjectAlternativeNames(); if (names == null) return false; + for (java.util.List name : names) if (name.size() == 2 && Integer.valueOf(6).equals(name.get(0)) && expected.equals(name.get(1))) return true; + return false; + } catch (Exception invalid) { return false; } + } + private void maybeRenewCredential() { + synchronized (renewal) { + Path directory = credentialDirectory; + if (directory == null) return; + if (pendingActivation != null) { + try { + HttpClientCredentialStore.activateReplacement(directory, pendingActivation.staged()); + profile = pendingActivation.staged().profile(); + client = pendingActivation.client(); + credential = pendingActivation.staged().credential(); + pendingActivation = null; + } catch (IOException unconfirmed) { return; } + } + if (!HttpTlsIdentity.needsRenewal(credential.certificate(), Clock.systemUTC())) return; + long now = System.nanoTime(); + if (nextRenewalCheckNanos != 0L && now - nextRenewalCheckNanos < 0L) return; + Duration retry = renewalRetryDelay(Duration.between(Instant.now(), + credential.certificate().getNotAfter().toInstant())); + try { + byte[] body = HttpTransportProtocol.renewalRequest(serverId); + HttpRequest request = HttpRequest.newBuilder(profile.endpoint().resolve("v1/renew")).timeout(CLIENT_TIMEOUT) + .header("Content-Type", "application/json").header("Cache-Control", "no-store") + .POST(HttpRequest.BodyPublishers.ofByteArray(body)).build(); + LimitedResponse response = sendLimited(client, request); + if (response.statusCode() != 201) { + nextRenewalCheckNanos = renewalDeadline(now, retry); + return; + } + HttpTlsIdentity.IssuedClientCertificate issued = HttpTransportProtocol.parseEnrollmentResponse(serverId, response.body()); + HttpClientCredentialStore.StagedCredential staged = HttpClientCredentialStore.stageReplacement(directory, issued); + HttpClientCredentialStore.ClientCredential replacement = staged.credential(); + HttpClientCredentialStore.HttpClientProfile replacementProfile = staged.profile(); + if (!matchesCredential(replacementProfile, replacement)) throw new IllegalArgumentException("Renewed HTTP certificate is invalid"); + HttpClient replacementClient = client(replacementProfile, replacement); + try { HttpClientCredentialStore.activateReplacement(directory, staged); } + catch (IOException unconfirmed) { + // A rename may already be visible while its parent fsync is unresolved. + // Keep using the restart-safe old identity and retry precisely this + // generation; requesting another renewal could revoke both identities. + pendingActivation = new PendingActivation(staged, replacementClient); + return; + } + profile = replacementProfile; + client = replacementClient; + credential = replacement; + nextRenewalCheckNanos = renewalDeadline(now, RENEWAL_SUCCESS_CHECK); + } catch (Exception ignored) { + // Keep the active generation and retry well before its remaining validity is consumed. + nextRenewalCheckNanos = renewalDeadline(now, retry); + } + } + } + private record PendingActivation(HttpClientCredentialStore.StagedCredential staged, HttpClient client) { } + static Duration renewalRetryDelay(Duration remainingValidity) { + if (remainingValidity == null || remainingValidity.isNegative() || remainingValidity.isZero()) + return Duration.ofSeconds(1); + Duration beforeExpiry = remainingValidity.dividedBy(4L); + if (beforeExpiry.isZero()) beforeExpiry = Duration.ofNanos(1L); + return beforeExpiry.compareTo(RENEWAL_FAILURE_RETRY) < 0 ? beforeExpiry : RENEWAL_FAILURE_RETRY; + } + private static long renewalDeadline(long now, Duration delay) { + long nanos; + try { nanos = delay.toNanos(); } + catch (ArithmeticException overflow) { nanos = Long.MAX_VALUE; } + return nanos > Long.MAX_VALUE - now ? Long.MAX_VALUE : now + nanos; + } + private static HttpClient client(HttpClientCredentialStore.HttpClientProfile profile, + HttpClientCredentialStore.ClientCredential credential) throws Exception { + return HttpClient.newBuilder().version(HttpClient.Version.HTTP_1_1).followRedirects(HttpClient.Redirect.NEVER) + .connectTimeout(Duration.ofSeconds(5)).sslContext(clientContext(profile, credential)).build(); + } + static LimitedResponse sendLimited(HttpClient client, HttpRequest request) throws IOException, InterruptedException { + CompletableFuture> exchange = client.sendAsync(request, + ignored -> new LimitedBodySubscriber(HttpTransportProtocol.MAX_BODY_BYTES)); + HttpResponse response; + try { + Duration timeout = request.timeout().orElse(CLIENT_TIMEOUT); + response = exchange.get(timeout.toMillis(), TimeUnit.MILLISECONDS); + } catch (TimeoutException timeout) { + exchange.cancel(true); + throw new HttpTimeoutException("HTTP transport response timed out"); + } catch (InterruptedException interrupted) { + exchange.cancel(true); + throw interrupted; + } catch (ExecutionException failure) { + Throwable cause = failure.getCause(); + if (cause instanceof IOException ioFailure) throw ioFailure; + throw new IOException("HTTP transport request failed", cause); + } + long declaredLength = response.headers().firstValueAsLong("Content-Length").orElse(-1L); + if (declaredLength > HttpTransportProtocol.MAX_BODY_BYTES) + throw new IOException("HTTP transport response exceeds its limit"); + return new LimitedResponse(response.statusCode(), response.body()); + } + + private static final class LimitedBodySubscriber implements HttpResponse.BodySubscriber { + private final int maximum; + private final ByteArrayOutputStream body = new ByteArrayOutputStream(); + private final CompletableFuture result = new CompletableFuture<>(); + private Flow.Subscription subscription; + + private LimitedBodySubscriber(int maximum) { + this.maximum = maximum; + } + + @Override + public CompletionStage getBody() { + return result; + } + + @Override + public void onSubscribe(Flow.Subscription subscription) { + if (this.subscription != null) { + subscription.cancel(); + return; + } + this.subscription = subscription; + subscription.request(1); + } + + @Override + public void onNext(List buffers) { + try { + for (ByteBuffer buffer : buffers) { + if (buffer.remaining() > maximum - body.size()) { + subscription.cancel(); + result.completeExceptionally(new IOException("HTTP transport response exceeds its limit")); + return; + } + byte[] chunk = new byte[buffer.remaining()]; + buffer.get(chunk); + body.writeBytes(chunk); + } + subscription.request(1); + } catch (RuntimeException failure) { + subscription.cancel(); + result.completeExceptionally(failure); + } + } + + @Override + public void onError(Throwable failure) { + result.completeExceptionally(failure); + } + + @Override + public void onComplete() { + result.complete(body.toByteArray()); + } + } + static byte[] readLimited(InputStream body) throws IOException { + byte[] bytes = body.readNBytes(HttpTransportProtocol.MAX_BODY_BYTES + 1); + if (bytes.length > HttpTransportProtocol.MAX_BODY_BYTES) + throw new IOException("HTTP transport response exceeds its limit"); + return bytes; + } + record LimitedResponse(int statusCode, byte[] body) { } + private static SSLContext clientContext(HttpClientCredentialStore.HttpClientProfile profile, HttpClientCredentialStore.ClientCredential credential) throws Exception { + String caPin = HttpTransportSecrets.certificatePin(credential.caCertificate()); + if (!HttpTransportSecrets.constantTimeEquals(profile.caCertificatePin().getBytes(StandardCharsets.US_ASCII), + caPin.getBytes(StandardCharsets.US_ASCII))) throw new IllegalArgumentException("HTTP authority does not match transport profile"); + char[] password = credential.password(); + try { + KeyStore store = KeyStore.getInstance("PKCS12"); store.load(null, new char[0]); + store.setKeyEntry("client", credential.privateKey(), password, + new java.security.cert.Certificate[] { credential.certificate(), credential.caCertificate() }); + KeyManagerFactory keys = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); keys.init(store, password); + KeyStore trustStore = KeyStore.getInstance(KeyStore.getDefaultType()); trustStore.load(null, new char[0]); + trustStore.setCertificateEntry("http-transport-ca", credential.caCertificate()); + javax.net.ssl.TrustManagerFactory trusts = javax.net.ssl.TrustManagerFactory.getInstance( + javax.net.ssl.TrustManagerFactory.getDefaultAlgorithm()); + trusts.init(trustStore); + SSLContext context = SSLContext.getInstance("TLS"); context.init(keys.getKeyManagers(), trusts.getTrustManagers(), null); return context; + } finally { java.util.Arrays.fill(password, '\0'); } + } +} diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpClientCredentialStore.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpClientCredentialStore.java new file mode 100644 index 0000000..7040e6c --- /dev/null +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpClientCredentialStore.java @@ -0,0 +1,459 @@ +package com.bencodez.simpleapi.servercomm.http; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.net.URI; +import java.security.KeyStore; +import java.security.PrivateKey; +import java.security.cert.X509Certificate; +import java.util.Base64; +import java.util.Properties; +import com.bencodez.simpleapi.file.DurableFiles; +import com.bencodez.simpleapi.file.PrivateFilePermissions; + +/** Owner-only persistence for the client certificate bundle returned by enrollment. */ +public final class HttpClientCredentialStore { + private static final String BUNDLE_FILE = "http-transport-client.p12"; + private static final String PASSWORD_FILE = "http-transport-client-password"; + private static final String PROFILE_FILE = "http-transport-profile.properties"; + private static final String CONNECTION_CODE_DIGEST_FILE = "http-transport-connection-code.sha256"; + private static final String GENERATIONS_DIRECTORY = "http-transport-client-generations"; + private static final String CURRENT_FILE = "http-transport-client-current"; + private HttpClientCredentialStore() { } + + public static void save(Path directory, HttpTlsIdentity.IssuedClientCertificate issued) throws IOException { + if (issued == null) throw new IllegalArgumentException("Issued credential is required"); + directory = credentialRoot(directory, true); + byte[] bundle = issued.pkcs12(); + try { writePrivate(safe(directory.resolve(BUNDLE_FILE)), bundle); } + finally { java.util.Arrays.fill(bundle, (byte) 0); } + char[] password = issued.password(); + try { writePrivate(safe(directory.resolve(PASSWORD_FILE)), asciiBytes(password)); } + finally { java.util.Arrays.fill(password, '\0'); } + } + + /** Persists the certificate plus the non-secret normal-transport profile after enrollment. */ + public static void saveEnrolled(Path directory, HttpConnectionCode code, HttpTlsIdentity.IssuedClientCertificate issued) + throws IOException { + if (code == null || issued == null) throw new IllegalArgumentException("Connection code and credential are required"); + HttpClientProfile profile = new HttpClientProfile(HttpTlsIdentity.canonicalServerId(issued.serverId()), code.endpoint(), + code.serverCertificatePin(), code.caCertificatePin()); + try { + StagedCredential staged = stage(directory, issued, profile, connectionCodeDigest(code), null); + activateReplacement(directory, staged); + } catch (IOException failure) { throw failure; + } catch (Exception failure) { throw new IOException("Could not persist HTTP client credential", failure); } + } + + private static void writeProfile(Path directory, HttpClientProfile profile) throws IOException { + Properties properties = new Properties(); + properties.setProperty("version", "1"); + properties.setProperty("serverId", profile.serverId()); + properties.setProperty("endpoint", profile.endpoint().toASCIIString()); + properties.setProperty("serverPin", profile.serverCertificatePin()); + properties.setProperty("caPin", profile.caCertificatePin()); + java.io.ByteArrayOutputStream bytes = new java.io.ByteArrayOutputStream(); + properties.store(bytes, "VotingPlugin HTTP transport profile"); + writePrivate(safe(directory.resolve(PROFILE_FILE)), bytes.toByteArray()); + } + + public static ClientCredential load(Path directory) throws Exception { + return loadCredential(activeDirectory(directory)); + } + + private static ClientCredential loadCredential(Path directory) throws Exception { + Path bundle = safe(directory.resolve(BUNDLE_FILE)); + Path passwordFile = safe(directory.resolve(PASSWORD_FILE)); + if (!Files.isRegularFile(bundle, LinkOption.NOFOLLOW_LINKS) || !Files.isRegularFile(passwordFile, LinkOption.NOFOLLOW_LINKS)) + throw new IOException("HTTP client certificate has not been enrolled"); + PrivateFilePermissions.ownerOnlyFile(bundle); + PrivateFilePermissions.ownerOnlyFile(passwordFile); + long passwordSize = Files.size(passwordFile); + if (passwordSize < 40L || passwordSize > 128L) throw new IOException("HTTP client password is invalid"); + byte[] passwordBytes; + try (var input = Files.newInputStream(passwordFile, LinkOption.NOFOLLOW_LINKS)) { + passwordBytes = input.readNBytes(129); + } + char[] password; + try { + if (passwordBytes.length < 40 || passwordBytes.length > 128) throw new IOException("HTTP client password is invalid"); + password = new String(passwordBytes, StandardCharsets.US_ASCII).toCharArray(); + } finally { java.util.Arrays.fill(passwordBytes, (byte) 0); } + try { + KeyStore store = KeyStore.getInstance("PKCS12"); + try (var input = Files.newInputStream(bundle, LinkOption.NOFOLLOW_LINKS)) { store.load(input, password); } + PrivateKey privateKey = (PrivateKey) store.getKey("client", password); + java.security.cert.Certificate[] chain = store.getCertificateChain("client"); + if (privateKey == null || chain == null || chain.length != 2 + || !(chain[0] instanceof X509Certificate client) || !(chain[1] instanceof X509Certificate authority)) + throw new IOException("HTTP client certificate bundle is invalid"); + verifyKeyPair(privateKey, client); + return new ClientCredential(privateKey, client, authority, password); + } finally { java.util.Arrays.fill(password, '\0'); } + } + + private static void verifyKeyPair(PrivateKey privateKey, X509Certificate certificate) throws Exception { + String algorithm = switch (certificate.getPublicKey().getAlgorithm()) { + case "EC" -> "SHA256withECDSA"; + case "RSA" -> "SHA256withRSA"; + case "DSA" -> "SHA256withDSA"; + case "Ed25519", "Ed448" -> certificate.getPublicKey().getAlgorithm(); + case "EdDSA" -> ((java.security.interfaces.EdECKey) certificate.getPublicKey()).getParams().getName(); + default -> throw new IOException("HTTP client key algorithm is unsupported"); + }; + byte[] challenge = new byte[32]; + new java.security.SecureRandom().nextBytes(challenge); + java.security.Signature signature = java.security.Signature.getInstance(algorithm); + signature.initSign(privateKey); + signature.update(challenge); + byte[] proof = signature.sign(); + signature.initVerify(certificate.getPublicKey()); + signature.update(challenge); + if (!signature.verify(proof)) throw new IOException("HTTP client private key does not match its certificate"); + } + + /** Writes and validates a replacement generation without touching the active credential. */ + static StagedCredential stageReplacement(Path directory, HttpTlsIdentity.IssuedClientCertificate issued) throws Exception { + Path active = activeDirectory(directory); + EnrolledClient enrolled = loadEnrolledDirectory(active); + return stage(directory, issued, enrolled.profile(), readConnectionCodeDigest(active), + enrolled.credential().caCertificate().getPublicKey()); + } + + private static StagedCredential stage(Path directory, HttpTlsIdentity.IssuedClientCertificate issued, + HttpClientProfile profile, String connectionCodeDigest, java.security.PublicKey trustedCaKey) throws Exception { + if (directory == null || issued == null) throw new IllegalArgumentException("Credential replacement is required"); + Path credentialDirectory = credentialRoot(directory, true); + Path generations = credentialDirectory.resolve(GENERATIONS_DIRECTORY); + if (Files.isSymbolicLink(generations)) throw new IOException("HTTP credential generation directory is unsafe"); + Files.createDirectories(generations); + if (Files.isSymbolicLink(generations) || !Files.isDirectory(generations, LinkOption.NOFOLLOW_LINKS)) + throw new IOException("HTTP credential generation directory is unsafe"); + PrivateFilePermissions.ownerOnlyDirectory(generations); + // Retry publication durability if an earlier staging attempt left this + // directory behind after its parent fsync failed. + DurableFiles.forceDirectory(credentialDirectory); + String name = java.util.UUID.randomUUID().toString(); + Path generation = generations.resolve(name); + Files.createDirectory(generation); + PrivateFilePermissions.ownerOnlyDirectory(generation); + try { + save(generation, issued); + ClientCredential replacement = loadCredential(generation); + // Initial enrollment retains the code's CA pin. Renewal may refresh the CA + // certificate, but must preserve the previously trusted CA public key. + if (trustedCaKey != null) { + if (!java.security.MessageDigest.isEqual(trustedCaKey.getEncoded(), + replacement.caCertificate().getPublicKey().getEncoded())) + throw new IOException("Renewed HTTP credential changes the trusted CA key"); + profile = new HttpClientProfile(profile.serverId(), profile.endpoint(), + profile.serverCertificatePin(), HttpTransportSecrets.certificatePin(replacement.caCertificate())); + } + writeProfile(generation, profile); + if (connectionCodeDigest != null) writePrivate(safe(generation.resolve(CONNECTION_CODE_DIGEST_FILE)), + connectionCodeDigest.getBytes(StandardCharsets.US_ASCII)); + EnrolledClient enrolled = loadEnrolled(generation); + // Each file is durable within the generation, but the generation name is + // published by its parent. Persist it before CURRENT can activate it. + DurableFiles.forceDirectory(generations); + return new StagedCredential(name, enrolled.credential(), enrolled.profile()); + } catch (Exception failure) { + try { Files.deleteIfExists(generation.resolve(BUNDLE_FILE)); Files.deleteIfExists(generation.resolve(PASSWORD_FILE)); + Files.deleteIfExists(generation.resolve(PROFILE_FILE)); Files.deleteIfExists(generation.resolve(CONNECTION_CODE_DIGEST_FILE)); + Files.deleteIfExists(generation); } + catch (IOException cleanup) { failure.addSuppressed(cleanup); } + throw failure; + } + } + + /** Atomically makes a fully validated generation durable and active. */ + static void activateReplacement(Path directory, StagedCredential staged) throws IOException { + if (directory == null || staged == null || !staged.name().matches("[0-9a-f-]{36}")) + throw new IllegalArgumentException("Staged credential is invalid"); + directory = credentialRoot(directory, false); + Path generations = directory.resolve(GENERATIONS_DIRECTORY); + if (Files.isSymbolicLink(generations) || !Files.isDirectory(generations, LinkOption.NOFOLLOW_LINKS)) + throw new IOException("HTTP credential generation directory is unsafe"); + Path generation = generations.resolve(staged.name()).normalize(); + if (!generation.getParent().equals(generations) || Files.isSymbolicLink(generation) + || !Files.isRegularFile(generation.resolve(BUNDLE_FILE), LinkOption.NOFOLLOW_LINKS) + || !Files.isRegularFile(generation.resolve(PASSWORD_FILE), LinkOption.NOFOLLOW_LINKS) + || !Files.isRegularFile(generation.resolve(PROFILE_FILE), LinkOption.NOFOLLOW_LINKS)) + throw new IOException("Staged HTTP credential is incomplete"); + writePrivate(safe(directory.resolve(CURRENT_FILE)), staged.name().getBytes(StandardCharsets.US_ASCII)); + } + + static record StagedCredential(String name, ClientCredential credential, HttpClientProfile profile) { } + + public static HttpClientProfile loadProfile(Path directory) throws IOException { + return loadProfileFile(activeDirectory(directory)); + } + + private static HttpClientProfile loadProfileFile(Path directory) throws IOException { + Path profile = safe(directory.resolve(PROFILE_FILE)); + if (!Files.isRegularFile(profile, LinkOption.NOFOLLOW_LINKS) || Files.size(profile) > 8192) + throw new IOException("HTTP transport profile has not been enrolled"); + Properties properties = new Properties(); + try (var input = Files.newInputStream(profile, LinkOption.NOFOLLOW_LINKS)) { properties.load(input); } + if (properties.size() != 5 || !"1".equals(properties.getProperty("version"))) + throw new IOException("HTTP transport profile is invalid"); + try { + return new HttpClientProfile(properties.getProperty("serverId"), URI.create(properties.getProperty("endpoint")), + properties.getProperty("serverPin"), properties.getProperty("caPin")); + } catch (IllegalArgumentException failure) { throw new IOException("HTTP transport profile is invalid", failure); } + } + + public static boolean hasEnrolledProfile(Path directory) { + try { loadEnrolled(directory); return true; } + catch (Exception unavailable) { return false; } + } + + /** Returns whether this exact one-time code created the active credential, without persisting the code itself. */ + public static boolean matchesEnrollmentCode(Path directory, HttpConnectionCode code) throws IOException { + if (code == null) throw new IllegalArgumentException("Connection code is required"); + String stored = readConnectionCodeDigest(activeDirectory(directory)); + return stored != null && matchesConnectionCodeDigest(stored, code); + } + + /** Confirms and returns an already-published initial enrollment before its one-time code is sent again. */ + static ClientCredential recoverPublishedEnrollment(Path directory, HttpConnectionCode code, + HttpClientProfile expectedProfile) throws Exception { + if (directory == null || code == null || expectedProfile == null) + throw new IllegalArgumentException("Enrollment recovery is required"); + Path candidate = directory.toAbsolutePath().normalize(); + if (!Files.exists(candidate, LinkOption.NOFOLLOW_LINKS)) return null; + Path root = credentialRoot(candidate, false); + Path active = activeDirectory(root); + if (!Files.isRegularFile(active.resolve(BUNDLE_FILE), LinkOption.NOFOLLOW_LINKS) + || !Files.isRegularFile(active.resolve(PASSWORD_FILE), LinkOption.NOFOLLOW_LINKS) + || !Files.isRegularFile(active.resolve(PROFILE_FILE), LinkOption.NOFOLLOW_LINKS) + || !Files.isRegularFile(active.resolve(CONNECTION_CODE_DIGEST_FILE), LinkOption.NOFOLLOW_LINKS)) + return null; + EnrolledClient enrolled = loadEnrolledDirectory(active); + String stored = readConnectionCodeDigest(active); + if (!expectedProfile.equals(enrolled.profile()) || stored == null || !matchesConnectionCodeDigest(stored, code)) + return null; + // CURRENT and every generation file were already forced before publication. + // Re-forcing the root confirms a pointer whose post-rename directory force failed. + DurableFiles.forceDirectory(root); + return enrolled.credential(); + } + + private static boolean matchesConnectionCodeDigest(String stored, HttpConnectionCode code) { + byte[] storedBytes = stored.getBytes(StandardCharsets.US_ASCII); + return HttpTransportSecrets.constantTimeEquals(storedBytes, + connectionCodeDigest(code.encode()).getBytes(StandardCharsets.US_ASCII)) + || HttpTransportSecrets.constantTimeEquals(storedBytes, + connectionCodeDigest(code.encodeLegacy()).getBytes(StandardCharsets.US_ASCII)); + } + + /** Loads and cross-checks the persisted client key material and bound normal-transport profile. */ + public static EnrolledClient loadEnrolled(Path directory) throws Exception { + Path active = activeDirectory(directory); + return loadEnrolledDirectory(active); + } + + private static EnrolledClient loadEnrolledDirectory(Path active) throws Exception { + ClientCredential credential = loadCredential(active); + HttpClientProfile profile = loadProfileFile(active); + if (!matchesProfile(credential, profile)) throw new IOException("HTTP client certificate does not match its profile"); + return new EnrolledClient(profile, credential); + } + + /** Captures the exact credential generation used by a transport before a staged replacement starts. */ + public static ActiveCredentialGeneration snapshotActiveGeneration(Path directory) throws Exception { + Path root = directory.toAbsolutePath().normalize(); + Path active = activeDirectory(root); + EnrolledClient enrolled = loadEnrolledDirectory(active); + return new ActiveCredentialGeneration(active.equals(root) ? "" : active.getFileName().toString(), + enrolled.profile(), readConnectionCodeDigest(active)); + } + + /** + * Restores a pre-replacement generation unless the replacement already activated a + * newer credential for the same backend endpoint. A successful renewal or same-endpoint + * re-enrollment may revoke the snapshotted certificate at the proxy, so that newer + * generation is the only safe rollback identity. + */ + public static void restoreActiveGenerationAfterReplacement(Path directory, + ActiveCredentialGeneration snapshot) throws Exception { + if (directory == null || snapshot == null) throw new IllegalArgumentException("Credential rollback is required"); + HttpClientProfile previous = snapshot.profile(); + Path active = null; + HttpClientProfile current = null; + if (previous != null) try { + active = activeDirectory(directory); + current = loadEnrolledDirectory(active).profile(); + } catch (Exception unavailable) { active = null; current = null; } + if (active != null && previous.serverId().equals(current.serverId()) + && previous.endpoint().equals(current.endpoint())) { + restoreConnectionCodeDigest(active, snapshot.connectionCodeDigest()); + return; + } + restoreActiveGeneration(directory, snapshot); + } + + /** Atomically restores a previously validated credential generation after replacement rollback. */ + public static void restoreActiveGeneration(Path directory, ActiveCredentialGeneration snapshot) throws Exception { + if (directory == null || snapshot == null) throw new IllegalArgumentException("Credential rollback is required"); + Path root = credentialRoot(directory, false); + Path generations = root.resolve(GENERATIONS_DIRECTORY); + if (!snapshot.name().isEmpty() && (Files.isSymbolicLink(generations) + || !Files.isDirectory(generations, LinkOption.NOFOLLOW_LINKS))) + throw new IOException("HTTP credential generation directory is unsafe"); + Path target = snapshot.name().isEmpty() ? root : generations.resolve(snapshot.name()).normalize(); + if (!snapshot.name().isEmpty() && (!target.getParent().equals(generations) + || Files.isSymbolicLink(target) || !Files.isDirectory(target, LinkOption.NOFOLLOW_LINKS))) + throw new IOException("HTTP client credential generation is invalid"); + loadEnrolledDirectory(target); + Path current = safe(root.resolve(CURRENT_FILE)); + if (snapshot.name().isEmpty()) { + Files.deleteIfExists(current); + DurableFiles.forceDirectory(root); + } else { + writePrivate(current, snapshot.name().getBytes(StandardCharsets.US_ASCII)); + } + } + + public record ClientCredential(PrivateKey privateKey, X509Certificate certificate, X509Certificate caCertificate, char[] password) { + public ClientCredential { password = password.clone(); } + @Override public char[] password() { return password.clone(); } + } + + public record HttpClientProfile(String serverId, URI endpoint, String serverCertificatePin, String caCertificatePin) { + public HttpClientProfile { + serverId = HttpTlsIdentity.canonicalServerId(serverId); + HttpConnectionCode validation = new HttpConnectionCode(serverId, endpoint, serverCertificatePin, caCertificatePin, + java.time.Instant.now().plusSeconds(1), HttpTransportSecrets.randomToken()); + endpoint = validation.endpoint(); + serverCertificatePin = validation.serverCertificatePin(); + caCertificatePin = validation.caCertificatePin(); + } + } + + public record EnrolledClient(HttpClientProfile profile, ClientCredential credential) { } + + public record ActiveCredentialGeneration(String name, HttpClientProfile profile, String connectionCodeDigest) { + public ActiveCredentialGeneration(String name) { this(name, null, null); } + public ActiveCredentialGeneration { + if (name == null || (!name.isEmpty() && !name.matches("[0-9a-f-]{36}"))) + throw new IllegalArgumentException("HTTP client credential generation is invalid"); + if (connectionCodeDigest != null && !connectionCodeDigest.matches("[0-9a-f]{64}")) + throw new IllegalArgumentException("HTTP connection-code marker is invalid"); + } + } + + private static boolean matchesProfile(ClientCredential credential, HttpClientProfile profile) { + try { + String authorityPin = HttpTransportSecrets.certificatePin(credential.caCertificate()); + if (!HttpTransportSecrets.constantTimeEquals(profile.caCertificatePin().getBytes(StandardCharsets.US_ASCII), + authorityPin.getBytes(StandardCharsets.US_ASCII))) return false; + credential.certificate().checkValidity(); + credential.certificate().verify(credential.caCertificate().getPublicKey()); + java.util.List usage = credential.certificate().getExtendedKeyUsage(); + boolean[] keyUsage = credential.certificate().getKeyUsage(); + if (usage == null || !usage.contains(org.bouncycastle.asn1.x509.KeyPurposeId.id_kp_clientAuth.getId()) + || keyUsage == null || !keyUsage[0]) return false; + String expected = "urn:votingplugin:http-backend:" + profile.serverId(); + var names = credential.certificate().getSubjectAlternativeNames(); + if (names == null) return false; + for (java.util.List name : names) if (name.size() == 2 + && Integer.valueOf(6).equals(name.get(0)) && expected.equals(name.get(1))) return true; + return false; + } catch (Exception invalid) { return false; } + } + + private static Path safe(Path file) throws IOException { + if (Files.isSymbolicLink(file)) throw new IOException("Refusing unsafe HTTP credential path"); + return file.toAbsolutePath().normalize(); + } + + private static Path activeDirectory(Path directory) throws IOException { + Path root = credentialRoot(directory, false); + Path current = safe(root.resolve(CURRENT_FILE)); + if (!Files.exists(current, LinkOption.NOFOLLOW_LINKS)) return root; + if (!Files.isRegularFile(current, LinkOption.NOFOLLOW_LINKS) || Files.size(current) > 64) + throw new IOException("HTTP client credential pointer is invalid"); + String name = Files.readString(current, StandardCharsets.US_ASCII); + if (!name.matches("[0-9a-f-]{36}")) throw new IOException("HTTP client credential pointer is invalid"); + Path generations = root.resolve(GENERATIONS_DIRECTORY); + if (Files.isSymbolicLink(generations) || !Files.isDirectory(generations, LinkOption.NOFOLLOW_LINKS)) + throw new IOException("HTTP credential generation directory is unsafe"); + Path generation = generations.resolve(name).normalize(); + if (!generation.getParent().equals(generations) || Files.isSymbolicLink(generation) || !Files.isDirectory(generation, LinkOption.NOFOLLOW_LINKS)) + throw new IOException("HTTP client credential generation is invalid"); + PrivateFilePermissions.ownerOnlyDirectory(generation); + return generation; + } + + private static String readConnectionCodeDigest(Path directory) throws IOException { + Path digest = safe(directory.resolve(CONNECTION_CODE_DIGEST_FILE)); + if (!Files.exists(digest, LinkOption.NOFOLLOW_LINKS)) return null; + if (!Files.isRegularFile(digest, LinkOption.NOFOLLOW_LINKS) || Files.size(digest) != 64) + throw new IOException("HTTP connection-code marker is invalid"); + String value = Files.readString(digest, StandardCharsets.US_ASCII); + if (!value.matches("[0-9a-f]{64}")) throw new IOException("HTTP connection-code marker is invalid"); + return value; + } + + private static Path credentialRoot(Path directory, boolean create) throws IOException { + if (directory == null) throw new IllegalArgumentException("Credential directory is required"); + Path root = directory.toAbsolutePath().normalize(); + if (Files.isSymbolicLink(root)) throw new IOException("HTTP credential directory is unsafe"); + if (create) Files.createDirectories(root); + if (Files.isSymbolicLink(root) || !Files.isDirectory(root, LinkOption.NOFOLLOW_LINKS)) + throw new IOException("HTTP credential directory is unsafe"); + PrivateFilePermissions.ownerOnlyDirectory(root); + if (create) { + // Existing can mean create succeeded but publishing it durably did not. + DurableFiles.forceDirectory(root.getParent()); + } + return root; + } + + private static String connectionCodeDigest(HttpConnectionCode code) { + return connectionCodeDigest(code.encode()); + } + + private static String connectionCodeDigest(String encoded) { + return HttpTransportSecrets.sha256Hex(encoded.getBytes(StandardCharsets.US_ASCII)); + } + + private static void restoreConnectionCodeDigest(Path directory, String digest) throws IOException { + Path marker = safe(directory.resolve(CONNECTION_CODE_DIGEST_FILE)); + if (digest == null) { + Files.deleteIfExists(marker); + DurableFiles.forceDirectory(directory); + } else { + writePrivate(marker, digest.getBytes(StandardCharsets.US_ASCII)); + } + } + + private static void writePrivate(Path file, byte[] contents) throws IOException { + Path temporary = Files.createTempFile(file.getParent(), file.getFileName().toString(), ".tmp"); + try { + PrivateFilePermissions.ownerOnlyFile(temporary); + Files.write(temporary, contents, StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING); + DurableFiles.forceFile(temporary); + try { Files.move(temporary, file, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); } + catch (java.nio.file.AtomicMoveNotSupportedException unsupported) { Files.move(temporary, file, StandardCopyOption.REPLACE_EXISTING); } + try { + PrivateFilePermissions.ownerOnlyFile(file); + DurableFiles.forceDirectory(file.getParent()); + } catch (IOException postPublicationFailure) { + throw new DurableFiles.PublishedException(postPublicationFailure); + } + } finally { Files.deleteIfExists(temporary); } + } + + private static byte[] asciiBytes(char[] characters) { + byte[] output = new byte[characters.length]; + for (int index = 0; index < characters.length; index++) output[index] = (byte) characters[index]; + return output; + } +} diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpConnectionCode.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpConnectionCode.java new file mode 100644 index 0000000..463729d --- /dev/null +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpConnectionCode.java @@ -0,0 +1,125 @@ +package com.bencodez.simpleapi.servercomm.http; + +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.charset.StandardCharsets; +import java.time.Clock; +import java.time.Instant; +import java.util.Base64; +import java.util.Locale; + +/** + * A copy/paste connection code. It deliberately includes no long-lived credential: + * its only secret is a short-lived, single-use enrollment token. The trailing MAC is a + * corruption check keyed by that included token; it is not a proxy signature and cannot stop + * someone who can replace the whole code from replacing it with another valid code. + */ +public record HttpConnectionCode(String serverId, URI endpoint, String serverCertificatePin, String caCertificatePin, + Instant expiresAt, String enrollmentToken) { + private static final String LEGACY_VERSION = "VPH1"; + private static final String VERSION = "VPH2"; + private static final int MAX_CODE_LENGTH = 4096; + + public HttpConnectionCode { + serverId = HttpTlsIdentity.canonicalServerId(serverId); + endpoint = validateEndpoint(endpoint); + serverCertificatePin = validatePin(serverCertificatePin, "server certificate pin"); + caCertificatePin = validatePin(caCertificatePin, "CA certificate pin"); + if (expiresAt == null) throw new IllegalArgumentException("Expiry is required"); + enrollmentToken = validateToken(enrollmentToken); + } + + public String encode() { + String serverPart = Base64.getUrlEncoder().withoutPadding() + .encodeToString(serverId.getBytes(StandardCharsets.UTF_8)); + return encode(VERSION, serverPart); + } + + String encodeLegacy() { + return encode(LEGACY_VERSION, serverId); + } + + private String encode(String version, String serverPart) { + String endpointPart = Base64.getUrlEncoder().withoutPadding() + .encodeToString(endpoint.toASCIIString().getBytes(StandardCharsets.UTF_8)); + String unsigned = String.join(".", version, serverPart, endpointPart, serverCertificatePin, caCertificatePin, + Long.toString(expiresAt.getEpochSecond()), enrollmentToken); + byte[] token = Base64.getUrlDecoder().decode(enrollmentToken); + return unsigned + "." + HttpTransportSecrets.hmacSha256Url(token, unsigned); + } + + public boolean isActive(Clock clock) { + return expiresAt.isAfter(clock.instant()); + } + + public void requireActive(Clock clock) { + if (!isActive(clock)) throw new IllegalArgumentException("Connection code has expired"); + } + + public static HttpConnectionCode parse(String code) { + if (code == null || code.length() > MAX_CODE_LENGTH || code.indexOf('\n') >= 0 || code.indexOf('\r') >= 0) + throw new IllegalArgumentException("Connection code is invalid"); + String[] parts = code.split("\\.", -1); + if (parts.length < 8 || (!VERSION.equals(parts[0]) && !LEGACY_VERSION.equals(parts[0]))) + throw new IllegalArgumentException("Connection code is invalid"); + if (LEGACY_VERSION.equals(parts[0]) && parts.length > 8) { + int endpointIndex = parts.length - 6; + StringBuilder serverId = new StringBuilder(parts[1]); + for (int index = 2; index < endpointIndex; index++) serverId.append('.').append(parts[index]); + String[] normalized = new String[8]; + normalized[0] = parts[0]; + normalized[1] = serverId.toString(); + System.arraycopy(parts, endpointIndex, normalized, 2, 6); + parts = normalized; + } + if (parts.length != 8) throw new IllegalArgumentException("Connection code is invalid"); + try { + String serverId = LEGACY_VERSION.equals(parts[0]) ? parts[1] + : new String(Base64.getUrlDecoder().decode(parts[1]), StandardCharsets.UTF_8); + String endpoint = new String(Base64.getUrlDecoder().decode(parts[2]), StandardCharsets.UTF_8); + String unsigned = String.join(".", parts[0], parts[1], parts[2], parts[3], parts[4], parts[5], parts[6]); + byte[] token = Base64.getUrlDecoder().decode(parts[6]); + String expected = HttpTransportSecrets.hmacSha256Url(token, unsigned); + if (!HttpTransportSecrets.constantTimeEquals(expected.getBytes(StandardCharsets.US_ASCII), + parts[7].getBytes(StandardCharsets.US_ASCII))) throw new IllegalArgumentException("Connection code is invalid"); + return new HttpConnectionCode(serverId, new URI(endpoint), parts[3], parts[4], Instant.ofEpochSecond(Long.parseLong(parts[5])), + parts[6]); + } catch (IllegalArgumentException | URISyntaxException failure) { + throw new IllegalArgumentException("Connection code is invalid", failure); + } + } + + private static URI validateEndpoint(URI value) { + if (value == null || !"https".equalsIgnoreCase(value.getScheme()) || value.getHost() == null + || value.getUserInfo() != null || value.getFragment() != null || value.getRawQuery() != null) + throw new IllegalArgumentException("Endpoint must be an absolute HTTPS URL without credentials or query"); + if (value.getPort() == 0 || value.getPort() > 65535 || value.getPort() < -1) + throw new IllegalArgumentException("Endpoint port is invalid"); + String path = value.getRawPath(); + if (path == null || path.isEmpty()) path = "/"; + if (!path.endsWith("/")) path += "/"; + try { + URI authority = new URI("https", null, value.getHost().toLowerCase(Locale.ROOT), + value.getPort(), null, null, null); + return new URI(authority.toASCIIString() + path); + } catch (URISyntaxException failure) { + throw new IllegalArgumentException("Endpoint is invalid", failure); + } + } + + private static String validatePin(String pin, String name) { + if (pin == null || !pin.matches("[0-9a-fA-F]{64}")) throw new IllegalArgumentException(name + " is invalid"); + return pin.toLowerCase(Locale.ROOT); + } + + private static String validateToken(String token) { + if (token == null || token.length() < 43 || token.length() > 128 || !token.matches("[A-Za-z0-9_-]+")) + throw new IllegalArgumentException("Enrollment token is invalid"); + try { + if (Base64.getUrlDecoder().decode(token).length < 32) throw new IllegalArgumentException("Enrollment token is invalid"); + return token; + } catch (IllegalArgumentException failure) { + throw new IllegalArgumentException("Enrollment token is invalid", failure); + } + } +} diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentAuthority.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentAuthority.java new file mode 100644 index 0000000..79abd78 --- /dev/null +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentAuthority.java @@ -0,0 +1,461 @@ +package com.bencodez.simpleapi.servercomm.http; + +import java.net.URI; +import java.nio.channels.FileChannel; +import java.nio.channels.FileLock; +import java.nio.channels.OverlappingFileLockException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; +import java.util.Base64; +import com.bencodez.simpleapi.file.DurableFiles; +import com.bencodez.simpleapi.file.PrivateFilePermissions; + +/** + * Single-activation enrollment tokens and client-certificate binding. A token may retry certificate + * issuance until possession is proved, but can activate only one binding. Token material is never + * retained; only SHA-256 hashes are kept until expiry. This type is thread-safe. + */ +public final class HttpEnrollmentAuthority { + private static final Object PROCESS_STATE_LOCK = new Object(); + private static final String STATE_LOCK_FILE = ".http-transport-authority.lock"; + private static final Duration MAX_ENROLLMENT_LIFETIME = Duration.ofMinutes(15); + private static final Duration MIN_RENEWAL_INTERVAL = Duration.ofMinutes(1); + private static final int MAX_PENDING_ENROLLMENTS = 128; + private static final int MAX_BINDINGS = 128; + private static final long MAX_STATE_BYTES = 65536; + private final HttpTlsIdentity identity; + private final Clock clock; + private final Path stateFile; + private final Map enrollments = new HashMap<>(); + private final Map bindings = new HashMap<>(); + private final Map renewalNotBefore = new HashMap<>(); + private boolean persistenceFailure; + private boolean rollbackStateAvailable; + private boolean revocationRetryRequired; + private String revocationRetryServerId; + + /** Creates a restart-safe authority. State contains public certificate pins plus bounded hashes of pending tokens. */ + public HttpEnrollmentAuthority(HttpTlsIdentity identity, Path stateDirectory) throws java.io.IOException { + this(identity, Clock.systemUTC(), stateFile(stateDirectory)); + withStateLock(() -> null); + } + + HttpEnrollmentAuthority(HttpTlsIdentity identity, Clock clock) { + this(identity, clock, null); + } + + HttpEnrollmentAuthority(HttpTlsIdentity identity, Clock clock, Path stateFile) { + if (identity == null || clock == null) throw new IllegalArgumentException("Identity and clock are required"); + this.identity = identity; + this.clock = clock; + this.stateFile = stateFile; + } + + public synchronized HttpConnectionCode createConnectionCode(String serverId, URI endpoint, Duration lifetime) { + try { return withMutationLock(() -> createConnectionCodeLocked(serverId, endpoint, lifetime)); } + catch (java.io.IOException failure) { throw new IllegalStateException("Could not read HTTP enrollment state", failure); } + } + + private HttpConnectionCode createConnectionCodeLocked(String serverId, URI endpoint, Duration lifetime) { + if (revocationRetryRequired) + throw new IllegalStateException("HTTP certificate revocation durability must be retried"); + serverId = HttpTlsIdentity.canonicalServerId(serverId); + if (lifetime == null || lifetime.compareTo(Duration.ofSeconds(1)) < 0 || lifetime.compareTo(MAX_ENROLLMENT_LIFETIME) > 0) + throw new IllegalArgumentException("Enrollment lifetime must be between one second and fifteen minutes"); + Instant expiresAt = clock.instant().plus(lifetime); + String token = HttpTransportSecrets.randomToken(); + // Validate the complete code before reserving or persisting a pending slot. + HttpConnectionCode code = new HttpConnectionCode(serverId, endpoint, identity.serverCertificatePin(), + identity.caCertificatePin(), expiresAt, token); + expireEnrollments(); + if (enrollments.size() >= MAX_PENDING_ENROLLMENTS) + throw new IllegalStateException("Too many pending HTTP enrollments"); + byte[] tokenHash = HttpTransportSecrets.sha256(token.getBytes(StandardCharsets.US_ASCII)); + String lookup = java.util.Base64.getUrlEncoder().withoutPadding().encodeToString(tokenHash); + enrollments.put(lookup, new Enrollment(tokenHash, expiresAt, serverId, null)); + try { persistState(); persistenceFailure = false; rollbackStateAvailable = false; } + catch (java.io.IOException failure) { + enrollments.remove(lookup); + rollbackStateAvailable = !(failure instanceof DurableFiles.PublishedException); + throw new IllegalStateException("Could not persist HTTP enrollment", failure); + } + return code; + } + + public synchronized HttpTlsIdentity.IssuedClientCertificate enroll(String serverId, String enrollmentToken) throws Exception { + return withMutationLock(() -> enrollLocked(serverId, enrollmentToken)); + } + + private HttpTlsIdentity.IssuedClientCertificate enrollLocked(String serverId, String enrollmentToken) throws Exception { + // A failed revoke may have restored a still-valid pending token in memory. + // Do not let enrollment persist that stale state before the revoke is retried. + if (revocationRetryRequired) + throw new java.io.IOException("HTTP certificate revocation durability must be retried"); + serverId = HttpTlsIdentity.canonicalServerId(serverId); + if (enrollmentToken == null || enrollmentToken.length() > 128) throw new IllegalArgumentException("Enrollment was rejected"); + expireEnrollments(); + byte[] suppliedHash = HttpTransportSecrets.sha256(enrollmentToken.getBytes(StandardCharsets.US_ASCII)); + String lookup = java.util.Base64.getUrlEncoder().withoutPadding().encodeToString(suppliedHash); + Enrollment enrollment = enrollments.get(lookup); + if (enrollment == null || !HttpTransportSecrets.constantTimeEquals(enrollment.tokenHash(), suppliedHash)) + throw new IllegalArgumentException("Enrollment was rejected"); + if (!serverId.equals(enrollment.serverId())) throw new IllegalArgumentException("Enrollment was rejected"); + ClientBinding existing = bindings.get(serverId); + if (existing != null && !existing.revoked()) throw new IllegalStateException("Server id is already enrolled"); + if (existing == null && !hasPendingCertificate(serverId) && reservedBindingCount() >= MAX_BINDINGS) + throw new IllegalStateException("Too many enrolled HTTP backends"); + HttpTlsIdentity.IssuedClientCertificate issued = identity.issueClientCertificate(serverId); + enrollments.put(lookup, new Enrollment(enrollment.tokenHash(), enrollment.expiresAt(), serverId, + HttpTransportSecrets.certificatePin(issued.certificate()))); + try { persistState(); persistenceFailure = false; rollbackStateAvailable = false; } + catch (java.io.IOException failure) { + if (!(failure instanceof DurableFiles.PublishedException)) { + enrollments.put(lookup, enrollment); + rollbackStateAvailable = true; + } + else { persistenceFailure = true; rollbackStateAvailable = false; } + throw failure; + } + return issued; + } + + public synchronized boolean authenticate(String serverId, java.security.cert.X509Certificate certificate) { + try { return withStateLock(() -> authenticateLocked(serverId, certificate), rollbackStateAvailable); } + catch (java.io.IOException failure) { return false; } + } + + private boolean authenticateLocked(String serverId, java.security.cert.X509Certificate certificate) { + if (persistenceFailure || serverId == null || certificate == null) return false; + try { serverId = HttpTlsIdentity.canonicalServerId(serverId); } + catch (IllegalArgumentException invalid) { return false; } + if (!identity.validClientCertificate(serverId, certificate)) return false; + ClientBinding binding = bindings.get(serverId); + String pin = HttpTransportSecrets.certificatePin(certificate); + if (binding != null && !binding.revoked()) { + if (samePin(binding.certificatePin(), pin)) return true; + if (!samePin(binding.pendingCertificatePin(), pin)) return false; + bindings.put(serverId, new ClientBinding(pin, null, false)); + try { persistState(); rollbackStateAvailable = false; return true; } + catch (DurableFiles.PublishedException published) { + rollbackStateAvailable = false; + // The replacement is visible. If publication is lost on a crash, the + // previous durable pending binding can promote this certificate again. + return true; + } + catch (java.io.IOException failure) { bindings.put(serverId, binding); rollbackStateAvailable = true; return false; } + } + Map.Entry pending = pendingCertificate(serverId, pin); + if (pending == null || !pending.getValue().expiresAt().isAfter(clock.instant()) + || bindings.size() >= MAX_BINDINGS) return false; + enrollments.remove(pending.getKey()); + bindings.put(serverId, new ClientBinding(pin, null, false)); + try { persistState(); rollbackStateAvailable = false; return true; } + catch (DurableFiles.PublishedException published) { + rollbackStateAvailable = false; + // The visible state is active. If its directory entry is lost on a crash, the + // prior pending state can promote this same certificate again after restart. + return true; + } + catch (java.io.IOException failure) { + bindings.remove(serverId); + enrollments.put(pending.getKey(), pending.getValue()); + rollbackStateAvailable = true; + return false; + } + } + + /** Issues a replacement while the currently bound certificate is still valid. The old binding remains active + * until the replacement successfully authenticates, making a lost renewal response safe to retry. */ + public synchronized HttpTlsIdentity.IssuedClientCertificate renew(String serverId, + java.security.cert.X509Certificate currentCertificate) throws Exception { + return withMutationLock(() -> renewLocked(serverId, currentCertificate)); + } + + private HttpTlsIdentity.IssuedClientCertificate renewLocked(String serverId, + java.security.cert.X509Certificate currentCertificate) throws Exception { + serverId = HttpTlsIdentity.canonicalServerId(serverId); + Instant now = clock.instant(); + Instant nextAllowed = renewalNotBefore.get(serverId); + if (nextAllowed != null && now.isBefore(nextAllowed)) throw new RenewalRateLimitException(); + if (!authenticateLocked(serverId, currentCertificate)) throw new IllegalArgumentException("Certificate renewal was rejected"); + // Bound the limiter by active bindings; failed issuance still consumes the window. + renewalNotBefore.keySet().retainAll(bindings.keySet()); + renewalNotBefore.put(serverId, now.plus(MIN_RENEWAL_INTERVAL)); + ClientBinding binding = bindings.get(serverId); + HttpTlsIdentity.IssuedClientCertificate issued = identity.issueClientCertificate(serverId); + bindings.put(serverId, new ClientBinding(binding.certificatePin(), + HttpTransportSecrets.certificatePin(issued.certificate()), false)); + try { persistState(); rollbackStateAvailable = false; } + catch (DurableFiles.PublishedException published) { + rollbackStateAvailable = false; + // The old certificate remains active in both the old and newly visible state, + // so a lost response can safely retry and republish the complete state. + throw published; + } + catch (java.io.IOException failure) { bindings.put(serverId, binding); rollbackStateAvailable = true; throw failure; } + return issued; + } + + static final class RenewalRateLimitException extends IllegalStateException { + private static final long serialVersionUID = 1L; + RenewalRateLimitException() { super("HTTP certificate renewal is rate limited"); } + } + + public synchronized void revoke(String serverId) { + try { withMutationLock(() -> { revokeLocked(serverId); return null; }); } + catch (java.io.IOException failure) { throw new IllegalStateException("Could not read HTTP enrollment state", failure); } + } + + private void revokeLocked(String serverId) { + try { serverId = HttpTlsIdentity.canonicalServerId(serverId); } + catch (IllegalArgumentException invalid) { return; } + if (revocationRetryRequired && !serverId.equals(revocationRetryServerId)) + throw new IllegalStateException("A different HTTP certificate revocation must be retried first"); + final String revokedServer = serverId; + Map removedEnrollments = new HashMap<>(); + enrollments.entrySet().removeIf(entry -> { + if (!revokedServer.equals(entry.getValue().serverId())) return false; + removedEnrollments.put(entry.getKey(), entry.getValue()); + return true; + }); + // Absence is the durable revocation fence: authentication always requires an exact active binding. + ClientBinding removedBinding = bindings.remove(serverId); + if (removedBinding != null || !removedEnrollments.isEmpty() || revocationRetryRequired) try { + persistState(); + persistenceFailure = false; + rollbackStateAvailable = false; + revocationRetryRequired = false; + revocationRetryServerId = null; + } + catch (java.io.IOException failure) { + // Before publication, restore the exact disk-backed state so a retry still has work to persist. + // After publication, retain the fail-closed new state and let a retry force it durably again. + if (!(failure instanceof DurableFiles.PublishedException)) { + if (removedBinding != null) bindings.put(serverId, removedBinding); + enrollments.putAll(removedEnrollments); + rollbackStateAvailable = true; + } + persistenceFailure = true; + if (failure instanceof DurableFiles.PublishedException) rollbackStateAvailable = false; + revocationRetryRequired = true; + revocationRetryServerId = serverId; + throw new IllegalStateException("Could not persist HTTP certificate revocation", failure); + } + } + + @FunctionalInterface + private interface StateOperation { T run() throws E; } + + private T withMutationLock(StateOperation operation) throws java.io.IOException, E { + return withStateLock(operation, true); + } + + /** Serializes every durable-state operation across both authority instances and processes, then + * adopts the latest complete file before making a decision or rewriting it. */ + private T withStateLock(StateOperation operation) throws java.io.IOException, E { + return withStateLock(operation, false); + } + + private T withStateLock(StateOperation operation, + boolean allowDirectoryFailure) throws java.io.IOException, E { + if (stateFile == null) return operation.run(); + synchronized (PROCESS_STATE_LOCK) { + Path sidecar = stateFile.resolveSibling(STATE_LOCK_FILE); + if (Files.isSymbolicLink(sidecar) || Files.exists(sidecar, LinkOption.NOFOLLOW_LINKS) + && !Files.isRegularFile(sidecar, LinkOption.NOFOLLOW_LINKS)) + throw new java.io.IOException("HTTP enrollment state lock is unsafe"); + try (FileChannel channel = FileChannel.open(sidecar, StandardOpenOption.CREATE, + StandardOpenOption.WRITE, LinkOption.NOFOLLOW_LINKS)) { + PrivateFilePermissions.ownerOnlyFile(sidecar); + try (FileLock ignored = lock(channel)) { + refreshState(allowDirectoryFailure); + return operation.run(); + } + } + } + } + + private static FileLock lock(FileChannel channel) throws java.io.IOException { + try { return channel.lock(); } + catch (OverlappingFileLockException alreadyOwned) { + throw new java.io.IOException("HTTP enrollment state is already being updated", alreadyOwned); + } + } + + private void refreshState(boolean allowDirectoryFailure) throws java.io.IOException { + // A pre-publication failure may temporarily leave no state file while this instance + // retains the exact rollback state needed for a retry. Any successful peer mutation + // recreates the file under this same lock and will therefore be adopted here. + if (!Files.exists(stateFile, LinkOption.NOFOLLOW_LINKS)) return; + // Tests and recovery callers may expose a failed replacement as a directory at + // the target path. Mutations must reach persistState() so its existing rollback + // and fail-closed retry semantics run; reads still reject the invalid state. + if (allowDirectoryFailure && Files.isDirectory(stateFile, LinkOption.NOFOLLOW_LINKS)) return; + Map previousEnrollments = new HashMap<>(enrollments); + Map previousBindings = new HashMap<>(bindings); + enrollments.clear(); + bindings.clear(); + try { loadState(); } + catch (java.io.IOException failure) { + enrollments.putAll(previousEnrollments); + bindings.putAll(previousBindings); + throw failure; + } + } + + private synchronized void loadState() throws java.io.IOException { + if (stateFile == null || !Files.exists(stateFile, LinkOption.NOFOLLOW_LINKS)) return; + if (!Files.isRegularFile(stateFile, LinkOption.NOFOLLOW_LINKS) || Files.size(stateFile) > MAX_STATE_BYTES) + throw new java.io.IOException("HTTP enrollment state is invalid"); + PrivateFilePermissions.ownerOnlyFile(stateFile); + Properties properties = new Properties(); + try (var input = Files.newInputStream(stateFile, LinkOption.NOFOLLOW_LINKS)) { properties.load(input); } + String version = properties.getProperty("version"); + if (!("1".equals(version) || "2".equals(version) || "3".equals(version) || "4".equals(version))) + throw new java.io.IOException("HTTP enrollment state is invalid"); + for (String key : properties.stringPropertyNames()) { + if (key.startsWith("binding.")) { + String serverId = new String(Base64.getUrlDecoder().decode(key.substring("binding.".length())), StandardCharsets.UTF_8); + serverId = HttpTlsIdentity.canonicalServerId(serverId); + String[] value = properties.getProperty(key, "").split(":", -1); + if (!((value.length == 2 && "1".equals(version)) + || (value.length == 3 && ("2".equals(version) || "3".equals(version) || "4".equals(version)))) + || !value[0].matches("[0-9a-f]{64}")) + throw new java.io.IOException("HTTP enrollment state is invalid"); + String pending = value.length == 3 && !"-".equals(value[1]) ? value[1] : null; + String revoked = value[value.length - 1]; + if ((pending != null && !pending.matches("[0-9a-f]{64}")) || !("0".equals(revoked) || "1".equals(revoked))) + throw new java.io.IOException("HTTP enrollment state is invalid"); + if ("0".equals(revoked)) { + if (bindings.size() >= MAX_BINDINGS) throw new java.io.IOException("HTTP enrollment state exceeds its bound"); + bindings.put(serverId, new ClientBinding(value[0], pending, false)); + } + } else if (key.startsWith("enrollment.") && ("3".equals(version) || "4".equals(version))) { + String lookup = key.substring("enrollment.".length()); + if (!lookup.matches("[A-Za-z0-9_-]{43}")) throw new java.io.IOException("HTTP enrollment state is invalid"); + byte[] tokenHash; + try { tokenHash = Base64.getUrlDecoder().decode(lookup); } + catch (IllegalArgumentException invalid) { throw new java.io.IOException("HTTP enrollment state is invalid", invalid); } + if (tokenHash.length != 32) throw new java.io.IOException("HTTP enrollment state is invalid"); + String[] value = properties.getProperty(key, "").split(":", -1); + if (!(value.length == 2 && "3".equals(version)) && !(value.length == 3 && "4".equals(version))) + throw new java.io.IOException("HTTP enrollment state is invalid"); + Instant expiresAt; + String serverId; + String pendingPin = value.length == 3 && !"-".equals(value[2]) ? value[2] : null; + try { + expiresAt = Instant.ofEpochMilli(Long.parseLong(value[0])); + serverId = HttpTlsIdentity.canonicalServerId(new String(Base64.getUrlDecoder().decode(value[1]), StandardCharsets.UTF_8)); + } catch (RuntimeException invalid) { throw new java.io.IOException("HTTP enrollment state is invalid", invalid); } + if (pendingPin != null && !pendingPin.matches("[0-9a-f]{64}")) + throw new java.io.IOException("HTTP enrollment state is invalid"); + if (expiresAt.isAfter(clock.instant())) { + if (enrollments.size() >= MAX_PENDING_ENROLLMENTS) + throw new java.io.IOException("HTTP enrollment state exceeds its bound"); + enrollments.put(lookup, new Enrollment(tokenHash, expiresAt, serverId, pendingPin)); + } + } else if (!"version".equals(key)) throw new java.io.IOException("HTTP enrollment state is invalid"); + } + if (reservedBindingCount() > MAX_BINDINGS) + throw new java.io.IOException("HTTP enrollment state exceeds its bound"); + } + + private synchronized void persistState() throws java.io.IOException { + if (stateFile == null) return; + if (reservedBindingCount() > MAX_BINDINGS || enrollments.size() > MAX_PENDING_ENROLLMENTS) + throw new java.io.IOException("HTTP enrollment state exceeds its bound"); + Properties properties = new Properties(); + properties.setProperty("version", "4"); + for (Map.Entry entry : bindings.entrySet()) { + String key = Base64.getUrlEncoder().withoutPadding().encodeToString(entry.getKey().getBytes(StandardCharsets.UTF_8)); + properties.setProperty("binding." + key, entry.getValue().certificatePin() + ":" + + (entry.getValue().pendingCertificatePin() == null ? "-" : entry.getValue().pendingCertificatePin()) + + ":" + (entry.getValue().revoked() ? "1" : "0")); + } + for (Map.Entry entry : enrollments.entrySet()) { + String server = Base64.getUrlEncoder().withoutPadding().encodeToString( + entry.getValue().serverId().getBytes(StandardCharsets.UTF_8)); + properties.setProperty("enrollment." + entry.getKey(), + entry.getValue().expiresAt().toEpochMilli() + ":" + server + ":" + + (entry.getValue().pendingCertificatePin() == null ? "-" : entry.getValue().pendingCertificatePin())); + } + java.io.ByteArrayOutputStream bytes = new java.io.ByteArrayOutputStream(); + properties.store(bytes, "VotingPlugin HTTP transport authority state"); + if (bytes.size() > MAX_STATE_BYTES) throw new java.io.IOException("HTTP enrollment state exceeds its byte bound"); + Path temporary = Files.createTempFile(stateFile.getParent(), stateFile.getFileName().toString(), ".tmp"); + try { + PrivateFilePermissions.ownerOnlyFile(temporary); + Files.write(temporary, bytes.toByteArray(), StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING); + DurableFiles.forceFile(temporary); + try { Files.move(temporary, stateFile, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); } + catch (java.nio.file.AtomicMoveNotSupportedException unsupported) { Files.move(temporary, stateFile, StandardCopyOption.REPLACE_EXISTING); } + try { + PrivateFilePermissions.ownerOnlyFile(stateFile); + DurableFiles.forceDirectory(stateFile.getParent()); + } catch (java.io.IOException postPublicationFailure) { + throw new DurableFiles.PublishedException(postPublicationFailure); + } + } finally { Files.deleteIfExists(temporary); } + } + + private static Path stateFile(Path directory) throws java.io.IOException { + if (directory == null) throw new IllegalArgumentException("State directory is required"); + Path stateDirectory = directory.toAbsolutePath().normalize(); + Files.createDirectories(stateDirectory); + if (Files.isSymbolicLink(stateDirectory) || !Files.isDirectory(stateDirectory, LinkOption.NOFOLLOW_LINKS)) + throw new java.io.IOException("HTTP enrollment state directory is unsafe"); + PrivateFilePermissions.ownerOnlyDirectory(stateDirectory); + // Existing can mean a previous create succeeded but its parent fsync did not. + DurableFiles.forceDirectory(stateDirectory.getParent()); + Path file = stateDirectory.resolve("http-transport-clients.properties"); + if (Files.isSymbolicLink(file)) throw new java.io.IOException("Refusing unsafe HTTP enrollment state path"); + return file; + } + + private void expireEnrollments() { + Instant now = clock.instant(); + enrollments.entrySet().removeIf(entry -> !entry.getValue().expiresAt().isAfter(now)); + } + + private int reservedBindingCount() { + java.util.Set reserved = new java.util.HashSet<>(bindings.keySet()); + for (Enrollment enrollment : enrollments.values()) + if (enrollment.pendingCertificatePin() != null) reserved.add(enrollment.serverId()); + return reserved.size(); + } + + private boolean hasPendingCertificate(String serverId) { + for (Enrollment enrollment : enrollments.values()) + if (serverId.equals(enrollment.serverId()) && enrollment.pendingCertificatePin() != null) return true; + return false; + } + + private Map.Entry pendingCertificate(String serverId, String pin) { + for (Map.Entry entry : enrollments.entrySet()) { + Enrollment enrollment = entry.getValue(); + if (serverId.equals(enrollment.serverId()) && samePin(enrollment.pendingCertificatePin(), pin)) return entry; + } + return null; + } + + private record Enrollment(byte[] tokenHash, Instant expiresAt, String serverId, String pendingCertificatePin) { + private Enrollment { tokenHash = tokenHash.clone(); } + @Override public byte[] tokenHash() { return tokenHash.clone(); } + } + private static boolean samePin(String expected, String actual) { + return expected != null && actual != null && HttpTransportSecrets.constantTimeEquals( + expected.getBytes(StandardCharsets.US_ASCII), actual.getBytes(StandardCharsets.US_ASCII)); + } + + private record ClientBinding(String certificatePin, String pendingCertificatePin, boolean revoked) { } +} diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpInboundDeliveryStore.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpInboundDeliveryStore.java new file mode 100644 index 0000000..3fbb140 --- /dev/null +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpInboundDeliveryStore.java @@ -0,0 +1,428 @@ +package com.bencodez.simpleapi.servercomm.http; + +import com.bencodez.simpleapi.file.DurableFiles; +import com.bencodez.simpleapi.file.PrivateFilePermissions; +import java.io.IOException; +import java.nio.channels.FileChannel; +import java.nio.channels.FileLock; +import java.nio.channels.OverlappingFileLockException; +import java.nio.charset.StandardCharsets; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; +import java.util.UUID; + +/** Crash-durable state for proxy deliveries around a non-transactional application callback. */ +final class HttpInboundDeliveryStore { + private static final String DIRECTORY = "http-transport-inbound-deliveries"; + // This sidecar is deliberately outside the deletable journal directory. Never remove it: + // deleting and recreating a locked file could split ownership across two inodes. + private static final String OWNER_LOCK_PREFIX = ".http-inbound-owner-"; + private static final String OWNER_LOCK_SUFFIX = ".lock"; + private static final int MAX_ENTRIES = HttpTransportProtocol.MAX_QUEUE; + private final Path root; + private final boolean readOnly; + private final Map entries = new LinkedHashMap<>(); + // A rename completed, but its directory entry still needs a successful fsync. + private final Set unconfirmedReservations = new HashSet<>(); + private final Set unconfirmedCompletions = new HashSet<>(); + // RUNNING was published before the callback, but restoring RESERVED was not yet confirmed. + // This is process-local evidence only: a RUNNING entry loaded after a restart remains ambiguous. + private final Set pendingRunningRollbacks = new HashSet<>(); + private FileChannel ownershipChannel; + private FileLock ownershipLock; + private boolean sealed; + private boolean retirementRecoveryRequired; + + HttpInboundDeliveryStore(Path credentialDirectory) throws IOException { + this(credentialDirectory, DIRECTORY, false); + } + + static HttpInboundDeliveryStore open(Path parent, String directoryName) throws IOException { + return new HttpInboundDeliveryStore(parent, directoryName, false); + } + + /** Opens an existing journal for state inspection without claiming its writer ownership. */ + static HttpInboundDeliveryStore inspect(Path parent, String directoryName) throws IOException { + return new HttpInboundDeliveryStore(parent, directoryName, true); + } + + static HttpInboundDeliveryStore inspect(Path credentialDirectory) throws IOException { + return new HttpInboundDeliveryStore(credentialDirectory, DIRECTORY, true); + } + + /** Returns validated journal directory names while tolerating their persistent ownership sidecars. */ + static Set discover(Path parent) throws IOException { + Path root = parent.toAbsolutePath().normalize(); + if (Files.isSymbolicLink(root) || !Files.isDirectory(root, LinkOption.NOFOLLOW_LINKS)) + throw new IOException("HTTP inbound delivery parent is unsafe"); + PrivateFilePermissions.ownerOnlyDirectory(root); + Set directories = new TreeSet<>(); + try (DirectoryStream entries = Files.newDirectoryStream(root)) { + for (Path entry : entries) { + String name = entry.getFileName().toString(); + if (!Files.isSymbolicLink(entry) && Files.isDirectory(entry, LinkOption.NOFOLLOW_LINKS)) { + if (!name.matches("[A-Za-z0-9][A-Za-z0-9._-]{0,63}")) + throw new IOException("HTTP inbound delivery directory name is invalid"); + PrivateFilePermissions.ownerOnlyDirectory(entry); + directories.add(name); + continue; + } + if (!name.startsWith(OWNER_LOCK_PREFIX) || !name.endsWith(OWNER_LOCK_SUFFIX) + || Files.isSymbolicLink(entry) || !Files.isRegularFile(entry, LinkOption.NOFOLLOW_LINKS)) + throw new IOException("HTTP inbound delivery parent contains an invalid entry"); + String journal = name.substring(OWNER_LOCK_PREFIX.length(), name.length() - OWNER_LOCK_SUFFIX.length()); + if (!journal.matches("[A-Za-z0-9][A-Za-z0-9._-]{0,63}")) + throw new IOException("HTTP inbound delivery ownership lock name is invalid"); + PrivateFilePermissions.ownerOnlyFile(entry); + } + } + return directories; + } + + private HttpInboundDeliveryStore(Path parent, String directoryName, boolean readOnly) throws IOException { + Path credentials = parent.toAbsolutePath().normalize(); + if (Files.isSymbolicLink(credentials) || !Files.isDirectory(credentials, LinkOption.NOFOLLOW_LINKS)) + throw new IOException("HTTP credential directory is unsafe"); + if (directoryName == null || !directoryName.matches("[A-Za-z0-9][A-Za-z0-9._-]{0,63}")) + throw new IOException("HTTP inbound delivery directory name is invalid"); + this.readOnly = readOnly; + if (!readOnly) PrivateFilePermissions.ownerOnlyDirectory(credentials); + root = credentials.resolve(directoryName).normalize(); + if (!root.getParent().equals(credentials)) throw new IOException("HTTP inbound delivery directory is invalid"); + if (!readOnly) { + try { + claimOwnership(credentials.resolve(OWNER_LOCK_PREFIX + directoryName + OWNER_LOCK_SUFFIX)); + // Claim before creating or checking the journal root: a retiring owner may be + // between its empty check and deletion, and only one owner may cross that boundary. + try { Files.createDirectory(root); } + catch (java.nio.file.FileAlreadyExistsException existing) { } + requireRoot(); + PrivateFilePermissions.ownerOnlyDirectory(root); + // Retry a parent fsync that may have failed after creating this root. + DurableFiles.forceDirectory(credentials); + load(); + } catch (IOException | RuntimeException failure) { + releaseOwnership(); + throw failure; + } + } else { + requireRoot(); + load(); + } + } + + synchronized State state(String id) { return entries.get(canonical(id)); } + + synchronized void reserve(String id) throws IOException { + requireWritable(); + id = canonical(id); + State existing = entries.get(id); + if (existing == State.RESERVED) { + confirmReserved(id); + return; + } + if (existing != null) throw new IOException("HTTP inbound delivery fence is already active"); + if (entries.size() >= MAX_ENTRIES) throw new IOException("HTTP inbound delivery fence is full"); + requireRoot(); + Path target = file(id, State.RESERVED); + if (Files.exists(target, LinkOption.NOFOLLOW_LINKS)) + throw new IOException("HTTP inbound delivery fence is inconsistent"); + Path temporary = Files.createTempFile(root, ".pending-", ".tmp"); + try { + PrivateFilePermissions.ownerOnlyFile(temporary); + Files.writeString(temporary, id, StandardCharsets.US_ASCII, StandardOpenOption.TRUNCATE_EXISTING); + DurableFiles.forceFile(temporary); + move(temporary, target); + try { + PrivateFilePermissions.ownerOnlyFile(target); + DurableFiles.forceDirectory(root); + } catch (IOException postPublicationFailure) { + entries.put(id, State.RESERVED); + unconfirmedReservations.add(id); + throw new DurableFiles.PublishedException(postPublicationFailure); + } + entries.put(id, State.RESERVED); + } finally { Files.deleteIfExists(temporary); } + } + + /** Records receipt of a remote acknowledgement before its confirmation is sent. */ + synchronized void recordCompleted(String id) throws IOException { + requireWritable(); + id = canonical(id); + if (entries.get(id) == State.COMPLETED) { + confirmCompleted(id); + return; + } + if (entries.containsKey(id) || entries.size() >= MAX_ENTRIES) + throw new IOException("HTTP acknowledgement confirmation fence is full"); + requireRoot(); + Path target = file(id, State.COMPLETED); + Path temporary = Files.createTempFile(root, ".pending-", ".tmp"); + try { + PrivateFilePermissions.ownerOnlyFile(temporary); + Files.writeString(temporary, id, StandardCharsets.US_ASCII, StandardOpenOption.TRUNCATE_EXISTING); + DurableFiles.forceFile(temporary); + move(temporary, target); + try { + PrivateFilePermissions.ownerOnlyFile(target); + DurableFiles.forceDirectory(root); + } + catch (IOException postPublicationFailure) { + entries.put(id, State.COMPLETED); + unconfirmedCompletions.add(id); + throw new DurableFiles.PublishedException(postPublicationFailure); + } + entries.put(id, State.COMPLETED); + } finally { Files.deleteIfExists(temporary); } + } + + synchronized void markRunning(String id) throws IOException { + id = canonical(id); + if (entries.get(id) == State.RUNNING && pendingRunningRollbacks.contains(id)) + recoverKnownNotStartedRunning(id); + transition(id, State.RESERVED, State.RUNNING); + } + synchronized void markCompleted(String id) throws IOException { transition(id, State.RUNNING, State.COMPLETED); } + synchronized void seal() { + if (sealed) return; + sealed = true; + releaseOwnership(); + } + synchronized void sealAndDeleteIfEmpty() throws IOException { + requireWritable(); + if (!entries.isEmpty()) throw new IOException("HTTP inbound delivery store is not empty"); + requireRoot(); + try (DirectoryStream files = Files.newDirectoryStream(root)) { + if (files.iterator().hasNext()) throw new IOException("HTTP inbound delivery directory is not empty"); + } + Path parent = root.getParent(); + // Keep exclusive ownership if deletion or its publication cannot be confirmed. + // The live backend may retry retirement or reconnect and resume journal writes. + retirementRecoveryRequired = true; + Files.delete(root); + DurableFiles.forceDirectory(parent); + seal(); + } + + synchronized void remove(String id) throws IOException { + requireWritable(); + id = canonical(id); + State state = entries.get(id); + if (state == null) return; + requireRoot(); + DurableFiles.deleteIfExists(file(id, state)); + entries.remove(id); + unconfirmedReservations.remove(id); + unconfirmedCompletions.remove(id); + pendingRunningRollbacks.remove(id); + } + + synchronized Map snapshot() { return Map.copyOf(entries); } + + private void transition(String id, State expected, State replacement) throws IOException { + requireWritable(); + id = canonical(id); + if (expected == State.RESERVED) confirmReserved(id); + if (replacement == State.COMPLETED && entries.get(id) == replacement) { + confirmCompleted(id); + return; + } + if (entries.get(id) != expected) throw new IOException("HTTP inbound delivery fence state is invalid"); + requireRoot(); + Path source = file(id, expected), target = file(id, replacement); + if (Files.isSymbolicLink(source) || !Files.isRegularFile(source, LinkOption.NOFOLLOW_LINKS) + || Files.exists(target, LinkOption.NOFOLLOW_LINKS)) + throw new IOException("HTTP inbound delivery fence state is unsafe"); + move(source, target); + try { DurableFiles.forceDirectory(root); } + catch (IOException postPublicationFailure) { + if (replacement == State.RUNNING) { + // The callback has not been exposed yet. Restore the durable reservation + // when possible, so a later redelivery can safely try the transition again. + entries.put(id, State.RUNNING); + pendingRunningRollbacks.add(id); + try { recoverKnownNotStartedRunning(id); } + catch (IOException rollbackFailure) { postPublicationFailure.addSuppressed(rollbackFailure); } + throw new DurableFiles.PublishedException(postPublicationFailure); + } + entries.put(id, replacement); + if (replacement == State.COMPLETED) unconfirmedCompletions.add(id); + throw new DurableFiles.PublishedException(postPublicationFailure); + } + entries.put(id, replacement); + } + + /** + * Retries a rollback known to have happened before this process could enter the callback. + * Entries loaded from disk are deliberately absent from this set: their callback is ambiguous. + */ + synchronized boolean recoverKnownNotStartedRunning(String id) throws IOException { + id = canonical(id); + if (!pendingRunningRollbacks.contains(id)) return false; + requireWritable(); + if (entries.get(id) != State.RUNNING) throw new IOException("HTTP inbound delivery fence state is invalid"); + requireRoot(); + Path reserved = file(id, State.RESERVED), running = file(id, State.RUNNING); + boolean hasReserved = Files.exists(reserved, LinkOption.NOFOLLOW_LINKS); + boolean hasRunning = Files.exists(running, LinkOption.NOFOLLOW_LINKS); + if (hasReserved == hasRunning) throw new IOException("HTTP inbound delivery fence rollback is unsafe"); + if (hasRunning) { + verifyStateFile(running, id); + move(running, reserved); + PrivateFilePermissions.ownerOnlyFile(reserved); + } else verifyStateFile(reserved, id); + DurableFiles.forceDirectory(root); + entries.put(id, State.RESERVED); + pendingRunningRollbacks.remove(id); + return true; + } + + /** Retries the directory fsync required before exposing a reservation to a callback. */ + synchronized void confirmReserved(String id) throws IOException { + requireWritable(); + id = canonical(id); + if (entries.get(id) != State.RESERVED) throw new IOException("HTTP inbound delivery fence state is invalid"); + if (!unconfirmedReservations.contains(id)) return; + requireRoot(); + Path target = file(id, State.RESERVED); + if (Files.isSymbolicLink(target) || !Files.isRegularFile(target, LinkOption.NOFOLLOW_LINKS) + || Files.size(target) > 64L + || !Files.readString(target, StandardCharsets.US_ASCII).equals(id)) + throw new IOException("HTTP inbound delivery fence state is unsafe"); + DurableFiles.forceDirectory(root); + unconfirmedReservations.remove(id); + } + + /** Retries the directory fsync required before exposing a completed delivery for acknowledgement. */ + synchronized void confirmCompleted(String id) throws IOException { + requireWritable(); + id = canonical(id); + if (entries.get(id) != State.COMPLETED) throw new IOException("HTTP inbound delivery fence state is invalid"); + if (!unconfirmedCompletions.contains(id)) return; + requireRoot(); + Path target = file(id, State.COMPLETED); + if (Files.isSymbolicLink(target) || !Files.isRegularFile(target, LinkOption.NOFOLLOW_LINKS) + || Files.size(target) > 64L + || !Files.readString(target, StandardCharsets.US_ASCII).equals(id)) + throw new IOException("HTTP inbound delivery fence state is unsafe"); + DurableFiles.forceDirectory(root); + unconfirmedCompletions.remove(id); + } + + private void load() throws IOException { + try (DirectoryStream files = Files.newDirectoryStream(root)) { + for (Path file : files) { + String name = file.getFileName().toString(); + if (name.startsWith(".pending-") && name.endsWith(".tmp") && !Files.isSymbolicLink(file) + && Files.isRegularFile(file, LinkOption.NOFOLLOW_LINKS)) { + if (!readOnly) DurableFiles.deleteIfExists(file); + continue; + } + State state = State.fromFileName(name); + if (state == null || Files.isSymbolicLink(file) || !Files.isRegularFile(file, LinkOption.NOFOLLOW_LINKS) + || Files.size(file) > 64L) + throw new IOException("HTTP inbound delivery fence contains an invalid entry"); + if (!readOnly) PrivateFilePermissions.ownerOnlyFile(file); + String id; + try { id = canonical(name.substring(0, name.length() - state.suffix.length())); } + catch (IllegalArgumentException invalid) { + throw new IOException("HTTP inbound delivery fence entry is invalid", invalid); + } + if (!name.equals(id + state.suffix) || !Files.readString(file, StandardCharsets.US_ASCII).equals(id)) + throw new IOException("HTTP inbound delivery fence entry is invalid"); + State existing = entries.get(id); + if (existing == null) entries.put(id, state); + else { + // A provider without atomic moves may expose both names after an + // interrupted transition. Preserve the furthest fail-closed state: + // RUNNING never replays, and COMPLETED alone may be acknowledged. + State retained = existing.ordinal() >= state.ordinal() ? existing : state; + State obsolete = retained == existing ? state : existing; + if (!readOnly) DurableFiles.deleteIfExists(file(id, obsolete)); + entries.put(id, retained); + } + if (entries.get(id) == State.COMPLETED) unconfirmedCompletions.add(id); + if (entries.get(id) == State.RESERVED) unconfirmedReservations.add(id); + if (entries.size() > MAX_ENTRIES) throw new IOException("HTTP inbound delivery fence exceeds its bound"); + } + } + } + + private Path file(String id, State state) { return root.resolve(id + state.suffix); } + private void verifyStateFile(Path file, String id) throws IOException { + if (Files.isSymbolicLink(file) || !Files.isRegularFile(file, LinkOption.NOFOLLOW_LINKS) + || Files.size(file) > 64L || !Files.readString(file, StandardCharsets.US_ASCII).equals(id)) + throw new IOException("HTTP inbound delivery fence state is unsafe"); + } + private void requireWritable() throws IOException { + if (readOnly || sealed || ownershipLock == null || !ownershipLock.isValid()) + throw new IOException("HTTP inbound delivery store ownership has ended"); + if (retirementRecoveryRequired) { + try { Files.createDirectory(root); } + catch (java.nio.file.FileAlreadyExistsException existing) { } + requireRoot(); + PrivateFilePermissions.ownerOnlyDirectory(root); + DurableFiles.forceDirectory(root.getParent()); + retirementRecoveryRequired = false; + } + } + private void claimOwnership(Path sidecar) throws IOException { + if (Files.isSymbolicLink(sidecar) || Files.exists(sidecar, LinkOption.NOFOLLOW_LINKS) + && !Files.isRegularFile(sidecar, LinkOption.NOFOLLOW_LINKS)) + throw new IOException("HTTP inbound delivery ownership lock is unsafe"); + FileChannel channel = FileChannel.open(sidecar, StandardOpenOption.CREATE, StandardOpenOption.WRITE, + LinkOption.NOFOLLOW_LINKS); + try { + PrivateFilePermissions.ownerOnlyFile(sidecar); + FileLock lock; + try { lock = channel.tryLock(); } + catch (OverlappingFileLockException alreadyOwned) { throw new IOException("HTTP inbound delivery store is already owned", alreadyOwned); } + if (lock == null) throw new IOException("HTTP inbound delivery store is already owned"); + ownershipChannel = channel; + ownershipLock = lock; + } catch (IOException | RuntimeException failure) { + try { channel.close(); } catch (IOException closeFailure) { failure.addSuppressed(closeFailure); } + throw failure; + } + } + private void releaseOwnership() { + FileLock lock = ownershipLock; + FileChannel channel = ownershipChannel; + ownershipLock = null; + ownershipChannel = null; + if (lock != null) try { lock.release(); } catch (IOException ignored) { } + if (channel != null) try { channel.close(); } catch (IOException ignored) { } + } + private static void move(Path source, Path target) throws IOException { + try { Files.move(source, target, StandardCopyOption.ATOMIC_MOVE); } + catch (java.nio.file.AtomicMoveNotSupportedException unsupported) { Files.move(source, target); } + } + private static String canonical(String id) { + if (id == null) throw new IllegalArgumentException("HTTP delivery id is invalid"); + String canonical = UUID.fromString(id).toString(); + if (!canonical.equals(id)) throw new IllegalArgumentException("HTTP delivery id is not canonical"); + return canonical; + } + private void requireRoot() throws IOException { + if (Files.isSymbolicLink(root) || !Files.isDirectory(root, LinkOption.NOFOLLOW_LINKS)) + throw new IOException("HTTP inbound delivery directory is unsafe"); + } + enum State { + RESERVED(".reserved"), RUNNING(".running"), COMPLETED(".completed"); + private final String suffix; + State(String suffix) { this.suffix = suffix; } + private static State fromFileName(String name) { + for (State state : values()) if (name.endsWith(state.suffix)) return state; + return null; + } + } +} diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpPinnedTls.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpPinnedTls.java new file mode 100644 index 0000000..4240897 --- /dev/null +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpPinnedTls.java @@ -0,0 +1,103 @@ +package com.bencodez.simpleapi.servercomm.http; + +import java.security.cert.X509Certificate; +import java.security.KeyStore; +import javax.net.ssl.KeyManagerFactory; +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLParameters; +import javax.net.ssl.TrustManager; +import javax.net.ssl.X509TrustManager; + +/** Builds the backend TLS context. A public CA store is intentionally not consulted. */ +public final class HttpPinnedTls { + private HttpPinnedTls() { } + + public static SSLContext clientContext(HttpConnectionCode code) throws Exception { + if (code == null) throw new IllegalArgumentException("Connection code is required"); + SSLContext context = SSLContext.getInstance("TLS"); + context.init(null, new TrustManager[] { new PinnedServerTrustManager(code.serverCertificatePin(), code.caCertificatePin()) }, null); + return context; + } + + /** + * Normal transport context: presents the enrolled client certificate and trusts only the + * pinned private authority. Callers must not override the HttpClient default endpoint-identification + * settings; hostname verification remains enabled and the proxy leaf may renew under the same CA. + */ + public static SSLContext mutualTlsContext(HttpConnectionCode code, HttpClientCredentialStore.ClientCredential credential) + throws Exception { + if (code == null || credential == null || credential.privateKey() == null || credential.certificate() == null || credential.caCertificate() == null) + throw new IllegalArgumentException("Enrolled client credential is required"); + char[] password = credential.password(); + try { + String authorityPin = HttpTransportSecrets.certificatePin(credential.caCertificate()); + if (!HttpTransportSecrets.constantTimeEquals(code.caCertificatePin().getBytes(java.nio.charset.StandardCharsets.US_ASCII), + authorityPin.getBytes(java.nio.charset.StandardCharsets.US_ASCII))) + throw new IllegalArgumentException("HTTP authority does not match connection code"); + KeyStore store = KeyStore.getInstance("PKCS12"); + store.load(null, new char[0]); + store.setKeyEntry("client", credential.privateKey(), password, + new java.security.cert.Certificate[] { credential.certificate(), credential.caCertificate() }); + KeyManagerFactory managers = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); + managers.init(store, password); + KeyStore trustStore = KeyStore.getInstance(KeyStore.getDefaultType()); + trustStore.load(null, new char[0]); + trustStore.setCertificateEntry("http-transport-ca", credential.caCertificate()); + javax.net.ssl.TrustManagerFactory trusts = javax.net.ssl.TrustManagerFactory.getInstance( + javax.net.ssl.TrustManagerFactory.getDefaultAlgorithm()); + trusts.init(trustStore); + SSLContext context = SSLContext.getInstance("TLS"); + context.init(managers.getKeyManagers(), trusts.getTrustManagers(), null); + return context; + } finally { java.util.Arrays.fill(password, '\0'); } + } + + public static boolean matchesServerPin(HttpConnectionCode code, X509Certificate certificate) { + if (code == null || certificate == null) return false; + String actual = HttpTransportSecrets.certificatePin(certificate); + return HttpTransportSecrets.constantTimeEquals(code.serverCertificatePin() + .getBytes(java.nio.charset.StandardCharsets.US_ASCII), actual.getBytes(java.nio.charset.StandardCharsets.US_ASCII)); + } + + /** TLS 1.3 is used where the runtime exposes it; hostname verification is deliberately left enabled. */ + public static SSLParameters secureParameters(SSLContext context) { + SSLParameters parameters = context.getDefaultSSLParameters(); + for (String protocol : context.getSupportedSSLParameters().getProtocols()) { + if ("TLSv1.3".equals(protocol)) { + parameters.setProtocols(new String[] { "TLSv1.3" }); + break; + } + } + return parameters; + } + + private static final class PinnedServerTrustManager implements X509TrustManager { + private final String expectedPin; + private final String expectedCaPin; + private PinnedServerTrustManager(String expectedPin, String expectedCaPin) { + this.expectedPin = expectedPin; + this.expectedCaPin = expectedCaPin; + } + @Override public void checkClientTrusted(X509Certificate[] chain, String authType) { throw new UnsupportedOperationException(); } + @Override public void checkServerTrusted(X509Certificate[] chain, String authType) throws java.security.cert.CertificateException { + if (chain == null || chain.length < 2) throw new java.security.cert.CertificateException("Server certificate chain is incomplete"); + chain[0].checkValidity(); + chain[chain.length - 1].checkValidity(); + try { chain[0].verify(chain[chain.length - 1].getPublicKey()); } + catch (java.security.GeneralSecurityException invalid) { + throw new java.security.cert.CertificateException("Server certificate signature is invalid", invalid); + } + if (chain[chain.length - 1].getBasicConstraints() < 0) + throw new java.security.cert.CertificateException("Server certificate authority is invalid"); + String actual = HttpTransportSecrets.certificatePin(chain[0]); + if (!HttpTransportSecrets.constantTimeEquals(expectedPin.getBytes(java.nio.charset.StandardCharsets.US_ASCII), + actual.getBytes(java.nio.charset.StandardCharsets.US_ASCII))) + throw new java.security.cert.CertificateException("Server certificate pin does not match"); + String issuer = HttpTransportSecrets.certificatePin(chain[chain.length - 1]); + if (!HttpTransportSecrets.constantTimeEquals(expectedCaPin.getBytes(java.nio.charset.StandardCharsets.US_ASCII), + issuer.getBytes(java.nio.charset.StandardCharsets.US_ASCII))) + throw new java.security.cert.CertificateException("Server certificate authority pin does not match"); + } + @Override public X509Certificate[] getAcceptedIssuers() { return new X509Certificate[0]; } + } +} diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java new file mode 100644 index 0000000..caaa2c7 --- /dev/null +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpProxyTransportServer.java @@ -0,0 +1,1031 @@ +package com.bencodez.simpleapi.servercomm.http; + +import com.bencodez.simpleapi.servercomm.codec.JsonEnvelope; +import com.bencodez.simpleapi.file.DurableFiles; +import com.bencodez.simpleapi.file.PrivateFilePermissions; +import com.sun.net.httpserver.Headers; +import com.sun.net.httpserver.HttpsConfigurator; +import com.sun.net.httpserver.HttpsExchange; +import com.sun.net.httpserver.HttpsParameters; +import com.sun.net.httpserver.HttpsServer; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.InetSocketAddress; +import java.net.URI; +import java.nio.channels.FileChannel; +import java.nio.channels.FileLock; +import java.nio.channels.OverlappingFileLockException; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.security.cert.Certificate; +import java.security.cert.X509Certificate; +import java.time.Duration; +import java.util.ArrayDeque; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Semaphore; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; +import java.util.function.LongSupplier; +import javax.net.ssl.SSLParameters; +import javax.net.ssl.SSLPeerUnverifiedException; + +/** + * One HTTPS listener for enrollment and the backend-to-proxy long-poll transport. + * Every normal request is certificate-authenticated in the handler, rather than relying on TLS WANT auth. + */ +public final class HttpProxyTransportServer implements AutoCloseable { + private static final int MAX_BACKENDS = 128; + private static final long BACKEND_REPLAY_RETENTION_NANOS = + TimeUnit.MILLISECONDS.toNanos(HttpTransportProtocol.MAX_CLOCK_SKEW_MILLIS) + 1L; + static { + // JDK HttpServer reads these once when its internal server configuration is initialized. + // Set conservative process-wide bounds before this transport creates its listener. + setDefault("sun.net.httpserver.maxReqTime", "10"); + setDefault("sun.net.httpserver.maxRspTime", "10"); + setDefault("jdk.httpserver.maxConnections", "144"); + setDefault("sun.net.httpserver.maxReqHeaders", "32"); + setDefault("sun.net.httpserver.maxReqHeaderSize", "16384"); + } + // Keep an idle request open long enough to reuse the TLS connection, but bound backend-origin + // latency when a message is queued immediately after the request body has already been sent. + public static final Duration LONG_POLL = Duration.ofSeconds(2); + private final HttpTlsIdentity identity; + private final HttpEnrollmentAuthority authority; + private final HttpsServer server; + private final ThreadPoolExecutor listenerExecutor; + private final ThreadPoolExecutor handlerExecutor; + private final AtomicReference handlerWorker = new AtomicReference<>(); + private final Object closeMonitor = new Object(); + private final Semaphore admission = new Semaphore(64); + private final Map backends = new HashMap<>(); + private final DurableOutgoingQueue durableOutgoing; + private final Path durableIncomingRoot; + private final Consumer onEnvelope; + private final DeliveryAcknowledgement onAcknowledged; + private final LongSupplier nanoTime; + private volatile boolean closed; + private boolean closeFinalizing, closeFinalized; + + /** In-memory constructor for tests; production callers must supply a durable state directory. */ + HttpProxyTransportServer(InetSocketAddress bind, HttpTlsIdentity identity, HttpEnrollmentAuthority authority, + Consumer onEnvelope) throws Exception { + this(bind, identity, authority, null, onEnvelope, (serverId, deliveryId) -> { }, System::nanoTime); + } + + public HttpProxyTransportServer(InetSocketAddress bind, HttpTlsIdentity identity, HttpEnrollmentAuthority authority, + Path outgoingDirectory, Consumer onEnvelope) throws Exception { + this(bind, identity, authority, outgoingDirectory, onEnvelope, (serverId, deliveryId) -> { }); + } + + public HttpProxyTransportServer(InetSocketAddress bind, HttpTlsIdentity identity, HttpEnrollmentAuthority authority, + Path outgoingDirectory, Consumer onEnvelope, + DeliveryAcknowledgement onAcknowledged) throws Exception { + this(bind, identity, authority, Objects.requireNonNull(outgoingDirectory, "outgoingDirectory is required"), + onEnvelope, onAcknowledged, System::nanoTime); + } + + HttpProxyTransportServer(InetSocketAddress bind, HttpTlsIdentity identity, HttpEnrollmentAuthority authority, + Path outgoingDirectory, Consumer onEnvelope, + DeliveryAcknowledgement onAcknowledged, LongSupplier nanoTime) throws Exception { + if (bind == null || identity == null || authority == null || onEnvelope == null || onAcknowledged == null) + throw new IllegalArgumentException("HTTP transport configuration is required"); + if (nanoTime == null) throw new IllegalArgumentException("HTTP transport clock is required"); + this.identity = identity; this.authority = authority; this.onEnvelope = onEnvelope; + this.onAcknowledged = onAcknowledged; this.nanoTime = nanoTime; + durableOutgoing = outgoingDirectory == null ? null : new DurableOutgoingQueue(outgoingDirectory); + HttpsServer createdServer = null; + ThreadPoolExecutor createdListener = null, createdHandler = null; + try { + durableIncomingRoot = outgoingDirectory == null ? null : incomingRoot(outgoingDirectory); + if (durableIncomingRoot != null) for (String serverId : HttpInboundDeliveryStore.discover(durableIncomingRoot)) { + String canonical; + try { canonical = HttpTlsIdentity.canonicalServerId(serverId); } + catch (IllegalArgumentException invalid) { throw new IOException("HTTP inbound backend id is invalid", invalid); } + if (!canonical.equals(serverId)) throw new IOException("HTTP inbound backend id is not canonical"); + HttpInboundDeliveryStore inbound = HttpInboundDeliveryStore.open(durableIncomingRoot, serverId); + if (inbound.snapshot().isEmpty()) { + try { inbound.sealAndDeleteIfEmpty(); } + catch (IOException cleanupFailure) { inbound.seal(); throw cleanupFailure; } + continue; + } + if (backends.size() >= MAX_BACKENDS) { + inbound.seal(); + throw new IOException("HTTP backend state exceeds its bound"); + } + backends.put(serverId, new BackendState(serverId, durableOutgoing, inbound, onAcknowledged, nanoTime)); + } + if (durableOutgoing != null) for (Map.Entry> pending + : durableOutgoing.load().entrySet()) { + BackendState state = backendState(pending.getKey()); + state.restore(pending.getValue()); + } + createdServer = HttpsServer.create(bind, 32); + createdServer.setHttpsConfigurator(new HttpsConfigurator(identity.serverContext()) { + @Override public void configure(HttpsParameters parameters) { + SSLParameters ssl = HttpPinnedTls.secureParameters(getSSLContext()); + ssl.setWantClientAuth(true); parameters.setSSLParameters(ssl); + } + }); + // Long polls are blocking by design. Capacity is bounded by admission, while enough workers + // remain available for all admitted polls plus setup requests. + createdListener = executor("SimpleAPI-HTTP-listener", 72, 72); + // The proxy router mutates shared presence, vote, and reward state. A separate + // bounded FIFO lane keeps wire order without blocking long-poll workers. + createdHandler = executor("SimpleAPI-HTTP-handler", 1, + HttpBackendTransportConnector.CALLBACK_QUEUE_CAPACITY, handlerWorker); + createdServer.setExecutor(createdListener); + createdServer.createContext("/v1/enroll", exchange -> enroll((HttpsExchange) exchange)); + createdServer.createContext("/v1/renew", exchange -> renew((HttpsExchange) exchange)); + createdServer.createContext("/v1/transport", exchange -> transport((HttpsExchange) exchange)); + server = createdServer; + listenerExecutor = createdListener; + handlerExecutor = createdHandler; + } catch (Exception | Error setupFailure) { + if (createdServer != null) createdServer.stop(0); + if (createdHandler != null) shutdown(createdHandler); + if (createdListener != null) shutdown(createdListener); + releaseBackendOwnership(); + if (durableOutgoing != null) durableOutgoing.close(); + throw setupFailure; + } + } + + private void releaseBackendOwnership() { + synchronized (backends) { + for (BackendState backend : backends.values()) backend.seal(); + backends.clear(); + } + } + + private void renew(HttpsExchange exchange) throws IOException { + if (!"/v1/renew".equals(exchange.getRequestURI().getPath()) || exchange.getRequestURI().getRawQuery() != null) { reply(exchange, 404, new byte[0]); return; } + if (!"POST".equals(exchange.getRequestMethod())) { reply(exchange, 405, new byte[0]); return; } + if (!json(exchange)) { reply(exchange, 415, new byte[0]); return; } + int bodyError = fixedBodyErrorStatus(exchange.getRequestHeaders(), 1024); + if (bodyError != 0) { reply(exchange, bodyError, new byte[0]); return; } + if (!admission.tryAcquire()) { reply(exchange, 429, new byte[0]); return; } + try { + String serverId = HttpTransportProtocol.parseRenewal(read(exchange.getRequestBody(), 1024)); + X509Certificate certificate = peerCertificate(exchange); + if (certificate == null || !authority.authenticate(serverId, certificate)) { reply(exchange, 401, new byte[0]); return; } + HttpTlsIdentity.IssuedClientCertificate issued = authority.renew(serverId, certificate); + reply(exchange, 201, HttpTransportProtocol.enrollmentResponse(issued)); + } catch (HttpEnrollmentAuthority.RenewalRateLimitException limited) { + exchange.getResponseHeaders().set("Retry-After", "60"); + reply(exchange, 429, new byte[0]); + } catch (IllegalArgumentException rejected) { reply(exchange, 403, new byte[0]); + } catch (Exception failure) { reply(exchange, 503, new byte[0]); + } finally { admission.release(); } + } + + public void start() { if (closed) throw new IllegalStateException("HTTP transport is closed"); server.start(); } + public int port() { return server.getAddress().getPort(); } + public URI endpoint(String host) { + if (host == null || host.isBlank()) throw new IllegalArgumentException("Advertised host is required"); + try { + URI endpoint = new URI("https", null, host, port(), "/", null, null); + if (endpoint.getHost() == null) throw new IllegalArgumentException("Advertised host is invalid"); + return endpoint; + } catch (java.net.URISyntaxException invalid) { + throw new IllegalArgumentException("Advertised host is invalid", invalid); + } + } + + /** + * Queues a proxy-origin envelope durably before reporting acceptance. + * @throws DeliveryRetryException if publication needs recovery; persist its delivery ID + * and retry the same envelope through the stable-ID overload + */ + public boolean send(String serverId, JsonEnvelope envelope) { + return send(serverId, UUID.randomUUID().toString(), envelope, true); + } + + /** Carries the generated ID needed to recover a quarantined or indeterminate send. */ + @SuppressWarnings("serial") + public static final class DeliveryRetryException extends IllegalStateException { + private final String deliveryId; + private DeliveryRetryException(String deliveryId, Throwable cause) { + super("HTTP delivery requires a same-ID retry: " + deliveryId, cause); + this.deliveryId = deliveryId; + } + public String deliveryId() { return deliveryId; } + } + + /** + * Queues a proxy-origin envelope with a stable, caller-persisted delivery ID. + * Callers recovering {@link DeliveryRetryException} must persist its ID and retry + * the identical envelope through this overload until it returns {@code true}. + */ + public boolean send(String serverId, String deliveryId, JsonEnvelope envelope) { + return send(serverId, deliveryId, envelope, false); + } + + private boolean send(String serverId, String deliveryId, JsonEnvelope envelope, boolean generatedId) { + if (closed || serverId == null || envelope == null) return false; + try { + serverId = HttpTlsIdentity.canonicalServerId(serverId); + HttpTransportProtocol.validId(deliveryId); + HttpTransportProtocol.validateEnvelope(envelope); + } + catch (IllegalArgumentException invalid) { return false; } + BackendState backend; + final String canonicalServerId = serverId; + try { backend = backendState(canonicalServerId); } + catch (IOException persistenceFailure) { return false; } + try { return backend.enqueue(new HttpTransportProtocol.Delivery(deliveryId, envelope), generatedId); } + catch (DeliveryRetryException failure) { throw failure; } + catch (IllegalStateException indeterminate) { + if (generatedId) throw new DeliveryRetryException(deliveryId, indeterminate); + throw indeterminate; + } + } + + @Override public void close() { + boolean callbackWorker = Thread.currentThread() == handlerWorker.get(); + boolean finalizeHere = false; + synchronized (closeMonitor) { + if (!closed) { + closed = true; + server.stop(1); + closeFinalizing = true; + if (callbackWorker) { + Thread finalizer = new Thread(this::finishClose, "SimpleAPI-HTTP-proxy-close"); + finalizer.setDaemon(true); + finalizer.start(); + return; + } + finalizeHere = true; + } else if (callbackWorker || closeFinalized) { + return; + } + } + if (finalizeHere) finishClose(); else awaitClose(); + } + + private void finishClose() { + try { + try { shutdown(handlerExecutor); } + finally { shutdown(listenerExecutor); } + } finally { + try { + synchronized (backends) { + for (BackendState backend : backends.values()) { backend.seal(); backend.signal(); } + backends.clear(); + } + } finally { try { if (durableOutgoing != null) durableOutgoing.close(); } + finally { + synchronized (closeMonitor) { closeFinalizing = false; closeFinalized = true; closeMonitor.notifyAll(); } + } } + } + } + + private void awaitClose() { + boolean interrupted = false; + synchronized (closeMonitor) { + while (closeFinalizing && !closeFinalized) try { closeMonitor.wait(); } + catch (InterruptedException stopRequested) { interrupted = true; } + } + if (interrupted) Thread.currentThread().interrupt(); + } + + private void enroll(HttpsExchange exchange) throws IOException { + if (!"/v1/enroll".equals(exchange.getRequestURI().getPath()) || exchange.getRequestURI().getRawQuery() != null) { reply(exchange, 404, new byte[0]); return; } + if (!"POST".equals(exchange.getRequestMethod())) { reply(exchange, 405, new byte[0]); return; } + if (!json(exchange)) { reply(exchange, 415, new byte[0]); return; } + int bodyError = fixedBodyErrorStatus(exchange.getRequestHeaders(), 8192); + if (bodyError != 0) { reply(exchange, bodyError, new byte[0]); return; } + if (!admission.tryAcquire()) { reply(exchange, 429, new byte[0]); return; } + try { + HttpTransportProtocol.Enrollment request = HttpTransportProtocol.parseEnrollment(read(exchange.getRequestBody(), 8192)); + HttpTlsIdentity.IssuedClientCertificate issued = authority.enroll(request.server(), request.token()); + reply(exchange, 201, HttpTransportProtocol.enrollmentResponse(issued)); + } catch (IllegalArgumentException rejected) { reply(exchange, 403, new byte[0]); + } catch (Exception failure) { reply(exchange, 503, new byte[0]); } + finally { admission.release(); } + } + + private void transport(HttpsExchange exchange) throws IOException { + if (!"/v1/transport".equals(exchange.getRequestURI().getPath()) || exchange.getRequestURI().getRawQuery() != null) { reply(exchange, 404, new byte[0]); return; } + if (!"POST".equals(exchange.getRequestMethod())) { reply(exchange, 405, new byte[0]); return; } + if (!json(exchange)) { reply(exchange, 415, new byte[0]); return; } + int bodyError = fixedBodyErrorStatus(exchange.getRequestHeaders(), HttpTransportProtocol.MAX_BODY_BYTES); + if (bodyError != 0) { reply(exchange, bodyError, new byte[0]); return; } + if (!admission.tryAcquire()) { reply(exchange, 429, new byte[0]); return; } + try { + HttpTransportProtocol.Packet packet = HttpTransportProtocol.parsePacket(read(exchange.getRequestBody(), HttpTransportProtocol.MAX_BODY_BYTES)); + X509Certificate certificate = peerCertificate(exchange); + if (certificate == null || !authority.authenticate(packet.server(), certificate)) { reply(exchange, 401, new byte[0]); return; } + BackendState backend; + backend = backendState(packet.server()); + if (!backend.beginPoll(packet.session())) { reply(exchange, 409, new byte[0]); return; } + try { + handlePacket(packet, backend); + Response response = backend.await(packet.server(), packet.session(), packet.sequence(), packet.acks()); + reply(exchange, 200, HttpTransportProtocol.response(packet.server(), packet.session(), packet.sequence(), + response.acks(), packet.acks(), response.messages())); + } finally { backend.endPoll(); } + } catch (RateLimitException rateLimited) { reply(exchange, 429, new byte[0]); + } catch (IllegalArgumentException rejected) { reply(exchange, 400, new byte[0]); + } catch (Exception failure) { reply(exchange, 503, new byte[0]); + } finally { admission.release(); } + } + + private void handlePacket(HttpTransportProtocol.Packet packet, BackendState backend) throws IOException { + List accepted; + synchronized (backend) { + if (!backend.allowRequest()) throw new RateLimitException("transport rate limited"); + if (!backend.acceptSession(packet.session(), packet.sequence())) throw new IllegalArgumentException("stale session request"); + } + backend.confirmIncoming(packet.ackConfirmations()); + backend.acknowledge(packet.acks()); + synchronized (backend) { accepted = backend.acceptIncoming(packet.messages()); } + for (HttpTransportProtocol.Delivery delivery : accepted) dispatch(packet.server(), backend, delivery); + } + private void dispatch(String serverId, BackendState backend, HttpTransportProtocol.Delivery delivery) { + Runnable callback = () -> { + boolean success = false; + try { + backend.beginIncoming(delivery.id()); + onEnvelope.accept(new ReceivedEnvelope(serverId, delivery.id(), normalizeBackendIdentity(serverId, delivery.envelope()))); + backend.completeIncomingDurably(delivery.id()); + success = true; + } + catch (IOException persistenceFailure) { } + catch (RuntimeException ignored) { } + synchronized (backend) { backend.completeIncoming(delivery.id(), success); } + }; + if (!HttpBackendTransportConnector.executeOrdered(handlerExecutor, callback)) + synchronized (backend) { backend.completeIncoming(delivery.id(), false); } + } + private BackendState backendState(String serverId) throws IOException { + synchronized (backends) { + if (closed) throw new IOException("HTTP proxy transport is closed"); + BackendState existing = backends.get(serverId); + if (existing != null) return existing; + if (backends.size() >= MAX_BACKENDS) reclaimInactiveBackend(); + if (backends.size() >= MAX_BACKENDS) throw new IOException("HTTP backend state exceeds its bound"); + HttpInboundDeliveryStore inbound = null; + try { + inbound = durableIncomingRoot == null ? null + : HttpInboundDeliveryStore.open(durableIncomingRoot, serverId); + BackendState created = new BackendState(serverId, durableOutgoing, inbound, onAcknowledged, nanoTime); + backends.put(serverId, created); + return created; + } catch (Exception | Error setupFailure) { + if (inbound != null) inbound.seal(); + throw setupFailure; + } + } + } + private void reclaimInactiveBackend() throws IOException { + long now = nanoTime.getAsLong(); + for (Iterator> iterator = backends.entrySet().iterator(); iterator.hasNext();) { + BackendState state = iterator.next().getValue(); + if (!state.retireIfQuiescent(now, BACKEND_REPLAY_RETENTION_NANOS)) continue; + iterator.remove(); + return; + } + } + BackendState backendStateForTest(String serverId) throws IOException { return backendState(serverId); } + int backendCountForTest() { synchronized (backends) { return backends.size(); } } + private static Path incomingRoot(Path outgoingDirectory) throws IOException { + Path outgoing = outgoingDirectory.toAbsolutePath().normalize(); + Path parent = outgoing.getParent(); + if (parent == null || outgoing.getFileName() == null) throw new IOException("HTTP incoming queue path is invalid"); + Path root = parent.resolve(outgoing.getFileName().toString() + "-incoming"); + try { Files.createDirectory(root); } + catch (java.nio.file.FileAlreadyExistsException existing) { } + if (Files.isSymbolicLink(root) || !Files.isDirectory(root, LinkOption.NOFOLLOW_LINKS)) + throw new IOException("HTTP incoming queue directory is invalid"); + PrivateFilePermissions.ownerOnlyDirectory(root); + // Retry publication durability even when an earlier attempt created the + // directory but failed before its parent could be forced. + DurableFiles.forceDirectory(parent); + return root; + } + private static JsonEnvelope normalizeBackendIdentity(String serverId, JsonEnvelope envelope) { + // The authenticated TLS identity is authoritative; never forward a forged `server` field. + return envelope.toBuilder().put("server", serverId).build(); + } + private static X509Certificate peerCertificate(HttpsExchange exchange) { + try { Certificate[] peer = exchange.getSSLSession().getPeerCertificates(); + return peer.length > 0 && peer[0] instanceof X509Certificate certificate ? certificate : null; + } catch (SSLPeerUnverifiedException absent) { return null; } + } + private static byte[] read(InputStream input, int maximum) throws IOException { + ByteArrayOutputStream output = new ByteArrayOutputStream(); byte[] buffer = new byte[4096]; int total = 0, read; + while ((read = input.read(buffer)) >= 0) { total += read; if (total > maximum) throw new IllegalArgumentException("HTTP body is too large"); output.write(buffer, 0, read); } + return output.toByteArray(); + } + private static void reply(HttpsExchange exchange, int status, byte[] body) throws IOException { + Headers headers = exchange.getResponseHeaders(); headers.set("Cache-Control", "no-store"); headers.set("Content-Type", "application/json; charset=utf-8"); + exchange.sendResponseHeaders(status, body.length); try (var output = exchange.getResponseBody()) { output.write(body); } + } + private static boolean json(HttpsExchange exchange) { + String contentType = exchange.getRequestHeaders().getFirst("Content-Type"); + return contentType != null && contentType.toLowerCase(java.util.Locale.ROOT).matches("application/json(?:\\s*;.*)?"); + } + static int fixedBodyErrorStatus(Headers headers, int maximum) { + if (headers.getFirst("Transfer-Encoding") != null) return 400; + String value = headers.getFirst("Content-Length"); + if (value == null) return 411; + try { + long length = Long.parseLong(value); + if (length <= 0L) return 400; + return length <= maximum ? 0 : 413; + } catch (NumberFormatException invalid) { return 400; } + } + private static ThreadPoolExecutor executor(String name, int threads, int queue) { + return executor(name, threads, queue, null); + } + private static ThreadPoolExecutor executor(String name, int threads, int queue, AtomicReference worker) { + ThreadFactory factory = task -> { + Thread thread = new Thread(() -> { + if (worker != null) worker.set(Thread.currentThread()); + task.run(); + }, name); + thread.setDaemon(true); + return thread; + }; + return new ThreadPoolExecutor(threads, threads, 0L, TimeUnit.MILLISECONDS, new ArrayBlockingQueue<>(queue), factory, new ThreadPoolExecutor.AbortPolicy()); + } + private static void setDefault(String name, String value) { if (System.getProperty(name) == null) System.setProperty(name, value); } + static void shutdown(ExecutorService executor) { + executor.shutdown(); + boolean interrupted = false; + try { + if (!executor.awaitTermination(5, TimeUnit.SECONDS)) { + executor.shutdownNow(); + executor.awaitTermination(1, TimeUnit.SECONDS); + } + } catch (InterruptedException stopRequested) { + interrupted = true; + executor.shutdownNow(); + try { executor.awaitTermination(1, TimeUnit.SECONDS); } + catch (InterruptedException repeated) { interrupted = true; } + } + if (interrupted) Thread.currentThread().interrupt(); + } + + public record ReceivedEnvelope(String serverId, String messageId, JsonEnvelope envelope) { } + + @FunctionalInterface + public interface DeliveryAcknowledgement { + void confirm(String serverId, String deliveryId) throws IOException; + } + static record Response(Collection acks, Collection messages) { } + static final class RateLimitException extends IllegalArgumentException { + RateLimitException(String message) { super(message); } + } + static final class BackendState { + private final String serverId; + private final DurableOutgoingQueue durableOutgoing; + private final HttpInboundDeliveryStore durableIncoming; + private final DeliveryAcknowledgement onAcknowledged; + private final LongSupplier nanoTime; + private String session; private long sequence = -1L; + private final LinkedHashMap outgoing = new LinkedHashMap<>(); + private final Set seen = new LinkedHashSet<>(); private final Set processing = new LinkedHashSet<>(); + private final ArrayDeque acknowledgements = new ArrayDeque<>(); + private final Map deliveredAtNanos = new HashMap<>(); + private double requestTokens = 24.0d; + private long lastTokenNanos; + private long lastActivityNanos; + private boolean activePoll, retired; + BackendState() { this(null, null, null, (serverId, deliveryId) -> { }, System::nanoTime); } + BackendState(LongSupplier nanoTime) { this(null, null, null, (serverId, deliveryId) -> { }, nanoTime); } + private BackendState(String serverId, DurableOutgoingQueue durableOutgoing) { + this(serverId, durableOutgoing, null, (ignoredServer, ignoredDelivery) -> { }, System::nanoTime); + } + BackendState(String serverId, DurableOutgoingQueue durableOutgoing, + DeliveryAcknowledgement onAcknowledged) { + this(serverId, durableOutgoing, null, onAcknowledged, System::nanoTime); + } + BackendState(String serverId, DurableOutgoingQueue durableOutgoing, HttpInboundDeliveryStore durableIncoming, + DeliveryAcknowledgement onAcknowledged) { + this(serverId, durableOutgoing, durableIncoming, onAcknowledged, System::nanoTime); + } + private BackendState(String serverId, DurableOutgoingQueue durableOutgoing, + HttpInboundDeliveryStore durableIncoming, DeliveryAcknowledgement onAcknowledged, LongSupplier nanoTime) { + this.serverId = serverId; this.durableOutgoing = durableOutgoing; + this.durableIncoming = durableIncoming; this.onAcknowledged = onAcknowledged; this.nanoTime = nanoTime; + lastTokenNanos = lastActivityNanos = nanoTime.getAsLong(); + if (durableIncoming != null) for (Map.Entry entry + : durableIncoming.snapshot().entrySet()) { + if (entry.getValue() == HttpInboundDeliveryStore.State.COMPLETED) { + try { + durableIncoming.confirmCompleted(entry.getKey()); + seen.add(entry.getKey()); queueAck(entry.getKey()); + } catch (IOException unconfirmed) { } + } + } + } + private synchronized void restore(Collection deliveries) { + for (HttpTransportProtocol.Delivery delivery : deliveries) outgoing.put(delivery.id(), delivery); + } + private boolean beginPoll(String requestedSession) { synchronized (this) { if (retired || activePoll) return false; activePoll = true; touch(); return true; } } + private void endPoll() { synchronized (this) { activePoll = false; touch(); notifyAll(); } } + boolean beginPollForTest() { return beginPoll("test"); } + void endPollForTest() { endPoll(); } + private boolean allowRequest() { + long now = nanoTime.getAsLong(); requestTokens = Math.min(24.0d, requestTokens + ((now - lastTokenNanos) / 1_000_000_000.0d) * 2.0d); + lastTokenNanos = now; touch(); if (requestTokens < 1.0d) return false; requestTokens -= 1.0d; return true; + } + boolean acceptSession(String requested, long requestedSequence) { + if (!requested.equals(session)) { session = requested; sequence = -1L; deliveredAtNanos.clear(); } + // The connector allocates a fresh monotonic sequence for every attempt. Rejecting equality + // prevents a captured request from being replayed with altered ACKs or a new payload. + if (requestedSequence <= sequence) return false; sequence = requestedSequence; return true; + } + synchronized boolean enqueue(HttpTransportProtocol.Delivery delivery) { + return enqueue(delivery, false); + } + private synchronized boolean enqueue(HttpTransportProtocol.Delivery delivery, boolean generatedId) { + if (retired) return false; + HttpTransportProtocol.Delivery existing = outgoing.get(delivery.id()); + if (existing != null) { + if (!Arrays.equals(HttpTransportProtocol.storedDelivery(existing), + HttpTransportProtocol.storedDelivery(delivery))) return false; + if (durableOutgoing != null) try { durableOutgoing.confirm(serverId, delivery.id()); } + catch (IOException failure) { return false; } + return true; + } + if (outgoing.size() >= HttpTransportProtocol.MAX_QUEUE) return false; + if (durableOutgoing != null) try { durableOutgoing.persist(serverId, delivery); } + catch (DurableFiles.PublishedException quarantined) { + if (generatedId) throw new DeliveryRetryException(delivery.id(), quarantined); + return false; + } + catch (IOException failure) { return false; } + outgoing.put(delivery.id(), delivery); touch(); signal(); return true; + } + void acknowledge(Collection acks) throws IOException { + for (String id : acks) { + synchronized (this) { if (!outgoing.containsKey(id)) continue; } + onAcknowledged.confirm(serverId, id); + synchronized (this) { + if (!outgoing.containsKey(id)) continue; + // A 200 response is the backend's proof that its durable replay fence may + // be deleted. Never return success while the proxy delivery still exists. + if (durableOutgoing != null) durableOutgoing.remove(serverId, id); + outgoing.remove(id); deliveredAtNanos.remove(id); + } + } + } + List acceptIncoming(List received) { + List accepted = new java.util.ArrayList<>(); + for (HttpTransportProtocol.Delivery delivery : received) { + HttpInboundDeliveryStore.State persisted = durableIncoming == null ? null : durableIncoming.state(delivery.id()); + if (persisted == HttpInboundDeliveryStore.State.RUNNING) try { + if (durableIncoming.recoverKnownNotStartedRunning(delivery.id())) persisted = durableIncoming.state(delivery.id()); + } catch (IOException rollbackUnconfirmed) { continue; } + if (persisted == HttpInboundDeliveryStore.State.COMPLETED) { + try { durableIncoming.confirmCompleted(delivery.id()); } + catch (IOException unconfirmed) { continue; } + seen.add(delivery.id()); queueAck(delivery.id()); continue; + } + if (seen.contains(delivery.id())) { + seen.add(delivery.id()); queueAck(delivery.id()); continue; + } + if (persisted == HttpInboundDeliveryStore.State.RUNNING) continue; + if (!processing.contains(delivery.id())) { + processing.add(delivery.id()); accepted.add(delivery); + } + } + return accepted; + } + void beginIncoming(String id) throws IOException { + if (durableIncoming == null) return; + if (durableIncoming.state(id) == null) durableIncoming.reserve(id); + durableIncoming.markRunning(id); + } + void completeIncomingDurably(String id) throws IOException { + if (durableIncoming != null) durableIncoming.markCompleted(id); + } + synchronized void completeIncoming(String id, boolean success) { processing.remove(id); if (success) { seen.add(id); while (seen.size() > HttpTransportProtocol.MAX_QUEUE) seen.remove(seen.iterator().next()); queueAck(id); signal(); } } + synchronized void confirmIncoming(Collection ids) throws IOException { + for (String id : ids) { + if (durableIncoming != null && durableIncoming.state(id) == HttpInboundDeliveryStore.State.COMPLETED) + durableIncoming.remove(id); + seen.remove(id); + acknowledgements.removeIf(id::equals); + } + } + private void seal() { if (durableIncoming != null) durableIncoming.seal(); } + private synchronized boolean retireIfQuiescent(long now, long retentionNanos) throws IOException { + if (retired || activePoll || !outgoing.isEmpty() || !deliveredAtNanos.isEmpty() || !seen.isEmpty() + || !processing.isEmpty() || !acknowledgements.isEmpty() || now - lastActivityNanos < retentionNanos + || durableIncoming != null && !durableIncoming.snapshot().isEmpty() + || durableOutgoing != null && durableOutgoing.hasQuarantined(serverId)) return false; + if (durableIncoming != null) durableIncoming.sealAndDeleteIfEmpty(); + retired = true; + return true; + } + private void touch() { lastActivityNanos = nanoTime.getAsLong(); } + private void queueAck(String id) { if (acknowledgements.size() < HttpTransportProtocol.MAX_QUEUE && !acknowledgements.contains(id)) acknowledgements.add(id); } + synchronized Response await(String serverId, String requestedSession, long requestedSequence) { + return await(serverId, requestedSession, requestedSequence, List.of()); + } + synchronized Response await(String serverId, String requestedSession, long requestedSequence, + Collection ackConfirmations) { + long deadline = System.nanoTime() + LONG_POLL.toNanos(); + while (acknowledgements.isEmpty() && !hasUndelivered()) { + long retryRemaining = nanosUntilRedelivery(nanoTime.getAsLong()); + if (retryRemaining <= 0L) break; + long requestRemaining = deadline - System.nanoTime(); if (requestRemaining <= 0L) break; + long wait = Math.min(requestRemaining, retryRemaining); + try { TimeUnit.NANOSECONDS.timedWait(this, wait); } catch (InterruptedException interrupted) { Thread.currentThread().interrupt(); break; } + } + List acks = new java.util.ArrayList<>(); while (!acknowledgements.isEmpty() && acks.size() < HttpTransportProtocol.MAX_BATCH) acks.add(acknowledgements.remove()); + List candidates = new java.util.ArrayList<>(); + long now = nanoTime.getAsLong(); + if (hasUndelivered() || redeliveryDue(now)) for (HttpTransportProtocol.Delivery delivery : outgoing.values()) { + if (!deliveredAtNanos.containsKey(delivery.id()) || redeliveryDue(delivery.id(), now)) candidates.add(delivery); + if (candidates.size() == HttpTransportProtocol.MAX_BATCH) break; + } + List messages = HttpTransportProtocol.fittingMessages(serverId, requestedSession, + requestedSequence, acks, ackConfirmations, candidates); + long deliveredAt = nanoTime.getAsLong(); + for (HttpTransportProtocol.Delivery delivery : messages) deliveredAtNanos.put(delivery.id(), deliveredAt); + return new Response(acks, messages); + } + private boolean hasUndelivered() { for (String id : outgoing.keySet()) if (!deliveredAtNanos.containsKey(id)) return true; return false; } + private boolean redeliveryDue(long now) { + for (String id : outgoing.keySet()) if (redeliveryDue(id, now)) return true; + return false; + } + private boolean redeliveryDue(String id, long now) { + Long deliveredAt = deliveredAtNanos.get(id); + return deliveredAt != null && now - deliveredAt >= LONG_POLL.toNanos(); + } + private long nanosUntilRedelivery(long now) { + long remaining = Long.MAX_VALUE; + for (String id : outgoing.keySet()) { + Long deliveredAt = deliveredAtNanos.get(id); + if (deliveredAt == null) continue; + long candidate = LONG_POLL.toNanos() - (now - deliveredAt); + if (candidate <= 0L) return 0L; + remaining = Math.min(remaining, candidate); + } + return remaining; + } + private synchronized void signal() { notifyAll(); } + } + + static final class DurableOutgoingQueue implements AutoCloseable { + @FunctionalInterface + interface DirectoryForcer { void force(Path directory) throws IOException; } + private static final String FILE_PATTERN = "[0-9]{20}-[0-9a-f-]{36}\\.json"; + private final Path root; + private final DirectoryForcer directoryForcer; + private final Map> files = new HashMap<>(); + private final Map> quarantinedFiles = new HashMap<>(); + private FileChannel ownershipChannel; + private FileLock ownershipLock; + private long sequence; + + private DurableOutgoingQueue(Path root) throws IOException { + this(root, DurableFiles::forceDirectory); + } + + DurableOutgoingQueue(Path root, DirectoryForcer directoryForcer) throws IOException { + if (root == null || directoryForcer == null) + throw new IllegalArgumentException("HTTP outgoing queue configuration is required"); + this.directoryForcer = directoryForcer; + this.root = root.toAbsolutePath().normalize(); + Path parent = this.root.getParent(); + if (parent == null || this.root.getFileName() == null) + throw new IOException("HTTP outgoing queue directory is invalid"); + try { + claimOwnership(parent.resolve("." + this.root.getFileName() + ".http-outgoing-owner.lock")); + try { Files.createDirectory(this.root); } + catch (java.nio.file.FileAlreadyExistsException existing) { } + if (Files.isSymbolicLink(this.root) || !Files.isDirectory(this.root, LinkOption.NOFOLLOW_LINKS)) + throw new IOException("HTTP outgoing queue directory is invalid"); + PrivateFilePermissions.ownerOnlyDirectory(this.root); + // A failed parent fsync can leave the directory present but not durable. + // Reopening must retry it before the queue can accept work. + directoryForcer.force(parent); + } catch (IOException | RuntimeException setupFailure) { + releaseOwnership(); + throw setupFailure; + } + } + + synchronized Map> load() throws IOException { + requireOwnership(); + Map> loaded = new LinkedHashMap<>(); + int serverDirectories = 0; + try (DirectoryStream servers = Files.newDirectoryStream(root)) { + for (Path directory : servers) { + if (Files.isSymbolicLink(directory) || !Files.isDirectory(directory, LinkOption.NOFOLLOW_LINKS)) + throw new IOException("HTTP outgoing queue contains an invalid entry"); + PrivateFilePermissions.ownerOnlyDirectory(directory); + String serverId; + try { serverId = HttpTlsIdentity.canonicalServerId(directory.getFileName().toString()); } + catch (IllegalArgumentException invalid) { throw new IOException("HTTP outgoing queue server is invalid", invalid); } + if (!serverId.equals(directory.getFileName().toString())) + throw new IOException("HTTP outgoing queue server is not canonical"); + List entries = new java.util.ArrayList<>(); + try (DirectoryStream messages = Files.newDirectoryStream(directory)) { + for (Path message : messages) entries.add(message); + } + entries.sort(java.util.Comparator.comparing(path -> path.getFileName().toString())); + List deliveries = new java.util.ArrayList<>(); + Map serverFiles = files.computeIfAbsent(serverId, ignored -> new HashMap<>()); + Map quarantined = quarantinedFiles.computeIfAbsent(serverId, ignored -> new HashMap<>()); + int durableEntries = 0; + for (Path message : entries) { + String name = message.getFileName().toString(); + if (name.startsWith(".pending-") && name.endsWith(".tmp") + && !Files.isSymbolicLink(message) && Files.isRegularFile(message, LinkOption.NOFOLLOW_LINKS)) { + DurableFiles.deleteIfExists(message); + continue; + } + if (name.startsWith(".pending-") && name.endsWith(".json")) { + if (Files.isSymbolicLink(message) || !Files.isRegularFile(message, LinkOption.NOFOLLOW_LINKS) + || Files.size(message) > HttpTransportProtocol.MAX_ENVELOPE_BYTES * 2L) + throw new IOException("HTTP outgoing queue quarantine is invalid"); + PrivateFilePermissions.ownerOnlyFile(message); + HttpTransportProtocol.Delivery delivery; + try { delivery = HttpTransportProtocol.parseStoredDelivery(Files.readAllBytes(message)); } + catch (IllegalArgumentException invalid) { throw new IOException("HTTP outgoing queue quarantine is invalid", invalid); } + if (!name.equals(".pending-" + delivery.id() + ".json") + || serverFiles.containsKey(delivery.id()) || quarantined.put(delivery.id(), message) != null) + throw new IOException("HTTP outgoing queue quarantine id is invalid"); + if (++durableEntries > HttpTransportProtocol.MAX_QUEUE) + throw new IOException("HTTP outgoing queue exceeds its bound"); + continue; + } + if (Files.isSymbolicLink(message) || !Files.isRegularFile(message, LinkOption.NOFOLLOW_LINKS) + || !name.matches(FILE_PATTERN) || Files.size(message) > HttpTransportProtocol.MAX_ENVELOPE_BYTES * 2L) + throw new IOException("HTTP outgoing queue message is invalid"); + PrivateFilePermissions.ownerOnlyFile(message); + HttpTransportProtocol.Delivery delivery; + try { delivery = HttpTransportProtocol.parseStoredDelivery(Files.readAllBytes(message)); } + catch (IllegalArgumentException invalid) { throw new IOException("HTTP outgoing queue message is invalid", invalid); } + if (!name.endsWith("-" + delivery.id() + ".json") || quarantined.containsKey(delivery.id()) + || serverFiles.put(delivery.id(), message) != null) + throw new IOException("HTTP outgoing queue message id is invalid"); + deliveries.add(delivery); + if (++durableEntries > HttpTransportProtocol.MAX_QUEUE) + throw new IOException("HTTP outgoing queue exceeds its bound"); + sequence = Math.max(sequence, Long.parseLong(name.substring(0, 20))); + } + if (durableEntries == 0 && hasNoIndexedDeliveries(serverId)) { + deleteVerifiedEmptyDirectory(directory); + files.remove(serverId); + quarantinedFiles.remove(serverId); + continue; + } + if (++serverDirectories > MAX_BACKENDS) + throw new IOException("HTTP outgoing queue exceeds its backend bound"); + // Quarantine-only queues still reserve this backend's runtime state. Omitting + // them can consume the cap with unrelated inbound journals and make the + // identical retry unable to reach persist()'s quarantine recovery path. + loaded.put(serverId, deliveries); + } + } + return loaded; + } + + private synchronized void persist(String serverId, HttpTransportProtocol.Delivery delivery) throws IOException { + requireOwnership(); + Path directory = root.resolve(serverId).normalize(); + if (!directory.getParent().equals(root)) throw new IOException("HTTP outgoing queue server is invalid"); + if (!Files.exists(directory, LinkOption.NOFOLLOW_LINKS) && serverDirectoryCount() >= MAX_BACKENDS) + throw new IOException("HTTP outgoing queue exceeds its backend bound"); + boolean created = false; + try { Files.createDirectory(directory); created = true; } + catch (java.nio.file.FileAlreadyExistsException existing) { } + try { + if (Files.isSymbolicLink(directory) || !Files.isDirectory(directory, LinkOption.NOFOLLOW_LINKS)) + throw new IOException("HTTP outgoing queue server directory is invalid"); + PrivateFilePermissions.ownerOnlyDirectory(directory); + // The child fsync below cannot make this published name durable in its + // parent. Repeat it so a prior failed attempt is recoverable. + directoryForcer.force(root); + } catch (IOException setupFailure) { + if (created) try { deleteVerifiedEmptyDirectory(directory); } + catch (IOException cleanupFailure) { setupFailure.addSuppressed(cleanupFailure); } + throw setupFailure; + } + Map serverFiles = files.computeIfAbsent(serverId, ignored -> new HashMap<>()); + Map quarantined = quarantinedFiles.computeIfAbsent(serverId, ignored -> new HashMap<>()); + Path existing = serverFiles.get(delivery.id()); + if (existing != null) { + if (Files.isSymbolicLink(existing) || !Files.isRegularFile(existing, LinkOption.NOFOLLOW_LINKS) + || Files.size(existing) > HttpTransportProtocol.MAX_ENVELOPE_BYTES * 2L + || !Arrays.equals(Files.readAllBytes(existing), HttpTransportProtocol.storedDelivery(delivery))) + throw new IOException("HTTP outgoing queue delivery id conflicts with persisted data"); + PrivateFilePermissions.ownerOnlyFile(existing); + directoryForcer.force(directory); + return; + } + Path pending = quarantined.get(delivery.id()); + if (pending != null) { + if (Files.isSymbolicLink(pending) || !Files.isRegularFile(pending, LinkOption.NOFOLLOW_LINKS) + || Files.size(pending) > HttpTransportProtocol.MAX_ENVELOPE_BYTES * 2L + || !Arrays.equals(Files.readAllBytes(pending), HttpTransportProtocol.storedDelivery(delivery))) + throw new IOException("HTTP outgoing queue delivery id conflicts with persisted data"); + if (sequence == Long.MAX_VALUE) throw new IOException("HTTP outgoing queue sequence is exhausted"); + String name = String.format(java.util.Locale.ROOT, "%020d-%s.json", ++sequence, delivery.id()); + Path target = directory.resolve(name); + try { Files.move(pending, target, StandardCopyOption.ATOMIC_MOVE); } + catch (java.nio.file.AtomicMoveNotSupportedException unsupported) { Files.move(pending, target); } + try { PrivateFilePermissions.ownerOnlyFile(target); directoryForcer.force(directory); } + catch (IOException postPublicationFailure) { + quarantinePublished(directory, target, pending, serverFiles, quarantined, delivery.id(), postPublicationFailure); + } + quarantined.remove(delivery.id()); + serverFiles.put(delivery.id(), target); + return; + } + if (serverFiles.size() + quarantined.size() >= HttpTransportProtocol.MAX_QUEUE) + throw new IOException("HTTP outgoing queue exceeds its bound"); + if (sequence == Long.MAX_VALUE) throw new IOException("HTTP outgoing queue sequence is exhausted"); + String name = String.format(java.util.Locale.ROOT, "%020d-%s.json", ++sequence, delivery.id()); + Path target = directory.resolve(name); + Path temporary = Files.createTempFile(directory, ".pending-", ".tmp"); + try { + PrivateFilePermissions.ownerOnlyFile(temporary); + Files.write(temporary, HttpTransportProtocol.storedDelivery(delivery), StandardOpenOption.TRUNCATE_EXISTING); + DurableFiles.forceFile(temporary); + try { Files.move(temporary, target, StandardCopyOption.ATOMIC_MOVE); } + catch (java.nio.file.AtomicMoveNotSupportedException unsupported) { Files.move(temporary, target); } + try { PrivateFilePermissions.ownerOnlyFile(target); directoryForcer.force(directory); } + catch (IOException postPublicationFailure) { + pending = directory.resolve(".pending-" + delivery.id() + ".json"); + quarantinePublished(directory, target, pending, serverFiles, quarantined, delivery.id(), postPublicationFailure); + } + serverFiles.put(delivery.id(), target); + } finally { Files.deleteIfExists(temporary); } + } + + /** Counts durable backend directories, including quarantine-only queues omitted from load's deliverable map. */ + private int serverDirectoryCount() throws IOException { + int count = 0; + try (DirectoryStream directories = Files.newDirectoryStream(root)) { + for (Path directory : directories) { + if (Files.isSymbolicLink(directory) || !Files.isDirectory(directory, LinkOption.NOFOLLOW_LINKS)) + throw new IOException("HTTP outgoing queue contains an invalid entry"); + String serverId; + try { serverId = HttpTlsIdentity.canonicalServerId(directory.getFileName().toString()); } + catch (IllegalArgumentException invalid) { throw new IOException("HTTP outgoing queue server is invalid", invalid); } + if (!serverId.equals(directory.getFileName().toString())) + throw new IOException("HTTP outgoing queue server is not canonical"); + PrivateFilePermissions.ownerOnlyDirectory(directory); + if (isEmptyDirectory(directory) && hasNoIndexedDeliveries(serverId)) { + deleteVerifiedEmptyDirectory(directory); + continue; + } + if (++count > MAX_BACKENDS) throw new IOException("HTTP outgoing queue exceeds its backend bound"); + } + } + return count; + } + + private boolean isEmptyDirectory(Path directory) throws IOException { + try (DirectoryStream entries = Files.newDirectoryStream(directory)) { return !entries.iterator().hasNext(); } + } + + private boolean hasNoIndexedDeliveries(String serverId) { + Map serverFiles = files.get(serverId); + Map quarantined = quarantinedFiles.get(serverId); + return (serverFiles == null || serverFiles.isEmpty()) && (quarantined == null || quarantined.isEmpty()); + } + + private synchronized boolean hasQuarantined(String serverId) throws IOException { + requireOwnership(); + Map quarantined = quarantinedFiles.get(serverId); + return quarantined != null && !quarantined.isEmpty(); + } + + /** Deletes only a validated, observed-empty backend directory and makes its removal durable. */ + private void deleteVerifiedEmptyDirectory(Path directory) throws IOException { + if (Files.isSymbolicLink(directory) || !Files.isDirectory(directory, LinkOption.NOFOLLOW_LINKS) + || !isEmptyDirectory(directory)) + throw new IOException("HTTP outgoing queue server directory is no longer empty"); + Files.delete(directory); + directoryForcer.force(root); + } + + private void quarantinePublished(Path directory, Path target, Path pending, Map serverFiles, + Map quarantined, String id, IOException publicationFailure) throws IOException { + try { + try { Files.move(target, pending, StandardCopyOption.ATOMIC_MOVE); } + catch (java.nio.file.AtomicMoveNotSupportedException unsupported) { Files.move(target, pending); } + // Record the observed rename before metadata writeback. If the force is + // indeterminate, a same-process retry must still find this quarantine. + quarantined.put(id, pending); + PrivateFilePermissions.ownerOnlyFile(pending); + directoryForcer.force(directory); + } catch (IOException quarantineFailure) { + // A failed quarantine rename leaves the original published name in place on + // ordinary filesystems. Preserve that observed target for a same-ID retry so + // persist() confirms it rather than publishing a second file for the ID. + if (Files.isRegularFile(target, LinkOption.NOFOLLOW_LINKS)) { + quarantined.remove(id); + serverFiles.put(id, target); + } else if (Files.isRegularFile(pending, LinkOption.NOFOLLOW_LINKS)) { + serverFiles.remove(id); + quarantined.put(id, pending); + } + quarantineFailure.addSuppressed(publicationFailure); + throw new IllegalStateException("HTTP outgoing queue publication could not be quarantined", quarantineFailure); + } + throw new DurableFiles.PublishedException(publicationFailure); + } + + private synchronized void confirm(String serverId, String id) throws IOException { + requireOwnership(); + Map serverFiles = files.get(serverId); + Path file = serverFiles == null ? null : serverFiles.get(id); + if (file == null || Files.isSymbolicLink(file) || !Files.isRegularFile(file, LinkOption.NOFOLLOW_LINKS)) + throw new IOException("HTTP outgoing queue delivery is unavailable"); + PrivateFilePermissions.ownerOnlyFile(file); + directoryForcer.force(file.getParent()); + } + + private synchronized void remove(String serverId, String id) throws IOException { + requireOwnership(); + Map serverFiles = files.get(serverId); + if (serverFiles == null) throw new IOException("HTTP outgoing queue acknowledgement is unknown"); + Path file = serverFiles.get(id); + if (file != null) { + Files.deleteIfExists(file); + // Also makes a retried deletion durable if an earlier directory force failed after unlinking the file. + DurableFiles.forceDirectory(file.getParent()); + serverFiles.remove(id); + } + if (serverFiles.isEmpty()) { + Path directory = root.resolve(serverId).normalize(); + if (!directory.getParent().equals(root) || Files.isSymbolicLink(directory)) + throw new IOException("HTTP outgoing queue server directory is invalid"); + try { + Files.deleteIfExists(directory); + directoryForcer.force(root); + files.remove(serverId); + } catch (java.nio.file.DirectoryNotEmptyException unexpectedEntry) { + // The acknowledged delivery is already durably removed; unrelated/tampered entries + // must not make its acknowledgement permanently unprocessable. + } + } + } + + @Override public synchronized void close() { releaseOwnership(); } + + private void requireOwnership() throws IOException { + if (ownershipLock == null || !ownershipLock.isValid()) + throw new IOException("HTTP outgoing queue ownership has ended"); + } + private void claimOwnership(Path sidecar) throws IOException { + if (Files.isSymbolicLink(sidecar) || Files.exists(sidecar, LinkOption.NOFOLLOW_LINKS) + && !Files.isRegularFile(sidecar, LinkOption.NOFOLLOW_LINKS)) + throw new IOException("HTTP outgoing queue ownership lock is unsafe"); + FileChannel channel = FileChannel.open(sidecar, StandardOpenOption.CREATE, StandardOpenOption.WRITE, + LinkOption.NOFOLLOW_LINKS); + try { + PrivateFilePermissions.ownerOnlyFile(sidecar); + FileLock lock; + try { lock = channel.tryLock(); } + catch (OverlappingFileLockException alreadyOwned) { throw new IOException("HTTP outgoing queue is already owned", alreadyOwned); } + if (lock == null) throw new IOException("HTTP outgoing queue is already owned"); + ownershipChannel = channel; + ownershipLock = lock; + } catch (IOException | RuntimeException failure) { + try { channel.close(); } catch (IOException closeFailure) { failure.addSuppressed(closeFailure); } + throw failure; + } + } + private void releaseOwnership() { + FileLock lock = ownershipLock; + FileChannel channel = ownershipChannel; + ownershipLock = null; + ownershipChannel = null; + if (lock != null) try { lock.release(); } catch (IOException ignored) { } + if (channel != null) try { channel.close(); } catch (IOException ignored) { } + } + + } +} diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpTlsIdentity.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpTlsIdentity.java new file mode 100644 index 0000000..3b42748 --- /dev/null +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpTlsIdentity.java @@ -0,0 +1,539 @@ +package com.bencodez.simpleapi.servercomm.http; + +import java.io.IOException; +import java.io.OutputStream; +import java.math.BigInteger; +import java.nio.channels.FileChannel; +import java.nio.channels.FileLock; +import java.nio.channels.OverlappingFileLockException; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.security.KeyPair; +import java.security.KeyPairGenerator; +import java.security.KeyStore; +import java.security.PrivateKey; +import java.security.Principal; +import java.security.Security; +import java.security.cert.Certificate; +import java.security.cert.X509Certificate; +import java.time.Instant; +import java.time.Clock; +import java.time.Duration; +import java.util.Date; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.net.Socket; +import javax.net.ssl.KeyManager; +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLEngine; +import javax.net.ssl.TrustManager; +import javax.net.ssl.TrustManagerFactory; +import javax.net.ssl.X509ExtendedKeyManager; +import org.bouncycastle.asn1.x500.X500Name; +import org.bouncycastle.asn1.x509.BasicConstraints; +import org.bouncycastle.asn1.x509.Extension; +import org.bouncycastle.asn1.x509.GeneralName; +import org.bouncycastle.asn1.x509.GeneralNames; +import org.bouncycastle.asn1.x509.KeyUsage; +import org.bouncycastle.asn1.x509.ExtendedKeyUsage; +import org.bouncycastle.asn1.x509.KeyPurposeId; +import org.bouncycastle.cert.X509CertificateHolder; +import org.bouncycastle.cert.X509v3CertificateBuilder; +import org.bouncycastle.cert.jcajce.JcaX509CertificateConverter; +import org.bouncycastle.cert.jcajce.JcaX509v3CertificateBuilder; +import org.bouncycastle.jce.provider.BouncyCastleProvider; +import org.bouncycastle.operator.ContentSigner; +import org.bouncycastle.operator.jcajce.JcaContentSignerBuilder; +import com.bencodez.simpleapi.file.DurableFiles; +import com.bencodez.simpleapi.file.PrivateFilePermissions; + +/** Durable private CA plus server identity used by the proxy HTTP listener. */ +public final class HttpTlsIdentity { + private static final String CA_FILE = "http-transport-ca.p12"; + private static final String SERVER_FILE = "http-transport-server.p12"; + private static final String PASSWORD_FILE = "http-transport-password"; + private static final String INITIALIZING_FILE = "http-transport-initializing"; + private static final String OWNERSHIP_LOCK_FILE = ".http-transport-identity.lock"; + private static final String ENROLLMENT_STATE_FILE = "http-transport-clients.properties"; + private static final String OUTGOING_DIRECTORY = "outgoing-v1"; + private static final char[] EMPTY_PASSWORD = new char[0]; + private static final Object IDENTITY_LOCK_MONITOR = new Object(); + static final Duration RENEW_BEFORE = Duration.ofDays(30); + static final Duration CA_RENEW_BEFORE = Duration.ofDays(365); + private final PrivateKey caKey; + private volatile X509Certificate caCertificate; + private volatile PrivateKey serverKey; + private volatile X509Certificate serverCertificate; + private final char[] password; + private final Path caFile; + private final Path serverFile; + private final String advertisedHost; + + private HttpTlsIdentity(PrivateKey caKey, X509Certificate caCertificate, PrivateKey serverKey, + X509Certificate serverCertificate, char[] password, Path caFile, Path serverFile, String advertisedHost) { + this.caKey = caKey; + this.caCertificate = caCertificate; + this.serverKey = serverKey; + this.serverCertificate = serverCertificate; + this.password = password.clone(); + this.caFile = caFile; + this.serverFile = serverFile; + this.advertisedHost = advertisedHost; + } + + public static HttpTlsIdentity loadOrCreate(Path directory, String advertisedHost) throws Exception { + return loadOrCreate(directory, advertisedHost, Clock.systemUTC()); + } + + static HttpTlsIdentity loadOrCreate(Path directory, String advertisedHost, Clock clock) throws Exception { + if (directory == null) throw new IllegalArgumentException("Identity directory is required"); + if (advertisedHost == null || advertisedHost.isBlank() || advertisedHost.length() > 253) + throw new IllegalArgumentException("Advertised HTTPS host is invalid"); + if (clock == null) throw new IllegalArgumentException("Clock is required"); + Path identityDirectory = directory.toAbsolutePath().normalize(); + if (Files.isSymbolicLink(identityDirectory)) throw new IOException("HTTP TLS identity directory is unsafe"); + Files.createDirectories(identityDirectory); + if (Files.isSymbolicLink(identityDirectory) || !Files.isDirectory(identityDirectory, LinkOption.NOFOLLOW_LINKS)) + throw new IOException("HTTP TLS identity directory is unsafe"); + PrivateFilePermissions.ownerOnlyDirectory(identityDirectory); + // The identity files cannot make the newly created directory entry durable. + // Persist its parent before the TLS identity is returned for listener use. + DurableFiles.forceDirectory(identityDirectory.getParent()); + synchronized (IDENTITY_LOCK_MONITOR) { + try (IdentityLock ignored = claimIdentityLock(identityDirectory)) { + directory = identityDirectory; + Path caFile = safe(directory.resolve(CA_FILE)); + Path serverFile = safe(directory.resolve(SERVER_FILE)); + Path passwordFile = safe(directory.resolve(PASSWORD_FILE)); + Path initializingFile = safe(directory.resolve(INITIALIZING_FILE)); + boolean caExists = Files.exists(caFile, LinkOption.NOFOLLOW_LINKS); + boolean serverExists = Files.exists(serverFile, LinkOption.NOFOLLOW_LINKS); + boolean passwordExists = Files.exists(passwordFile, LinkOption.NOFOLLOW_LINKS); + boolean initializing = Files.exists(initializingFile, LinkOption.NOFOLLOW_LINKS); + boolean anyIdentityFile = caExists || serverExists || passwordExists; + boolean completeIdentity = caExists && serverExists && passwordExists; + boolean persistentTransportState = hasPersistentTransportState(directory); + if (anyIdentityFile && !completeIdentity && !initializing) + throw new IOException("HTTP TLS identity files are incomplete"); + if (initializing) { + if (persistentTransportState) + throw new IOException("HTTP TLS identity files are incomplete"); + discardUncommittedIdentity(caFile, serverFile, passwordFile); + caExists = false; + serverExists = false; + passwordExists = false; + initializing = true; + } + if (caExists || serverExists || passwordExists) { + if (!completeIdentity) throw new IOException("HTTP TLS identity files are incomplete"); + PrivateFilePermissions.ownerOnlyFile(caFile); + PrivateFilePermissions.ownerOnlyFile(serverFile); + PrivateFilePermissions.ownerOnlyFile(passwordFile); + char[] password = readPassword(passwordFile); + try { + KeyStore ca = load(caFile, password); + KeyStore server = load(serverFile, password); + PrivateKey caKey = (PrivateKey) ca.getKey("ca", password); + X509Certificate caCertificate = (X509Certificate) ca.getCertificate("ca"); + PrivateKey serverKey = (PrivateKey) server.getKey("server", password); + X509Certificate serverCertificate = (X509Certificate) server.getCertificate("server"); + if (caKey == null || caCertificate == null || serverKey == null || serverCertificate == null) + throw new IOException("HTTP TLS identity files are invalid"); + boolean caRenewed = needsCaRenewal(caCertificate, clock); + if (caRenewed) { + ensureBouncyCastle(); + KeyPair caPair = new KeyPair(caCertificate.getPublicKey(), caKey); + caCertificate = certificate("CN=VotingPlugin HTTP private CA", caPair, null, null, + CertificateRole.CA, null, clock.instant()); + ca = KeyStore.getInstance("PKCS12"); + ca.load(null, EMPTY_PASSWORD); + ca.setKeyEntry("ca", caKey, password, new Certificate[] { caCertificate }); + writeStore(caFile, ca, password); + } + if (caRenewed || !hasServerName(serverCertificate, advertisedHost) || needsRenewal(serverCertificate, clock)) { + ensureBouncyCastle(); + KeyPair serverPair = keyPair(); + serverCertificate = certificate("CN=" + certificateName(advertisedHost), serverPair, caCertificate, caKey, + CertificateRole.SERVER, advertisedHost, clock.instant()); + serverKey = serverPair.getPrivate(); + server = KeyStore.getInstance("PKCS12"); + server.load(null, EMPTY_PASSWORD); + server.setKeyEntry("server", serverKey, password, new Certificate[] { serverCertificate, caCertificate }); + writeStore(serverFile, server, password); + } + return new HttpTlsIdentity(caKey, caCertificate, serverKey, serverCertificate, password, caFile, serverFile, + advertisedHost); + } finally { Arrays.fill(password, '\0'); } + } + if (persistentTransportState) + throw new IOException("HTTP TLS identity files are missing"); + if (!initializing) writeInitializationMarker(initializingFile); + ensureBouncyCastle(); + char[] password = HttpTransportSecrets.randomToken().toCharArray(); + try { + KeyPair caPair = keyPair(); + X509Certificate caCertificate = certificate("CN=VotingPlugin HTTP private CA", caPair, null, null, CertificateRole.CA, null, + clock.instant()); + KeyPair serverPair = keyPair(); + X509Certificate serverCertificate = certificate("CN=" + certificateName(advertisedHost), serverPair, caCertificate, + caPair.getPrivate(), CertificateRole.SERVER, advertisedHost, clock.instant()); + KeyStore ca = KeyStore.getInstance("PKCS12"); + ca.load(null, EMPTY_PASSWORD); + ca.setKeyEntry("ca", caPair.getPrivate(), password, new Certificate[] { caCertificate }); + KeyStore server = KeyStore.getInstance("PKCS12"); + server.load(null, EMPTY_PASSWORD); + server.setKeyEntry("server", serverPair.getPrivate(), password, new Certificate[] { serverCertificate, caCertificate }); + writeStore(caFile, ca, password); + writeStore(serverFile, server, password); + byte[] passwordBytes = asciiBytes(password); + try { writePrivate(passwordFile, passwordBytes); } + finally { Arrays.fill(passwordBytes, (byte) 0); } + DurableFiles.deleteIfExists(initializingFile); + return new HttpTlsIdentity(caPair.getPrivate(), caCertificate, serverPair.getPrivate(), serverCertificate, password, + caFile, serverFile, advertisedHost); + } finally { Arrays.fill(password, '\0'); } + } + } + } + + public String serverCertificatePin() { + refreshIdentity(); + return HttpTransportSecrets.certificatePin(serverCertificate); + } + public String caCertificatePin() { refreshIdentity(); return HttpTransportSecrets.certificatePin(caCertificate); } + public X509Certificate caCertificate() { return caCertificate; } + public X509Certificate serverCertificate() { return serverCertificate; } + + /** + * The listener requests an optional client certificate so enrollment can share the same port. + * Any certificate that is presented must chain to this transport's private CA; normal requests + * additionally validate the certificate's persisted backend binding in the HTTP handler. + */ + public SSLContext serverContext() throws Exception { + renewIdentityIfNeeded(); + SSLContext context = SSLContext.getInstance("TLS"); + context.init(new KeyManager[] { new RotatingServerKeyManager() }, trustManagers(caCertificate), null); + return context; + } + + static TrustManager[] trustManagers(X509Certificate caCertificate) throws Exception { + KeyStore trustStore = KeyStore.getInstance(KeyStore.getDefaultType()); + trustStore.load(null, EMPTY_PASSWORD); + trustStore.setCertificateEntry("http-transport-ca", caCertificate); + TrustManagerFactory factory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); + factory.init(trustStore); + return factory.getTrustManagers(); + } + + private void refreshIdentity() { + try { renewIdentityIfNeeded(); } + catch (Exception failure) { throw new IllegalStateException("Could not renew HTTP TLS identity", failure); } + } + + private synchronized void renewIdentityIfNeeded() throws Exception { + Clock clock = Clock.systemUTC(); + boolean renewCa = needsCaRenewal(caCertificate, clock); + if (!renewCa && !needsRenewal(serverCertificate, clock)) return; + synchronized (IDENTITY_LOCK_MONITOR) { + try (IdentityLock ignored = claimIdentityLock(caFile.getParent())) { + // Another process or long-lived instance may have renewed while this object + // waited for the identity lock. Adopt the durable state before deciding whether + // another renewal is still necessary. + KeyStore persistedCaStore = load(caFile, password); + KeyStore persistedServerStore = load(serverFile, password); + PrivateKey persistedCaKey = (PrivateKey) persistedCaStore.getKey("ca", password); + X509Certificate persistedCa = (X509Certificate) persistedCaStore.getCertificate("ca"); + PrivateKey persistedServerKey = (PrivateKey) persistedServerStore.getKey("server", password); + X509Certificate persistedServer = (X509Certificate) persistedServerStore.getCertificate("server"); + if (persistedCaKey == null || persistedCa == null || persistedServerKey == null || persistedServer == null + || !java.security.MessageDigest.isEqual(caKey.getEncoded(), persistedCaKey.getEncoded())) + throw new IOException("HTTP TLS identity files are invalid"); + persistedServer.verify(persistedCa.getPublicKey()); + if (!hasServerName(persistedServer, advertisedHost)) + throw new IOException("HTTP TLS server identity does not match its advertised host"); + caCertificate = persistedCa; + serverKey = persistedServerKey; + serverCertificate = persistedServer; + renewCa = needsCaRenewal(caCertificate, clock); + if (!renewCa && !needsRenewal(serverCertificate, clock)) return; + ensureBouncyCastle(); + X509Certificate replacementCa = caCertificate; + if (renewCa) { + KeyPair caPair = new KeyPair(caCertificate.getPublicKey(), caKey); + replacementCa = certificate("CN=VotingPlugin HTTP private CA", caPair, null, null, + CertificateRole.CA, null, clock.instant()); + KeyStore caStore = KeyStore.getInstance("PKCS12"); + caStore.load(null, EMPTY_PASSWORD); + caStore.setKeyEntry("ca", caKey, password, new Certificate[] { replacementCa }); + writeStore(caFile, caStore, password); + } + KeyPair pair = keyPair(); + X509Certificate replacement = certificate("CN=" + certificateName(advertisedHost), pair, replacementCa, caKey, + CertificateRole.SERVER, advertisedHost, clock.instant()); + KeyStore store = KeyStore.getInstance("PKCS12"); + store.load(null, EMPTY_PASSWORD); + store.setKeyEntry("server", pair.getPrivate(), password, new Certificate[] { replacement, replacementCa }); + writeStore(serverFile, store, password); + caCertificate = replacementCa; + serverKey = pair.getPrivate(); + serverCertificate = replacement; + } + } + } + + public IssuedClientCertificate issueClientCertificate(String serverId) throws Exception { + return issueClientCertificate(serverId, Instant.now()); + } + + IssuedClientCertificate issueClientCertificate(String serverId, Instant issuedAt) throws Exception { + serverId = canonicalServerId(serverId); + if (issuedAt == null) throw new IllegalArgumentException("Certificate issuance time is required"); + ensureBouncyCastle(); + KeyPair pair = keyPair(); + X509Certificate certificate = certificate("CN=" + serverId, pair, caCertificate, caKey, CertificateRole.CLIENT, + "urn:votingplugin:http-backend:" + serverId, issuedAt); + char[] clientPassword = HttpTransportSecrets.randomToken().toCharArray(); + try { + KeyStore store = KeyStore.getInstance("PKCS12"); + store.load(null, EMPTY_PASSWORD); + store.setKeyEntry("client", pair.getPrivate(), clientPassword, new Certificate[] { certificate, caCertificate }); + java.io.ByteArrayOutputStream bytes = new java.io.ByteArrayOutputStream(); + store.store(bytes, clientPassword); + return new IssuedClientCertificate(serverId, certificate, bytes.toByteArray(), clientPassword); + } finally { Arrays.fill(clientPassword, '\0'); } + } + + public static String canonicalServerId(String serverId) { + if (serverId == null || !serverId.matches("[A-Za-z0-9][A-Za-z0-9._-]{0,63}")) + throw new IllegalArgumentException("Server id is invalid"); + return serverId.toLowerCase(java.util.Locale.ROOT); + } + + public boolean issuedByThisCa(X509Certificate certificate) { + if (certificate == null) return false; + try { + certificate.checkValidity(); + certificate.verify(caCertificate.getPublicKey()); + return true; + } catch (Exception failure) { + return false; + } + } + + public boolean validClientCertificate(String expectedServerId, X509Certificate certificate) { + if (!issuedByThisCa(certificate)) return false; + try { + List usage = certificate.getExtendedKeyUsage(); + boolean[] keyUsage = certificate.getKeyUsage(); + if (usage == null || !usage.contains(KeyPurposeId.id_kp_clientAuth.getId()) || keyUsage == null || !keyUsage[0]) return false; + String expectedUri = "urn:votingplugin:http-backend:" + canonicalServerId(expectedServerId); + Collection> names = certificate.getSubjectAlternativeNames(); + if (names == null) return false; + for (List name : names) { + if (name.size() == 2 && Integer.valueOf(GeneralName.uniformResourceIdentifier).equals(name.get(0)) + && expectedUri.equals(name.get(1))) return true; + } + return false; + } catch (Exception failure) { return false; } + } + + public record IssuedClientCertificate(String serverId, X509Certificate certificate, byte[] pkcs12, char[] password) { + public IssuedClientCertificate { + pkcs12 = pkcs12.clone(); + password = password.clone(); + } + @Override public byte[] pkcs12() { return pkcs12.clone(); } + @Override public char[] password() { return password.clone(); } + } + + private static KeyPair keyPair() throws Exception { + KeyPairGenerator generator = KeyPairGenerator.getInstance("EC"); + generator.initialize(new java.security.spec.ECGenParameterSpec("secp256r1")); + return generator.generateKeyPair(); + } + + private static X509Certificate certificate(String subject, KeyPair subjectKey, X509Certificate issuer, PrivateKey issuerKey, + CertificateRole role, String subjectAlternativeName, Instant now) throws Exception { + X500Name issuerName = issuer == null ? new X500Name(subject) : new X500Name(issuer.getSubjectX500Principal().getName()); + X509v3CertificateBuilder builder = new JcaX509v3CertificateBuilder(issuerName, + new BigInteger(160, new java.security.SecureRandom()).setBit(159), Date.from(now.minusSeconds(300)), + Date.from(now.plusSeconds(role == CertificateRole.CA ? 315360000L : 31536000L)), new X500Name(subject), subjectKey.getPublic()); + builder.addExtension(Extension.basicConstraints, true, new BasicConstraints(role == CertificateRole.CA)); + builder.addExtension(Extension.keyUsage, true, new KeyUsage(role == CertificateRole.CA ? KeyUsage.keyCertSign | KeyUsage.cRLSign + : KeyUsage.digitalSignature)); + if (role == CertificateRole.SERVER) builder.addExtension(Extension.extendedKeyUsage, false, + new ExtendedKeyUsage(KeyPurposeId.id_kp_serverAuth)); + if (role == CertificateRole.CLIENT) builder.addExtension(Extension.extendedKeyUsage, false, + new ExtendedKeyUsage(KeyPurposeId.id_kp_clientAuth)); + if (role == CertificateRole.SERVER && subjectAlternativeName != null) { + GeneralName name; + if (subjectAlternativeName.matches("(?:\\d{1,3}\\.){3}\\d{1,3}") || subjectAlternativeName.indexOf(':') >= 0) + name = new GeneralName(GeneralName.iPAddress, subjectAlternativeName); + else name = new GeneralName(GeneralName.dNSName, subjectAlternativeName); + builder.addExtension(Extension.subjectAlternativeName, false, new GeneralNames(name)); + } + if (role == CertificateRole.CLIENT) builder.addExtension(Extension.subjectAlternativeName, false, + new GeneralNames(new GeneralName(GeneralName.uniformResourceIdentifier, subjectAlternativeName))); + ContentSigner signer = new JcaContentSignerBuilder("SHA256withECDSA").setProvider("BC") + .build(issuerKey == null ? subjectKey.getPrivate() : issuerKey); + X509CertificateHolder holder = builder.build(signer); + return new JcaX509CertificateConverter().setProvider("BC").getCertificate(holder); + } + + static boolean needsRenewal(X509Certificate certificate, Clock clock) { + return certificate == null || !certificate.getNotAfter().toInstant().isAfter(clock.instant().plus(RENEW_BEFORE)); + } + + static boolean needsCaRenewal(X509Certificate certificate, Clock clock) { + return certificate == null || !certificate.getNotAfter().toInstant().isAfter(clock.instant().plus(CA_RENEW_BEFORE)); + } + + private static void ensureBouncyCastle() { + if (Security.getProvider("BC") == null) Security.addProvider(new BouncyCastleProvider()); + } + + private static String certificateName(String host) { + return host.replaceAll("[^A-Za-z0-9 ._-]", "_"); + } + + private static boolean hasServerName(X509Certificate certificate, String advertisedHost) { + try { + Collection> names = certificate.getSubjectAlternativeNames(); + if (names == null) return false; + for (List name : names) { + if (name.size() != 2 || !(name.get(1) instanceof String value)) continue; + if ((Integer.valueOf(GeneralName.dNSName).equals(name.get(0)) || Integer.valueOf(GeneralName.iPAddress).equals(name.get(0))) + && advertisedHost.equalsIgnoreCase(value)) return true; + } + return false; + } catch (Exception failure) { return false; } + } + + private static Path safe(Path file) throws IOException { + Path parent = file.toAbsolutePath().normalize().getParent(); + if (parent == null || Files.isSymbolicLink(file)) throw new IOException("Refusing unsafe HTTP TLS identity path"); + return file.toAbsolutePath().normalize(); + } + private static IdentityLock claimIdentityLock(Path directory) throws IOException { + Path sidecar = safe(directory.resolve(OWNERSHIP_LOCK_FILE)); + if (Files.exists(sidecar, LinkOption.NOFOLLOW_LINKS) + && !Files.isRegularFile(sidecar, LinkOption.NOFOLLOW_LINKS)) + throw new IOException("HTTP TLS identity ownership lock is unsafe"); + FileChannel channel = FileChannel.open(sidecar, StandardOpenOption.CREATE, StandardOpenOption.WRITE, + LinkOption.NOFOLLOW_LINKS); + try { + PrivateFilePermissions.ownerOnlyFile(sidecar); + FileLock lock; + try { lock = channel.tryLock(); } + catch (OverlappingFileLockException alreadyOwned) { throw new IOException("HTTP TLS identity is already initializing", alreadyOwned); } + if (lock == null) throw new IOException("HTTP TLS identity is already initializing"); + return new IdentityLock(channel, lock); + } catch (IOException | RuntimeException failure) { + try { channel.close(); } catch (IOException closeFailure) { failure.addSuppressed(closeFailure); } + throw failure; + } + } + private static final class IdentityLock implements AutoCloseable { + private final FileChannel channel; + private final FileLock lock; + private IdentityLock(FileChannel channel, FileLock lock) { this.channel = channel; this.lock = lock; } + @Override public void close() { + try { lock.release(); } catch (IOException ignored) { } + try { channel.close(); } catch (IOException ignored) { } + } + } + + private static KeyStore load(Path path, char[] password) throws Exception { + KeyStore store = KeyStore.getInstance("PKCS12"); + try (var input = Files.newInputStream(path, LinkOption.NOFOLLOW_LINKS)) { store.load(input, password); } + return store; + } + + private static char[] readPassword(Path path) throws IOException { + if (!Files.isRegularFile(path, LinkOption.NOFOLLOW_LINKS)) + throw new IOException("HTTP TLS password file is invalid"); + long size = Files.size(path); + if (size < 40L || size > 128L) throw new IOException("HTTP TLS password file is invalid"); + byte[] bytes; + try (var input = Files.newInputStream(path, LinkOption.NOFOLLOW_LINKS)) { + bytes = input.readNBytes(129); + } + try { + if (bytes.length < 40 || bytes.length > 128) throw new IOException("HTTP TLS password file is invalid"); + return new String(bytes, java.nio.charset.StandardCharsets.US_ASCII).toCharArray(); + } + finally { Arrays.fill(bytes, (byte) 0); } + } + + private static void writeStore(Path file, KeyStore store, char[] password) throws Exception { + java.io.ByteArrayOutputStream bytes = new java.io.ByteArrayOutputStream(); + store.store(bytes, password); + byte[] contents = bytes.toByteArray(); + try { writePrivate(file, contents); } + finally { Arrays.fill(contents, (byte) 0); } + } + + private static void writePrivate(Path file, byte[] contents) throws IOException { + Path temporary = Files.createTempFile(file.getParent(), file.getFileName().toString(), ".tmp"); + try { + PrivateFilePermissions.ownerOnlyFile(temporary); + Files.write(temporary, contents, StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING); + DurableFiles.forceFile(temporary); + try { Files.move(temporary, file, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); } + catch (java.nio.file.AtomicMoveNotSupportedException unsupported) { Files.move(temporary, file, StandardCopyOption.REPLACE_EXISTING); } + PrivateFilePermissions.ownerOnlyFile(file); + DurableFiles.forceDirectory(file.getParent()); + } finally { Files.deleteIfExists(temporary); } + } + + private static void writeInitializationMarker(Path file) throws IOException { + writePrivate(file, "initializing\n".getBytes(java.nio.charset.StandardCharsets.US_ASCII)); + } + + private static void discardUncommittedIdentity(Path caFile, Path serverFile, Path passwordFile) throws IOException { + DurableFiles.deleteIfExists(caFile); + DurableFiles.deleteIfExists(serverFile); + DurableFiles.deleteIfExists(passwordFile); + } + + private static boolean hasPersistentTransportState(Path directory) { + return Files.exists(directory.resolve(ENROLLMENT_STATE_FILE), LinkOption.NOFOLLOW_LINKS) + || Files.exists(directory.resolve(OUTGOING_DIRECTORY), LinkOption.NOFOLLOW_LINKS); + } + + private static byte[] asciiBytes(char[] characters) { + byte[] output = new byte[characters.length]; + for (int index = 0; index < characters.length; index++) output[index] = (byte) characters[index]; + return output; + } + + private final class RotatingServerKeyManager extends X509ExtendedKeyManager { + private static final String ALIAS = "server"; + private void refresh() { + refreshIdentity(); + } + private String alias(String keyType) { + refresh(); + return keyType != null && ("EC".equalsIgnoreCase(keyType) || keyType.toUpperCase(java.util.Locale.ROOT).startsWith("EC_")) + ? ALIAS : null; + } + @Override public String[] getClientAliases(String keyType, Principal[] issuers) { return null; } + @Override public String chooseClientAlias(String[] keyTypes, Principal[] issuers, Socket socket) { return null; } + @Override public String[] getServerAliases(String keyType, Principal[] issuers) { + return alias(keyType) == null ? null : new String[] { ALIAS }; + } + @Override public String chooseServerAlias(String keyType, Principal[] issuers, Socket socket) { return alias(keyType); } + @Override public String chooseEngineServerAlias(String keyType, Principal[] issuers, SSLEngine engine) { return alias(keyType); } + @Override public X509Certificate[] getCertificateChain(String alias) { + refresh(); + return ALIAS.equals(alias) ? new X509Certificate[] { serverCertificate, caCertificate } : null; + } + @Override public PrivateKey getPrivateKey(String alias) { refresh(); return ALIAS.equals(alias) ? serverKey : null; } + } + + private enum CertificateRole { CA, SERVER, CLIENT } +} diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpTransportProtocol.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpTransportProtocol.java new file mode 100644 index 0000000..e1aae28 --- /dev/null +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpTransportProtocol.java @@ -0,0 +1,234 @@ +package com.bencodez.simpleapi.servercomm.http; + +import com.bencodez.simpleapi.servercomm.codec.JsonEnvelope; +import com.bencodez.simpleapi.servercomm.codec.JsonEnvelopeCodec; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Base64; +import java.util.Collection; +import java.util.List; +import java.util.UUID; + +/** Strict, versioned HTTP transport envelope. Payloads remain canonical JsonEnvelopeCodec values. */ +final class HttpTransportProtocol { + static final int VERSION = 1; + static final int MAX_BODY_BYTES = 256 * 1024; + static final int MAX_BATCH = 64; + static final int MAX_ENVELOPE_BYTES = 48 * 1024; + static final int MAX_QUEUE = 1024; + static final long MAX_CLOCK_SKEW_MILLIS = 90_000L; + + private HttpTransportProtocol() { } + + static void validateEnvelope(JsonEnvelope envelope) { + if (envelope == null || JsonEnvelopeCodec.encode(envelope).getBytes(StandardCharsets.UTF_8).length > MAX_ENVELOPE_BYTES) throw bad(); + } + + static byte[] request(String server, String session, long sequence, Collection acks, + Collection ackConfirmations, Collection messages) { + JsonObject root = base(server, session, sequence); + root.add("acks", ids(acks)); + root.add("ackConfirmations", ids(ackConfirmations)); + root.add("messages", messages(messages)); + byte[] encoded = root.toString().getBytes(StandardCharsets.UTF_8); + if (encoded.length > MAX_BODY_BYTES) throw bad(); + return encoded; + } + + static List fittingMessages(String server, String session, long sequence, Collection acks, + Collection ackConfirmations, Collection candidates) { + List output = new ArrayList<>(); + for (Delivery candidate : candidates) { + if (output.size() == MAX_BATCH) break; + output.add(candidate); + try { request(server, session, sequence, acks, ackConfirmations, output); } + catch (IllegalArgumentException tooLarge) { output.remove(output.size() - 1); break; } + } + return output; + } + + static byte[] storedDelivery(Delivery delivery) { + validId(delivery.id()); + validateEnvelope(delivery.envelope()); + JsonObject root = new JsonObject(); + root.addProperty("v", VERSION); + root.addProperty("id", delivery.id()); + root.addProperty("payload", Base64.getUrlEncoder().withoutPadding().encodeToString( + JsonEnvelopeCodec.encode(delivery.envelope()).getBytes(StandardCharsets.UTF_8))); + return root.toString().getBytes(StandardCharsets.UTF_8); + } + + static Delivery parseStoredDelivery(byte[] body) { + if (body == null || body.length == 0 || body.length > MAX_ENVELOPE_BYTES * 2) throw bad(); + try { + JsonObject root = JsonParser.parseString(new String(body, StandardCharsets.UTF_8)).getAsJsonObject(); + requireOnly(root, "v", "id", "payload"); + if (integer(root, "v") != VERSION) throw bad(); + String id = string(root, "id", 64); validId(id); + byte[] payload = Base64.getUrlDecoder().decode(string(root, "payload", MAX_ENVELOPE_BYTES * 2)); + if (payload.length == 0 || payload.length > MAX_ENVELOPE_BYTES) throw bad(); + return new Delivery(id, JsonEnvelopeCodec.decode(new String(payload, StandardCharsets.UTF_8))); + } catch (RuntimeException invalid) { throw bad(); } + } + + static byte[] response(String server, String session, long sequence, Collection acks, + Collection ackConfirmations, Collection messages) { + return request(server, session, sequence, acks, ackConfirmations, messages); + } + + static Packet parsePacket(byte[] body) { + if (body == null || body.length == 0 || body.length > MAX_BODY_BYTES) throw bad(); + try { + JsonElement parsed = JsonParser.parseString(new String(body, StandardCharsets.UTF_8)); + if (!parsed.isJsonObject()) throw bad(); + JsonObject root = parsed.getAsJsonObject(); + requireOnly(root, "v", "server", "session", "sequence", "timestamp", "acks", "ackConfirmations", "messages"); + if (integer(root, "v") != VERSION) throw bad(); + String server = HttpTlsIdentity.canonicalServerId(string(root, "server", 64)); + String session = uuid(root, "session"); + long sequence = nonNegative(root, "sequence"); + long timestamp = integer(root, "timestamp"); + long now = Instant.now().toEpochMilli(); + if (timestamp < now - MAX_CLOCK_SKEW_MILLIS || timestamp > now + MAX_CLOCK_SKEW_MILLIS) throw bad(); + List acks = parseIds(root.get("acks")); + List ackConfirmations = parseIds(root.get("ackConfirmations")); + List messages = parseMessages(root.get("messages")); + return new Packet(server, session, sequence, acks, ackConfirmations, messages); + } catch (RuntimeException invalid) { throw bad(); } + } + + static byte[] enrollmentResponse(HttpTlsIdentity.IssuedClientCertificate certificate) { + JsonObject output = new JsonObject(); + byte[] bundle = certificate.pkcs12(); + try { output.addProperty("bundle", Base64.getUrlEncoder().withoutPadding().encodeToString(bundle)); } + finally { java.util.Arrays.fill(bundle, (byte) 0); } + char[] password = certificate.password(); + try { output.addProperty("password", new String(password)); } + finally { java.util.Arrays.fill(password, '\0'); } + return output.toString().getBytes(StandardCharsets.UTF_8); + } + + static Enrollment parseEnrollment(byte[] body) { + if (body == null || body.length == 0 || body.length > 8192) throw bad(); + try { + JsonElement parsed = JsonParser.parseString(new String(body, StandardCharsets.UTF_8)); + if (!parsed.isJsonObject()) throw bad(); + JsonObject root = parsed.getAsJsonObject(); + requireOnly(root, "server", "token"); + String server = HttpTlsIdentity.canonicalServerId(string(root, "server", 64)); + String token = string(root, "token", 128); + if (!token.matches("[A-Za-z0-9_-]{43,128}")) throw bad(); + return new Enrollment(server, token); + } catch (RuntimeException invalid) { throw bad(); } + } + + static byte[] renewalRequest(String server) { + JsonObject root = new JsonObject(); + root.addProperty("server", HttpTlsIdentity.canonicalServerId(server)); + return root.toString().getBytes(StandardCharsets.UTF_8); + } + + static String parseRenewal(byte[] body) { + if (body == null || body.length == 0 || body.length > 1024) throw bad(); + try { + JsonElement parsed = JsonParser.parseString(new String(body, StandardCharsets.UTF_8)); + if (!parsed.isJsonObject()) throw bad(); + JsonObject root = parsed.getAsJsonObject(); + requireOnly(root, "server"); + return HttpTlsIdentity.canonicalServerId(string(root, "server", 64)); + } catch (RuntimeException invalid) { throw bad(); } + } + + static HttpTlsIdentity.IssuedClientCertificate parseEnrollmentResponse(String server, byte[] body) { + if (body == null || body.length == 0 || body.length > MAX_BODY_BYTES) throw bad(); + try { + JsonObject root = JsonParser.parseString(new String(body, StandardCharsets.UTF_8)).getAsJsonObject(); + requireOnly(root, "bundle", "password"); + byte[] bundle = Base64.getUrlDecoder().decode(string(root, "bundle", MAX_BODY_BYTES * 2)); + char[] password = string(root, "password", 128).toCharArray(); + if (bundle.length == 0 || password.length < 40) throw bad(); + try { return new HttpTlsIdentity.IssuedClientCertificate(HttpTlsIdentity.canonicalServerId(server), null, bundle, password); } + finally { java.util.Arrays.fill(bundle, (byte) 0); java.util.Arrays.fill(password, '\0'); } + } catch (RuntimeException invalid) { throw bad(); } + } + + private static JsonObject base(String server, String session, long sequence) { + JsonObject root = new JsonObject(); + root.addProperty("v", VERSION); root.addProperty("server", server); root.addProperty("session", session); + root.addProperty("sequence", sequence); root.addProperty("timestamp", Instant.now().toEpochMilli()); + return root; + } + private static JsonArray ids(Collection values) { + if (values == null || values.size() > MAX_BATCH) throw bad(); + JsonArray output = new JsonArray(); + for (String value : values) { validId(value); output.add(value); } + return output; + } + private static JsonArray messages(Collection values) { + if (values == null || values.size() > MAX_BATCH) throw bad(); + JsonArray output = new JsonArray(); + for (Delivery delivery : values) { + validId(delivery.id()); + String encoded = JsonEnvelopeCodec.encode(delivery.envelope()); + byte[] bytes = encoded.getBytes(StandardCharsets.UTF_8); + if (bytes.length > MAX_ENVELOPE_BYTES) throw bad(); + JsonObject item = new JsonObject(); item.addProperty("id", delivery.id()); + item.addProperty("payload", Base64.getUrlEncoder().withoutPadding().encodeToString(bytes)); output.add(item); + } + return output; + } + private static List parseIds(JsonElement value) { + if (value == null || !value.isJsonArray() || value.getAsJsonArray().size() > MAX_BATCH) throw bad(); + List output = new ArrayList<>(); + for (JsonElement item : value.getAsJsonArray()) { if (!item.isJsonPrimitive()) throw bad(); String id = item.getAsString(); validId(id); output.add(id); } + return output; + } + private static List parseMessages(JsonElement value) { + if (value == null || !value.isJsonArray() || value.getAsJsonArray().size() > MAX_BATCH) throw bad(); + List output = new ArrayList<>(); + for (JsonElement item : value.getAsJsonArray()) { + if (!item.isJsonObject()) throw bad(); JsonObject object = item.getAsJsonObject(); requireOnly(object, "id", "payload"); + String id = string(object, "id", 64); validId(id); + byte[] payload = Base64.getUrlDecoder().decode(string(object, "payload", MAX_ENVELOPE_BYTES * 2)); + if (payload.length == 0 || payload.length > MAX_ENVELOPE_BYTES) throw bad(); + JsonEnvelope envelope = JsonEnvelopeCodec.decode(new String(payload, StandardCharsets.UTF_8)); + output.add(new Delivery(id, envelope)); + } + return output; + } + private static void requireOnly(JsonObject object, String... names) { + for (String name : object.keySet()) { boolean found = false; for (String allowed : names) if (allowed.equals(name)) { found = true; break; } if (!found) throw bad(); } + for (String name : names) if (!object.has(name) || object.get(name).isJsonNull()) throw bad(); + } + private static String string(JsonObject object, String name, int max) { JsonElement v = object.get(name); if (!v.isJsonPrimitive() || !v.getAsJsonPrimitive().isString()) throw bad(); String value = v.getAsString(); if (value.isEmpty() || value.length() > max) throw bad(); return value; } + private static long integer(JsonObject object, String name) { + try { + JsonElement value = object.get(name); + if (!value.isJsonPrimitive() || !value.getAsJsonPrimitive().isNumber()) throw bad(); + String token = value.getAsString(); + if (!token.matches("-?(?:0|[1-9][0-9]*)")) throw bad(); + return Long.parseLong(token); + } catch (RuntimeException failure) { throw bad(); } + } + private static long nonNegative(JsonObject object, String name) { long n = integer(object, name); if (n < 0) throw bad(); return n; } + private static String uuid(JsonObject object, String name) { return canonicalUuid(string(object, name, 64)); } + static void validId(String id) { if (id == null || id.length() > 64) throw bad(); canonicalUuid(id); } + private static String canonicalUuid(String value) { + try { + UUID parsed = UUID.fromString(value); + if (!parsed.toString().equals(value)) throw bad(); + return value; + } catch (IllegalArgumentException invalid) { throw bad(); } + } + private static IllegalArgumentException bad() { return new IllegalArgumentException("Invalid HTTP transport message"); } + + record Delivery(String id, JsonEnvelope envelope) { } + record Packet(String server, String session, long sequence, List acks, + List ackConfirmations, List messages) { } + record Enrollment(String server, String token) { } +} diff --git a/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpTransportSecrets.java b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpTransportSecrets.java new file mode 100644 index 0000000..efd0b3b --- /dev/null +++ b/SimpleAPI/src/main/java/com/bencodez/simpleapi/servercomm/http/HttpTransportSecrets.java @@ -0,0 +1,64 @@ +package com.bencodez.simpleapi.servercomm.http; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.security.SecureRandom; +import java.security.cert.X509Certificate; +import java.util.Base64; +import javax.crypto.Mac; +import javax.crypto.spec.SecretKeySpec; + +/** Small, deliberately dependency-free cryptographic helpers for the HTTP transport. */ +final class HttpTransportSecrets { + private static final SecureRandom RANDOM = new SecureRandom(); + + private HttpTransportSecrets() { } + + static byte[] randomBytes(int length) { + if (length < 16) throw new IllegalArgumentException("Secret length is too small"); + byte[] value = new byte[length]; + RANDOM.nextBytes(value); + return value; + } + + static String randomToken() { + return Base64.getUrlEncoder().withoutPadding().encodeToString(randomBytes(32)); + } + + static byte[] sha256(byte[] value) { + try { + return MessageDigest.getInstance("SHA-256").digest(value); + } catch (NoSuchAlgorithmException impossible) { + throw new IllegalStateException("SHA-256 is unavailable", impossible); + } + } + + static String sha256Hex(byte[] value) { + StringBuilder output = new StringBuilder(64); + for (byte part : sha256(value)) output.append(String.format("%02x", part & 0xff)); + return output.toString(); + } + + static String certificatePin(X509Certificate certificate) { + try { + return sha256Hex(certificate.getEncoded()); + } catch (Exception failure) { + throw new IllegalArgumentException("Could not encode certificate", failure); + } + } + + static boolean constantTimeEquals(byte[] first, byte[] second) { + return first != null && second != null && MessageDigest.isEqual(first, second); + } + + static String hmacSha256Url(byte[] key, String value) { + try { + Mac mac = Mac.getInstance("HmacSHA256"); + mac.init(new SecretKeySpec(key, "HmacSHA256")); + return Base64.getUrlEncoder().withoutPadding().encodeToString(mac.doFinal(value.getBytes(StandardCharsets.US_ASCII))); + } catch (Exception failure) { + throw new IllegalStateException("HMAC-SHA-256 is unavailable", failure); + } + } +} diff --git a/SimpleAPI/src/test/java/com/bencodez/simpleapi/file/PrivateFilePermissionsTest.java b/SimpleAPI/src/test/java/com/bencodez/simpleapi/file/PrivateFilePermissionsTest.java new file mode 100644 index 0000000..dcad178 --- /dev/null +++ b/SimpleAPI/src/test/java/com/bencodez/simpleapi/file/PrivateFilePermissionsTest.java @@ -0,0 +1,180 @@ +package com.bencodez.simpleapi.file; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.CALLS_REAL_METHODS; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.AclEntry; +import java.nio.file.attribute.AclEntryPermission; +import java.nio.file.attribute.AclEntryType; +import java.nio.file.attribute.AclFileAttributeView; +import java.nio.file.attribute.GroupPrincipal; +import java.nio.file.attribute.PosixFileAttributeView; +import java.nio.file.attribute.PosixFilePermission; +import java.nio.file.attribute.UserPrincipal; +import java.util.EnumSet; +import java.util.List; +import java.util.Set; + +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class PrivateFilePermissionsTest { + + @TempDir + Path tempDir; + + @Test + void fileIsOwnerReadWriteOnlyOnPosixProvider() throws Exception { + Path file = Files.createFile(tempDir.resolve("secret")); + Assumptions.assumeTrue(Files.getFileAttributeView(file, PosixFileAttributeView.class) != null); + + PrivateFilePermissions.ownerOnlyFile(file); + + assertEquals(Set.of(PosixFilePermission.OWNER_READ, PosixFilePermission.OWNER_WRITE), + Files.getPosixFilePermissions(file)); + } + + @Test + void directoryIsOwnerReadWriteExecuteOnlyOnPosixProvider() throws Exception { + Path directory = Files.createDirectory(tempDir.resolve("private")); + Assumptions.assumeTrue(Files.getFileAttributeView(directory, PosixFileAttributeView.class) != null); + + PrivateFilePermissions.ownerOnlyDirectory(directory); + + assertEquals(Set.of(PosixFilePermission.OWNER_READ, PosixFilePermission.OWNER_WRITE, + PosixFilePermission.OWNER_EXECUTE), Files.getPosixFilePermissions(directory)); + } + + @Test + void aclReplacementLeavesOnlyOwnerAllowEntry() throws Exception { + Path file = tempDir.resolve("acl-secret"); + UserPrincipal owner = mock(UserPrincipal.class); + AclFileAttributeView acl = mock(AclFileAttributeView.class); + when(acl.getOwner()).thenReturn(owner); + when(acl.getAcl()).thenReturn(List.of(allow(owner), allow(mock(UserPrincipal.class)))); + try (var files = mockStatic(Files.class, CALLS_REAL_METHODS)) { + stubAcl(files, file, acl); + doAnswer(invocation -> { + when(acl.getAcl()).thenReturn(invocation.getArgument(0)); + return null; + }).when(acl).setAcl(org.mockito.ArgumentMatchers.anyList()); + + PrivateFilePermissions.ownerOnlyFile(file); + + var replacement = org.mockito.ArgumentCaptor.forClass(List.class); + verify(acl).setAcl(replacement.capture()); + assertEquals(List.of(allow(owner)), replacement.getValue()); + } + } + + @Test + void ignoredSetAclOrRemainingNonOwnerAccessIsRejected() throws Exception { + Path file = tempDir.resolve("bad-acl"); + UserPrincipal owner = mock(UserPrincipal.class); + AclFileAttributeView acl = mock(AclFileAttributeView.class); + when(acl.getOwner()).thenReturn(owner); + when(acl.getAcl()).thenReturn(List.of(allow(owner), allow(mock(UserPrincipal.class)))); + doAnswer(invocation -> null).when(acl).setAcl(org.mockito.ArgumentMatchers.anyList()); + try (var files = mockStatic(Files.class, CALLS_REAL_METHODS)) { + stubAcl(files, file, acl); + + assertThrows(IOException.class, () -> PrivateFilePermissions.ownerOnlyFile(file)); + } + } + + @Test + void missingAclAndPosixSupportIsRejected() throws Exception { + Path file = tempDir.resolve("unsupported"); + try (var files = mockStatic(Files.class, CALLS_REAL_METHODS)) { + files.when(() -> Files.setPosixFilePermissions(eq(file), org.mockito.ArgumentMatchers.anySet())) + .thenThrow(new UnsupportedOperationException()); + files.when(() -> Files.getFileAttributeView(file, PosixFileAttributeView.class)).thenReturn(null); + files.when(() -> Files.getFileAttributeView(file, AclFileAttributeView.class)).thenReturn(null); + files.when(() -> Files.getFileAttributeView(file, AclFileAttributeView.class, + java.nio.file.LinkOption.NOFOLLOW_LINKS)).thenReturn(null); + assertThrows(IOException.class, () -> PrivateFilePermissions.ownerOnlyFile(file)); + } + } + + @Test + void groupOwnerIsRejected() throws Exception { + Path file = tempDir.resolve("group-owner"); + AclFileAttributeView acl = mock(AclFileAttributeView.class); + when(acl.getOwner()).thenReturn(mock(GroupPrincipal.class)); + try (var files = mockStatic(Files.class, CALLS_REAL_METHODS)) { + stubAcl(files, file, acl); + assertThrows(IOException.class, () -> PrivateFilePermissions.ownerOnlyFile(file)); + } + } + + @Test + void aclReadbackFailureIsRejected() throws Exception { + Path file = tempDir.resolve("readback-failure"); + UserPrincipal owner = mock(UserPrincipal.class); + AclFileAttributeView acl = mock(AclFileAttributeView.class); + when(acl.getOwner()).thenReturn(owner); + when(acl.getAcl()).thenThrow(new IOException("readback")); + try (var files = mockStatic(Files.class, CALLS_REAL_METHODS)) { + stubAcl(files, file, acl); + assertThrows(IOException.class, () -> PrivateFilePermissions.ownerOnlyFile(file)); + } + } + + @Test + void unsupportedAclMutationIsRejectedAsIoException() throws Exception { + Path file = tempDir.resolve("acl-unsupported-mutation"); + UserPrincipal owner = mock(UserPrincipal.class); + AclFileAttributeView acl = mock(AclFileAttributeView.class); + when(acl.getOwner()).thenReturn(owner); + doAnswer(invocation -> { throw new UnsupportedOperationException("ACL mutation"); }) + .when(acl).setAcl(org.mockito.ArgumentMatchers.anyList()); + try (var files = mockStatic(Files.class, CALLS_REAL_METHODS)) { + stubAcl(files, file, acl); + assertThrows(IOException.class, () -> PrivateFilePermissions.ownerOnlyFile(file)); + } + } + + @Test + void symlinkIsRejectedWithoutChangingReferentPermissions() throws Exception { + Path target = Files.createFile(tempDir.resolve("referent")); + Assumptions.assumeTrue(Files.getFileAttributeView(target, PosixFileAttributeView.class) != null); + Set original = Set.of(PosixFilePermission.OWNER_READ, + PosixFilePermission.OWNER_WRITE, PosixFilePermission.GROUP_READ); + Files.setPosixFilePermissions(target, original); + Path link = tempDir.resolve("referent-link"); + try { + Files.createSymbolicLink(link, target.getFileName()); + } catch (UnsupportedOperationException | IOException unavailable) { + Assumptions.assumeTrue(false, "symbolic links unavailable: " + unavailable); + return; + } + + assertThrows(IOException.class, () -> PrivateFilePermissions.ownerOnlyFile(link)); + assertEquals(original, Files.getPosixFilePermissions(target)); + } + + private static AclEntry allow(UserPrincipal principal) { + return AclEntry.newBuilder().setType(AclEntryType.ALLOW).setPrincipal(principal) + .setPermissions(EnumSet.allOf(AclEntryPermission.class)).setFlags(Set.of()).build(); + } + + private static void stubAcl(org.mockito.MockedStatic files, Path path, AclFileAttributeView acl) { + files.when(() -> Files.setPosixFilePermissions(eq(path), org.mockito.ArgumentMatchers.anySet())) + .thenThrow(new UnsupportedOperationException()); + files.when(() -> Files.getFileAttributeView(path, PosixFileAttributeView.class)).thenReturn(null); + files.when(() -> Files.getFileAttributeView(path, AclFileAttributeView.class)).thenReturn(acl); + files.when(() -> Files.getFileAttributeView(path, AclFileAttributeView.class, + java.nio.file.LinkOption.NOFOLLOW_LINKS)).thenReturn(acl); + } +} diff --git a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpBackendLifecycleRecoveryTest.java b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpBackendLifecycleRecoveryTest.java new file mode 100644 index 0000000..c6e90e9 --- /dev/null +++ b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpBackendLifecycleRecoveryTest.java @@ -0,0 +1,51 @@ +package com.bencodez.simpleapi.servercomm.http; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.bencodez.simpleapi.servercomm.codec.JsonEnvelope; +import java.net.URI; +import java.nio.file.Path; +import java.time.Instant; +import java.util.UUID; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class HttpBackendLifecycleRecoveryTest { + @TempDir Path directory; + + @Test + void callbackOwnedCloseCompletesItsJournalBeforeAsynchronousSealing() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); + HttpTlsIdentity.IssuedClientCertificate issued = identity.issueClientCertificate("lobby-1"); + HttpConnectionCode code = new HttpConnectionCode("lobby-1", URI.create("https://localhost:8443/"), + identity.serverCertificatePin(), identity.caCertificatePin(), Instant.now().plusSeconds(300), "A".repeat(43)); + Path clientDirectory = directory.resolve("client"); + HttpClientCredentialStore.saveEnrolled(clientDirectory, code, issued); + AtomicReference connector = new AtomicReference<>(); + CountDownLatch callbackReturned = new CountDownLatch(1); + connector.set(new HttpBackendTransportConnector(clientDirectory, ignored -> { + connector.get().close(); + callbackReturned.countDown(); + })); + String id = UUID.randomUUID().toString(); + connector.get().dispatch(new HttpTransportProtocol.Delivery(id, JsonEnvelope.builder("self-close").build())); + assertTrue(callbackReturned.await(2, TimeUnit.SECONDS), "callback-owned close must not drain its own worker"); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(2); + HttpInboundDeliveryStore.State state = null; + while (System.nanoTime() < deadline) { + state = HttpInboundDeliveryStore.inspect(clientDirectory).state(id); + if (state == HttpInboundDeliveryStore.State.COMPLETED) break; + Thread.sleep(5); + } + assertEquals(HttpInboundDeliveryStore.State.COMPLETED, state, + "the callback wrapper must mark completed before close seals the journal"); + connector.get().start(); + assertFalse(connector.get().pollerAlive(), "a closed connector must not restart its poller"); + connector.get().close(); + } +} diff --git a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpBackendSendLifecycleTest.java b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpBackendSendLifecycleTest.java new file mode 100644 index 0000000..ca7d64a --- /dev/null +++ b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpBackendSendLifecycleTest.java @@ -0,0 +1,136 @@ +package com.bencodez.simpleapi.servercomm.http; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.bencodez.simpleapi.servercomm.codec.JsonEnvelope; +import java.lang.reflect.Field; +import java.net.InetSocketAddress; +import java.nio.file.Path; +import java.time.Duration; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class HttpBackendSendLifecycleTest { + @TempDir Path directory; + + @Test + void pausedSendCannotEnterAfterCloseAndFlushCutsOffNewSends() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, directory.resolve("authority")); + try (HttpProxyTransportServer server = new HttpProxyTransportServer(new InetSocketAddress("localhost", 0), identity, + authority, ignored -> { })) { + server.start(); + HttpConnectionCode code = authority.createConnectionCode("lobby-1", server.endpoint("localhost"), Duration.ofMinutes(5)); + Path firstDirectory = directory.resolve("close-client"); + HttpBackendTransportConnector.enroll(code, "lobby-1", firstDirectory); + HttpBackendTransportConnector first = new HttpBackendTransportConnector(firstDirectory, ignored -> { }); + try { + first.start(); + assertTrue(first.awaitFirstResponse(System.nanoTime() + TimeUnit.SECONDS.toNanos(8))); + assertPausedSendIsRejectedByClose(first); + } finally { first.close(); } + + Path secondDirectory = directory.resolve("flush-client"); + HttpConnectionCode secondCode = authority.createConnectionCode("lobby-2", server.endpoint("localhost"), Duration.ofMinutes(5)); + HttpBackendTransportConnector.enroll(secondCode, "lobby-2", secondDirectory); + try (HttpBackendTransportConnector second = new HttpBackendTransportConnector(secondDirectory, ignored -> { })) { + second.start(); + assertTrue(second.awaitFirstResponse(System.nanoTime() + TimeUnit.SECONDS.toNanos(8))); + assertTrue(second.send(JsonEnvelope.builder("accepted-before-flush").build())); + assertPausedSendIsRejectedByFlush(second); + assertFalse(second.send(JsonEnvelope.builder("rejected-after-flush").build())); + assertEquals(0, second.queuedOutgoing()); + second.start(); + assertTrue(second.send(JsonEnvelope.builder("accepted-after-restart").build())); + } + } + } + + @Test + void restartedConnectorRequiresAResponseFromItsNewRun() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("restart-proxy"), "localhost"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, directory.resolve("restart-authority")); + try (HttpProxyTransportServer server = new HttpProxyTransportServer(new InetSocketAddress("localhost", 0), identity, + authority, ignored -> { })) { + server.start(); + HttpConnectionCode code = authority.createConnectionCode("lobby-1", server.endpoint("localhost"), Duration.ofMinutes(5)); + Path clientDirectory = directory.resolve("restart-client"); + HttpBackendTransportConnector.enroll(code, "lobby-1", clientDirectory); + try (HttpBackendTransportConnector connector = new HttpBackendTransportConnector(clientDirectory, ignored -> { })) { + connector.start(); + assertTrue(connector.awaitFirstResponse(System.nanoTime() + TimeUnit.SECONDS.toNanos(8))); + assertTrue(connector.flushOutgoing(System.nanoTime() + TimeUnit.SECONDS.toNanos(5))); + server.close(); + + connector.start(); + assertFalse(connector.awaitFirstResponse(System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(250)), + "a completed flush must not satisfy the restarted run's first-response wait"); + } + } + } + + private static void assertPausedSendIsRejectedByClose(HttpBackendTransportConnector connector) throws Exception { + Object state = field("state").get(connector); + AtomicBoolean closing = (AtomicBoolean) field("closing").get(connector); + AtomicBoolean accepted = new AtomicBoolean(true); + CountDownLatch senderStarted = new CountDownLatch(1); + Thread sender = new Thread(() -> { + senderStarted.countDown(); + accepted.set(connector.send(JsonEnvelope.builder("paused-send").build())); + }, "paused-backend-send"); + Thread closer = new Thread(connector::close, "backend-close"); + synchronized (state) { + sender.start(); + assertTrue(senderStarted.await(1, TimeUnit.SECONDS)); + closer.start(); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(2); + while (!closing.get() && System.nanoTime() < deadline) Thread.onSpinWait(); + assertTrue(closing.get(), "close must transition running before it waits for state"); + } + sender.join(3000); + closer.join(3000); + assertFalse(sender.isAlive()); + assertFalse(closer.isAlive()); + assertFalse(accepted.get()); + assertEquals(0, connector.queuedOutgoing()); + } + + private static void assertPausedSendIsRejectedByFlush(HttpBackendTransportConnector connector) throws Exception { + Object state = field("state").get(connector); + AtomicBoolean running = (AtomicBoolean) field("running").get(connector); + AtomicBoolean accepted = new AtomicBoolean(true), flushed = new AtomicBoolean(); + CountDownLatch senderStarted = new CountDownLatch(1); + Thread sender = new Thread(() -> { + senderStarted.countDown(); + accepted.set(connector.send(JsonEnvelope.builder("paused-flush-send").build())); + }, "paused-flush-send"); + Thread flusher = new Thread(() -> flushed.set(connector.flushOutgoing( + System.nanoTime() + TimeUnit.SECONDS.toNanos(5))), "backend-flush"); + synchronized (state) { + sender.start(); + assertTrue(senderStarted.await(1, TimeUnit.SECONDS)); + flusher.start(); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(2); + while (running.get() && System.nanoTime() < deadline) Thread.onSpinWait(); + assertFalse(running.get(), "flush must cut off admission before it waits for state"); + } + sender.join(3000); + flusher.join(6000); + assertFalse(sender.isAlive()); + assertFalse(flusher.isAlive()); + assertFalse(accepted.get()); + assertTrue(flushed.get()); + assertEquals(0, connector.queuedOutgoing()); + } + + private static Field field(String name) throws Exception { + Field field = HttpBackendTransportConnector.class.getDeclaredField(name); + field.setAccessible(true); + return field; + } +} diff --git a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpBackendTransportConnectorConstructorTest.java b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpBackendTransportConnectorConstructorTest.java new file mode 100644 index 0000000..5e61ad8 --- /dev/null +++ b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpBackendTransportConnectorConstructorTest.java @@ -0,0 +1,34 @@ +package com.bencodez.simpleapi.servercomm.http; + +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.net.InetSocketAddress; +import java.nio.file.Path; +import java.time.Duration; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class HttpBackendTransportConnectorConstructorTest { + @TempDir Path directory; + + @Test + void invalidConnectionCodeDoesNotPreventImmediateValidRetry() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, directory.resolve("authority")); + try (HttpProxyTransportServer server = new HttpProxyTransportServer(new InetSocketAddress("localhost", 0), + identity, authority, ignored -> { })) { + server.start(); + HttpConnectionCode code = authority.createConnectionCode("lobby-1", server.endpoint("localhost"), + Duration.ofMinutes(5)); + Path credentials = directory.resolve("client"); + HttpBackendTransportConnector.enroll(code, "lobby-1", credentials); + + assertThrows(IllegalArgumentException.class, + () -> new HttpBackendTransportConnector(null, "lobby-1", credentials, ignored -> { })); + try (HttpBackendTransportConnector ignored = new HttpBackendTransportConnector(code, "lobby-1", + credentials, ignoredMessage -> { })) { + // The valid retry must be able to claim both durable journal locks. + } + } + } +} diff --git a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentAuthorityOwnershipTest.java b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentAuthorityOwnershipTest.java new file mode 100644 index 0000000..11e8eaf --- /dev/null +++ b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentAuthorityOwnershipTest.java @@ -0,0 +1,36 @@ +package com.bencodez.simpleapi.servercomm.http; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +import java.net.URI; +import java.nio.file.Path; +import java.time.Duration; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class HttpEnrollmentAuthorityOwnershipTest { + @TempDir Path directory; + + @Test + void staleAuthorityCannotEraseAnotherEnrollmentOrRestoreARevokedBinding() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("identity"), "localhost"); + Path state = directory.resolve("authority"); + HttpEnrollmentAuthority first = new HttpEnrollmentAuthority(identity, state); + HttpEnrollmentAuthority stale = new HttpEnrollmentAuthority(identity, state); + URI endpoint = URI.create("https://localhost:8443/"); + + HttpConnectionCode firstCode = first.createConnectionCode("lobby-1", endpoint, Duration.ofMinutes(5)); + HttpConnectionCode secondCode = stale.createConnectionCode("lobby-2", endpoint, Duration.ofMinutes(5)); + assertNotNull(first.enroll("lobby-1", firstCode.enrollmentToken()), + "a stale authority rewrite must retain another instance's pending enrollment"); + var secondCertificate = stale.enroll("lobby-2", secondCode.enrollmentToken()); + assertNotNull(secondCertificate); + + first.revoke("lobby-2"); + stale.createConnectionCode("lobby-3", endpoint, Duration.ofMinutes(5)); + HttpEnrollmentAuthority reloaded = new HttpEnrollmentAuthority(identity, state); + assertFalse(reloaded.authenticate("lobby-2", secondCertificate.certificate()), + "a stale authority rewrite must not restore a revoked certificate"); + } +} diff --git a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentPinTest.java b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentPinTest.java new file mode 100644 index 0000000..6df6ae4 --- /dev/null +++ b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentPinTest.java @@ -0,0 +1,186 @@ +package com.bencodez.simpleapi.servercomm.http; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.net.URI; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.nio.file.Path; +import java.security.KeyPair; +import java.security.KeyPairGenerator; +import java.security.KeyStore; +import java.time.Duration; +import java.time.Instant; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class HttpEnrollmentPinTest { + @TempDir Path directory; + + @Test + void enrollmentLifetimeRequiresAtLeastOneSecond() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); + java.time.Clock clock = java.time.Clock.fixed(Instant.parse("2026-09-05T12:00:00.999Z"), + java.time.ZoneOffset.UTC); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, clock); + URI endpoint = URI.create("https://localhost:8443/"); + for (Duration lifetime : new Duration[] { Duration.ofNanos(1), Duration.ofMillis(999), + Duration.ofSeconds(1).minusNanos(1), Duration.ZERO, Duration.ofSeconds(-1) }) { + assertThrows(IllegalArgumentException.class, + () -> authority.createConnectionCode("lobby-1", endpoint, lifetime)); + } + HttpConnectionCode minimum = authority.createConnectionCode("lobby-1", endpoint, Duration.ofSeconds(1)); + org.junit.jupiter.api.Assertions.assertTrue(HttpConnectionCode.parse(minimum.encode()).expiresAt().isAfter(clock.instant())); + } + + @Test + void rejectsDifferentCaBundleDuringInitialEnrollment() throws Exception { + HttpTlsIdentity proxy = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); + HttpTlsIdentity foreign = HttpTlsIdentity.loadOrCreate(directory.resolve("foreign"), "localhost"); + HttpConnectionCode code = code(proxy, "lobby-1"); + + assertThrows(java.io.IOException.class, () -> HttpClientCredentialStore.saveEnrolled( + directory.resolve("client"), code, foreign.issueClientCertificate("lobby-1"))); + assertFalse(HttpClientCredentialStore.hasEnrolledProfile(directory.resolve("client"))); + } + + @Test + void rejectsDifferentCaBundleWithoutReplacingExistingEnrollment() throws Exception { + HttpTlsIdentity proxy = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); + HttpTlsIdentity foreign = HttpTlsIdentity.loadOrCreate(directory.resolve("foreign"), "localhost"); + Path client = directory.resolve("client"); + HttpConnectionCode code = code(proxy, "lobby-1"); + HttpTlsIdentity.IssuedClientCertificate original = proxy.issueClientCertificate("lobby-1"); + HttpClientCredentialStore.saveEnrolled(client, code, original); + HttpClientCredentialStore.EnrolledClient before = HttpClientCredentialStore.loadEnrolled(client); + + assertThrows(java.io.IOException.class, () -> HttpClientCredentialStore.saveEnrolled( + client, code, foreign.issueClientCertificate("lobby-1"))); + HttpClientCredentialStore.EnrolledClient after = HttpClientCredentialStore.loadEnrolled(client); + assertEquals(before.profile(), after.profile()); + assertArrayEquals(before.credential().certificate().getEncoded(), after.credential().certificate().getEncoded()); + } + + @Test + void acceptsBundleSignedByConnectionCodeCa() throws Exception { + HttpTlsIdentity proxy = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); + Path client = directory.resolve("client"); + HttpConnectionCode code = code(proxy, "lobby-1"); + + HttpClientCredentialStore.saveEnrolled(client, code, proxy.issueClientCertificate("lobby-1")); + HttpClientCredentialStore.EnrolledClient enrolled = HttpClientCredentialStore.loadEnrolled(client); + assertEquals(code.caCertificatePin(), enrolled.profile().caCertificatePin()); + assertEquals(code.caCertificatePin(), HttpTransportSecrets.certificatePin(enrolled.credential().caCertificate())); + } + + @Test + void rejectsPkcs12WithUnrelatedPrivateKeyDuringEnrollmentAndReplacement() throws Exception { + HttpTlsIdentity proxy = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); + HttpConnectionCode code = code(proxy, "lobby-1"); + HttpTlsIdentity.IssuedClientCertificate original = proxy.issueClientCertificate("lobby-1"); + HttpTlsIdentity.IssuedClientCertificate mismatched = withUnrelatedPrivateKey(original); + Path client = directory.resolve("client"); + + assertThrows(java.io.IOException.class, () -> HttpClientCredentialStore.saveEnrolled(client, code, mismatched)); + assertFalse(HttpClientCredentialStore.hasEnrolledProfile(client)); + + HttpClientCredentialStore.saveEnrolled(client, code, original); + HttpClientCredentialStore.EnrolledClient before = HttpClientCredentialStore.loadEnrolled(client); + assertThrows(java.io.IOException.class, () -> HttpClientCredentialStore.stageReplacement(client, mismatched)); + HttpClientCredentialStore.EnrolledClient after = HttpClientCredentialStore.loadEnrolled(client); + assertEquals(before.profile(), after.profile()); + assertArrayEquals(before.credential().certificate().getEncoded(), after.credential().certificate().getEncoded()); + } + + @Test + void stageReplacementRejectsForeignCaAndPreservesActiveEnrollment() throws Exception { + HttpTlsIdentity proxy = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); + HttpTlsIdentity foreign = HttpTlsIdentity.loadOrCreate(directory.resolve("foreign"), "localhost"); + Path client = directory.resolve("client"); + HttpConnectionCode code = code(proxy, "lobby-1"); + HttpTlsIdentity.IssuedClientCertificate originalCredential = proxy.issueClientCertificate("lobby-1"); + HttpClientCredentialStore.saveEnrolled(client, code, originalCredential); + HttpClientCredentialStore.EnrolledClient before = HttpClientCredentialStore.loadEnrolled(client); + + assertThrows(java.io.IOException.class, () -> HttpClientCredentialStore.stageReplacement( + client, foreign.issueClientCertificate("lobby-1"))); + HttpClientCredentialStore.EnrolledClient after = HttpClientCredentialStore.loadEnrolled(client); + assertEquals(before.profile(), after.profile()); + assertArrayEquals(before.credential().certificate().getEncoded(), after.credential().certificate().getEncoded()); + } + + @Test + void stageReplacementAcceptsCaRenewalWithSameAuthorityKey() throws Exception { + Instant now = Instant.now(); + java.time.Clock originalClock = java.time.Clock.fixed( + now.minus(Duration.ofDays(9 * 365L + 30L)), java.time.ZoneOffset.UTC); + Path proxyDirectory = directory.resolve("proxy"); + HttpTlsIdentity original = HttpTlsIdentity.loadOrCreate(proxyDirectory, "localhost", originalClock); + HttpTlsIdentity.IssuedClientCertificate originalCredential = original.issueClientCertificate("lobby-1", now); + String originalCaPin = HttpTransportSecrets.certificatePin(original.caCertificate()); + java.security.PublicKey originalCaKey = original.caCertificate().getPublicKey(); + Path client = directory.resolve("client"); + HttpConnectionCode code = new HttpConnectionCode("lobby-1", URI.create("https://localhost:8443/"), + HttpTransportSecrets.certificatePin(original.serverCertificate()), + HttpTransportSecrets.certificatePin(original.caCertificate()), now.plusSeconds(60), "A".repeat(43)); + HttpClientCredentialStore.saveEnrolled(client, code, originalCredential); + + HttpTlsIdentity renewed = HttpTlsIdentity.loadOrCreate(proxyDirectory, "localhost", + java.time.Clock.fixed(now, java.time.ZoneOffset.UTC)); + assertNotEquals(originalCaPin, renewed.caCertificatePin()); + assertEquals(originalCaKey, renewed.caCertificate().getPublicKey()); + + HttpClientCredentialStore.StagedCredential staged = HttpClientCredentialStore.stageReplacement( + client, renewed.issueClientCertificate("lobby-1", now)); + assertEquals(renewed.caCertificatePin(), staged.profile().caCertificatePin()); + HttpClientCredentialStore.activateReplacement(client, staged); + assertEquals(renewed.caCertificatePin(), HttpClientCredentialStore.loadProfile(client).caCertificatePin()); + } + + @Test + void invalidConnectionCodeEndpointsDoNotConsumePendingCapacityOrBreakReload() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); + Path state = directory.resolve("state"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, state); + for (int i = 0; i < 128; i++) { + final int index = i; + URI endpoint = (i & 1) == 0 ? null : URI.create("http://localhost:8443/"); + assertThrows(IllegalArgumentException.class, () -> authority.createConnectionCode( + "lobby-" + index, endpoint, Duration.ofMinutes(5))); + } + + HttpConnectionCode first = authority.createConnectionCode("lobby-1", URI.create("https://localhost:8443/"), + Duration.ofMinutes(5)); + assertEquals("lobby-1", first.serverId()); + HttpEnrollmentAuthority reloaded = new HttpEnrollmentAuthority(identity, state); + HttpConnectionCode second = reloaded.createConnectionCode("lobby-2", URI.create("https://localhost:8443/"), + Duration.ofMinutes(5)); + assertEquals("lobby-2", second.serverId()); + } + + private HttpConnectionCode code(HttpTlsIdentity identity, String serverId) { + return new HttpConnectionCode(serverId, URI.create("https://localhost:8443/"), + identity.serverCertificatePin(), identity.caCertificatePin(), Instant.now().plusSeconds(300), + HttpTransportSecrets.randomToken()); + } + + private HttpTlsIdentity.IssuedClientCertificate withUnrelatedPrivateKey( + HttpTlsIdentity.IssuedClientCertificate issued) throws Exception { + char[] password = issued.password(); + KeyStore source = KeyStore.getInstance("PKCS12"); + source.load(new ByteArrayInputStream(issued.pkcs12()), password); + KeyPairGenerator generator = KeyPairGenerator.getInstance("EC"); + generator.initialize(new java.security.spec.ECGenParameterSpec("secp256r1")); + KeyPair unrelated = generator.generateKeyPair(); + KeyStore replacement = KeyStore.getInstance("PKCS12"); + replacement.load(null, password); + replacement.setKeyEntry("client", unrelated.getPrivate(), password, source.getCertificateChain("client")); + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + replacement.store(bytes, password); + return new HttpTlsIdentity.IssuedClientCertificate(issued.serverId(), issued.certificate(), bytes.toByteArray(), password); + } +} diff --git a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentPublicationRecoveryTest.java b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentPublicationRecoveryTest.java new file mode 100644 index 0000000..a050fdd --- /dev/null +++ b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpEnrollmentPublicationRecoveryTest.java @@ -0,0 +1,51 @@ +package com.bencodez.simpleapi.servercomm.http; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.net.InetSocketAddress; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.time.Duration; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import com.bencodez.simpleapi.file.DurableFiles; + +class HttpEnrollmentPublicationRecoveryTest { + @TempDir Path directory; + + @Test + void publishedInitialCredentialIsConfirmedWithoutReusingTheEnrollmentToken() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, directory.resolve("authority")); + Path credentials = directory.resolve("client").toAbsolutePath().normalize(); + HttpConnectionCode code; + HttpClientCredentialStore.ClientCredential published; + try (HttpProxyTransportServer server = new HttpProxyTransportServer(new InetSocketAddress("localhost", 0), + identity, authority, ignored -> { })) { + server.start(); + code = authority.createConnectionCode("lobby-1", server.endpoint("localhost"), Duration.ofMinutes(5)); + Path current = credentials.resolve("http-transport-client-current"); + try (var forces = org.mockito.Mockito.mockStatic(DurableFiles.class, org.mockito.Mockito.CALLS_REAL_METHODS)) { + forces.when(() -> DurableFiles.forceDirectory(credentials)).thenAnswer(call -> { + if (Files.isRegularFile(current, LinkOption.NOFOLLOW_LINKS)) + throw new java.io.IOException("injected CURRENT publication failure"); + return call.callRealMethod(); + }); + assertThrows(DurableFiles.PublishedException.class, + () -> HttpBackendTransportConnector.enroll(code, "lobby-1", credentials)); + } + published = HttpClientCredentialStore.load(credentials); + } + + // The endpoint is now closed. Recovery can succeed only by confirming the + // already-published CURRENT pointer rather than sending the token again. + HttpClientCredentialStore.ClientCredential recovered = HttpBackendTransportConnector.enroll( + code, "lobby-1", credentials); + assertArrayEquals(published.certificate().getEncoded(), recovered.certificate().getEncoded()); + assertTrue(authority.authenticate("lobby-1", recovered.certificate())); + } +} diff --git a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpInboundJournalCapacityTest.java b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpInboundJournalCapacityTest.java new file mode 100644 index 0000000..47f7a2e --- /dev/null +++ b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpInboundJournalCapacityTest.java @@ -0,0 +1,110 @@ +package com.bencodez.simpleapi.servercomm.http; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.bencodez.simpleapi.servercomm.codec.JsonEnvelope; +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.UUID; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class HttpInboundJournalCapacityTest { + @TempDir Path directory; + + @Test + void restoresRunningInboundJournalsAndEnforcesCapacityAcrossRestart() throws Exception { + Path outgoing = directory.resolve("outgoing"); + Files.createDirectory(outgoing); + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("identity"), "localhost"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, directory.resolve("authority")); + seedRunningJournals(outgoing, 128); + + try (HttpProxyTransportServer proxy = new HttpProxyTransportServer(new InetSocketAddress("localhost", 0), + identity, authority, outgoing, ignored -> { })) { + assertEquals(128, proxy.backendCountForTest(), "validated journals must restore backend state"); + assertNotNull(proxy.backendStateForTest("server-0")); + assertThrows(java.io.IOException.class, () -> proxy.backendStateForTest("server-128"), + "a 129th backend must be rejected while durable journals occupy the cap"); + } + + try (HttpProxyTransportServer restarted = new HttpProxyTransportServer(new InetSocketAddress("localhost", 0), + identity, authority, outgoing, ignored -> { })) { + assertEquals(128, restarted.backendCountForTest(), "restart must retain the restored backend count"); + assertThrows(java.io.IOException.class, () -> restarted.backendStateForTest("server-128"), + "the cap must remain enforced after restart"); + } + } + + @Test + void prunesEmptySafeJournalsAndAllowsAReplacementBackend() throws Exception { + Path outgoing = directory.resolve("outgoing"); + Files.createDirectory(outgoing); + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("identity"), "localhost"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, directory.resolve("authority")); + seedEmptyJournals(outgoing, 128); + + try (HttpProxyTransportServer proxy = new HttpProxyTransportServer(new InetSocketAddress("localhost", 0), + identity, authority, outgoing, ignored -> { })) { + assertEquals(0, proxy.backendCountForTest(), "empty journals are safe to prune at startup"); + assertNotNull(proxy.backendStateForTest("replacement")); + assertEquals(1, proxy.backendCountForTest()); + assertFalse(Files.exists(outgoing.getParent().resolve("outgoing-incoming").resolve("server-0"))); + } + } + + @Test + void quarantineOnlyBackendIsIncludedInTheCombinedStartupCapacity() throws Exception { + Path outgoing = directory.resolve("quarantine-outgoing"); + Files.createDirectory(outgoing); + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("quarantine-identity"), "localhost"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, directory.resolve("quarantine-authority")); + seedRunningJournals(outgoing, 128); + String serverId = "server-128"; + String deliveryId = UUID.nameUUIDFromBytes(serverId.getBytes(StandardCharsets.UTF_8)).toString(); + HttpTransportProtocol.Delivery delivery = new HttpTransportProtocol.Delivery(deliveryId, + JsonEnvelope.builder("quarantined-retry").build()); + Path serverDirectory = Files.createDirectory(outgoing.resolve(serverId)); + Files.write(serverDirectory.resolve(".pending-" + deliveryId + ".json"), + HttpTransportProtocol.storedDelivery(delivery)); + + assertThrows(java.io.IOException.class, () -> new HttpProxyTransportServer( + new InetSocketAddress("localhost", 0), identity, authority, outgoing, ignored -> { }), + "129 distinct durable backend states must be rejected during startup rather than blocking a later retry"); + + try (HttpProxyTransportServer.DurableOutgoingQueue queue = new HttpProxyTransportServer.DurableOutgoingQueue( + outgoing, com.bencodez.simpleapi.file.DurableFiles::forceDirectory)) { + var loaded = queue.load(); + assertTrue(loaded.containsKey(serverId), "quarantine-only state must reserve its backend identity"); + assertTrue(loaded.get(serverId).isEmpty(), "an unconfirmed quarantine must remain hidden from delivery"); + } + } + + private static void seedRunningJournals(Path outgoing, int count) throws Exception { + Path incoming = outgoing.getParent().resolve(outgoing.getFileName() + "-incoming"); + Files.createDirectory(incoming); + for (int index = 0; index < count; index++) { + String serverId = "server-" + index; + HttpInboundDeliveryStore store = HttpInboundDeliveryStore.open(incoming, serverId); + String deliveryId = UUID.nameUUIDFromBytes(serverId.getBytes(StandardCharsets.UTF_8)).toString(); + store.reserve(deliveryId); + store.markRunning(deliveryId); + store.seal(); + } + } + + private static void seedEmptyJournals(Path outgoing, int count) throws Exception { + Path incoming = outgoing.getParent().resolve(outgoing.getFileName() + "-incoming"); + Files.createDirectory(incoming); + for (int index = 0; index < count; index++) { + HttpInboundDeliveryStore store = HttpInboundDeliveryStore.open(incoming, "server-" + index); + store.seal(); + } + } +} diff --git a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpInboundRetirementTest.java b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpInboundRetirementTest.java new file mode 100644 index 0000000..4941b09 --- /dev/null +++ b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpInboundRetirementTest.java @@ -0,0 +1,60 @@ +package com.bencodez.simpleapi.servercomm.http; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicBoolean; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class HttpInboundRetirementTest { + @TempDir Path directory; + + @Test + void failedRetirementKeepsOwnershipAndRestoresRootOnNextWrite() throws Exception { + Path parent = directory.resolve("incoming"); + Files.createDirectory(parent); + HttpInboundDeliveryStore store = HttpInboundDeliveryStore.open(parent, "lobby-1"); + AtomicBoolean fail = new AtomicBoolean(true); + try (var forces = org.mockito.Mockito.mockStatic(com.bencodez.simpleapi.file.DurableFiles.class, + org.mockito.Mockito.CALLS_REAL_METHODS)) { + forces.when(() -> com.bencodez.simpleapi.file.DurableFiles.forceDirectory(parent)).thenAnswer(call -> { + if (fail.getAndSet(false)) throw new IOException("injected retirement force failure"); + return call.callRealMethod(); + }); + assertThrows(IOException.class, store::sealAndDeleteIfEmpty); + assertThrows(IOException.class, () -> HttpInboundDeliveryStore.open(parent, "lobby-1")); + String id = UUID.randomUUID().toString(); + store.reserve(id); + store.markRunning(id); + assertEquals(HttpInboundDeliveryStore.State.RUNNING, store.state(id)); + } + store.seal(); + } + + @Test + void failedRetirementCanRetryAfterDeleteFailure() throws Exception { + Path parent = directory.resolve("incoming"); + Files.createDirectory(parent); + HttpInboundDeliveryStore store = HttpInboundDeliveryStore.open(parent, "lobby-1"); + Path root = parent.resolve("lobby-1"); + AtomicBoolean fail = new AtomicBoolean(true); + try (var files = org.mockito.Mockito.mockStatic(Files.class, org.mockito.Mockito.CALLS_REAL_METHODS)) { + files.when(() -> Files.delete(root)).thenAnswer(call -> { + if (fail.getAndSet(false)) throw new IOException("injected retirement delete failure"); + return call.callRealMethod(); + }); + // CALLS_REAL_METHODS executes the unstubbed delete while registering it. + Files.createDirectory(root); + assertThrows(IOException.class, store::sealAndDeleteIfEmpty); + assertThrows(IOException.class, () -> HttpInboundDeliveryStore.open(parent, "lobby-1")); + store.sealAndDeleteIfEmpty(); + } + HttpInboundDeliveryStore successor = HttpInboundDeliveryStore.open(parent, "lobby-1"); + successor.seal(); + } +} diff --git a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpOutgoingOwnershipTest.java b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpOutgoingOwnershipTest.java new file mode 100644 index 0000000..1fe22a8 --- /dev/null +++ b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpOutgoingOwnershipTest.java @@ -0,0 +1,88 @@ +package com.bencodez.simpleapi.servercomm.http; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.bencodez.simpleapi.servercomm.codec.JsonEnvelope; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.nio.file.Path; +import java.util.List; +import java.util.UUID; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class HttpOutgoingOwnershipTest { + @TempDir Path directory; + + @Test + void senderWaitingForBackendMonitorCannotReopenJournalAfterClose() throws Exception { + Path root = directory.resolve("outgoing"); + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, directory.resolve("authority")); + HttpProxyTransportServer server = new HttpProxyTransportServer(new InetSocketAddress("localhost", 0), + identity, authority, root, ignored -> { }); + var field = HttpProxyTransportServer.class.getDeclaredField("backends"); + field.setAccessible(true); + java.util.concurrent.atomic.AtomicBoolean sent = new java.util.concurrent.atomic.AtomicBoolean(true); + Thread sender = new Thread(() -> sent.set(server.send("lobby-1", JsonEnvelope.builder("close-race").build()))); + try { + synchronized (field.get(server)) { + sender.start(); + long deadline = System.nanoTime() + java.util.concurrent.TimeUnit.SECONDS.toNanos(3); + while (sender.getState() != Thread.State.BLOCKED && System.nanoTime() < deadline) Thread.sleep(5); + org.junit.jupiter.api.Assertions.assertEquals(Thread.State.BLOCKED, sender.getState()); + server.close(); + } + sender.join(3000); + assertFalse(sender.isAlive()); + assertFalse(sent.get()); + try (HttpProxyTransportServer successor = new HttpProxyTransportServer(new InetSocketAddress("localhost", 0), + identity, authority, root, ignored -> { })) { + org.junit.jupiter.api.Assertions.assertNotNull(successor.backendStateForTest("lobby-1")); + } + } finally { + server.close(); + sender.join(3000); + } + } + + @Test + void outgoingQueueAllowsOneOwnerAndFailsClosedAfterClose() throws Exception { + Path root = directory.resolve("outgoing"); + HttpProxyTransportServer.DurableOutgoingQueue owner = new HttpProxyTransportServer.DurableOutgoingQueue(root, + com.bencodez.simpleapi.file.DurableFiles::forceDirectory); + HttpProxyTransportServer.BackendState state = new HttpProxyTransportServer.BackendState("lobby-1", owner, + (server, id) -> { }); + HttpTransportProtocol.Delivery delivery = new HttpTransportProtocol.Delivery(UUID.randomUUID().toString(), + JsonEnvelope.builder("owner").build()); + assertTrue(state.enqueue(delivery)); + assertThrows(IOException.class, () -> new HttpProxyTransportServer.DurableOutgoingQueue(root, + com.bencodez.simpleapi.file.DurableFiles::forceDirectory)); + owner.close(); + assertThrows(IOException.class, owner::load); + assertFalse(state.enqueue(new HttpTransportProtocol.Delivery(UUID.randomUUID().toString(), + JsonEnvelope.builder("closed").build()))); + assertThrows(IOException.class, () -> state.acknowledge(List.of(delivery.id()))); + HttpProxyTransportServer.DurableOutgoingQueue successor = new HttpProxyTransportServer.DurableOutgoingQueue(root, + com.bencodez.simpleapi.file.DurableFiles::forceDirectory); + successor.close(); + } + + @Test + void proxyClaimsOutgoingOwnershipBeforeAnyBackendStateExists() throws Exception { + Path root = directory.resolve("outgoing"); + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, directory.resolve("authority")); + HttpProxyTransportServer owner = new HttpProxyTransportServer(new InetSocketAddress("localhost", 0), identity, + authority, root, ignored -> { }); + try { + assertThrows(IOException.class, () -> new HttpProxyTransportServer(new InetSocketAddress("localhost", 0), identity, + authority, root, ignored -> { }), + "even an idle proxy must own the durable outgoing queue exclusively"); + } finally { owner.close(); } + try (HttpProxyTransportServer successor = new HttpProxyTransportServer(new InetSocketAddress("localhost", 0), identity, + authority, root, ignored -> { })) { } + } +} diff --git a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpOutgoingQueueCapacityTest.java b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpOutgoingQueueCapacityTest.java new file mode 100644 index 0000000..422c3e4 --- /dev/null +++ b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpOutgoingQueueCapacityTest.java @@ -0,0 +1,78 @@ +package com.bencodez.simpleapi.servercomm.http; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.bencodez.simpleapi.servercomm.codec.JsonEnvelope; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashSet; +import java.util.Set; +import java.util.UUID; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class HttpOutgoingQueueCapacityTest { + @TempDir Path directory; + + @Test + void quarantineDirectoriesCountTowardCapacityWithoutBlockingSameServerRetry() throws Exception { + Path queueRoot = directory.resolve("outgoing"); + Files.createDirectory(queueRoot); + HttpTransportProtocol.Delivery existingDelivery = null; + for (int index = 0; index < 128; index++) { + String serverId = "server-" + index; + String deliveryId = UUID.nameUUIDFromBytes(serverId.getBytes(StandardCharsets.UTF_8)).toString(); + HttpTransportProtocol.Delivery delivery = new HttpTransportProtocol.Delivery(deliveryId, + JsonEnvelope.builder("quarantined-" + index).build()); + Path serverDirectory = Files.createDirectory(queueRoot.resolve(serverId)); + Files.write(serverDirectory.resolve(".pending-" + deliveryId + ".json"), + HttpTransportProtocol.storedDelivery(delivery)); + if (index == 0) existingDelivery = delivery; + } + + HttpProxyTransportServer.DurableOutgoingQueue queue = new HttpProxyTransportServer.DurableOutgoingQueue( + queueRoot, com.bencodez.simpleapi.file.DurableFiles::forceDirectory); + queue.load(); + + String newServerId = "server-128"; + HttpTransportProtocol.Delivery newDelivery = new HttpTransportProtocol.Delivery(UUID.randomUUID().toString(), + JsonEnvelope.builder("new-server").build()); + HttpProxyTransportServer.BackendState newServer = new HttpProxyTransportServer.BackendState( + newServerId, queue, (server, id) -> { }); + assertFalse(newServer.enqueue(newDelivery), "a new backend must be rejected at the durable directory cap"); + assertFalse(Files.exists(queueRoot.resolve(newServerId)), "rejected enqueue must not publish a backend directory"); + + HttpProxyTransportServer.BackendState existingServer = new HttpProxyTransportServer.BackendState( + "server-0", queue, (server, id) -> { }); + assertTrue(existingServer.enqueue(existingDelivery), "same-server retry must recover its quarantine at capacity"); + queue.close(); + + HttpProxyTransportServer.DurableOutgoingQueue restarted = new HttpProxyTransportServer.DurableOutgoingQueue( + queueRoot, com.bencodez.simpleapi.file.DurableFiles::forceDirectory); + restarted.load(); + + try (var servers = Files.list(queueRoot)) { + assertEquals(128L, servers.count(), "restart must not create a 129th backend directory"); + } + Set deliveryIds = new HashSet<>(); + long durableFiles = 0; + try (var servers = Files.list(queueRoot)) { + for (Path serverDirectory : servers.toList()) { + try (var messages = Files.list(serverDirectory)) { + for (Path message : messages.toList()) { + if (!message.getFileName().toString().endsWith(".json")) continue; + durableFiles++; + assertTrue(deliveryIds.add(HttpTransportProtocol.parseStoredDelivery(Files.readAllBytes(message)).id()), + "restart must not duplicate a durable delivery"); + } + } + } + } + assertEquals(128L, durableFiles, "each seeded delivery must remain represented exactly once"); + assertEquals(128, deliveryIds.size()); + restarted.close(); + } +} diff --git a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpPendingEnrollmentExpiryTest.java b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpPendingEnrollmentExpiryTest.java new file mode 100644 index 0000000..9d6cc1c --- /dev/null +++ b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpPendingEnrollmentExpiryTest.java @@ -0,0 +1,42 @@ +package com.bencodez.simpleapi.servercomm.http; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.net.URI; +import java.nio.file.Path; +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.util.concurrent.atomic.AtomicReference; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class HttpPendingEnrollmentExpiryTest { + @TempDir Path directory; + + @Test + void expiredPendingCertificateCannotActivateButActiveBindingRemainsValid() throws Exception { + AtomicReference now = new AtomicReference<>(Instant.now()); + Clock clock = new Clock() { + @Override public ZoneId getZone() { return ZoneOffset.UTC; } + @Override public Clock withZone(ZoneId zone) { return this; } + @Override public Instant instant() { return now.get(); } + }; + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("identity"), "localhost"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, clock); + URI endpoint = URI.create("https://localhost:8443/"); + HttpConnectionCode pendingCode = authority.createConnectionCode("pending", endpoint, Duration.ofSeconds(1)); + var pending = authority.enroll("pending", pendingCode.enrollmentToken()); + HttpConnectionCode activeCode = authority.createConnectionCode("active", endpoint, Duration.ofSeconds(1)); + var active = authority.enroll("active", activeCode.enrollmentToken()); + assertTrue(authority.authenticate("active", active.certificate())); + now.set(pendingCode.expiresAt()); + assertFalse(authority.authenticate("pending", pending.certificate())); + assertTrue(authority.authenticate("active", active.certificate())); + now.set(now.get().plusSeconds(1)); + assertFalse(authority.authenticate("pending", pending.certificate())); + } +} diff --git a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpProxyLifecycleRecoveryTest.java b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpProxyLifecycleRecoveryTest.java new file mode 100644 index 0000000..bdf8218 --- /dev/null +++ b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpProxyLifecycleRecoveryTest.java @@ -0,0 +1,141 @@ +package com.bencodez.simpleapi.servercomm.http; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.bencodez.simpleapi.servercomm.codec.JsonEnvelope; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.util.UUID; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class HttpProxyLifecycleRecoveryTest { + @TempDir Path directory; + + @Test + void callbackOwnedCloseCompletesItsInboundJournalBeforeItIsSealed() throws Exception { + Path outgoing = directory.resolve("outgoing"); + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, directory.resolve("authority")); + AtomicReference proxy = new AtomicReference<>(); + AtomicReference deliveryId = new AtomicReference<>(); + CountDownLatch callbackStarted = new CountDownLatch(1), callbackReturned = new CountDownLatch(1); + HttpProxyTransportServer server = new HttpProxyTransportServer(new InetSocketAddress("localhost", 0), identity, + authority, outgoing, received -> { + deliveryId.set(received.messageId()); + callbackStarted.countDown(); + proxy.get().close(); + callbackReturned.countDown(); + }); + proxy.set(server); + try { + server.start(); + HttpConnectionCode code = authority.createConnectionCode("lobby-1", server.endpoint("localhost"), + Duration.ofMinutes(5)); + HttpClientCredentialStore.ClientCredential credential = HttpBackendTransportConnector.enroll(code, "lobby-1", + directory.resolve("client")); + try (HttpBackendTransportConnector connector = new HttpBackendTransportConnector(code, "lobby-1", credential, + ignored -> { })) { + connector.start(); + assertTrue(connector.awaitFirstResponse(System.nanoTime() + TimeUnit.SECONDS.toNanos(8))); + assertTrue(connector.send(JsonEnvelope.builder("callback-close").build())); + assertTrue(callbackStarted.await(8, TimeUnit.SECONDS)); + assertTrue(callbackReturned.await(3, TimeUnit.SECONDS), + "a callback must not wait for shutdown of its own handler worker"); + } + server.close(); + assertNotNull(deliveryId.get()); + assertEquals(HttpInboundDeliveryStore.State.COMPLETED, + HttpInboundDeliveryStore.inspect(directory.resolve("outgoing-incoming"), "lobby-1").state(deliveryId.get()), + "the callback completion must be durable before close seals the journal"); + } finally { + server.close(); + } + } + + @Test + void failedNewBackendSetupCleansItsEmptyDirectoryBeforeRetry() throws Exception { + Path root = directory.resolve("outgoing"); + AtomicBoolean failFirstBackendPublication = new AtomicBoolean(true); + HttpProxyTransportServer.DurableOutgoingQueue queue = new HttpProxyTransportServer.DurableOutgoingQueue(root, + forced -> { + if (forced.equals(root) && failFirstBackendPublication.getAndSet(false)) + throw new IOException("injected backend directory publication failure"); + }); + HttpProxyTransportServer.BackendState state = new HttpProxyTransportServer.BackendState("lobby-1", queue, + (server, id) -> { }); + HttpTransportProtocol.Delivery delivery = new HttpTransportProtocol.Delivery(UUID.randomUUID().toString(), + JsonEnvelope.builder("retry-directory").build()); + + assertFalse(state.enqueue(delivery)); + assertFalse(Files.exists(root.resolve("lobby-1")), + "a failed setup must not leave an empty backend directory consuming the global bound"); + assertTrue(state.enqueue(delivery)); + queue.close(); + } + + @Test + void restartPrunesAccumulatedEmptyBackendDirectoriesBeforeApplyingTheCap() throws Exception { + Path root = directory.resolve("outgoing"); + HttpProxyTransportServer.DurableOutgoingQueue initial = new HttpProxyTransportServer.DurableOutgoingQueue(root, + ignored -> { }); + initial.close(); + for (int index = 0; index < 128; index++) Files.createDirectory(root.resolve("server-" + index)); + + HttpProxyTransportServer.DurableOutgoingQueue restarted = new HttpProxyTransportServer.DurableOutgoingQueue(root, + ignored -> { }); + assertTrue(restarted.load().isEmpty()); + HttpProxyTransportServer.BackendState state = new HttpProxyTransportServer.BackendState("replacement", restarted, + (server, id) -> { }); + assertTrue(state.enqueue(new HttpTransportProtocol.Delivery(UUID.randomUUID().toString(), + JsonEnvelope.builder("recovered-capacity").build()))); + assertTrue(Files.isDirectory(root.resolve("replacement"))); + restarted.close(); + } + + @Test + void capPruningPreservesAnEmptyDirectoryUntilItsUnconfirmedAckCanRetry() throws Exception { + Path root = directory.resolve("outgoing"); + Path lobbyDirectory = root.resolve("lobby-1"); + HttpProxyTransportServer.DurableOutgoingQueue queue = new HttpProxyTransportServer.DurableOutgoingQueue(root, + ignored -> { }); + HttpProxyTransportServer.BackendState lobby = new HttpProxyTransportServer.BackendState("lobby-1", queue, + (server, id) -> { }); + HttpTransportProtocol.Delivery delivery = new HttpTransportProtocol.Delivery(UUID.randomUUID().toString(), + JsonEnvelope.builder("ack-retry").build()); + assertTrue(lobby.enqueue(delivery)); + + AtomicBoolean failFinalAckForce = new AtomicBoolean(true); + try (var forces = org.mockito.Mockito.mockStatic(com.bencodez.simpleapi.file.DurableFiles.class, + org.mockito.Mockito.CALLS_REAL_METHODS)) { + forces.when(() -> com.bencodez.simpleapi.file.DurableFiles.forceDirectory(lobbyDirectory)).thenAnswer(call -> { + if (failFinalAckForce.getAndSet(false)) throw new IOException("injected acknowledgement force failure"); + return call.callRealMethod(); + }); + assertThrows(IOException.class, () -> lobby.acknowledge(List.of(delivery.id()))); + assertTrue(Files.isDirectory(lobbyDirectory), "the unlink has happened but its directory fsync is unresolved"); + + HttpProxyTransportServer.BackendState replacement = new HttpProxyTransportServer.BackendState("replacement", + queue, (server, id) -> { }); + assertTrue(replacement.enqueue(new HttpTransportProtocol.Delivery(UUID.randomUUID().toString(), + JsonEnvelope.builder("trigger-cap-count").build()))); + assertTrue(Files.isDirectory(lobbyDirectory), + "capacity pruning must retain the directory indexed by the failed acknowledgement"); + lobby.acknowledge(List.of(delivery.id())); + } + assertFalse(Files.exists(lobbyDirectory), "the same acknowledgement retry must be able to finish cleanup"); + queue.close(); + } +} diff --git a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpRenewalRateLimitTest.java b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpRenewalRateLimitTest.java new file mode 100644 index 0000000..5977f49 --- /dev/null +++ b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpRenewalRateLimitTest.java @@ -0,0 +1,47 @@ +package com.bencodez.simpleapi.servercomm.http; + +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.net.URI; +import java.nio.file.Path; +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.util.concurrent.atomic.AtomicReference; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class HttpRenewalRateLimitTest { + @TempDir Path directory; + + @Test + void repeatedRenewalIsLimitedPerBackendBeforeCertificateIssuance() throws Exception { + AtomicReference now = new AtomicReference<>(Instant.now()); + Clock clock = new Clock() { + @Override public ZoneId getZone() { return ZoneOffset.UTC; } + @Override public Clock withZone(ZoneId zone) { return this; } + @Override public Instant instant() { return now.get(); } + }; + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("identity"), "localhost"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, clock); + URI endpoint = URI.create("https://localhost:8443/"); + var code = authority.createConnectionCode("lobby-1", endpoint, Duration.ofMinutes(5)); + var original = authority.enroll("lobby-1", code.enrollmentToken()); + assertTrue(authority.authenticate("lobby-1", original.certificate())); + var replacement = authority.renew("lobby-1", original.certificate()); + for (int index = 0; index < 10; index++) assertThrows(HttpEnrollmentAuthority.RenewalRateLimitException.class, + () -> authority.renew("lobby-1", original.certificate())); + assertTrue(authority.authenticate("lobby-1", replacement.certificate())); + assertThrows(HttpEnrollmentAuthority.RenewalRateLimitException.class, + () -> authority.renew("lobby-1", replacement.certificate())); + var otherCode = authority.createConnectionCode("lobby-2", endpoint, Duration.ofMinutes(5)); + var other = authority.enroll("lobby-2", otherCode.enrollmentToken()); + assertNotNull(authority.renew("lobby-2", other.certificate())); + now.set(now.get().plusSeconds(60)); + assertNotNull(authority.renew("lobby-1", replacement.certificate())); + } +} diff --git a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpRevocationRetryIsolationTest.java b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpRevocationRetryIsolationTest.java new file mode 100644 index 0000000..ad4b783 --- /dev/null +++ b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpRevocationRetryIsolationTest.java @@ -0,0 +1,58 @@ +package com.bencodez.simpleapi.servercomm.http; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.net.URI; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class HttpRevocationRetryIsolationTest { + @TempDir Path directory; + + @Test + void failedRevocationRetryCannotClearOrReplaceAnotherServerBinding() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); + Path state = directory.resolve("authority"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, state); + URI endpoint = URI.create("https://localhost:8443/"); + + HttpConnectionCode codeA = authority.createConnectionCode("backend-a", endpoint, Duration.ofMinutes(5)); + HttpTlsIdentity.IssuedClientCertificate certificateA = authority.enroll("backend-a", codeA.enrollmentToken()); + HttpConnectionCode codeB = authority.createConnectionCode("backend-b", endpoint, Duration.ofMinutes(5)); + HttpTlsIdentity.IssuedClientCertificate certificateB = authority.enroll("backend-b", codeB.enrollmentToken()); + assertTrue(authority.authenticate("backend-a", certificateA.certificate())); + assertTrue(authority.authenticate("backend-b", certificateB.certificate())); + + Path stateFile = state.resolve("http-transport-clients.properties"); + Files.delete(stateFile); + Files.createDirectory(stateFile); + + assertThrows(IllegalStateException.class, () -> authority.revoke("backend-a")); + assertFalse(authority.authenticate("backend-a", certificateA.certificate()), + "authentication must fail closed while the revocation state is unresolved"); + assertFalse(authority.authenticate("backend-b", certificateB.certificate()), + "the global retry guard must fail closed for every backend"); + + assertThrows(IllegalStateException.class, () -> authority.revoke("nonexistent")); + assertThrows(IllegalStateException.class, () -> authority.revoke("backend-b")); + assertFalse(authority.authenticate("backend-a", certificateA.certificate())); + assertFalse(authority.authenticate("backend-b", certificateB.certificate()), + "unrelated revocation attempts must not clear the retry guard or restore state"); + + Files.delete(stateFile); + authority.revoke("backend-a"); + assertFalse(authority.authenticate("backend-a", certificateA.certificate())); + assertTrue(authority.authenticate("backend-b", certificateB.certificate()), + "retrying the original target must revoke only that target"); + + HttpEnrollmentAuthority restarted = new HttpEnrollmentAuthority(identity, state); + assertFalse(restarted.authenticate("backend-a", certificateA.certificate())); + assertTrue(restarted.authenticate("backend-b", certificateB.certificate()), + "the isolated revocation result must survive restart"); + } +} diff --git a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTlsIdentityOwnershipTest.java b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTlsIdentityOwnershipTest.java new file mode 100644 index 0000000..138821b --- /dev/null +++ b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTlsIdentityOwnershipTest.java @@ -0,0 +1,71 @@ +package com.bencodez.simpleapi.servercomm.http; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Clock; +import java.time.Instant; +import java.time.Duration; +import java.time.ZoneOffset; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class HttpTlsIdentityOwnershipTest { + @TempDir Path directory; + + @Test + void concurrentFirstOpenPublishesOneReloadableIdentity() throws Exception { + Path identityDirectory = directory.resolve("identity"); + CountDownLatch ready = new CountDownLatch(2), start = new CountDownLatch(1); + ExecutorService workers = Executors.newFixedThreadPool(2); + try { + Future first = workers.submit(() -> openTogether(identityDirectory, ready, start)); + Future second = workers.submit(() -> openTogether(identityDirectory, ready, start)); + assertTrue(ready.await(2, TimeUnit.SECONDS)); + start.countDown(); + HttpTlsIdentity firstIdentity = first.get(10, TimeUnit.SECONDS); + HttpTlsIdentity secondIdentity = second.get(10, TimeUnit.SECONDS); + assertEquals(firstIdentity.caCertificatePin(), secondIdentity.caCertificatePin()); + assertEquals(firstIdentity.serverCertificatePin(), secondIdentity.serverCertificatePin()); + HttpTlsIdentity reloaded = HttpTlsIdentity.loadOrCreate(identityDirectory, "localhost"); + assertEquals(firstIdentity.caCertificatePin(), reloaded.caCertificatePin()); + assertEquals(firstIdentity.serverCertificatePin(), reloaded.serverCertificatePin()); + assertTrue(Files.isRegularFile(identityDirectory.resolve(".http-transport-identity.lock"))); + } finally { + workers.shutdownNow(); + workers.awaitTermination(2, TimeUnit.SECONDS); + } + } + + @Test + void staleInstanceAdoptsAnotherInstancesRenewedServerCertificate() throws Exception { + Instant now = Instant.now(); + Path identityDirectory = directory.resolve("stale-renewal"); + Clock creationClock = Clock.fixed(now.minus(Duration.ofDays(340)), ZoneOffset.UTC); + HttpTlsIdentity first = HttpTlsIdentity.loadOrCreate(identityDirectory, "localhost", creationClock); + HttpTlsIdentity second = HttpTlsIdentity.loadOrCreate(identityDirectory, "localhost", creationClock); + String originalPin = HttpTransportSecrets.certificatePin(first.serverCertificate()); + + String renewedByFirst = first.serverCertificatePin(); + assertNotEquals(originalPin, renewedByFirst); + String adoptedBySecond = second.serverCertificatePin(); + assertEquals(renewedByFirst, adoptedBySecond, + "a stale instance must reload the persisted renewal instead of generating another server key"); + HttpTlsIdentity reloaded = HttpTlsIdentity.loadOrCreate(identityDirectory, "localhost"); + assertEquals(renewedByFirst, reloaded.serverCertificatePin()); + } + + private static HttpTlsIdentity openTogether(Path directory, CountDownLatch ready, CountDownLatch start) throws Exception { + ready.countDown(); + if (!start.await(5, TimeUnit.SECONDS)) throw new IllegalStateException("concurrent identity start timed out"); + return HttpTlsIdentity.loadOrCreate(directory, "localhost"); + } +} diff --git a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportRuntimeTest.java b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportRuntimeTest.java new file mode 100644 index 0000000..05bed61 --- /dev/null +++ b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportRuntimeTest.java @@ -0,0 +1,1540 @@ +package com.bencodez.simpleapi.servercomm.http; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTimeoutPreemptively; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.bencodez.simpleapi.servercomm.codec.JsonEnvelope; +import com.sun.net.httpserver.Headers; +import java.net.InetSocketAddress; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.time.Instant; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class HttpTransportRuntimeTest { + @TempDir Path directory; + + @Test + void publicConstructorsRequireDurableOutgoingDirectory() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("guard-proxy"), "localhost"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, directory.resolve("guard-authority")); + InetSocketAddress bind = new InetSocketAddress("localhost", 0); + assertThrows(NullPointerException.class, () -> new HttpProxyTransportServer(bind, identity, authority, + null, ignored -> { })); + assertThrows(NullPointerException.class, () -> new HttpProxyTransportServer(bind, identity, authority, + null, ignored -> { }, (serverId, deliveryId) -> { })); + } + + @Test + void endpointHelperSupportsIpv6Literals() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("ipv6-proxy"), "::1"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, directory.resolve("ipv6-authority")); + try (HttpProxyTransportServer server = new HttpProxyTransportServer(new InetSocketAddress("localhost", 0), + identity, authority, directory.resolve("ipv6-outgoing"), ignored -> { })) { + URI endpoint = server.endpoint("::1"); + assertTrue(endpoint.getHost() != null); + assertTrue(endpoint.toASCIIString().startsWith("https://[::1]:")); + assertDoesNotThrow(() -> new HttpConnectionCode("lobby-1", endpoint, identity.serverCertificatePin(), + identity.caCertificatePin(), Instant.now().plusSeconds(60), "A".repeat(43))); + } + } + + @Test + void enrollsThenDeliversBothDirectionsWithAuthenticatedIdentity() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, directory.resolve("authority")); + CountDownLatch proxyReceived = new CountDownLatch(1), backendReceived = new CountDownLatch(1); + AtomicReference received = new AtomicReference<>(); + Path proxyOutgoing = directory.resolve("proxy-outgoing"); + try (HttpProxyTransportServer server = new HttpProxyTransportServer(new InetSocketAddress("localhost", 0), identity, authority, + proxyOutgoing, + message -> { received.set(message); proxyReceived.countDown(); })) { + server.start(); + HttpConnectionCode code = authority.createConnectionCode("lobby-1", server.endpoint("localhost"), Duration.ofMinutes(5)); + HttpBackendTransportConnector.enroll(code, "lobby-1", directory.resolve("client")); + try (HttpBackendTransportConnector connector = new HttpBackendTransportConnector(directory.resolve("client"), + envelope -> backendReceived.countDown())) { + connector.start(); + assertTrue(connector.awaitFirstResponse(System.nanoTime() + TimeUnit.SECONDS.toNanos(8)), + "an authenticated transport response must make the connector ready"); + assertTrue(connector.send(JsonEnvelope.builder("to-proxy").put("server", "forged").build())); + assertTrue(proxyReceived.await(8, TimeUnit.SECONDS)); + assertEquals("lobby-1", received.get().serverId()); + assertEquals("lobby-1", received.get().envelope().getFields().get("server")); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(3); + while (connector.queuedOutgoing() != 0 && System.nanoTime() < deadline) Thread.sleep(10); + assertEquals(0, connector.queuedOutgoing(), "proxy ACK must remove the exact outbound delivery ID"); + Path proxyInboundFence = directory.resolve("proxy-outgoing-incoming").resolve("lobby-1"); + long confirmationDeadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(3); + while (countRegularFiles(proxyInboundFence) != 0L && System.nanoTime() < confirmationDeadline) Thread.sleep(10); + assertEquals(0L, countRegularFiles(proxyInboundFence), + "the backend must durably confirm receipt of the proxy ACK"); + assertTrue(server.send("lobby-1", JsonEnvelope.builder("to-backend").build())); + assertTrue(backendReceived.await(8, TimeUnit.SECONDS)); + long outgoingCleanupDeadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(3); + while (Files.exists(proxyOutgoing.resolve("lobby-1")) && System.nanoTime() < outgoingCleanupDeadline) + Thread.sleep(10); + assertFalse(Files.exists(proxyOutgoing.resolve("lobby-1")), + "acknowledging the final delivery must remove its empty backend directory"); + Path inboundFence = directory.resolve("client").resolve("http-transport-inbound-deliveries"); + long fenceDeadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (countRegularFiles(inboundFence) != 0L && System.nanoTime() < fenceDeadline) Thread.sleep(20); + assertEquals(0L, countRegularFiles(inboundFence), "a confirmed ACK must remove the backend replay fence"); + } + } + } + + @Test + void responseBodyConsumptionRemainsBoundedByRequestTimeout() throws Exception { + com.sun.net.httpserver.HttpServer server = com.sun.net.httpserver.HttpServer.create( + new InetSocketAddress("localhost", 0), 1); + CountDownLatch release = new CountDownLatch(1); + server.createContext("/stall", exchange -> { + exchange.sendResponseHeaders(200, 8); + try (var output = exchange.getResponseBody()) { + output.write(1); + output.flush(); + try { release.await(5, TimeUnit.SECONDS); } + catch (InterruptedException interrupted) { Thread.currentThread().interrupt(); } + } + }); + server.start(); + try { + HttpRequest request = HttpRequest.newBuilder(URI.create("http://localhost:" + server.getAddress().getPort() + + "/stall")).timeout(Duration.ofMillis(250)).GET().build(); + assertTimeoutPreemptively(Duration.ofSeconds(2), () -> assertThrows(java.io.IOException.class, + () -> HttpBackendTransportConnector.sendLimited(HttpClient.newHttpClient(), request))); + } finally { + release.countDown(); + server.stop(0); + } + } + + @Test + void stableProxyDeliveryIdsAreIdempotentAndAcknowledgedBeforeRemoval() throws Exception { + AtomicLong acknowledged = new AtomicLong(); + HttpProxyTransportServer.BackendState state = new HttpProxyTransportServer.BackendState("lobby-1", null, + (server, deliveryId) -> acknowledged.incrementAndGet()); + String deliveryId = java.util.UUID.randomUUID().toString(); + HttpTransportProtocol.Delivery delivery = new HttpTransportProtocol.Delivery(deliveryId, + JsonEnvelope.builder("vote-party").build()); + assertTrue(state.enqueue(delivery)); + assertTrue(state.enqueue(delivery)); + assertFalse(state.enqueue(new HttpTransportProtocol.Delivery(deliveryId, + JsonEnvelope.builder("different").build()))); + state.acknowledge(java.util.List.of(deliveryId)); + assertEquals(1L, acknowledged.get()); + assertTrue(state.await("lobby-1", java.util.UUID.randomUUID().toString(), 0).messages().isEmpty()); + } + + @Test + void outgoingQueueRetriesDirectoryPublicationAfterForceFailure() throws Exception { + Path queueRoot = directory.resolve("retry-outgoing-root"); + java.util.concurrent.atomic.AtomicBoolean failRootPublication = new java.util.concurrent.atomic.AtomicBoolean(true); + assertThrows(java.io.IOException.class, () -> new HttpProxyTransportServer.DurableOutgoingQueue(queueRoot, + ignored -> { + if (failRootPublication.getAndSet(false)) + throw new java.io.IOException("injected root publication failure"); + })); + assertTrue(Files.isDirectory(queueRoot), "the failed force occurs after the root name is published"); + AtomicLong parentForces = new AtomicLong(); + HttpProxyTransportServer.DurableOutgoingQueue queue = new HttpProxyTransportServer.DurableOutgoingQueue( + queueRoot, forced -> { + if (forced.equals(queueRoot.getParent())) parentForces.incrementAndGet(); + }); + assertEquals(1L, parentForces.get(), "reopening an existing root must retry its parent fsync"); + + AtomicLong serverRootForces = new AtomicLong(); + java.util.concurrent.atomic.AtomicBoolean failServerPublication = new java.util.concurrent.atomic.AtomicBoolean(true); + HttpProxyTransportServer.DurableOutgoingQueue serverQueue = new HttpProxyTransportServer.DurableOutgoingQueue( + directory.resolve("retry-outgoing-server"), forced -> { + if (forced.getFileName().toString().equals("retry-outgoing-server")) { + serverRootForces.incrementAndGet(); + if (failServerPublication.getAndSet(false)) + throw new java.io.IOException("injected backend publication failure"); + } + }); + HttpProxyTransportServer.BackendState state = new HttpProxyTransportServer.BackendState( + "lobby-1", serverQueue, (server, id) -> { }); + HttpTransportProtocol.Delivery delivery = new HttpTransportProtocol.Delivery( + java.util.UUID.randomUUID().toString(), JsonEnvelope.builder("retry-directory-force").build()); + assertFalse(state.enqueue(delivery)); + assertEquals(2L, serverRootForces.get(), "failed publication and empty-directory cleanup must both force the parent"); + assertTrue(state.enqueue(delivery)); + assertEquals(3L, serverRootForces.get(), "retrying backend directory creation must force its parent again"); + queue.close(); + serverQueue.close(); + } + + @Test + void generatedSendExposesRecoverableIdAfterPublicationFailure() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("generated-proxy"), "localhost"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, directory.resolve("generated-authority")); + for (boolean persistentFailure : new boolean[] { false, true }) { + Path queueRoot = directory.resolve("generated-outgoing-" + persistentFailure); + JsonEnvelope envelope = JsonEnvelope.builder("generated-retry").build(); + String retryId; + try (HttpProxyTransportServer server = new HttpProxyTransportServer(new InetSocketAddress("localhost", 0), + identity, authority, queueRoot, ignored -> { })) { + AtomicLong publicationForces = new AtomicLong(); + try (var forces = org.mockito.Mockito.mockStatic(com.bencodez.simpleapi.file.DurableFiles.class, + org.mockito.Mockito.CALLS_REAL_METHODS)) { + forces.when(() -> com.bencodez.simpleapi.file.DurableFiles.forceDirectory(queueRoot.resolve("lobby-1"))) + .thenAnswer(call -> { + if (publicationForces.incrementAndGet() == 1 || persistentFailure) + throw new java.io.IOException("injected publication failure"); + return call.callRealMethod(); + }); + HttpProxyTransportServer.DeliveryRetryException retry = assertThrows( + HttpProxyTransportServer.DeliveryRetryException.class, () -> server.send("lobby-1", envelope)); + retryId = retry.deliveryId(); + assertEquals(retryId, java.util.UUID.fromString(retryId).toString()); + } + assertTrue(server.backendStateForTest("lobby-1") + .await("lobby-1", java.util.UUID.randomUUID().toString(), 0).messages().isEmpty()); + } + try (HttpProxyTransportServer restarted = new HttpProxyTransportServer(new InetSocketAddress("localhost", 0), + identity, authority, queueRoot, ignored -> { })) { + assertTrue(restarted.backendStateForTest("lobby-1") + .await("lobby-1", java.util.UUID.randomUUID().toString(), 0).messages().isEmpty()); + assertTrue(restarted.send("lobby-1", retryId, envelope)); + assertEquals(1L, countRegularFiles(queueRoot)); + var messages = restarted.backendStateForTest("lobby-1") + .await("lobby-1", java.util.UUID.randomUUID().toString(), 0).messages(); + assertEquals(1, messages.size()); + assertEquals(retryId, messages.iterator().next().id()); + restarted.backendStateForTest("lobby-1").acknowledge(java.util.List.of(retryId)); + assertEquals(0L, countRegularFiles(queueRoot)); + } + } + } + + @Test + void outgoingQueueRecoversWhenPromotionQuarantineRenameFails() throws Exception { + Path queueRoot = directory.resolve("promotion-rename-failure"); + String deliveryId = java.util.UUID.randomUUID().toString(); + Path serverDirectory = queueRoot.resolve("lobby-1"); + Path pending = serverDirectory.resolve(".pending-" + deliveryId + ".json"); + AtomicLong serverForces = new AtomicLong(); + HttpProxyTransportServer.DurableOutgoingQueue queue = new HttpProxyTransportServer.DurableOutgoingQueue(queueRoot, + forced -> { + if (!forced.equals(serverDirectory)) return; + long attempt = serverForces.incrementAndGet(); + if (attempt == 1L) throw new java.io.IOException("injected initial publication failure"); + if (attempt == 3L) { + Files.createDirectory(pending); + throw new java.io.IOException("injected promotion publication failure"); + } + }); + HttpProxyTransportServer.BackendState state = new HttpProxyTransportServer.BackendState("lobby-1", queue, + (server, id) -> { }); + HttpTransportProtocol.Delivery delivery = new HttpTransportProtocol.Delivery(deliveryId, + JsonEnvelope.builder("promotion-rename-failure").build()); + assertFalse(state.enqueue(delivery)); + assertThrows(IllegalStateException.class, () -> state.enqueue(delivery)); + Files.delete(pending); + assertTrue(state.enqueue(delivery), "same-ID retry must confirm the observed target instead of duplicating it"); + var quarantinedFiles = HttpProxyTransportServer.DurableOutgoingQueue.class.getDeclaredField("quarantinedFiles"); + quarantinedFiles.setAccessible(true); + @SuppressWarnings("unchecked") + var quarantined = (java.util.Map>) quarantinedFiles.get(queue); + assertFalse(quarantined.get("lobby-1").containsKey(deliveryId), + "target fallback must clear the stale quarantine index"); + state.acknowledge(java.util.List.of(deliveryId)); + assertFalse(Files.exists(serverDirectory)); + queue.close(); + } + + @Test + void publishedOutgoingDeliveryRemainsTrackedUntilDurabilityCanBeConfirmed() throws Exception { + AtomicLong forceCalls = new AtomicLong(); + Path queueRoot = directory.resolve("uncertain-outgoing"); + HttpProxyTransportServer.DurableOutgoingQueue queue = new HttpProxyTransportServer.DurableOutgoingQueue( + queueRoot, ignored -> { + if (forceCalls.incrementAndGet() == 3L) throw new java.io.IOException("injected directory force failure"); + }); + HttpProxyTransportServer.BackendState state = new HttpProxyTransportServer.BackendState( + "lobby-1", queue, (server, id) -> { }); + String deliveryId = java.util.UUID.randomUUID().toString(); + HttpTransportProtocol.Delivery delivery = new HttpTransportProtocol.Delivery(deliveryId, + JsonEnvelope.builder("durable").build()); + + assertFalse(state.enqueue(delivery), "post-publication failure must not confirm durable acceptance"); + assertTrue(state.await("lobby-1", java.util.UUID.randomUUID().toString(), 0).messages().isEmpty(), + "an uncertain publication must remain hidden until its durability retry succeeds"); + assertEquals(1L, countRegularFiles(queueRoot)); + queue.close(); + HttpProxyTransportServer.DurableOutgoingQueue restartedQueue = new HttpProxyTransportServer.DurableOutgoingQueue( + queueRoot, com.bencodez.simpleapi.file.DurableFiles::forceDirectory); + assertEquals(java.util.List.of(), restartedQueue.load().get("lobby-1"), + "a quarantine must reserve its backend without exposing an operation whose sender observed rejection"); + HttpProxyTransportServer.BackendState restarted = new HttpProxyTransportServer.BackendState( + "lobby-1", restartedQueue, (server, id) -> { }); + assertTrue(restarted.await("lobby-1", java.util.UUID.randomUUID().toString(), 0).messages().isEmpty()); + assertTrue(restarted.enqueue(delivery), "same-ID retry must confirm the quarantined file"); + assertEquals(1L, countRegularFiles(queueRoot), "durability retry must not create a duplicate file"); + assertEquals(java.util.List.of(delivery), + restarted.await("lobby-1", java.util.UUID.randomUUID().toString(), 0).messages()); + restartedQueue.close(); + HttpProxyTransportServer.DurableOutgoingQueue confirmedQueue = new HttpProxyTransportServer.DurableOutgoingQueue( + queueRoot, com.bencodez.simpleapi.file.DurableFiles::forceDirectory); + java.util.List confirmed = confirmedQueue.load().get("lobby-1"); + assertEquals(1, confirmed.size()); + assertTrue(java.util.Arrays.equals(HttpTransportProtocol.storedDelivery(delivery), + HttpTransportProtocol.storedDelivery(confirmed.get(0))), + "a confirmed same-ID retry must become deliverable after restart"); + HttpProxyTransportServer.BackendState confirmedState = new HttpProxyTransportServer.BackendState( + "lobby-1", confirmedQueue, (server, id) -> { }); + assertTrue(confirmedState.enqueue(delivery)); + confirmedState.acknowledge(java.util.List.of(deliveryId)); + assertEquals(0L, countRegularFiles(queueRoot), "the tracked published file must be removable by ACK"); + confirmedQueue.close(); + } + + @Test + void unresolvedOutgoingPublicationDoesNotReportAFalseRejection() throws Exception { + AtomicLong forceCalls = new AtomicLong(); + java.util.concurrent.atomic.AtomicBoolean failForces = new java.util.concurrent.atomic.AtomicBoolean(true); + Path queueRoot = directory.resolve("unresolved-outgoing"); + HttpProxyTransportServer.DurableOutgoingQueue queue = new HttpProxyTransportServer.DurableOutgoingQueue( + queueRoot, ignored -> { + if (forceCalls.incrementAndGet() >= 3L && failForces.get()) + throw new java.io.IOException("persistent directory force failure"); + }); + HttpProxyTransportServer.BackendState state = new HttpProxyTransportServer.BackendState( + "lobby-1", queue, (server, id) -> { }); + HttpTransportProtocol.Delivery delivery = new HttpTransportProtocol.Delivery( + java.util.UUID.randomUUID().toString(), JsonEnvelope.builder("unresolved").build()); + assertThrows(IllegalStateException.class, () -> state.enqueue(delivery), + "an indeterminate rollback must not be reported as a definitive false result"); + failForces.set(false); + assertTrue(state.enqueue(delivery), "a same-ID retry must recover the observed quarantine"); + assertEquals(1L, countRegularFiles(queueRoot), "recovery must not leave duplicate queue entries"); + queue.close(); + HttpProxyTransportServer.DurableOutgoingQueue restarted = new HttpProxyTransportServer.DurableOutgoingQueue( + queueRoot, com.bencodez.simpleapi.file.DurableFiles::forceDirectory); + assertEquals(1, restarted.load().get("lobby-1").size()); + restarted.close(); + } + + @Test + void proxyInboundCompletionSurvivesRestartBeforeAcknowledgement() throws Exception { + Path root = directory.resolve("proxy-incoming"); + Files.createDirectory(root); + String deliveryId = java.util.UUID.randomUUID().toString(); + HttpTransportProtocol.Delivery delivery = new HttpTransportProtocol.Delivery(deliveryId, + JsonEnvelope.builder("backend-event").build()); + HttpInboundDeliveryStore firstStore = HttpInboundDeliveryStore.open(root, "lobby-1"); + HttpProxyTransportServer.BackendState first = new HttpProxyTransportServer.BackendState( + "lobby-1", null, firstStore, (server, id) -> { }); + assertEquals(java.util.List.of(delivery), first.acceptIncoming(java.util.List.of(delivery))); + first.beginIncoming(deliveryId); + first.completeIncomingDurably(deliveryId); + first.completeIncoming(deliveryId, true); + firstStore.seal(); + + HttpInboundDeliveryStore restartedStore = HttpInboundDeliveryStore.open(root, "lobby-1"); + HttpProxyTransportServer.BackendState restarted = new HttpProxyTransportServer.BackendState( + "lobby-1", null, restartedStore, (server, id) -> { }); + assertTrue(restarted.acceptIncoming(java.util.List.of(delivery)).isEmpty(), + "a completed callback must not run again after a lost response and proxy restart"); + HttpProxyTransportServer.Response response = restarted.await("lobby-1", + java.util.UUID.randomUUID().toString(), 0); + assertEquals(java.util.List.of(deliveryId), response.acks()); + restarted.confirmIncoming(response.acks()); + restartedStore.seal(); + HttpInboundDeliveryStore confirmedStore = HttpInboundDeliveryStore.open(root, "lobby-1"); + HttpProxyTransportServer.BackendState confirmed = new HttpProxyTransportServer.BackendState( + "lobby-1", null, confirmedStore, (server, id) -> { }); + assertEquals(java.util.List.of(delivery), confirmed.acceptIncoming(java.util.List.of(delivery)), + "only an acknowledgement confirmation may retire the durable replay fence"); + } + + @Test + void inboundJournalHasOneWriterWhileReadOnlyInspectionRemainsAvailable() throws Exception { + Path root = directory.resolve("exclusive-incoming"); + Files.createDirectory(root); + String deliveryId = java.util.UUID.randomUUID().toString(); + HttpInboundDeliveryStore owner = HttpInboundDeliveryStore.open(root, "lobby-1"); + owner.reserve(deliveryId); + assertThrows(java.io.IOException.class, () -> HttpInboundDeliveryStore.open(root, "lobby-1"), + "a live journal owner must exclude a stale in-memory writer"); + assertEquals(HttpInboundDeliveryStore.State.RESERVED, + HttpInboundDeliveryStore.inspect(root, "lobby-1").state(deliveryId)); + owner.seal(); + HttpInboundDeliveryStore successor = HttpInboundDeliveryStore.open(root, "lobby-1"); + successor.markRunning(deliveryId); + successor.seal(); + } + + @Test + void backendConnectorReleasesJournalOwnershipOnlyAfterClose() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("owner-proxy"), "localhost"); + HttpTlsIdentity.IssuedClientCertificate issued = identity.issueClientCertificate("lobby-1"); + HttpConnectionCode code = new HttpConnectionCode("lobby-1", URI.create("https://localhost:8443/"), + identity.serverCertificatePin(), identity.caCertificatePin(), Instant.now().plusSeconds(300), "A".repeat(43)); + Path credentials = directory.resolve("owner-client"); + HttpClientCredentialStore.saveEnrolled(credentials, code, issued); + try (HttpBackendTransportConnector owner = new HttpBackendTransportConnector(credentials, ignored -> { })) { + assertThrows(java.io.IOException.class, () -> new HttpBackendTransportConnector(credentials, ignored -> { }), + "two live connectors must not own the same inbound delivery journal"); + } + try (HttpBackendTransportConnector successor = new HttpBackendTransportConnector(credentials, ignored -> { })) { + assertFalse(successor.pollerAlive()); + } + } + + @Test + void completedInboundPublicationRetriesForceBeforeProxyAcknowledgement() throws Exception { + Path root = directory.resolve("proxy-incoming-force-retry"); + Files.createDirectory(root); + String deliveryId = java.util.UUID.randomUUID().toString(); + Path inboundDirectory = root.resolve("lobby-1"); + Path completed = inboundDirectory.resolve(deliveryId + ".completed"); + HttpTransportProtocol.Delivery delivery = new HttpTransportProtocol.Delivery(deliveryId, + JsonEnvelope.builder("backend-event").build()); + HttpInboundDeliveryStore store = HttpInboundDeliveryStore.open(root, "lobby-1"); + HttpProxyTransportServer.BackendState state = new HttpProxyTransportServer.BackendState( + "lobby-1", null, store, (server, id) -> { }); + try (var forces = org.mockito.Mockito.mockStatic(com.bencodez.simpleapi.file.DurableFiles.class, + org.mockito.Mockito.CALLS_REAL_METHODS)) { + forces.when(() -> com.bencodez.simpleapi.file.DurableFiles.forceDirectory(inboundDirectory)) + .thenAnswer(call -> { + if (Files.exists(completed)) throw new java.io.IOException("injected completed fsync failure"); + return call.callRealMethod(); + }); + assertEquals(java.util.List.of(delivery), state.acceptIncoming(java.util.List.of(delivery))); + state.beginIncoming(deliveryId); + assertThrows(com.bencodez.simpleapi.file.DurableFiles.PublishedException.class, + () -> state.completeIncomingDurably(deliveryId)); + state.completeIncoming(deliveryId, false); + assertEquals(HttpInboundDeliveryStore.State.COMPLETED, store.state(deliveryId)); + assertTrue(state.acceptIncoming(java.util.List.of(delivery)).isEmpty(), + "a visible completed target must never rerun its callback while fsync is unresolved"); + assertTrue(state.await("lobby-1", java.util.UUID.randomUUID().toString(), 0).acks().isEmpty(), + "a completed target must not be acknowledged before its fsync succeeds"); + } + assertTrue(state.acceptIncoming(java.util.List.of(delivery)).isEmpty(), + "recovered publication must be acknowledged rather than dispatched again"); + assertEquals(java.util.List.of(deliveryId), state.await("lobby-1", java.util.UUID.randomUUID().toString(), 0).acks()); + } + + @Test + void reservedInboundPublicationRetriesBeforeProxyCallback() throws Exception { + Path root = directory.resolve("reserved-force-proxy"); + Files.createDirectory(root); + String deliveryId = java.util.UUID.randomUUID().toString(); + Path inboundDirectory = root.resolve("lobby-1"); + Path reserved = inboundDirectory.resolve(deliveryId + ".reserved"); + CountDownLatch firstForce = new CountDownLatch(1), secondForce = new CountDownLatch(1); + java.util.concurrent.atomic.AtomicInteger forceCalls = new java.util.concurrent.atomic.AtomicInteger(); + java.util.concurrent.atomic.AtomicInteger callbacks = new java.util.concurrent.atomic.AtomicInteger(); + HttpTransportProtocol.Delivery delivery = new HttpTransportProtocol.Delivery(deliveryId, JsonEnvelope.builder("vote").build()); + HttpInboundDeliveryStore store = HttpInboundDeliveryStore.open(root, "lobby-1"); + HttpProxyTransportServer.BackendState state = new HttpProxyTransportServer.BackendState( + "lobby-1", null, store, (server, id) -> { }); + try (var forces = org.mockito.Mockito.mockStatic(com.bencodez.simpleapi.file.DurableFiles.class, + org.mockito.Mockito.CALLS_REAL_METHODS)) { + forces.when(() -> com.bencodez.simpleapi.file.DurableFiles.forceDirectory(inboundDirectory)) + .thenAnswer(call -> { + if (Files.exists(reserved)) { + if (forceCalls.incrementAndGet() == 1) firstForce.countDown(); else secondForce.countDown(); + throw new java.io.IOException("injected reservation fsync failure"); + } + return call.callRealMethod(); + }); + assertEquals(java.util.List.of(delivery), state.acceptIncoming(java.util.List.of(delivery))); + assertThrows(com.bencodez.simpleapi.file.DurableFiles.PublishedException.class, + () -> state.beginIncoming(deliveryId)); + state.completeIncoming(deliveryId, false); + assertTrue(firstForce.await(2, TimeUnit.SECONDS)); + assertEquals(java.util.List.of(delivery), state.acceptIncoming(java.util.List.of(delivery))); + assertThrows(java.io.IOException.class, () -> state.beginIncoming(deliveryId)); + state.completeIncoming(deliveryId, false); + assertTrue(secondForce.await(2, TimeUnit.SECONDS)); + assertEquals(0, callbacks.get(), "an unconfirmed reservation must not enter the callback"); + assertTrue(state.await("lobby-1", java.util.UUID.randomUUID().toString(), 0).acks().isEmpty()); + } + assertEquals(java.util.List.of(delivery), state.acceptIncoming(java.util.List.of(delivery))); + state.beginIncoming(deliveryId); + callbacks.incrementAndGet(); // Models the callback reached only after beginIncoming's durable RUNNING transition. + state.completeIncomingDurably(deliveryId); + state.completeIncoming(deliveryId, true); + assertEquals(java.util.List.of(deliveryId), state.await("lobby-1", java.util.UUID.randomUUID().toString(), 0).acks()); + assertEquals(1, callbacks.get()); + } + + @Test + void runningPublicationRollbackLeavesAReservationSafeAfterRestart() throws Exception { + Path root = directory.resolve("running-rollback-restart"); + Files.createDirectory(root); + String deliveryId = java.util.UUID.randomUUID().toString(); + Path inboundDirectory = root.resolve("lobby-1"); + Path running = inboundDirectory.resolve(deliveryId + ".running"); + HttpInboundDeliveryStore store = HttpInboundDeliveryStore.open(root, "lobby-1"); + try (var forces = org.mockito.Mockito.mockStatic(com.bencodez.simpleapi.file.DurableFiles.class, + org.mockito.Mockito.CALLS_REAL_METHODS)) { + java.util.concurrent.atomic.AtomicBoolean failed = new java.util.concurrent.atomic.AtomicBoolean(); + forces.when(() -> com.bencodez.simpleapi.file.DurableFiles.forceDirectory(inboundDirectory)) + .thenAnswer(call -> { + if (Files.exists(running) && !failed.getAndSet(true)) + throw new java.io.IOException("injected running fsync failure"); + return call.callRealMethod(); + }); + store.reserve(deliveryId); + assertThrows(com.bencodez.simpleapi.file.DurableFiles.PublishedException.class, + () -> store.markRunning(deliveryId)); + assertEquals(HttpInboundDeliveryStore.State.RESERVED, store.state(deliveryId)); + } + store.seal(); + assertEquals(HttpInboundDeliveryStore.State.RESERVED, + HttpInboundDeliveryStore.inspect(root, "lobby-1").state(deliveryId), + "a callback never exposed must be recoverable after a one-shot RUNNING force failure"); + } + + @Test + void proxyRetriesKnownNotStartedRunningRollbackBeforeDispatch() throws Exception { + Path root = directory.resolve("running-rollback-proxy"); + Files.createDirectory(root); + String deliveryId = java.util.UUID.randomUUID().toString(); + Path inboundDirectory = root.resolve("lobby-1"); + Path running = inboundDirectory.resolve(deliveryId + ".running"); + AtomicInteger failures = new AtomicInteger(2), callbacks = new AtomicInteger(); + java.util.concurrent.atomic.AtomicBoolean rollbackStarted = new java.util.concurrent.atomic.AtomicBoolean(); + HttpTransportProtocol.Delivery delivery = new HttpTransportProtocol.Delivery(deliveryId, JsonEnvelope.builder("vote").build()); + HttpInboundDeliveryStore store = HttpInboundDeliveryStore.open(root, "lobby-1"); + HttpProxyTransportServer.BackendState state = new HttpProxyTransportServer.BackendState( + "lobby-1", null, store, (server, id) -> { }); + try (var forces = org.mockito.Mockito.mockStatic(com.bencodez.simpleapi.file.DurableFiles.class, + org.mockito.Mockito.CALLS_REAL_METHODS)) { + forces.when(() -> com.bencodez.simpleapi.file.DurableFiles.forceDirectory(inboundDirectory)) + .thenAnswer(call -> { + if (Files.exists(running)) rollbackStarted.set(true); + if (rollbackStarted.get() && failures.getAndDecrement() > 0) + throw new java.io.IOException("injected running or rollback fsync failure"); + return call.callRealMethod(); + }); + assertEquals(java.util.List.of(delivery), state.acceptIncoming(java.util.List.of(delivery))); + assertThrows(com.bencodez.simpleapi.file.DurableFiles.PublishedException.class, + () -> state.beginIncoming(deliveryId)); + state.completeIncoming(deliveryId, false); + assertEquals(java.util.List.of(delivery), state.acceptIncoming(java.util.List.of(delivery)), + "the in-process known-not-started RUNNING state must recover once its rollback force succeeds"); + state.beginIncoming(deliveryId); + callbacks.incrementAndGet(); + state.completeIncomingDurably(deliveryId); + state.completeIncoming(deliveryId, true); + } + assertEquals(1, callbacks.get(), "the callback becomes eligible exactly once after RUNNING is durable"); + } + + @Test + void backendRetriesKnownNotStartedRunningRollbackBeforeDispatch() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("running-backend-proxy"), "localhost"); + HttpTlsIdentity.IssuedClientCertificate issued = identity.issueClientCertificate("lobby-1"); + HttpConnectionCode code = new HttpConnectionCode("lobby-1", java.net.URI.create("https://localhost:8443/"), + identity.serverCertificatePin(), identity.caCertificatePin(), Instant.now().plusSeconds(300), "D".repeat(43)); + Path clientDirectory = directory.resolve("running-backend-client"); + HttpClientCredentialStore.saveEnrolled(clientDirectory, code, issued); + String deliveryId = java.util.UUID.randomUUID().toString(); + Path inboundDirectory = clientDirectory.resolve("http-transport-inbound-deliveries"); + Path running = inboundDirectory.resolve(deliveryId + ".running"); + AtomicInteger failures = new AtomicInteger(2), callbacks = new AtomicInteger(); + java.util.concurrent.atomic.AtomicBoolean rollbackStarted = new java.util.concurrent.atomic.AtomicBoolean(); + HttpTransportProtocol.Delivery delivery = new HttpTransportProtocol.Delivery(deliveryId, JsonEnvelope.builder("vote").build()); + try (var forces = org.mockito.Mockito.mockStatic(com.bencodez.simpleapi.file.DurableFiles.class, + org.mockito.Mockito.CALLS_REAL_METHODS); + HttpBackendTransportConnector connector = new HttpBackendTransportConnector(clientDirectory, + ignored -> callbacks.incrementAndGet())) { + forces.when(() -> com.bencodez.simpleapi.file.DurableFiles.forceDirectory(inboundDirectory)) + .thenAnswer(call -> { + if (Files.exists(running)) rollbackStarted.set(true); + if (rollbackStarted.get() && failures.getAndDecrement() > 0) + throw new java.io.IOException("injected running or rollback fsync failure"); + return call.callRealMethod(); + }); + var inboundField = HttpBackendTransportConnector.class.getDeclaredField("inboundDeliveries"); + inboundField.setAccessible(true); + HttpInboundDeliveryStore store = (HttpInboundDeliveryStore) inboundField.get(connector); + store.reserve(deliveryId); + assertThrows(com.bencodez.simpleapi.file.DurableFiles.PublishedException.class, + () -> store.markRunning(deliveryId)); + java.util.List retried = connector.accept(java.util.List.of(delivery)); + assertEquals(java.util.List.of(delivery), retried); + connector.dispatch(retried.get(0)); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(2); + while (callbacks.get() == 0 && System.nanoTime() < deadline) Thread.sleep(5); + assertEquals(1, callbacks.get()); + } + } + + @Test + void failedAcknowledgementCallbackRetainsProxyDelivery() throws Exception { + HttpProxyTransportServer.BackendState state = new HttpProxyTransportServer.BackendState("lobby-1", null, + (server, deliveryId) -> { throw new java.io.IOException("cache save failed"); }); + String deliveryId = java.util.UUID.randomUUID().toString(); + assertTrue(state.enqueue(new HttpTransportProtocol.Delivery(deliveryId, + JsonEnvelope.builder("vote-party").build()))); + assertThrows(java.io.IOException.class, () -> state.acknowledge(java.util.List.of(deliveryId))); + assertEquals(deliveryId, state.await("lobby-1", java.util.UUID.randomUUID().toString(), 0) + .messages().iterator().next().id()); + } + + @Test + void oppositeDirectionIdsUseSeparateAcknowledgementNamespaces() throws Exception { + String deliveryId = java.util.UUID.randomUUID().toString(); + Path incomingRoot = directory.resolve("separate-ack-incoming"); + Files.createDirectory(incomingRoot); + HttpInboundDeliveryStore incoming = HttpInboundDeliveryStore.open(incomingRoot, "lobby-1"); + HttpProxyTransportServer.BackendState state = new HttpProxyTransportServer.BackendState( + "lobby-1", null, incoming, (server, id) -> { }); + assertTrue(state.enqueue(new HttpTransportProtocol.Delivery(deliveryId, + JsonEnvelope.builder("proxy-reply").build()))); + HttpTransportProtocol.Delivery backendMessage = new HttpTransportProtocol.Delivery(deliveryId, + JsonEnvelope.builder("backend-request").build()); + assertEquals(java.util.List.of(backendMessage), state.acceptIncoming(java.util.List.of(backendMessage))); + state.beginIncoming(deliveryId); + state.completeIncomingDurably(deliveryId); + state.completeIncoming(deliveryId, true); + + byte[] request = HttpTransportProtocol.request("lobby-1", java.util.UUID.randomUUID().toString(), 0, + java.util.List.of(), java.util.List.of(deliveryId), java.util.List.of()); + HttpTransportProtocol.Packet packet = HttpTransportProtocol.parsePacket(request); + state.confirmIncoming(packet.ackConfirmations()); + state.acknowledge(packet.acks()); + assertEquals(deliveryId, state.await("lobby-1", java.util.UUID.randomUUID().toString(), 0) + .messages().iterator().next().id(), + "confirming the backend-origin acknowledgement must not acknowledge a same-ID proxy reply"); + } + + @Test + void closeWaitsForTheCredentialOwningPollerToStop() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("close-proxy"), "localhost"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, directory.resolve("close-authority")); + try (HttpProxyTransportServer server = new HttpProxyTransportServer(new InetSocketAddress("localhost", 0), + identity, authority, ignored -> { })) { + server.start(); + Path clientDirectory = directory.resolve("close-client"); + HttpConnectionCode code = authority.createConnectionCode("lobby-1", server.endpoint("localhost"), + Duration.ofMinutes(5)); + HttpBackendTransportConnector.enroll(code, "lobby-1", clientDirectory); + try (HttpBackendTransportConnector connector = new HttpBackendTransportConnector(clientDirectory, ignored -> { })) { + connector.start(); + assertTrue(connector.awaitFirstResponse(System.nanoTime() + TimeUnit.SECONDS.toNanos(8))); + connector.close(); + assertFalse(connector.pollerAlive(), + "credential-directory ownership must outlive every poller filesystem mutation"); + } + } + } + + @Test + void closeDrainsRunningCallbacksBeforeSealingTheirJournal() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("close-callback-proxy"), "localhost"); + HttpTlsIdentity.IssuedClientCertificate issued = identity.issueClientCertificate("lobby-1"); + HttpConnectionCode code = new HttpConnectionCode("lobby-1", URI.create("https://localhost:8443/"), + identity.serverCertificatePin(), identity.caCertificatePin(), Instant.now().plusSeconds(300), "A".repeat(43)); + Path clientDirectory = directory.resolve("close-callback-client"); + HttpClientCredentialStore.saveEnrolled(clientDirectory, code, issued); + CountDownLatch started = new CountDownLatch(1), release = new CountDownLatch(1); + HttpBackendTransportConnector connector = new HttpBackendTransportConnector(clientDirectory, ignored -> { + started.countDown(); + try { release.await(); } + catch (InterruptedException interrupted) { Thread.currentThread().interrupt(); } + }); + String id = java.util.UUID.randomUUID().toString(); + connector.dispatch(new HttpTransportProtocol.Delivery(id, JsonEnvelope.builder("close-callback").build())); + assertTrue(started.await(2, TimeUnit.SECONDS)); + Thread closer = new Thread(connector::close, "close-callback-test"); + closer.start(); + try { + Thread.sleep(100); + assertTrue(closer.isAlive(), "close must wait for a running callback to finish its journal transition"); + } finally { release.countDown(); } + closer.join(3000); + assertFalse(closer.isAlive()); + assertEquals(HttpInboundDeliveryStore.State.COMPLETED, + HttpInboundDeliveryStore.inspect(clientDirectory).state(id)); + } + + @Test + void interruptedCloseRemainsBoundedWhenACallbackIgnoresInterruption() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("bounded-close-proxy"), "localhost"); + HttpTlsIdentity.IssuedClientCertificate issued = identity.issueClientCertificate("lobby-1"); + HttpConnectionCode code = new HttpConnectionCode("lobby-1", URI.create("https://localhost:8443/"), + identity.serverCertificatePin(), identity.caCertificatePin(), Instant.now().plusSeconds(300), "A".repeat(43)); + Path clientDirectory = directory.resolve("bounded-close-client"); + HttpClientCredentialStore.saveEnrolled(clientDirectory, code, issued); + CountDownLatch started = new CountDownLatch(1), release = new CountDownLatch(1), stopped = new CountDownLatch(1); + HttpBackendTransportConnector connector = new HttpBackendTransportConnector(clientDirectory, ignored -> { + started.countDown(); + while (release.getCount() != 0L) try { release.await(); } + catch (InterruptedException ignoredInterrupt) { } + stopped.countDown(); + }); + String id = java.util.UUID.randomUUID().toString(); + connector.dispatch(new HttpTransportProtocol.Delivery(id, JsonEnvelope.builder("bounded-close").build())); + assertTrue(started.await(2, TimeUnit.SECONDS)); + Thread closer = new Thread(connector::close, "bounded-close-test"); + closer.start(); + closer.interrupt(); + closer.join(2500); + assertFalse(closer.isAlive(), "a non-cooperative application callback must not hang connector shutdown"); + try { + release.countDown(); + assertTrue(stopped.await(2, TimeUnit.SECONDS)); + } finally { release.countDown(); } + assertEquals(HttpInboundDeliveryStore.State.RUNNING, + HttpInboundDeliveryStore.inspect(clientDirectory).state(id), + "an ambiguous callback must remain fail-closed after bounded shutdown"); + } + + @Test + void proxyShutdownGivesInterruptedCallbacksTimeToFinish() throws Exception { + java.util.concurrent.ThreadPoolExecutor executor = new java.util.concurrent.ThreadPoolExecutor(1, 1, 0L, + TimeUnit.MILLISECONDS, new java.util.concurrent.ArrayBlockingQueue<>(1)); + CountDownLatch started = new CountDownLatch(1), finished = new CountDownLatch(1); + executor.execute(() -> { + started.countDown(); + try { new CountDownLatch(1).await(); } + catch (InterruptedException stopRequested) { + try { Thread.sleep(100); } + catch (InterruptedException repeated) { Thread.currentThread().interrupt(); } + } finally { finished.countDown(); } + }); + assertTrue(started.await(2, TimeUnit.SECONDS)); + Thread closer = new Thread(() -> HttpProxyTransportServer.shutdown(executor), "proxy-shutdown-test"); + closer.start(); + closer.interrupt(); + closer.join(2500); + assertFalse(closer.isAlive()); + assertEquals(0L, finished.getCount(), + "forced shutdown must give a cooperative callback time to finish before journals are sealed"); + } + + @Test + void finalFlushDeliversMessagesQueuedBehindAnActiveLongPoll() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("flush-proxy"), "localhost"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, directory.resolve("flush-authority")); + CountDownLatch received = new CountDownLatch(1); + AtomicReference envelope = new AtomicReference<>(); + try (HttpProxyTransportServer server = new HttpProxyTransportServer(new InetSocketAddress("localhost", 0), + identity, authority, message -> { + envelope.set(message.envelope()); + received.countDown(); + })) { + server.start(); + Path clientDirectory = directory.resolve("flush-client"); + HttpConnectionCode code = authority.createConnectionCode("lobby-1", server.endpoint("localhost"), + Duration.ofMinutes(5)); + HttpBackendTransportConnector.enroll(code, "lobby-1", clientDirectory); + HttpBackendTransportConnector connector = new HttpBackendTransportConnector(clientDirectory, ignored -> { }); + connector.start(); + assertTrue(connector.awaitFirstResponse(System.nanoTime() + TimeUnit.SECONDS.toNanos(8))); + assertTrue(connector.send(JsonEnvelope.builder("backend-stopped").build())); + try { + assertTrue(connector.flushOutgoing(System.nanoTime() + TimeUnit.SECONDS.toNanos(5))); + assertTrue(received.await(1, TimeUnit.SECONDS)); + assertEquals("backend-stopped", envelope.get().getSubChannel()); + assertEquals(0, connector.queuedOutgoing()); + } finally { connector.close(); } + } + } + + @Test + void normalTransportRejectsAClientWithoutCertificate() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, directory.resolve("authority")); + try (HttpProxyTransportServer server = new HttpProxyTransportServer(new InetSocketAddress("localhost", 0), identity, authority, ignored -> { })) { + server.start(); + HttpConnectionCode code = authority.createConnectionCode("lobby-1", server.endpoint("localhost"), Duration.ofMinutes(5)); + HttpClient client = HttpClient.newBuilder().sslContext(HttpPinnedTls.clientContext(code)).build(); + byte[] body = HttpTransportProtocol.request("lobby-1", java.util.UUID.randomUUID().toString(), 0, + java.util.List.of(), java.util.List.of(), java.util.List.of()); + HttpResponse response = client.send(HttpRequest.newBuilder(code.endpoint().resolve("v1/transport")) + .timeout(Duration.ofSeconds(5)).header("Content-Type", "application/json").POST(HttpRequest.BodyPublishers.ofByteArray(body)).build(), HttpResponse.BodyHandlers.ofByteArray()); + assertEquals(401, response.statusCode()); + } + } + + @Test + void fixedRequestBodiesReportProtocolErrorsBeforeAdmissionErrors() { + Headers headers = new Headers(); + assertEquals(411, HttpProxyTransportServer.fixedBodyErrorStatus(headers, 128)); + headers.set("Content-Length", "invalid"); + assertEquals(400, HttpProxyTransportServer.fixedBodyErrorStatus(headers, 128)); + headers.set("Content-Length", "0"); + assertEquals(400, HttpProxyTransportServer.fixedBodyErrorStatus(headers, 128)); + headers.set("Content-Length", "129"); + assertEquals(413, HttpProxyTransportServer.fixedBodyErrorStatus(headers, 128)); + headers.set("Content-Length", "128"); + assertEquals(0, HttpProxyTransportServer.fixedBodyErrorStatus(headers, 128)); + headers.set("Transfer-Encoding", "chunked"); + assertEquals(400, HttpProxyTransportServer.fixedBodyErrorStatus(headers, 128)); + } + + @Test + void enrollmentPersistenceFailuresReturnServiceUnavailable() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("failed-enroll-proxy"), "localhost"); + Path authorityDirectory = directory.resolve("failed-enroll-authority"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, authorityDirectory); + try (HttpProxyTransportServer server = new HttpProxyTransportServer(new InetSocketAddress("localhost", 0), + identity, authority, ignored -> { })) { + server.start(); + HttpConnectionCode code = authority.createConnectionCode("lobby-1", server.endpoint("localhost"), + Duration.ofMinutes(5)); + Path state = authorityDirectory.resolve("http-transport-clients.properties"); + Files.delete(state); + Files.createDirectory(state); + byte[] body = ("{\"server\":\"lobby-1\",\"token\":\"" + code.enrollmentToken() + "\"}") + .getBytes(java.nio.charset.StandardCharsets.UTF_8); + HttpClient client = HttpClient.newBuilder().sslContext(HttpPinnedTls.clientContext(code)).build(); + HttpResponse response = client.send(HttpRequest.newBuilder(code.endpoint().resolve("v1/enroll")) + .timeout(Duration.ofSeconds(5)).header("Content-Type", "application/json") + .POST(HttpRequest.BodyPublishers.ofByteArray(body)).build(), HttpResponse.BodyHandlers.ofByteArray()); + assertEquals(503, response.statusCode()); + } + } + + @Test + void boundedQueuesFailClosed() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, directory.resolve("authority")); + try (HttpProxyTransportServer server = new HttpProxyTransportServer(new InetSocketAddress("localhost", 0), identity, authority, ignored -> { })) { + for (int i = 0; i < HttpTransportProtocol.MAX_QUEUE; i++) assertTrue(server.send("lobby-1", JsonEnvelope.builder("x").build())); + assertFalse(server.send("lobby-1", JsonEnvelope.builder("x").build())); + assertFalse(server.send("lobby-1", JsonEnvelope.builder("x").put("large", "x".repeat(HttpTransportProtocol.MAX_ENVELOPE_BYTES)).build())); + } + } + + @Test + void proxyBackendStateIsGloballyBounded() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("bounded-backend-proxy"), "localhost"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, + directory.resolve("bounded-backend-authority")); + try (HttpProxyTransportServer server = new HttpProxyTransportServer(new InetSocketAddress("localhost", 0), + identity, authority, ignored -> { })) { + for (int index = 0; index < 128; index++) + assertTrue(server.send("server-" + index, JsonEnvelope.builder("x").build())); + assertFalse(server.send("server-overflow", JsonEnvelope.builder("x").build())); + assertTrue(server.send("server-0", JsonEnvelope.builder("existing").build()), + "the global bound must not reject an existing backend state"); + } + } + + @Test + void proxyReclaimsOnlyQuiescentBackendStateAfterReplayWindow() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("reclaim-proxy"), "localhost"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, + directory.resolve("reclaim-authority")); + AtomicLong nanoTime = new AtomicLong(); + try (HttpProxyTransportServer server = new HttpProxyTransportServer(new InetSocketAddress("localhost", 0), + identity, authority, null, ignored -> { }, (serverId, deliveryId) -> { }, nanoTime::get)) { + for (int index = 0; index < 128; index++) { + HttpProxyTransportServer.BackendState state = server.backendStateForTest("server-" + index); + assertTrue(state.beginPollForTest()); + state.endPollForTest(); + } + assertFalse(server.send("replacement", JsonEnvelope.builder("x").build()), + "fresh state must retain its replay fence"); + nanoTime.addAndGet(TimeUnit.MILLISECONDS.toNanos(HttpTransportProtocol.MAX_CLOCK_SKEW_MILLIS) + 1L); + assertTrue(server.send("replacement", JsonEnvelope.builder("x").build()), + "a quiescent state must be reclaimable after captured requests expire"); + assertEquals(128, server.backendCountForTest()); + } + } + + @Test + void proxyDoesNotReclaimAQuarantineOnlyBackendAfterReplayWindow() throws Exception { + Path queueRoot = directory.resolve("quarantine-reclamation-outgoing"); + AtomicLong forceCalls = new AtomicLong(); + String serverId = "quarantined"; + HttpTransportProtocol.Delivery retry = new HttpTransportProtocol.Delivery( + java.util.UUID.randomUUID().toString(), JsonEnvelope.builder("retry").build()); + try (HttpProxyTransportServer.DurableOutgoingQueue queue = new HttpProxyTransportServer.DurableOutgoingQueue( + queueRoot, ignored -> { + if (forceCalls.incrementAndGet() == 3L) throw new java.io.IOException("injected directory force failure"); + })) { + HttpProxyTransportServer.BackendState state = new HttpProxyTransportServer.BackendState( + serverId, queue, (server, id) -> { }); + assertFalse(state.enqueue(retry)); + } + + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("quarantine-reclamation-proxy"), "localhost"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, + directory.resolve("quarantine-reclamation-authority")); + AtomicLong nanoTime = new AtomicLong(); + try (HttpProxyTransportServer server = new HttpProxyTransportServer(new InetSocketAddress("localhost", 0), + identity, authority, queueRoot, ignored -> { }, (backend, delivery) -> { }, nanoTime::get)) { + for (int index = 0; index < 127; index++) { + HttpProxyTransportServer.BackendState state = server.backendStateForTest("idle-" + index); + assertTrue(state.beginPollForTest()); + state.endPollForTest(); + } + nanoTime.addAndGet(TimeUnit.MILLISECONDS.toNanos(HttpTransportProtocol.MAX_CLOCK_SKEW_MILLIS) + 1L); + for (int index = 0; index < 127; index++) + assertTrue(server.send("replacement-" + index, JsonEnvelope.builder("replacement").build())); + assertFalse(server.send("overflow", JsonEnvelope.builder("overflow").build())); + assertTrue(server.backendStateForTest(serverId).enqueue(retry), + "the identical retry must still reach and promote its quarantined delivery at capacity"); + } + } + + @Test + void proxyOutgoingQueueSurvivesRestartUntilBackendAcknowledges() throws Exception { + Path proxyDirectory = directory.resolve("proxy"); + Path authorityDirectory = directory.resolve("authority"); + Path queueDirectory = directory.resolve("outgoing"); + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(proxyDirectory, "localhost"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, authorityDirectory); + HttpProxyTransportServer first = new HttpProxyTransportServer(new InetSocketAddress("localhost", 0), + identity, authority, queueDirectory, ignored -> { }); + assertTrue(first.send("lobby-1", JsonEnvelope.builder("durable").build())); + first.close(); + + CountDownLatch received = new CountDownLatch(1); + try (HttpProxyTransportServer restarted = new HttpProxyTransportServer( + new InetSocketAddress("localhost", 0), identity, authority, queueDirectory, ignored -> { })) { + restarted.start(); + HttpConnectionCode code = authority.createConnectionCode("lobby-1", restarted.endpoint("localhost"), + Duration.ofMinutes(5)); + HttpClientCredentialStore.ClientCredential credential = HttpBackendTransportConnector.enroll(code, + "lobby-1", directory.resolve("durable-client")); + try (HttpBackendTransportConnector connector = new HttpBackendTransportConnector(code, "lobby-1", + credential, envelope -> received.countDown())) { + connector.start(); + assertTrue(received.await(8, TimeUnit.SECONDS)); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (countRegularFiles(queueDirectory) != 0L && System.nanoTime() < deadline) Thread.sleep(20); + assertEquals(0L, countRegularFiles(queueDirectory), "backend ACK must durably remove the delivery"); + } + } + } + + @Test + void pollCreatedBackendStateUsesDurableOutgoingQueue() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("poll-proxy"), "localhost"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, directory.resolve("poll-authority")); + Path queueDirectory = directory.resolve("poll-outgoing"); + CountDownLatch proxyReceived = new CountDownLatch(1), backendReceived = new CountDownLatch(1); + CountDownLatch releaseBackendCallback = new CountDownLatch(1); + try (HttpProxyTransportServer server = new HttpProxyTransportServer(new InetSocketAddress("localhost", 0), + identity, authority, queueDirectory, ignored -> proxyReceived.countDown())) { + server.start(); + HttpConnectionCode code = authority.createConnectionCode("lobby-1", server.endpoint("localhost"), + Duration.ofMinutes(5)); + HttpClientCredentialStore.ClientCredential credential = HttpBackendTransportConnector.enroll(code, + "lobby-1", directory.resolve("poll-client")); + try (HttpBackendTransportConnector connector = new HttpBackendTransportConnector(code, "lobby-1", + credential, envelope -> { + backendReceived.countDown(); + try { releaseBackendCallback.await(5, TimeUnit.SECONDS); } + catch (InterruptedException interrupted) { Thread.currentThread().interrupt(); } + })) { + connector.start(); + assertTrue(connector.send(JsonEnvelope.builder("establish-poll").build())); + assertTrue(proxyReceived.await(8, TimeUnit.SECONDS)); + assertTrue(server.send("lobby-1", JsonEnvelope.builder("durable-after-poll").build())); + assertTrue(backendReceived.await(8, TimeUnit.SECONDS)); + assertEquals(1L, countRegularFiles(queueDirectory), + "a poll-created backend state must persist before reporting acceptance"); + releaseBackendCallback.countDown(); + } + } finally { + releaseBackendCallback.countDown(); + } + } + + private static long countRegularFiles(Path root) throws Exception { + try (java.util.stream.Stream paths = java.nio.file.Files.walk(root)) { + return paths.filter(path -> java.nio.file.Files.isRegularFile(path, java.nio.file.LinkOption.NOFOLLOW_LINKS)).count(); + } + } + + @Test + void aggregatePacketBudgetSplitsLargeValidEnvelopes() { + java.util.List candidates = new java.util.ArrayList<>(); + for (int index = 0; index < 12; index++) candidates.add(new HttpTransportProtocol.Delivery( + java.util.UUID.randomUUID().toString(), JsonEnvelope.builder("large").put("value", "x".repeat(40_000)).build())); + String session = java.util.UUID.randomUUID().toString(); + java.util.List fitted = HttpTransportProtocol.fittingMessages( + "lobby-1", session, 0, java.util.List.of(), java.util.List.of(), candidates); + assertTrue(fitted.size() > 0 && fitted.size() < candidates.size()); + assertTrue(HttpTransportProtocol.request("lobby-1", session, 0, java.util.List.of(), java.util.List.of(), fitted).length + <= HttpTransportProtocol.MAX_BODY_BYTES); + } + + @Test + void packetNumbersMustUseCanonicalJsonIntegerTokens() { + com.google.gson.JsonObject packet = com.google.gson.JsonParser.parseString(new String(HttpTransportProtocol.request( + "lobby-1", java.util.UUID.randomUUID().toString(), 0, java.util.List.of(), java.util.List.of(), java.util.List.of()), + java.nio.charset.StandardCharsets.UTF_8)).getAsJsonObject(); + assertDoesNotThrow(() -> HttpTransportProtocol.parsePacket(packet.toString() + .getBytes(java.nio.charset.StandardCharsets.UTF_8))); + String timestamp = packet.get("timestamp").getAsString(); + java.util.Map> invalid = java.util.Map.of( + "v", java.util.List.of("\"1\"", "1.0", "1e0"), + "sequence", java.util.List.of("\"0\"", "0.0", "0e0"), + "timestamp", java.util.List.of("\"" + timestamp + "\"", timestamp + ".0", timestamp + "e0")); + for (var field : invalid.entrySet()) for (String token : field.getValue()) { + com.google.gson.JsonObject rejected = packet.deepCopy(); + rejected.add(field.getKey(), com.google.gson.JsonParser.parseString(token)); + assertThrows(IllegalArgumentException.class, () -> HttpTransportProtocol.parsePacket( + rejected.toString().getBytes(java.nio.charset.StandardCharsets.UTF_8)), field.getKey() + "=" + token); + } + } + + @Test + void packetParsingRejectsNoncanonicalUuidForms() { + String deliveryId = java.util.UUID.randomUUID().toString(); + com.google.gson.JsonObject packet = com.google.gson.JsonParser.parseString(new String(HttpTransportProtocol.request( + "lobby-1", java.util.UUID.randomUUID().toString(), 0, java.util.List.of(deliveryId), + java.util.List.of(deliveryId), + java.util.List.of(new HttpTransportProtocol.Delivery(deliveryId, JsonEnvelope.builder("payload").build()))), + java.nio.charset.StandardCharsets.UTF_8)).getAsJsonObject(); + String abbreviated = "1-1-1-1-1"; + + com.google.gson.JsonObject invalidSession = packet.deepCopy(); + invalidSession.addProperty("session", abbreviated); + assertThrows(IllegalArgumentException.class, () -> HttpTransportProtocol.parsePacket( + invalidSession.toString().getBytes(java.nio.charset.StandardCharsets.UTF_8))); + + com.google.gson.JsonObject invalidAck = packet.deepCopy(); + invalidAck.getAsJsonArray("acks").set(0, new com.google.gson.JsonPrimitive(abbreviated)); + assertThrows(IllegalArgumentException.class, () -> HttpTransportProtocol.parsePacket( + invalidAck.toString().getBytes(java.nio.charset.StandardCharsets.UTF_8))); + com.google.gson.JsonObject invalidConfirmation = packet.deepCopy(); + invalidConfirmation.getAsJsonArray("ackConfirmations").set(0, new com.google.gson.JsonPrimitive(abbreviated)); + assertThrows(IllegalArgumentException.class, () -> HttpTransportProtocol.parsePacket( + invalidConfirmation.toString().getBytes(java.nio.charset.StandardCharsets.UTF_8))); + + com.google.gson.JsonObject invalidMessage = packet.deepCopy(); + invalidMessage.getAsJsonArray("messages").get(0).getAsJsonObject().addProperty("id", abbreviated); + assertThrows(IllegalArgumentException.class, () -> HttpTransportProtocol.parsePacket( + invalidMessage.toString().getBytes(java.nio.charset.StandardCharsets.UTF_8))); + } + + @Test + void persistedProfileStartsAfterTheEnrollmentCodeExpires() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, directory.resolve("authority")); + try (HttpProxyTransportServer server = new HttpProxyTransportServer(new InetSocketAddress("localhost", 0), identity, authority, ignored -> { })) { + server.start(); + HttpConnectionCode active = authority.createConnectionCode("lobby-1", server.endpoint("localhost"), Duration.ofMinutes(5)); + HttpBackendTransportConnector.enroll(active, "lobby-1", directory.resolve("client")); + HttpConnectionCode expired = new HttpConnectionCode(active.serverId(), active.endpoint(), active.serverCertificatePin(), active.caCertificatePin(), + java.time.Instant.now().minusSeconds(1), active.enrollmentToken()); + try (HttpBackendTransportConnector ignored = new HttpBackendTransportConnector(expired, "lobby-1", directory.resolve("client"), message -> { })) { + assertTrue(true); + } + try (HttpBackendTransportConnector ignored = new HttpBackendTransportConnector(directory.resolve("client"), message -> { })) { + assertTrue(true); + } + } + } + + @Test + void persistedBackendConnectsAfterAutomaticServerLeafRotation() throws Exception { + Instant now = Instant.now(); + Path proxyDirectory = directory.resolve("proxy"); + HttpTlsIdentity original = HttpTlsIdentity.loadOrCreate(proxyDirectory, "localhost", + java.time.Clock.fixed(now.minus(Duration.ofDays(340)), java.time.ZoneOffset.UTC)); + String originalServerPin = HttpTransportSecrets.certificatePin(original.serverCertificate()); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(original, directory.resolve("authority")); + HttpTlsIdentity rotated = HttpTlsIdentity.loadOrCreate(proxyDirectory, "localhost"); + CountDownLatch received = new CountDownLatch(1); + try (HttpProxyTransportServer server = new HttpProxyTransportServer(new InetSocketAddress("localhost", 0), rotated, + authority, ignored -> received.countDown())) { + HttpConnectionCode activeCode = authority.createConnectionCode("lobby-1", server.endpoint("localhost"), Duration.ofMinutes(5)); + HttpTlsIdentity.IssuedClientCertificate issued = authority.enroll("lobby-1", activeCode.enrollmentToken()); + HttpConnectionCode oldProfileCode = new HttpConnectionCode(activeCode.serverId(), activeCode.endpoint(), originalServerPin, + activeCode.caCertificatePin(), activeCode.expiresAt(), activeCode.enrollmentToken()); + HttpClientCredentialStore.saveEnrolled(directory.resolve("client"), oldProfileCode, issued); + assertFalse(oldProfileCode.serverCertificatePin().equals(rotated.serverCertificatePin())); + server.start(); + try (HttpBackendTransportConnector connector = new HttpBackendTransportConnector(directory.resolve("client"), ignored -> { })) { + connector.start(); + assertTrue(connector.send(JsonEnvelope.builder("after-rotation").build())); + assertTrue(received.await(8, TimeUnit.SECONDS)); + } + } + } + + @Test + void renewalRetainsOldCredentialUntilPublishedPointerIsConfirmed() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("pointer-proxy"), "localhost"); + HttpTlsIdentity.IssuedClientCertificate expiring = identity.issueClientCertificate("lobby-1", + Instant.now().minus(Duration.ofDays(340))); + String originalPin = HttpTransportSecrets.certificatePin(expiring.certificate()); + Path authorityDirectory = Files.createDirectory(directory.resolve("pointer-authority")); + String key = java.util.Base64.getUrlEncoder().withoutPadding() + .encodeToString("lobby-1".getBytes(java.nio.charset.StandardCharsets.UTF_8)); + Files.writeString(authorityDirectory.resolve("http-transport-clients.properties"), + "version=2\nbinding." + key + "=" + originalPin + ":-:0\n"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, authorityDirectory); + CountDownLatch received = new CountDownLatch(1); + try (HttpProxyTransportServer server = new HttpProxyTransportServer(new InetSocketAddress("localhost", 0), + identity, authority, ignored -> received.countDown())) { + Path clientDirectory = directory.resolve("pointer-client"); + HttpConnectionCode code = new HttpConnectionCode("lobby-1", server.endpoint("localhost"), + identity.serverCertificatePin(), identity.caCertificatePin(), Instant.now().plusSeconds(60), "A".repeat(43)); + HttpClientCredentialStore.saveEnrolled(clientDirectory, code, expiring); + server.start(); + try (HttpBackendTransportConnector connector = new HttpBackendTransportConnector(clientDirectory, ignored -> { })) { + var renew = HttpBackendTransportConnector.class.getDeclaredMethod("maybeRenewCredential"); + renew.setAccessible(true); + var pending = HttpBackendTransportConnector.class.getDeclaredField("pendingActivation"); + pending.setAccessible(true); + var credential = HttpBackendTransportConnector.class.getDeclaredField("credential"); + credential.setAccessible(true); + Path pointer = clientDirectory.resolve("http-transport-client-current"); + String generation; + try (var forces = org.mockito.Mockito.mockStatic(com.bencodez.simpleapi.file.DurableFiles.class, + org.mockito.Mockito.CALLS_REAL_METHODS)) { + AtomicLong clientDirectoryForces = new AtomicLong(); + forces.when(() -> com.bencodez.simpleapi.file.DurableFiles.forceDirectory(clientDirectory)) + .thenAnswer(call -> { + // Staging first confirms the existing directory. Subsequent forces + // are the CURRENT pointer publication and its same-generation retry. + if (clientDirectoryForces.incrementAndGet() > 1L) + throw new java.io.IOException("injected pointer force failure"); + return call.callRealMethod(); + }); + renew.invoke(connector); + generation = Files.readString(pointer); + var selected = HttpClientCredentialStore.load(clientDirectory); + String selectedPin = HttpTransportSecrets.certificatePin(selected.certificate()); + assertFalse(originalPin.equals(selectedPin)); + assertEquals(originalPin, HttpTransportSecrets.certificatePin( + ((HttpClientCredentialStore.ClientCredential) credential.get(connector)).certificate())); + assertTrue(pending.get(connector) != null); + assertTrue(authority.authenticate("lobby-1", expiring.certificate()), + "uncertain pointer publication must not invalidate the restart-safe old credential"); + renew.invoke(connector); + assertEquals(generation, Files.readString(pointer)); + assertTrue(pending.get(connector) != null, "failed retry must retain the same pending activation"); + } + renew.invoke(connector); + assertTrue(pending.get(connector) == null); + assertEquals(generation, Files.readString(pointer)); + var confirmed = HttpClientCredentialStore.load(clientDirectory); + assertEquals(HttpTransportSecrets.certificatePin(confirmed.certificate()), + HttpTransportSecrets.certificatePin(((HttpClientCredentialStore.ClientCredential) + credential.get(connector)).certificate())); + assertTrue(authority.authenticate("lobby-1", confirmed.certificate())); + assertFalse(authority.authenticate("lobby-1", expiring.certificate())); + connector.start(); + assertTrue(connector.send(JsonEnvelope.builder("after-pointer-recovery").build())); + assertTrue(received.await(8, TimeUnit.SECONDS), "transport must use the adopted TLS client"); + } + } + } + + @Test + void backendRenewsClientCertificateBeforeExpiryWithoutNewConnectionCode() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); + HttpTlsIdentity.IssuedClientCertificate expiring = identity.issueClientCertificate("lobby-1", + Instant.now().minus(Duration.ofDays(340))); + String originalPin = HttpTransportSecrets.certificatePin(expiring.certificate()); + Path authorityDirectory = directory.resolve("authority"); + java.nio.file.Files.createDirectories(authorityDirectory); + String key = java.util.Base64.getUrlEncoder().withoutPadding() + .encodeToString("lobby-1".getBytes(java.nio.charset.StandardCharsets.UTF_8)); + java.nio.file.Files.writeString(authorityDirectory.resolve("http-transport-clients.properties"), + "version=2\nbinding." + key + "=" + originalPin + ":-:0\n"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, authorityDirectory); + try (HttpProxyTransportServer server = new HttpProxyTransportServer(new InetSocketAddress("localhost", 0), identity, + authority, ignored -> { })) { + HttpConnectionCode profileCode = new HttpConnectionCode("lobby-1", server.endpoint("localhost"), + identity.serverCertificatePin(), identity.caCertificatePin(), Instant.now().plusSeconds(60), "A".repeat(43)); + Path clientDirectory = directory.resolve("client"); + HttpClientCredentialStore.saveEnrolled(clientDirectory, profileCode, expiring); + server.start(); + try (HttpBackendTransportConnector connector = new HttpBackendTransportConnector(clientDirectory, ignored -> { })) { + connector.start(); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(8); + String renewedPin = originalPin; + while (renewedPin.equals(originalPin) && System.nanoTime() < deadline) { + Thread.sleep(25); + renewedPin = HttpTransportSecrets.certificatePin(HttpClientCredentialStore.load(clientDirectory).certificate()); + } + assertFalse(renewedPin.equals(originalPin)); + HttpClientCredentialStore.ClientCredential renewed = HttpClientCredentialStore.load(clientDirectory); + long promotionDeadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5); + while (authority.authenticate("lobby-1", expiring.certificate()) && System.nanoTime() < promotionDeadline) Thread.sleep(25); + assertTrue(authority.authenticate("lobby-1", renewed.certificate())); + assertFalse(authority.authenticate("lobby-1", expiring.certificate())); + } + } + } + + @Test + void duplicateInboundDeliveryIsReAcknowledgedWithoutSecondDispatch() { + HttpProxyTransportServer.BackendState state = new HttpProxyTransportServer.BackendState(); + String session = java.util.UUID.randomUUID().toString(); + String id = java.util.UUID.randomUUID().toString(); + HttpTransportProtocol.Delivery delivery = new HttpTransportProtocol.Delivery(id, JsonEnvelope.builder("x").build()); + assertTrue(state.acceptSession(session, 0)); + assertEquals(1, state.acceptIncoming(java.util.List.of(delivery)).size()); + state.completeIncoming(id, true); + assertEquals(java.util.List.of(id), state.await("lobby-1", session, 0).acks()); + assertTrue(state.acceptSession(session, 1)); + assertTrue(state.acceptIncoming(java.util.List.of(delivery)).isEmpty()); + assertEquals(java.util.List.of(id), state.await("lobby-1", session, 1).acks()); + String replacementSession = java.util.UUID.randomUUID().toString(); + assertTrue(state.acceptSession(replacementSession, 0)); + assertTrue(state.acceptIncoming(java.util.List.of(delivery)).isEmpty()); + assertEquals(java.util.List.of(id), state.await("lobby-1", replacementSession, 0).acks()); + } + + @Test + void newerDeliveriesDoNotPostponeRetryOfOlderUnacknowledgedDelivery() { + AtomicLong nanoTime = new AtomicLong(1L); + HttpProxyTransportServer.BackendState state = new HttpProxyTransportServer.BackendState(nanoTime::get); + String session = java.util.UUID.randomUUID().toString(); + HttpTransportProtocol.Delivery first = new HttpTransportProtocol.Delivery( + java.util.UUID.randomUUID().toString(), JsonEnvelope.builder("first").build()); + HttpTransportProtocol.Delivery second = new HttpTransportProtocol.Delivery( + java.util.UUID.randomUUID().toString(), JsonEnvelope.builder("second").build()); + + assertTrue(state.acceptSession(session, 0)); + assertTrue(state.enqueue(first)); + assertEquals(java.util.List.of(first), state.await("lobby-1", session, 0).messages()); + + nanoTime.addAndGet(TimeUnit.SECONDS.toNanos(1)); + assertTrue(state.acceptSession(session, 1)); + assertTrue(state.enqueue(second)); + assertEquals(java.util.List.of(second), state.await("lobby-1", session, 1).messages()); + + nanoTime.addAndGet(TimeUnit.MILLISECONDS.toNanos(1100)); + assertTrue(state.acceptSession(session, 2)); + assertEquals(java.util.List.of(first), state.await("lobby-1", session, 2).messages(), + "sending newer traffic must not reset an older delivery's retry age"); + } + + @Test + void longPollWakesAtTheOldestDeliveryRetryDeadline() throws Exception { + AtomicLong nanoTime = new AtomicLong(1L); + HttpProxyTransportServer.BackendState state = new HttpProxyTransportServer.BackendState(nanoTime::get); + String session = java.util.UUID.randomUUID().toString(); + HttpTransportProtocol.Delivery first = new HttpTransportProtocol.Delivery( + java.util.UUID.randomUUID().toString(), JsonEnvelope.builder("first").build()); + HttpTransportProtocol.Delivery second = new HttpTransportProtocol.Delivery( + java.util.UUID.randomUUID().toString(), JsonEnvelope.builder("second").build()); + assertTrue(state.acceptSession(session, 0)); + assertTrue(state.enqueue(first)); + assertEquals(java.util.List.of(first), state.await("lobby-1", session, 0).messages()); + + nanoTime.addAndGet(HttpProxyTransportServer.LONG_POLL.minusMillis(100).toNanos()); + assertTrue(state.acceptSession(session, 1)); + assertTrue(state.enqueue(second)); + assertEquals(java.util.List.of(second), state.await("lobby-1", session, 1).messages()); + assertTrue(state.acceptSession(session, 2)); + Thread clock = new Thread(() -> { + try { Thread.sleep(50L); } + catch (InterruptedException interrupted) { Thread.currentThread().interrupt(); } + nanoTime.addAndGet(TimeUnit.MILLISECONDS.toNanos(100)); + }, "HTTP-retry-test-clock"); + clock.setDaemon(true); + long started = System.nanoTime(); + clock.start(); + assertEquals(java.util.List.of(first), state.await("lobby-1", session, 2).messages()); + clock.join(); + assertTrue(System.nanoTime() - started < TimeUnit.SECONDS.toNanos(1), + "the poll must wake at the oldest delivery deadline, not a fresh long-poll deadline"); + } + + @Test + void proxyDedupWindowEvictsOldestCompletedDeliveryAtCapacity() { + HttpProxyTransportServer.BackendState state = new HttpProxyTransportServer.BackendState(); + String oldest = null, newest = null; + for (int index = 0; index < HttpTransportProtocol.MAX_QUEUE + 1; index++) { + String id = java.util.UUID.randomUUID().toString(); + if (index == 0) oldest = id; + newest = id; + HttpTransportProtocol.Delivery delivery = new HttpTransportProtocol.Delivery(id, JsonEnvelope.builder("x").build()); + assertEquals(1, state.acceptIncoming(java.util.List.of(delivery)).size()); + state.completeIncoming(id, true); + } + HttpTransportProtocol.Delivery evicted = new HttpTransportProtocol.Delivery(oldest, JsonEnvelope.builder("x").build()); + HttpTransportProtocol.Delivery retained = new HttpTransportProtocol.Delivery(newest, JsonEnvelope.builder("x").build()); + assertEquals(1, state.acceptIncoming(java.util.List.of(evicted)).size()); + assertTrue(state.acceptIncoming(java.util.List.of(retained)).isEmpty()); + } + + @Test + void backendReAcknowledgesLostAckDuplicateWithoutSecondCallback() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); + HttpTlsIdentity.IssuedClientCertificate issued = identity.issueClientCertificate("lobby-1"); + HttpClientCredentialStore.save(directory.resolve("client"), issued); + HttpClientCredentialStore.HttpClientProfile profile = new HttpClientCredentialStore.HttpClientProfile("lobby-1", + java.net.URI.create("https://localhost:8443/"), identity.serverCertificatePin(), identity.caCertificatePin()); + CountDownLatch callback = new CountDownLatch(1); + try (HttpBackendTransportConnector connector = new HttpBackendTransportConnector(profile, + HttpClientCredentialStore.load(directory.resolve("client")), envelope -> callback.countDown())) { + String id = java.util.UUID.randomUUID().toString(); + HttpTransportProtocol.Delivery delivery = new HttpTransportProtocol.Delivery(id, JsonEnvelope.builder("x").build()); + connector.dispatch(delivery); + assertTrue(callback.await(2, TimeUnit.SECONDS)); + java.util.List acknowledgements = java.util.List.of(); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(2); + while (acknowledgements.isEmpty() && System.nanoTime() < deadline) { + acknowledgements = connector.drainAcknowledgements(); + if (acknowledgements.isEmpty()) Thread.sleep(5); + } + assertEquals(java.util.List.of(id), acknowledgements); + assertTrue(connector.accept(java.util.List.of(delivery)).isEmpty()); + assertEquals(java.util.List.of(id), connector.drainAcknowledgements()); + } + } + + @Test + void backendCallbacksAreSerializedInDeliveryOrder() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("ordered-proxy"), "localhost"); + HttpTlsIdentity.IssuedClientCertificate issued = identity.issueClientCertificate("lobby-1"); + HttpClientCredentialStore.save(directory.resolve("ordered-client"), issued); + HttpClientCredentialStore.HttpClientProfile profile = new HttpClientCredentialStore.HttpClientProfile("lobby-1", + java.net.URI.create("https://localhost:8443/"), identity.serverCertificatePin(), identity.caCertificatePin()); + CountDownLatch firstStarted = new CountDownLatch(1), releaseFirst = new CountDownLatch(1), secondStarted = new CountDownLatch(1); + java.util.List order = new java.util.concurrent.CopyOnWriteArrayList<>(); + try (HttpBackendTransportConnector connector = new HttpBackendTransportConnector(profile, + HttpClientCredentialStore.load(directory.resolve("ordered-client")), envelope -> { + String marker = String.valueOf(envelope.getFields().get("marker")); + order.add(marker); + if ("first".equals(marker)) { + firstStarted.countDown(); + try { releaseFirst.await(5, TimeUnit.SECONDS); } + catch (InterruptedException interrupted) { Thread.currentThread().interrupt(); } + } else secondStarted.countDown(); + })) { + connector.dispatch(new HttpTransportProtocol.Delivery(java.util.UUID.randomUUID().toString(), + JsonEnvelope.builder("x").put("marker", "first").build())); + connector.dispatch(new HttpTransportProtocol.Delivery(java.util.UUID.randomUUID().toString(), + JsonEnvelope.builder("x").put("marker", "second").build())); + assertTrue(firstStarted.await(2, TimeUnit.SECONDS)); + assertFalse(secondStarted.await(150, TimeUnit.MILLISECONDS)); + releaseFirst.countDown(); + assertTrue(secondStarted.await(2, TimeUnit.SECONDS)); + assertEquals(java.util.List.of("first", "second"), order); + } finally { releaseFirst.countDown(); } + } + + @Test + void backendCallbackQueueBackpressuresWithoutBreakingFifo() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("backpressure-proxy"), "localhost"); + HttpTlsIdentity.IssuedClientCertificate issued = identity.issueClientCertificate("lobby-1"); + HttpClientCredentialStore.save(directory.resolve("backpressure-client"), issued); + HttpClientCredentialStore.HttpClientProfile profile = new HttpClientCredentialStore.HttpClientProfile("lobby-1", + java.net.URI.create("https://localhost:8443/"), identity.serverCertificatePin(), identity.caCertificatePin()); + CountDownLatch firstStarted = new CountDownLatch(1), releaseFirst = new CountDownLatch(1); + int deliveries = HttpBackendTransportConnector.CALLBACK_QUEUE_CAPACITY + 2; + CountDownLatch completed = new CountDownLatch(deliveries), overflowSubmitted = new CountDownLatch(1); + java.util.List order = new java.util.concurrent.CopyOnWriteArrayList<>(); + try (HttpBackendTransportConnector connector = new HttpBackendTransportConnector(profile, + HttpClientCredentialStore.load(directory.resolve("backpressure-client")), envelope -> { + int marker = Integer.parseInt(envelope.getFields().get("marker")); + order.add(marker); + if (marker == 0) { + firstStarted.countDown(); + try { releaseFirst.await(5, TimeUnit.SECONDS); } + catch (InterruptedException interrupted) { Thread.currentThread().interrupt(); } + } + completed.countDown(); + })) { + connector.dispatch(delivery(0)); + assertTrue(firstStarted.await(2, TimeUnit.SECONDS)); + for (int marker = 1; marker <= HttpBackendTransportConnector.CALLBACK_QUEUE_CAPACITY; marker++) + connector.dispatch(delivery(marker)); + Thread overflow = new Thread(() -> { + connector.dispatch(delivery(deliveries - 1)); + overflowSubmitted.countDown(); + }, "HTTP-overflow-submitter"); + overflow.start(); + assertFalse(overflowSubmitted.await(150, TimeUnit.MILLISECONDS), "a full ordered lane must backpressure its producer"); + releaseFirst.countDown(); + assertTrue(overflowSubmitted.await(2, TimeUnit.SECONDS)); + assertTrue(completed.await(5, TimeUnit.SECONDS)); + assertEquals(java.util.stream.IntStream.range(0, deliveries).boxed().toList(), order); + } finally { releaseFirst.countDown(); } + } + + @Test + void durableBackendFencePreventsCallbackReplayAfterRestartBeforeAck() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("fence-proxy"), "localhost"); + HttpTlsIdentity.IssuedClientCertificate issued = identity.issueClientCertificate("lobby-1"); + HttpConnectionCode code = new HttpConnectionCode("lobby-1", java.net.URI.create("https://localhost:8443/"), + identity.serverCertificatePin(), identity.caCertificatePin(), Instant.now().plusSeconds(300), "A".repeat(43)); + Path clientDirectory = directory.resolve("fence-client"); + HttpClientCredentialStore.saveEnrolled(clientDirectory, code, issued); + java.util.concurrent.atomic.AtomicInteger callbacks = new java.util.concurrent.atomic.AtomicInteger(); + String id = java.util.UUID.randomUUID().toString(); + HttpTransportProtocol.Delivery delivery = new HttpTransportProtocol.Delivery(id, JsonEnvelope.builder("vote").build()); + try (HttpBackendTransportConnector first = new HttpBackendTransportConnector(clientDirectory, + ignored -> callbacks.incrementAndGet())) { + first.dispatch(delivery); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(2); + while (callbacks.get() != 1 && System.nanoTime() < deadline) Thread.sleep(5); + assertEquals(1, callbacks.get()); + } + try (HttpBackendTransportConnector restarted = new HttpBackendTransportConnector(clientDirectory, + ignored -> callbacks.incrementAndGet())) { + assertEquals(java.util.List.of(id), restarted.drainAcknowledgements(), + "restart must retain and acknowledge the pre-callback delivery fence"); + assertTrue(restarted.accept(java.util.List.of(delivery)).isEmpty()); + assertEquals(1, callbacks.get(), "a durable proxy replay must not award twice"); + } + } + + @Test + void failedBackendCallbackRemainsUnacknowledgedAndIsNotReplayed() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("retry-fence-proxy"), "localhost"); + HttpTlsIdentity.IssuedClientCertificate issued = identity.issueClientCertificate("lobby-1"); + HttpConnectionCode code = new HttpConnectionCode("lobby-1", java.net.URI.create("https://localhost:8443/"), + identity.serverCertificatePin(), identity.caCertificatePin(), Instant.now().plusSeconds(300), "B".repeat(43)); + Path clientDirectory = directory.resolve("retry-fence-client"); + HttpClientCredentialStore.saveEnrolled(clientDirectory, code, issued); + java.util.concurrent.atomic.AtomicInteger attempts = new java.util.concurrent.atomic.AtomicInteger(); + CountDownLatch failed = new CountDownLatch(1); + HttpTransportProtocol.Delivery delivery = new HttpTransportProtocol.Delivery(java.util.UUID.randomUUID().toString(), + JsonEnvelope.builder("vote").build()); + try (HttpBackendTransportConnector first = new HttpBackendTransportConnector(clientDirectory, ignored -> { + attempts.incrementAndGet(); failed.countDown(); throw new IllegalStateException("retry"); + })) { + first.dispatch(delivery); + assertTrue(failed.await(2, TimeUnit.SECONDS)); + assertTrue(first.drainAcknowledgements().isEmpty()); + } + try (HttpBackendTransportConnector restarted = new HttpBackendTransportConnector(clientDirectory, + ignored -> attempts.incrementAndGet())) { + assertTrue(restarted.drainAcknowledgements().isEmpty()); + java.util.List accepted = restarted.accept(java.util.List.of(delivery)); + assertTrue(accepted.isEmpty(), "an ambiguous callback must not be awarded twice"); + assertEquals(1, attempts.get()); + } + } + + @Test + void reservedButNotStartedDeliveryResumesAfterRestart() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("reserved-proxy"), "localhost"); + HttpTlsIdentity.IssuedClientCertificate issued = identity.issueClientCertificate("lobby-1"); + HttpConnectionCode code = new HttpConnectionCode("lobby-1", java.net.URI.create("https://localhost:8443/"), + identity.serverCertificatePin(), identity.caCertificatePin(), Instant.now().plusSeconds(300), "C".repeat(43)); + Path clientDirectory = directory.resolve("reserved-client"); + HttpClientCredentialStore.saveEnrolled(clientDirectory, code, issued); + String id = java.util.UUID.randomUUID().toString(); + HttpInboundDeliveryStore reservedStore = new HttpInboundDeliveryStore(clientDirectory); + reservedStore.reserve(id); + reservedStore.seal(); + CountDownLatch completed = new CountDownLatch(1); + HttpTransportProtocol.Delivery delivery = new HttpTransportProtocol.Delivery(id, JsonEnvelope.builder("vote").build()); + try (HttpBackendTransportConnector restarted = new HttpBackendTransportConnector(clientDirectory, ignored -> completed.countDown())) { + assertTrue(restarted.drainAcknowledgements().isEmpty(), "a reservation alone must never be acknowledged"); + java.util.List accepted = restarted.accept(java.util.List.of(delivery)); + assertEquals(1, accepted.size()); + restarted.dispatch(accepted.get(0)); + assertTrue(completed.await(2, TimeUnit.SECONDS)); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(2); + java.util.List acknowledgements = java.util.List.of(); + while (acknowledgements.isEmpty() && System.nanoTime() < deadline) { + acknowledgements = restarted.drainAcknowledgements(); + if (acknowledgements.isEmpty()) Thread.sleep(5); + } + assertEquals(java.util.List.of(id), acknowledgements); + } + } + + @Test + void interruptedStateRenameRetainsTheFurthestSafeState() throws Exception { + Path clientDirectory = directory.resolve("interrupted-state-client"); + String id = java.util.UUID.randomUUID().toString(); + String completedId = java.util.UUID.randomUUID().toString(); + Path states = clientDirectory.resolve("http-transport-inbound-deliveries"); + Files.createDirectories(states); + Files.writeString(states.resolve(id + ".reserved"), id); + Files.writeString(states.resolve(id + ".running"), id); + Files.writeString(states.resolve(completedId + ".running"), completedId); + Files.writeString(states.resolve(completedId + ".completed"), completedId); + HttpInboundDeliveryStore store = new HttpInboundDeliveryStore(clientDirectory); + assertEquals(HttpInboundDeliveryStore.State.RUNNING, store.state(id)); + assertEquals(HttpInboundDeliveryStore.State.COMPLETED, store.state(completedId)); + assertFalse(Files.exists(states.resolve(id + ".reserved"))); + assertTrue(Files.exists(states.resolve(id + ".running"))); + assertFalse(Files.exists(states.resolve(completedId + ".running"))); + assertTrue(Files.exists(states.resolve(completedId + ".completed"))); + } + + private static HttpTransportProtocol.Delivery delivery(int marker) { + return new HttpTransportProtocol.Delivery(java.util.UUID.randomUUID().toString(), + JsonEnvelope.builder("x").put("marker", marker).build()); + } + + @Test + void proxyCallbacksAreSerializedInDeliveryOrder() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("ordered-proxy-server"), "localhost"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, directory.resolve("ordered-authority")); + CountDownLatch firstStarted = new CountDownLatch(1), releaseFirst = new CountDownLatch(1), secondStarted = new CountDownLatch(1); + java.util.List order = new java.util.concurrent.CopyOnWriteArrayList<>(); + try (HttpProxyTransportServer server = new HttpProxyTransportServer(new InetSocketAddress("localhost", 0), identity, + authority, received -> { + String marker = String.valueOf(received.envelope().getFields().get("marker")); + order.add(marker); + if ("first".equals(marker)) { + firstStarted.countDown(); + try { releaseFirst.await(5, TimeUnit.SECONDS); } + catch (InterruptedException interrupted) { Thread.currentThread().interrupt(); } + } else secondStarted.countDown(); + })) { + server.start(); + HttpConnectionCode code = authority.createConnectionCode("lobby-1", server.endpoint("localhost"), Duration.ofMinutes(5)); + HttpClientCredentialStore.ClientCredential credential = HttpBackendTransportConnector.enroll(code, "lobby-1", + directory.resolve("ordered-proxy-client")); + try (HttpBackendTransportConnector connector = new HttpBackendTransportConnector(code, "lobby-1", credential, + ignored -> { })) { + connector.start(); + assertTrue(connector.send(JsonEnvelope.builder("x").put("marker", "first").build())); + assertTrue(connector.send(JsonEnvelope.builder("x").put("marker", "second").build())); + assertTrue(firstStarted.await(3, TimeUnit.SECONDS)); + assertFalse(secondStarted.await(150, TimeUnit.MILLISECONDS)); + releaseFirst.countDown(); + assertTrue(secondStarted.await(3, TimeUnit.SECONDS)); + assertEquals(java.util.List.of("first", "second"), order); + } + } finally { releaseFirst.countDown(); } + } + + @Test + void backendDedupWindowContinuesAfterCapacity() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); + HttpTlsIdentity.IssuedClientCertificate issued = identity.issueClientCertificate("lobby-1"); + HttpClientCredentialStore.save(directory.resolve("client"), issued); + HttpClientCredentialStore.HttpClientProfile profile = new HttpClientCredentialStore.HttpClientProfile("lobby-1", + java.net.URI.create("https://localhost:8443/"), identity.serverCertificatePin(), identity.caCertificatePin()); + try (HttpBackendTransportConnector connector = new HttpBackendTransportConnector(profile, + HttpClientCredentialStore.load(directory.resolve("client")), envelope -> { })) { + for (int index = 0; index < HttpTransportProtocol.MAX_QUEUE; index++) { + String id = java.util.UUID.randomUUID().toString(); + HttpTransportProtocol.Delivery delivery = new HttpTransportProtocol.Delivery(id, JsonEnvelope.builder("x").build()); + assertEquals(1, connector.accept(java.util.List.of(delivery)).size()); + connector.completeIncoming(id, true); + } + HttpTransportProtocol.Delivery next = new HttpTransportProtocol.Delivery(java.util.UUID.randomUUID().toString(), + JsonEnvelope.builder("x").build()); + assertEquals(1, connector.accept(java.util.List.of(next)).size()); + } + } +} diff --git a/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportSecurityTest.java b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportSecurityTest.java new file mode 100644 index 0000000..0c0352d --- /dev/null +++ b/SimpleAPI/src/test/java/com/bencodez/simpleapi/servercomm/http/HttpTransportSecurityTest.java @@ -0,0 +1,697 @@ +package com.bencodez.simpleapi.servercomm.http; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.net.URI; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.cert.X509Certificate; +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.time.ZoneOffset; +import java.util.Arrays; +import javax.net.ssl.X509TrustManager; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class HttpTransportSecurityTest { + @Test + void failedRenewalRetriesBeforeActiveCertificateExpires() { + assertEquals(Duration.ofMinutes(5), + HttpBackendTransportConnector.renewalRetryDelay(Duration.ofHours(6))); + assertEquals(Duration.ofMinutes(1), + HttpBackendTransportConnector.renewalRetryDelay(Duration.ofMinutes(4))); + assertTrue(HttpBackendTransportConnector.renewalRetryDelay(Duration.ofSeconds(3)) + .compareTo(Duration.ofSeconds(3)) < 0); + } + + @Test + void connectionCodeRejectsExplicitZeroPort() { + assertThrows(IllegalArgumentException.class, + () -> new HttpConnectionCode("lobby", URI.create("https://proxy.example.test:0/"), pin('a'), + pin('b'), Instant.now().plusSeconds(60), "token")); + } + + @TempDir Path directory; + + @Test + void backendResponseReaderRejectsBodiesBeyondTheWireLimit() throws Exception { + byte[] maximum = new byte[HttpTransportProtocol.MAX_BODY_BYTES]; + assertEquals(maximum.length, HttpBackendTransportConnector.readLimited( + new java.io.ByteArrayInputStream(maximum)).length); + assertThrows(java.io.IOException.class, () -> HttpBackendTransportConnector.readLimited( + new java.io.ByteArrayInputStream(new byte[HttpTransportProtocol.MAX_BODY_BYTES + 1]))); + } + + @Test + void connectionCodeRoundTripsAndRejectsAccidentalCorruption() { + HttpConnectionCode original = new HttpConnectionCode("lobby.eu", URI.create("https://Proxy.Example.test:8443/http"), pin('a'), pin('b'), + Instant.parse("2030-01-01T00:00:00Z"), HttpTransportSecrets.randomToken()); + String encoded = original.encode(); + HttpConnectionCode parsed = HttpConnectionCode.parse(encoded); + assertEquals("lobby.eu", parsed.serverId()); + assertEquals(URI.create("https://proxy.example.test:8443/http/"), parsed.endpoint()); + assertEquals(original.serverCertificatePin(), parsed.serverCertificatePin()); + char last = encoded.charAt(encoded.length() - 1); + assertThrows(IllegalArgumentException.class, () -> HttpConnectionCode.parse(encoded.substring(0, encoded.length() - 1) + + (last == 'A' ? 'B' : 'A'))); + assertThrows(IllegalArgumentException.class, () -> HttpConnectionCode.parse("http://not-a-code")); + } + + @Test + void connectionCodePreservesEscapedEndpointPaths() { + HttpConnectionCode original = new HttpConnectionCode("lobby.eu", + URI.create("https://Proxy.Example.test:8443/api%20root/%2F"), pin('a'), pin('b'), + Instant.parse("2030-01-01T00:00:00Z"), HttpTransportSecrets.randomToken()); + URI expected = URI.create("https://proxy.example.test:8443/api%20root/%2F/"); + assertEquals(expected, original.endpoint()); + assertEquals(expected, HttpConnectionCode.parse(original.encode()).endpoint()); + } + + @Test + void legacyConnectionCodesAndConsumedMarkersRemainCompatible() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("legacy-code-proxy"), "proxy.example.test"); + HttpConnectionCode legacy = new HttpConnectionCode("lobby", URI.create("https://proxy.example.test:8443/"), + identity.serverCertificatePin(), identity.caCertificatePin(), + Instant.now().plusSeconds(60).truncatedTo(java.time.temporal.ChronoUnit.SECONDS), + HttpTransportSecrets.randomToken()); + assertEquals(legacy, HttpConnectionCode.parse(legacy.encodeLegacy())); + + Path client = directory.resolve("legacy-code-client"); + HttpClientCredentialStore.saveEnrolled(client, legacy, identity.issueClientCertificate("lobby")); + Path active = client.resolve("http-transport-client-generations") + .resolve(Files.readString(client.resolve("http-transport-client-current"))); + Files.writeString(active.resolve("http-transport-connection-code.sha256"), + HttpTransportSecrets.sha256Hex(legacy.encodeLegacy().getBytes(java.nio.charset.StandardCharsets.US_ASCII))); + assertTrue(HttpClientCredentialStore.matchesEnrollmentCode(client, + HttpConnectionCode.parse(legacy.encodeLegacy()))); + } + + @Test + void legacyConnectionCodesSupportDottedServerIds() { + HttpConnectionCode original = new HttpConnectionCode("lobby.eu", URI.create("https://proxy.example.test:8443/"), + pin('a'), pin('b'), Instant.parse("2030-01-01T00:00:00Z"), HttpTransportSecrets.randomToken()); + assertEquals(original, HttpConnectionCode.parse(original.encodeLegacy())); + } + + @Test + void expiredCodesAreNotActive() { + HttpConnectionCode code = new HttpConnectionCode("lobby", URI.create("https://proxy.example.test/"), pin('a'), pin('b'), + Instant.parse("2029-12-31T23:59:59Z"), HttpTransportSecrets.randomToken()); + assertFalse(code.isActive(Clock.fixed(Instant.parse("2030-01-01T00:00:00Z"), ZoneOffset.UTC))); + assertThrows(IllegalArgumentException.class, () -> code.requireActive(Clock.fixed(Instant.parse("2030-01-01T00:00:00Z"), ZoneOffset.UTC))); + } + + @Test + void inboundDeliveryFenceRejectsCorruptionAndPathReplacement() throws Exception { + Path corruptCredentials = directory.resolve("corrupt-client"); + Path corruptFence = corruptCredentials.resolve("http-transport-inbound-deliveries"); + Files.createDirectories(corruptFence); + Files.writeString(corruptFence.resolve("not-a-delivery.seen"), "not-a-delivery"); + assertThrows(java.io.IOException.class, () -> new HttpInboundDeliveryStore(corruptCredentials)); + + Path replacedCredentials = directory.resolve("replaced-client"); + Files.createDirectories(replacedCredentials); + HttpInboundDeliveryStore store = new HttpInboundDeliveryStore(replacedCredentials); + Path fence = replacedCredentials.resolve("http-transport-inbound-deliveries"); + Path outside = directory.resolve("outside-fence"); + Files.createDirectory(outside); + Files.delete(fence); + Files.createSymbolicLink(fence, outside); + assertThrows(java.io.IOException.class, () -> store.reserve(java.util.UUID.randomUUID().toString())); + } + + @Test + void sealedInboundStoreCannotChangeAfterOwnershipHandoff() throws Exception { + Path credentials = directory.resolve("sealed-client"); + Files.createDirectories(credentials); + String id = java.util.UUID.randomUUID().toString(); + HttpInboundDeliveryStore store = new HttpInboundDeliveryStore(credentials); + store.reserve(id); + store.markRunning(id); + store.seal(); + assertThrows(java.io.IOException.class, () -> store.markCompleted(id)); + assertEquals(HttpInboundDeliveryStore.State.RUNNING, new HttpInboundDeliveryStore(credentials).state(id)); + } + + @Test + void identityIsDurableAndPinsRejectTheWrongServer() throws Exception { + HttpTlsIdentity created = HttpTlsIdentity.loadOrCreate(directory, "localhost"); + HttpTlsIdentity loaded = HttpTlsIdentity.loadOrCreate(directory, "localhost"); + assertEquals(created.serverCertificatePin(), loaded.serverCertificatePin()); + assertEquals(created.caCertificatePin(), loaded.caCertificatePin()); + HttpConnectionCode correct = new HttpConnectionCode("lobby", URI.create("https://localhost:8443/"), created.serverCertificatePin(), + created.caCertificatePin(), Instant.now().plusSeconds(60), HttpTransportSecrets.randomToken()); + HttpConnectionCode incorrect = new HttpConnectionCode("lobby", URI.create("https://localhost:8443/"), pin('0'), created.caCertificatePin(), + Instant.now().plusSeconds(60), HttpTransportSecrets.randomToken()); + assertTrue(HttpPinnedTls.matchesServerPin(correct, created.serverCertificate())); + assertFalse(HttpPinnedTls.matchesServerPin(incorrect, created.serverCertificate())); + assertTrue(Files.exists(directory.resolve("http-transport-ca.p12"))); + HttpTlsIdentity rotated = HttpTlsIdentity.loadOrCreate(directory, "127.0.0.1"); + assertEquals(created.caCertificatePin(), rotated.caCertificatePin()); + assertNotEquals(created.serverCertificatePin(), rotated.serverCertificatePin()); + } + + @Test + void oversizedTlsAndClientPasswordFilesFailClosed() throws Exception { + Path identityDirectory = directory.resolve("oversized-password-proxy"); + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(identityDirectory, "localhost"); + Files.writeString(identityDirectory.resolve("http-transport-password"), "x".repeat(129)); + assertThrows(java.io.IOException.class, () -> HttpTlsIdentity.loadOrCreate(identityDirectory, "localhost")); + + Path clientDirectory = directory.resolve("oversized-password-client"); + HttpClientCredentialStore.save(clientDirectory, identity.issueClientCertificate("lobby-1")); + Files.writeString(clientDirectory.resolve("http-transport-client-password"), "x".repeat(129)); + assertThrows(java.io.IOException.class, () -> HttpClientCredentialStore.load(clientDirectory)); + } + + @Test + void privateCredentialRootsRejectSymbolicLinks() throws Exception { + Path identityTarget = directory.resolve("identity-target"); + Path identityLink = directory.resolve("identity-link"); + Files.createDirectory(identityTarget); + Files.createSymbolicLink(identityLink, identityTarget); + assertThrows(java.io.IOException.class, () -> HttpTlsIdentity.loadOrCreate(identityLink, "localhost")); + assertFalse(Files.exists(identityTarget.resolve("http-transport-ca.p12"))); + + HttpTlsIdentity authority = HttpTlsIdentity.loadOrCreate(directory.resolve("safe-identity"), "localhost"); + HttpTlsIdentity.IssuedClientCertificate issued = authority.issueClientCertificate("lobby-1"); + Path credentialTarget = directory.resolve("credential-target"); + Path credentialLink = directory.resolve("credential-link"); + Files.createDirectory(credentialTarget); + Files.createSymbolicLink(credentialLink, credentialTarget); + assertThrows(java.io.IOException.class, () -> HttpClientCredentialStore.save(credentialLink, issued)); + HttpConnectionCode code = new HttpConnectionCode("lobby-1", URI.create("https://localhost:8443/"), + authority.serverCertificatePin(), authority.caCertificatePin(), Instant.now().plusSeconds(60), "A".repeat(43)); + assertThrows(java.io.IOException.class, () -> HttpClientCredentialStore.saveEnrolled(credentialLink, code, issued)); + try (var files = Files.list(credentialTarget)) { + assertTrue(files.findAny().isEmpty(), "a symlinked credential root must receive no private files"); + } + } + + @Test + void markedIncompleteFirstRunTlsProvisioningRecoversWithoutManualCleanup() throws Exception { + Path source = directory.resolve("complete-identity"); + HttpTlsIdentity original = HttpTlsIdentity.loadOrCreate(source, "localhost"); + Path interrupted = directory.resolve("interrupted-identity"); + Files.createDirectories(interrupted); + Files.writeString(interrupted.resolve("http-transport-initializing"), "initializing\n"); + Files.copy(source.resolve("http-transport-ca.p12"), interrupted.resolve("http-transport-ca.p12")); + Files.copy(source.resolve("http-transport-server.p12"), interrupted.resolve("http-transport-server.p12")); + + HttpTlsIdentity recovered = HttpTlsIdentity.loadOrCreate(interrupted, "localhost"); + + assertNotEquals(original.caCertificatePin(), recovered.caCertificatePin()); + assertTrue(Files.exists(interrupted.resolve("http-transport-ca.p12"))); + assertTrue(Files.exists(interrupted.resolve("http-transport-server.p12"))); + assertTrue(Files.exists(interrupted.resolve("http-transport-password"))); + assertFalse(Files.exists(interrupted.resolve("http-transport-initializing"))); + } + + @Test + void unmarkedPartialIdentityFailsClosedWithExternalTransportState() throws Exception { + Path source = directory.resolve("external-state-source"); + HttpTlsIdentity.loadOrCreate(source, "localhost"); + Path partial = directory.resolve("external-state-partial"); + Files.createDirectories(partial); + Files.copy(source.resolve("http-transport-ca.p12"), partial.resolve("http-transport-ca.p12")); + byte[] retainedCa = Files.readAllBytes(partial.resolve("http-transport-ca.p12")); + Path externalState = directory.resolve("external-authority"); + Files.createDirectories(externalState); + Files.writeString(externalState.resolve("http-transport-clients.properties"), "version=3\n"); + + assertThrows(java.io.IOException.class, () -> HttpTlsIdentity.loadOrCreate(partial, "localhost")); + assertTrue(Arrays.equals(retainedCa, Files.readAllBytes(partial.resolve("http-transport-ca.p12"))), + "fail-closed recovery must preserve the surviving CA bytes"); + } + + @Test + void completedFirstRunFilesRecoverWhenInitializationMarkerSurvives() throws Exception { + Path interrupted = directory.resolve("marked-identity"); + HttpTlsIdentity original = HttpTlsIdentity.loadOrCreate(interrupted, "localhost"); + String originalCaPin = original.caCertificatePin(); + Files.writeString(interrupted.resolve("http-transport-initializing"), "initializing\n"); + + HttpTlsIdentity recovered = HttpTlsIdentity.loadOrCreate(interrupted, "localhost"); + + assertNotEquals(originalCaPin, recovered.caCertificatePin()); + assertFalse(Files.exists(interrupted.resolve("http-transport-initializing"))); + } + + @Test + void incompleteEstablishedTlsIdentityFailsClosed() throws Exception { + Path established = directory.resolve("established-identity"); + HttpTlsIdentity.loadOrCreate(established, "localhost"); + Files.writeString(established.resolve("http-transport-clients.properties"), "version=2\n"); + Files.delete(established.resolve("http-transport-server.p12")); + + assertThrows(java.io.IOException.class, () -> HttpTlsIdentity.loadOrCreate(established, "localhost")); + assertTrue(Files.exists(established.resolve("http-transport-ca.p12"))); + assertTrue(Files.exists(established.resolve("http-transport-password"))); + } + + @Test + void enrollmentIsSingleUseBoundToServerAndRevocable() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory, "localhost"); + Clock clock = Clock.fixed(Instant.parse("2030-01-01T00:00:00Z"), ZoneOffset.UTC); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, clock); + HttpConnectionCode wrongTargetCode = authority.createConnectionCode("lobby-1", URI.create("https://localhost:8443/"), Duration.ofMinutes(5)); + assertThrows(IllegalArgumentException.class, () -> authority.enroll("attacker", wrongTargetCode.enrollmentToken())); + assertTrue(authority.authenticate("lobby-1", authority.enroll("lobby-1", wrongTargetCode.enrollmentToken()).certificate()), + "a wrong backend must not consume another backend's connection code"); + authority.revoke("lobby-1"); + HttpConnectionCode code = authority.createConnectionCode("lobby-1", URI.create("https://localhost:8443/"), Duration.ofMinutes(5)); + HttpTlsIdentity.IssuedClientCertificate issued = authority.enroll("lobby-1", code.enrollmentToken()); + assertTrue(authority.authenticate("lobby-1", issued.certificate())); + assertTrue(identity.validClientCertificate("LOBBY-1", issued.certificate())); + assertFalse(authority.authenticate("lobby-2", issued.certificate())); + assertThrows(IllegalArgumentException.class, () -> authority.enroll("lobby-2", code.enrollmentToken())); + authority.revoke("lobby-1"); + assertFalse(authority.authenticate("lobby-1", issued.certificate())); + HttpConnectionCode replacementCode = authority.createConnectionCode("lobby-1", URI.create("https://localhost:8443/"), Duration.ofMinutes(5)); + HttpTlsIdentity.IssuedClientCertificate replacement = authority.enroll("lobby-1", replacementCode.enrollmentToken()); + assertTrue(authority.authenticate("lobby-1", replacement.certificate())); + assertFalse(authority.authenticate("lobby-1", issued.certificate())); + HttpClientCredentialStore.saveEnrolled(directory.resolve("client"), code, issued); + HttpClientCredentialStore.ClientCredential restored = HttpClientCredentialStore.load(directory.resolve("client")); + assertEquals(HttpTransportSecrets.certificatePin(issued.certificate()), HttpTransportSecrets.certificatePin(restored.certificate())); + HttpClientCredentialStore.HttpClientProfile profile = HttpClientCredentialStore.loadProfile(directory.resolve("client")); + assertEquals("lobby-1", profile.serverId()); + assertEquals(code.endpoint(), profile.endpoint()); + assertEquals("lobby-1", HttpClientCredentialStore.loadEnrolled(directory.resolve("client")).profile().serverId()); + assertNotEquals(null, HttpPinnedTls.mutualTlsContext(code, restored)); + Path clientDirectory = directory.resolve("client"); + String generation = Files.readString(clientDirectory.resolve("http-transport-client-current")); + Files.writeString(clientDirectory.resolve("http-transport-client-generations").resolve(generation) + .resolve("http-transport-profile.properties"), "version=1\nserverId=lobby-1\n"); + assertThrows(java.io.IOException.class, () -> HttpClientCredentialStore.loadProfile(directory.resolve("client"))); + HttpEnrollmentAuthority durable = new HttpEnrollmentAuthority(identity, directory.resolve("state")); + HttpConnectionCode durableCode = durable.createConnectionCode("survival", URI.create("https://localhost:8443/"), Duration.ofMinutes(5)); + HttpTlsIdentity.IssuedClientCertificate durableIssued = durable.enroll("survival", durableCode.enrollmentToken()); + assertTrue(new HttpEnrollmentAuthority(identity, directory.resolve("state")).authenticate("survival", durableIssued.certificate())); + durable.revoke("survival"); + assertFalse(new HttpEnrollmentAuthority(identity, directory.resolve("state")).authenticate("survival", durableIssued.certificate())); + } + + @Test + void revocationInvalidatesEveryPendingCodeForTheBackend() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("revoke-pending"), "localhost"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, + Clock.fixed(Instant.parse("2030-01-01T00:00:00Z"), ZoneOffset.UTC)); + URI endpoint = URI.create("https://localhost:8443/"); + HttpConnectionCode beforeEnrollment = authority.createConnectionCode("lobby-1", endpoint, Duration.ofMinutes(5)); + authority.revoke("LOBBY-1"); + assertThrows(IllegalArgumentException.class, + () -> authority.enroll("lobby-1", beforeEnrollment.enrollmentToken())); + + HttpConnectionCode active = authority.createConnectionCode("lobby-1", endpoint, Duration.ofMinutes(5)); + authority.enroll("lobby-1", active.enrollmentToken()); + HttpConnectionCode firstPending = authority.createConnectionCode("lobby-1", endpoint, Duration.ofMinutes(5)); + HttpConnectionCode secondPending = authority.createConnectionCode("lobby-1", endpoint, Duration.ofMinutes(5)); + authority.revoke("lobby-1"); + assertThrows(IllegalArgumentException.class, + () -> authority.enroll("lobby-1", firstPending.enrollmentToken())); + assertThrows(IllegalArgumentException.class, + () -> authority.enroll("lobby-1", secondPending.enrollmentToken())); + } + + @Test + void failedRevocationPersistenceCanBeRetriedWithoutLosingState() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("retry-revoke-proxy"), "localhost"); + Path stateDirectory = directory.resolve("retry-revoke-state"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, stateDirectory); + URI endpoint = URI.create("https://localhost:8443/"); + HttpConnectionCode active = authority.createConnectionCode("lobby-1", endpoint, Duration.ofMinutes(5)); + HttpTlsIdentity.IssuedClientCertificate issued = authority.enroll("lobby-1", active.enrollmentToken()); + HttpConnectionCode pending = authority.createConnectionCode("lobby-1", endpoint, Duration.ofMinutes(5)); + HttpConnectionCode unaffected = authority.createConnectionCode("lobby-2", endpoint, Duration.ofMinutes(5)); + HttpTlsIdentity.IssuedClientCertificate unaffectedIssued = authority.enroll("lobby-2", unaffected.enrollmentToken()); + Path stateFile = stateDirectory.resolve("http-transport-clients.properties"); + Files.delete(stateFile); + Files.createDirectory(stateFile); + + assertThrows(IllegalStateException.class, () -> authority.revoke("lobby-1")); + assertFalse(authority.authenticate("lobby-1", issued.certificate()), + "an unpersisted revocation must fail authentication closed"); + assertFalse(authority.authenticate("lobby-2", unaffectedIssued.certificate()), + "all authentication must fail closed while persistence is unresolved"); + Files.delete(stateFile); + assertThrows(java.io.IOException.class, () -> authority.enroll("lobby-1", pending.enrollmentToken()), + "a restored pending code must not republish an unresolved revocation"); + assertThrows(IllegalStateException.class, + () -> authority.createConnectionCode("lobby-3", endpoint, Duration.ofMinutes(5)), + "new codes must not persist the pre-revocation state either"); + authority.revoke("lobby-1"); + assertThrows(IllegalArgumentException.class, () -> authority.enroll("lobby-1", pending.enrollmentToken()), + "the revocation retry must permanently consume every prior code"); + assertTrue(authority.authenticate("lobby-2", unaffectedIssued.certificate()), + "a successful full-state retry must restore authentication availability"); + + HttpEnrollmentAuthority restarted = new HttpEnrollmentAuthority(identity, stateDirectory); + assertFalse(restarted.authenticate("lobby-1", issued.certificate())); + assertThrows(IllegalArgumentException.class, + () -> restarted.enroll("lobby-1", pending.enrollmentToken())); + } + + @Test + void pendingEnrollmentSurvivesRestartAndRevocationRemainsDurable() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("pending-restart-proxy"), "localhost"); + Path state = directory.resolve("pending-restart-state"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, state); + HttpConnectionCode code = authority.createConnectionCode("lobby-1", + URI.create("https://localhost:8443/"), Duration.ofMinutes(5)); + assertFalse(Files.readString(state.resolve("http-transport-clients.properties")) + .contains(code.enrollmentToken()), "raw enrollment tokens must never be persisted"); + + HttpEnrollmentAuthority restarted = new HttpEnrollmentAuthority(identity, state); + HttpTlsIdentity.IssuedClientCertificate issued = restarted.enroll("lobby-1", code.enrollmentToken()); + assertTrue(restarted.authenticate("lobby-1", issued.certificate())); + + restarted.revoke("lobby-1"); + HttpConnectionCode revokedPending = restarted.createConnectionCode("lobby-1", + URI.create("https://localhost:8443/"), Duration.ofMinutes(5)); + restarted.revoke("lobby-1"); + HttpEnrollmentAuthority afterRevocation = new HttpEnrollmentAuthority(identity, state); + assertThrows(IllegalArgumentException.class, + () -> afterRevocation.enroll("lobby-1", revokedPending.enrollmentToken())); + } + + @Test + void lostEnrollmentResponseCanRetryUntilCertificatePossessionIsProved() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("retry-enrollment-proxy"), "localhost"); + Path state = directory.resolve("retry-enrollment-state"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, state); + HttpConnectionCode code = authority.createConnectionCode("lobby-1", + URI.create("https://localhost:8443/"), Duration.ofMinutes(5)); + HttpTlsIdentity.IssuedClientCertificate lost = authority.enroll("lobby-1", code.enrollmentToken()); + + HttpEnrollmentAuthority restarted = new HttpEnrollmentAuthority(identity, state); + HttpTlsIdentity.IssuedClientCertificate retried = restarted.enroll("lobby-1", code.enrollmentToken()); + HttpEnrollmentAuthority beforeProof = new HttpEnrollmentAuthority(identity, state); + assertFalse(beforeProof.authenticate("lobby-1", lost.certificate()), + "retrying enrollment must supersede the certificate from the lost response"); + assertTrue(beforeProof.authenticate("lobby-1", retried.certificate()), + "the first authenticated request must promote the certificate durably"); + + HttpEnrollmentAuthority afterProof = new HttpEnrollmentAuthority(identity, state); + assertTrue(afterProof.authenticate("lobby-1", retried.certificate())); + assertThrows(IllegalArgumentException.class, + () -> afterProof.enroll("lobby-1", code.enrollmentToken()), + "proof of possession must consume the one-time enrollment token"); + } + + @Test + void failedPendingCertificateWriteLeavesEnrollmentTokenRetryable() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("failed-enrollment-proxy"), "localhost"); + Path state = directory.resolve("failed-enrollment-state"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, state); + HttpConnectionCode code = authority.createConnectionCode("lobby-1", + URI.create("https://localhost:8443/"), Duration.ofMinutes(5)); + Path stateFile = state.resolve("http-transport-clients.properties"); + Files.delete(stateFile); + Files.createDirectory(stateFile); + + assertThrows(java.io.IOException.class, () -> authority.enroll("lobby-1", code.enrollmentToken())); + Files.delete(stateFile); + HttpTlsIdentity.IssuedClientCertificate retried = authority.enroll("lobby-1", code.enrollmentToken()); + assertTrue(authority.authenticate("lobby-1", retried.certificate())); + } + + @Test + void enrollmentPublicationFailureDisablesAuthenticationUntilCodePersistenceRecovers() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("published-enrollment-proxy"), "localhost"); + Path stateDirectory = directory.resolve("published-enrollment-state"); + Files.createDirectories(stateDirectory); + java.util.concurrent.atomic.AtomicReference now = new java.util.concurrent.atomic.AtomicReference<>( + Instant.parse("2030-01-01T00:00:00Z")); + Clock clock = new Clock() { + @Override public ZoneOffset getZone() { return ZoneOffset.UTC; } + @Override public Clock withZone(java.time.ZoneId zone) { return this; } + @Override public Instant instant() { return now.get(); } + }; + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, clock, + stateDirectory.resolve("http-transport-clients.properties")); + URI endpoint = URI.create("https://localhost:8443/"); + HttpConnectionCode unrelatedCode = authority.createConnectionCode("unrelated", endpoint, Duration.ofMinutes(5)); + HttpTlsIdentity.IssuedClientCertificate unrelated = authority.enroll("unrelated", unrelatedCode.enrollmentToken()); + assertTrue(authority.authenticate("unrelated", unrelated.certificate())); + + HttpConnectionCode targetCode = authority.createConnectionCode("lobby-1", endpoint, Duration.ofMinutes(5)); + try (var forces = org.mockito.Mockito.mockStatic(com.bencodez.simpleapi.file.DurableFiles.class, + org.mockito.Mockito.CALLS_REAL_METHODS)) { + forces.when(() -> com.bencodez.simpleapi.file.DurableFiles.forceDirectory(stateDirectory)) + .thenThrow(new java.io.IOException("injected authority publication failure")); + assertThrows(com.bencodez.simpleapi.file.DurableFiles.PublishedException.class, + () -> authority.enroll("lobby-1", targetCode.enrollmentToken())); + } + + assertFalse(authority.authenticate("unrelated", unrelated.certificate()), + "authentication must fail closed while the published enrollment state is unresolved"); + now.set(Instant.parse("2030-01-01T00:06:00Z")); + HttpConnectionCode recoveryCode = authority.createConnectionCode("recovery", endpoint, Duration.ofMinutes(5)); + assertFalse(recoveryCode.enrollmentToken().isEmpty()); + assertTrue(authority.authenticate("unrelated", unrelated.certificate()), + "a successful full-state rewrite must restore authentication availability"); + } + + @Test + void authorityStatePrunesRevocationsAndBoundsActiveBindings() throws Exception { + Path proxy = directory.resolve("bounded-proxy"); + Path state = directory.resolve("bounded-state"); + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(proxy, "localhost"); + Files.createDirectories(state); + java.util.Properties properties = new java.util.Properties(); + properties.setProperty("version", "3"); + String firstServer = boundedServerId(0); + for (int index = 0; index < 128; index++) { + String serverId = boundedServerId(index); + String encodedServer = java.util.Base64.getUrlEncoder().withoutPadding() + .encodeToString(serverId.getBytes(java.nio.charset.StandardCharsets.UTF_8)); + properties.setProperty("binding." + encodedServer, pin('a') + ":-:0"); + byte[] hash = new byte[32]; + java.nio.ByteBuffer.wrap(hash).putInt(index); + properties.setProperty("enrollment." + java.util.Base64.getUrlEncoder().withoutPadding().encodeToString(hash), + Instant.parse("2099-01-01T00:00:00Z").toEpochMilli() + ":" + java.util.Base64.getUrlEncoder() + .withoutPadding().encodeToString(firstServer.getBytes(java.nio.charset.StandardCharsets.UTF_8))); + } + Path stateFile = state.resolve("http-transport-clients.properties"); + try (var output = Files.newOutputStream(stateFile)) { properties.store(output, "bounded authority state"); } + assertTrue(Files.size(stateFile) < 65536, "the maximum supported state must fit the read bound"); + + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, state); + authority.revoke(firstServer); + assertFalse(Files.readString(stateFile).contains("binding." + java.util.Base64.getUrlEncoder().withoutPadding() + .encodeToString(firstServer.getBytes(java.nio.charset.StandardCharsets.UTF_8))), + "revoked bindings must not accumulate in durable state"); + HttpEnrollmentAuthority restarted = new HttpEnrollmentAuthority(identity, state); + HttpConnectionCode replacement = restarted.createConnectionCode("replacement", URI.create("https://localhost:8443/"), + Duration.ofMinutes(5)); + restarted.enroll("replacement", replacement.enrollmentToken()); + HttpConnectionCode overflow = restarted.createConnectionCode("overflow", URI.create("https://localhost:8443/"), + Duration.ofMinutes(5)); + assertThrows(IllegalStateException.class, () -> restarted.enroll("overflow", overflow.enrollmentToken())); + restarted.revoke(boundedServerId(1)); + assertDoesNotThrow(() -> restarted.enroll("overflow", overflow.enrollmentToken()), + "a capacity rejection must not consume the enrollment token"); + assertTrue(Files.size(stateFile) <= 65536); + assertDoesNotThrow(() -> new HttpEnrollmentAuthority(identity, state)); + } + + @Test + void renewalKeepsOldCredentialUntilReplacementAuthenticates() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, directory.resolve("state")); + HttpConnectionCode code = authority.createConnectionCode("lobby-1", URI.create("https://localhost:8443/"), Duration.ofMinutes(5)); + HttpTlsIdentity.IssuedClientCertificate original = authority.enroll("lobby-1", code.enrollmentToken()); + HttpTlsIdentity.IssuedClientCertificate replacement = authority.renew("lobby-1", original.certificate()); + + assertTrue(authority.authenticate("lobby-1", original.certificate()), "lost renewal responses must leave the old credential usable"); + assertTrue(authority.authenticate("lobby-1", replacement.certificate()), "first replacement request promotes the pending binding"); + assertFalse(authority.authenticate("lobby-1", original.certificate()), "promotion revokes the superseded credential"); + assertTrue(new HttpEnrollmentAuthority(identity, directory.resolve("state")) + .authenticate("lobby-1", replacement.certificate()), "promoted renewal must survive restart"); + } + + @Test + void failedRenewalPersistenceRestoresTheActiveBindingAndCanRetry() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("retry-renewal-proxy"), "localhost"); + Path stateDirectory = directory.resolve("retry-renewal-state"); + Files.createDirectory(stateDirectory); + java.util.concurrent.atomic.AtomicReference now = new java.util.concurrent.atomic.AtomicReference<>(Instant.now()); + Clock clock = new Clock() { + @Override public java.time.ZoneId getZone() { return ZoneOffset.UTC; } + @Override public Clock withZone(java.time.ZoneId zone) { return this; } + @Override public Instant instant() { return now.get(); } + }; + HttpEnrollmentAuthority authority = new HttpEnrollmentAuthority(identity, clock, + stateDirectory.resolve("http-transport-clients.properties")); + HttpConnectionCode code = authority.createConnectionCode("lobby-1", URI.create("https://localhost:8443/"), + Duration.ofMinutes(5)); + HttpTlsIdentity.IssuedClientCertificate original = authority.enroll("lobby-1", code.enrollmentToken()); + assertTrue(authority.authenticate("lobby-1", original.certificate())); + Path stateFile = stateDirectory.resolve("http-transport-clients.properties"); + Files.delete(stateFile); + Files.createDirectory(stateFile); + + assertThrows(java.io.IOException.class, () -> authority.renew("lobby-1", original.certificate())); + assertTrue(authority.authenticate("lobby-1", original.certificate()), + "a pre-publication renewal failure must leave the active credential usable"); + Files.delete(stateFile); + now.set(now.get().plusSeconds(60)); + HttpTlsIdentity.IssuedClientCertificate retried = authority.renew("lobby-1", original.certificate()); + assertTrue(authority.authenticate("lobby-1", retried.certificate()), + "renewal must remain retryable after persistence recovers"); + } + + @Test + void serverLeafRotatesInsideRenewalWindowAndPreservesAuthority() throws Exception { + Instant now = Instant.now(); + HttpTlsIdentity original = HttpTlsIdentity.loadOrCreate(directory, "localhost", + Clock.fixed(now.minus(Duration.ofDays(340)), ZoneOffset.UTC)); + String originalPin = HttpTransportSecrets.certificatePin(original.serverCertificate()); + HttpTlsIdentity renewed = HttpTlsIdentity.loadOrCreate(directory, "localhost", Clock.fixed(now, ZoneOffset.UTC)); + assertNotEquals(originalPin, renewed.serverCertificatePin()); + assertEquals(original.caCertificatePin(), renewed.caCertificatePin()); + assertFalse(HttpTlsIdentity.needsRenewal(renewed.serverCertificate(), Clock.fixed(now, ZoneOffset.UTC))); + } + + @Test + void runningPrivateCaRollsOverBeforeExpiryWithoutStrandingExistingClients() throws Exception { + Instant now = Instant.now(); + Clock originalClock = Clock.fixed(now.minus(Duration.ofDays(9 * 365L + 30L)), ZoneOffset.UTC); + HttpTlsIdentity original = HttpTlsIdentity.loadOrCreate(directory, "localhost", originalClock); + X509Certificate originalCa = original.caCertificate(); + HttpTlsIdentity.IssuedClientCertificate existingClient = original.issueClientCertificate("lobby-1", now); + Path client = directory.resolve("client"); + HttpConnectionCode oldCode = new HttpConnectionCode("lobby-1", URI.create("https://localhost:8443/"), + HttpTransportSecrets.certificatePin(original.serverCertificate()), HttpTransportSecrets.certificatePin(originalCa), + now.plusSeconds(60), "A".repeat(43)); + HttpClientCredentialStore.saveEnrolled(client, oldCode, existingClient); + + String renewedPin = original.caCertificatePin(); + assertNotEquals(HttpTransportSecrets.certificatePin(originalCa), renewedPin); + assertEquals(originalCa.getPublicKey(), original.caCertificate().getPublicKey(), + "certificate rollover keeps the private authority key so old and new trust anchors overlap"); + assertFalse(HttpTlsIdentity.needsCaRenewal(original.caCertificate(), Clock.fixed(now, ZoneOffset.UTC))); + assertTrue(original.validClientCertificate("lobby-1", existingClient.certificate())); + + X509TrustManager oldClientTrust = Arrays.stream(HttpTlsIdentity.trustManagers(originalCa)) + .filter(X509TrustManager.class::isInstance).map(X509TrustManager.class::cast).findFirst().orElseThrow(); + assertDoesNotThrow(() -> oldClientTrust.checkServerTrusted( + new X509Certificate[] { original.serverCertificate(), original.caCertificate() }, "ECDHE_ECDSA")); + HttpClientCredentialStore.StagedCredential staged = HttpClientCredentialStore.stageReplacement(client, + original.issueClientCertificate("lobby-1", now)); + assertEquals(HttpTransportSecrets.certificatePin(originalCa), HttpClientCredentialStore.loadProfile(client).caCertificatePin()); + assertEquals(renewedPin, staged.profile().caCertificatePin()); + HttpClientCredentialStore.activateReplacement(client, staged); + assertEquals(renewedPin, HttpClientCredentialStore.loadProfile(client).caCertificatePin()); + assertEquals(renewedPin, HttpTlsIdentity.loadOrCreate(directory, "localhost").caCertificatePin(), + "live CA rollover must survive restart"); + } + + @Test + void activeTlsContextRotatesServerLeafInsideRenewalWindow() throws Exception { + Instant now = Instant.now(); + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory, "localhost", + Clock.fixed(now.minus(Duration.ofDays(340)), ZoneOffset.UTC)); + String expiringPin = HttpTransportSecrets.certificatePin(identity.serverCertificate()); + identity.serverContext(); + assertNotEquals(expiringPin, identity.serverCertificatePin()); + assertFalse(HttpTlsIdentity.needsRenewal(identity.serverCertificate(), Clock.systemUTC())); + } + + @Test + void serverTlsUsesPrivateCaTrustAndRejectsForeignClients() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); + HttpTlsIdentity foreign = HttpTlsIdentity.loadOrCreate(directory.resolve("foreign"), "localhost"); + X509TrustManager trust = Arrays.stream(HttpTlsIdentity.trustManagers(identity.caCertificate())) + .filter(X509TrustManager.class::isInstance).map(X509TrustManager.class::cast).findFirst().orElseThrow(); + HttpTlsIdentity.IssuedClientCertificate accepted = identity.issueClientCertificate("lobby-1"); + HttpTlsIdentity.IssuedClientCertificate rejected = foreign.issueClientCertificate("lobby-1"); + assertDoesNotThrow(() -> trust.checkClientTrusted( + new java.security.cert.X509Certificate[] { accepted.certificate(), identity.caCertificate() }, "EC")); + assertThrows(java.security.cert.CertificateException.class, () -> trust.checkClientTrusted( + new java.security.cert.X509Certificate[] { rejected.certificate(), foreign.caCertificate() }, "EC")); + } + + @Test + void stagedCredentialDoesNotReplaceActiveGenerationUntilAtomicActivation() throws Exception { + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("proxy"), "localhost"); + Path client = directory.resolve("client"); + HttpTlsIdentity.IssuedClientCertificate original = identity.issueClientCertificate("lobby-1"); + HttpConnectionCode code = new HttpConnectionCode("lobby-1", URI.create("https://localhost:8443/"), + identity.serverCertificatePin(), identity.caCertificatePin(), Instant.now().plusSeconds(60), "A".repeat(43)); + HttpClientCredentialStore.saveEnrolled(client, code, original); + String originalPin = HttpTransportSecrets.certificatePin(HttpClientCredentialStore.load(client).certificate()); + HttpTlsIdentity.IssuedClientCertificate replacement = identity.issueClientCertificate("lobby-1"); + HttpClientCredentialStore.StagedCredential staged = HttpClientCredentialStore.stageReplacement(client, replacement); + assertEquals(originalPin, HttpTransportSecrets.certificatePin(HttpClientCredentialStore.load(client).certificate())); + HttpClientCredentialStore.activateReplacement(client, staged); + assertNotEquals(originalPin, HttpTransportSecrets.certificatePin(HttpClientCredentialStore.load(client).certificate())); + assertTrue(HttpClientCredentialStore.matchesEnrollmentCode(client, code), + "automatic certificate renewal must retain the consumed-code marker"); + HttpTlsIdentity.IssuedClientCertificate manuallyReenrolled = identity.issueClientCertificate("lobby-1"); + HttpClientCredentialStore.saveEnrolled(client, code, manuallyReenrolled); + assertEquals(HttpTransportSecrets.certificatePin(manuallyReenrolled.certificate()), + HttpTransportSecrets.certificatePin(HttpClientCredentialStore.loadEnrolled(client).credential().certificate())); + } + + @Test + void restoresCredentialGenerationAfterFailedReenrollment() throws Exception { + Path client = directory.resolve("client-rollback"); + HttpTlsIdentity oldIdentity = HttpTlsIdentity.loadOrCreate(directory.resolve("old-proxy"), "old.example.test"); + HttpConnectionCode oldCode = new HttpConnectionCode("lobby-1", URI.create("https://old.example.test:1297/"), + oldIdentity.serverCertificatePin(), oldIdentity.caCertificatePin(), Instant.now().plusSeconds(60), + "R".repeat(43)); + HttpClientCredentialStore.saveEnrolled(client, oldCode, oldIdentity.issueClientCertificate("lobby-1")); + HttpClientCredentialStore.ActiveCredentialGeneration previous = + HttpClientCredentialStore.snapshotActiveGeneration(client); + + HttpTlsIdentity replacementIdentity = HttpTlsIdentity.loadOrCreate(directory.resolve("new-proxy"), "new.example.test"); + HttpConnectionCode replacementCode = new HttpConnectionCode("lobby-1", URI.create("https://new.example.test:1297/"), + replacementIdentity.serverCertificatePin(), replacementIdentity.caCertificatePin(), + Instant.now().plusSeconds(60), "S".repeat(43)); + HttpClientCredentialStore.saveEnrolled(client, replacementCode, + replacementIdentity.issueClientCertificate("lobby-1")); + assertEquals(replacementCode.endpoint(), HttpClientCredentialStore.loadProfile(client).endpoint()); + + HttpClientCredentialStore.restoreActiveGeneration(client, previous); + assertEquals(oldCode.endpoint(), HttpClientCredentialStore.loadProfile(client).endpoint()); + assertTrue(HttpClientCredentialStore.matchesEnrollmentCode(client, oldCode)); + } + + @Test + void rollbackRetainsNewerCredentialForTheSameEndpoint() throws Exception { + Path client = directory.resolve("client-renewal-rollback"); + HttpTlsIdentity identity = HttpTlsIdentity.loadOrCreate(directory.resolve("renewal-proxy"), "renew.example.test"); + HttpConnectionCode code = new HttpConnectionCode("lobby-1", URI.create("https://renew.example.test:1297/"), + identity.serverCertificatePin(), identity.caCertificatePin(), Instant.now().plusSeconds(60), + "T".repeat(43)); + HttpTlsIdentity.IssuedClientCertificate original = identity.issueClientCertificate("lobby-1"); + HttpClientCredentialStore.saveEnrolled(client, code, original); + HttpClientCredentialStore.ActiveCredentialGeneration previous = + HttpClientCredentialStore.snapshotActiveGeneration(client); + + HttpConnectionCode replacementCode = new HttpConnectionCode("lobby-1", code.endpoint(), + identity.serverCertificatePin(), identity.caCertificatePin(), Instant.now().plusSeconds(60), + "U".repeat(43)); + HttpTlsIdentity.IssuedClientCertificate renewed = identity.issueClientCertificate("lobby-1"); + HttpClientCredentialStore.saveEnrolled(client, replacementCode, renewed); + HttpClientCredentialStore.restoreActiveGenerationAfterReplacement(client, previous); + + assertEquals(HttpTransportSecrets.certificatePin(renewed.certificate()), + HttpTransportSecrets.certificatePin(HttpClientCredentialStore.load(client).certificate()), + "rollback must not reactivate a same-endpoint certificate that renewal may have revoked"); + assertTrue(HttpClientCredentialStore.matchesEnrollmentCode(client, code), + "the retained credential must recognize the connection code restored in YAML"); + } + + private static String pin(char character) { return String.valueOf(character).repeat(64); } + private static String boundedServerId(int index) { return String.format("s%03d", index) + "x".repeat(60); } +} diff --git a/SimpleAPI/src/test/java/com/bencodez/simpleapi/tests/file/VelocityJsonFileTest.java b/SimpleAPI/src/test/java/com/bencodez/simpleapi/tests/file/VelocityJsonFileTest.java index 4826268..0b94f6c 100644 --- a/SimpleAPI/src/test/java/com/bencodez/simpleapi/tests/file/VelocityJsonFileTest.java +++ b/SimpleAPI/src/test/java/com/bencodez/simpleapi/tests/file/VelocityJsonFileTest.java @@ -8,10 +8,13 @@ import java.nio.file.Files; import java.nio.file.Path; +import java.util.ArrayList; import java.util.Comparator; import java.util.List; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; @@ -118,18 +121,25 @@ void concurrentSavesAreSafeEnough() throws Exception { Path file = tmpRoot.resolve("conc/config.json"); VelocityJSONFile v = new VelocityJSONFile(file); - ExecutorService pool = Executors.newFixedThreadPool(4); + ExecutorService pool = Executors.newFixedThreadPool(4); + List> saves = new ArrayList<>(); try { for (int i = 0; i < 20; i++) { final int n = i; - pool.submit(() -> { + saves.add(pool.submit(() -> { v.set(new Object[] { "counter" }, n); v.save(); - }); + })); + } + for (Future save : saves) { + save.get(); } } finally { pool.shutdown(); - Thread.sleep(200); // small settle time + if (!pool.awaitTermination(10, TimeUnit.SECONDS)) { + pool.shutdownNow(); + assertTrue(pool.awaitTermination(10, TimeUnit.SECONDS), "save workers did not terminate"); + } } VelocityJSONFile re = new VelocityJSONFile(file);