From 41ecc50487d9b8193ad34c9f21eb032c31fe4749 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Tue, 8 Sep 2026 11:31:07 +0800 Subject: [PATCH 1/5] [kafka] Serve ApiVersions from registered capabilities Route requests through the dispatcher and advertise only implemented APIs. Return version-aware errors for unsupported APIs and invalid requests. Validated with mvn -o -pl fluss-kafka verify (23 unit tests and 1 IT). Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 313/313 AI-Contributed/UT: 137/137 --- .../fluss/kafka/KafkaProtocolPlugin.java | 3 +- .../fluss/kafka/KafkaRequestHandler.java | 232 ++---------------- .../api/versions/ApiVersionsHandler.java | 78 ++++++ .../fluss/kafka/KafkaRequestHandlerTest.java | 137 ++++++++--- 4 files changed, 207 insertions(+), 243 deletions(-) create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/api/versions/ApiVersionsHandler.java diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaProtocolPlugin.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaProtocolPlugin.java index 2bd2f5d81b..9a9e2e9ff6 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaProtocolPlugin.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaProtocolPlugin.java @@ -66,7 +66,6 @@ public RequestHandler createRequestHandler(RpcGatewayService service) { "Kafka protocol endpoints can only be enabled on TabletServers, but the service is " + service.getClass().getSimpleName()); } - TabletServerGateway gateway = (TabletServerGateway) service; - return new KafkaRequestHandler(gateway); + return new KafkaRequestHandler(); } } diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java index 73555093ff..2df16a8bd7 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java @@ -17,27 +17,24 @@ package org.apache.fluss.kafka; -import org.apache.fluss.rpc.gateway.TabletServerGateway; +import org.apache.fluss.kafka.api.versions.ApiVersionsHandler; +import org.apache.fluss.kafka.dispatcher.KafkaApiRegistry; +import org.apache.fluss.kafka.dispatcher.KafkaRequestDispatcher; +import org.apache.fluss.kafka.error.KafkaErrorMapper; import org.apache.fluss.rpc.netty.server.RequestHandler; import org.apache.fluss.rpc.protocol.RequestType; -import org.apache.kafka.common.message.ApiVersionsResponseData; -import org.apache.kafka.common.protocol.ApiKeys; -import org.apache.kafka.common.protocol.Errors; -import org.apache.kafka.common.record.RecordBatch; -import org.apache.kafka.common.requests.AbstractRequest; -import org.apache.kafka.common.requests.AbstractResponse; -import org.apache.kafka.common.requests.ApiVersionsResponse; - -/** Kafka protocol implementation for request handler. */ +/** Entry point that dispatches Kafka protocol requests to registered API handlers. */ public class KafkaRequestHandler implements RequestHandler { - // TODO: we may need a new abstraction between TabletService and ReplicaManager to avoid - // affecting Fluss protocol when supporting compatibility with Kafka. - private final TabletServerGateway gateway; + private final KafkaRequestDispatcher dispatcher; - public KafkaRequestHandler(TabletServerGateway gateway) { - this.gateway = gateway; + /** Creates a Kafka request handler with the implemented server capabilities. */ + public KafkaRequestHandler() { + KafkaApiRegistry registry = new KafkaApiRegistry(); + registry.register(new ApiVersionsHandler(registry)); + registry.freeze(); + this.dispatcher = new KafkaRequestDispatcher(registry, new KafkaErrorMapper()); } @Override @@ -47,200 +44,15 @@ public RequestType requestType() { @Override public void processRequest(KafkaRequest request) { - // See kafka.server.KafkaApis#handle - switch (request.apiKey()) { - case API_VERSIONS: - handleApiVersionsRequest(request); - break; - case METADATA: - handleMetadataRequest(request); - break; - case PRODUCE: - handleProducerRequest(request); - break; - case FIND_COORDINATOR: - handleFindCoordinatorRequest(request); - break; - case LIST_OFFSETS: - handleListOffsetRequest(request); - break; - case OFFSET_FETCH: - handleOffsetFetchRequest(request); - break; - case OFFSET_COMMIT: - handleOffsetCommitRequest(request); - break; - case FETCH: - handleFetchRequest(request); - break; - case JOIN_GROUP: - handleJoinGroupRequest(request); - break; - case SYNC_GROUP: - handleSyncGroupRequest(request); - break; - case HEARTBEAT: - handleHeartbeatRequest(request); - break; - case LEAVE_GROUP: - handleLeaveGroupRequest(request); - break; - case DESCRIBE_GROUPS: - handleDescribeGroupsRequest(request); - break; - case LIST_GROUPS: - handleListGroupsRequest(request); - break; - case DELETE_GROUPS: - handleDeleteGroupsRequest(request); - break; - case SASL_HANDSHAKE: - handleSaslHandshakeRequest(request); - break; - case SASL_AUTHENTICATE: - handleSaslAuthenticateRequest(request); - break; - case CREATE_TOPICS: - handleCreateTopicsRequest(request); - break; - case INIT_PRODUCER_ID: - handleInitProducerIdRequest(request); - break; - case ADD_PARTITIONS_TO_TXN: - handleAddPartitionsToTxnRequest(request); - break; - case ADD_OFFSETS_TO_TXN: - handleAddOffsetsToTxnRequest(request); - break; - case TXN_OFFSET_COMMIT: - handleTxnOffsetCommitRequest(request); - break; - case END_TXN: - handleEndTxnRequest(request); - break; - case WRITE_TXN_MARKERS: - handleWriteTxnMarkersRequest(request); - break; - case DESCRIBE_CONFIGS: - handleDescribeConfigsRequest(request); - break; - case ALTER_CONFIGS: - handleAlterConfigsRequest(request); - break; - case DELETE_TOPICS: - handleDeleteTopicsRequest(request); - break; - case DELETE_RECORDS: - handleDeleteRecordsRequest(request); - break; - case OFFSET_DELETE: - handleOffsetDeleteRequest(request); - break; - case CREATE_PARTITIONS: - handleCreatePartitionsRequest(request); - break; - case DESCRIBE_CLUSTER: - handleDescribeClusterRequest(request); - break; - default: - handleUnsupportedRequest(request); - } - } - - private void handleUnsupportedRequest(KafkaRequest request) { - String message = String.format("Unsupported request with api key %s", request.apiKey()); - AbstractRequest abstractRequest = request.request(); - AbstractResponse response = - abstractRequest.getErrorResponse(new UnsupportedOperationException(message)); - request.complete(response); - } - - void handleApiVersionsRequest(KafkaRequest request) { - short apiVersion = request.apiVersion(); - if (!ApiKeys.API_VERSIONS.isVersionSupported(apiVersion)) { - request.fail(Errors.UNSUPPORTED_VERSION.exception()); - return; - } - ApiVersionsResponseData data = new ApiVersionsResponseData(); - for (ApiKeys apiKey : ApiKeys.values()) { - if (apiKey.minRequiredInterBrokerMagic <= RecordBatch.CURRENT_MAGIC_VALUE) { - ApiVersionsResponseData.ApiVersion apiVersionData = - new ApiVersionsResponseData.ApiVersion() - .setApiKey(apiKey.id) - .setMinVersion(apiKey.oldestVersion()) - .setMaxVersion(apiKey.latestVersion()); - if (apiKey.equals(ApiKeys.METADATA)) { - // Not support TopicId - short v = apiKey.latestVersion() > 11 ? 11 : apiKey.latestVersion(); - apiVersionData.setMaxVersion(v); - } else if (apiKey.equals(ApiKeys.FETCH)) { - // Not support TopicId - short v = apiKey.latestVersion() > 12 ? 12 : apiKey.latestVersion(); - apiVersionData.setMaxVersion(v); - } - data.apiKeys().add(apiVersionData); - } - } - request.complete(new ApiVersionsResponse(data)); + dispatcher + .dispatch(request) + .whenComplete( + (response, failure) -> { + if (failure == null) { + request.complete(response); + } else { + request.fail(failure); + } + }); } - - void handleProducerRequest(KafkaRequest request) {} - - void handleMetadataRequest(KafkaRequest request) {} - - void handleFindCoordinatorRequest(KafkaRequest request) {} - - void handleListOffsetRequest(KafkaRequest request) {} - - void handleOffsetFetchRequest(KafkaRequest request) {} - - void handleOffsetCommitRequest(KafkaRequest request) {} - - void handleFetchRequest(KafkaRequest request) {} - - void handleJoinGroupRequest(KafkaRequest request) {} - - void handleSyncGroupRequest(KafkaRequest request) {} - - void handleHeartbeatRequest(KafkaRequest request) {} - - void handleLeaveGroupRequest(KafkaRequest request) {} - - void handleDescribeGroupsRequest(KafkaRequest request) {} - - void handleListGroupsRequest(KafkaRequest request) {} - - void handleDeleteGroupsRequest(KafkaRequest request) {} - - void handleSaslHandshakeRequest(KafkaRequest request) {} - - void handleSaslAuthenticateRequest(KafkaRequest request) {} - - void handleCreateTopicsRequest(KafkaRequest request) {} - - void handleInitProducerIdRequest(KafkaRequest request) {} - - void handleAddPartitionsToTxnRequest(KafkaRequest request) {} - - void handleAddOffsetsToTxnRequest(KafkaRequest request) {} - - void handleTxnOffsetCommitRequest(KafkaRequest request) {} - - void handleEndTxnRequest(KafkaRequest request) {} - - void handleWriteTxnMarkersRequest(KafkaRequest request) {} - - void handleDescribeConfigsRequest(KafkaRequest request) {} - - void handleAlterConfigsRequest(KafkaRequest request) {} - - void handleDeleteTopicsRequest(KafkaRequest request) {} - - void handleDeleteRecordsRequest(KafkaRequest request) {} - - void handleOffsetDeleteRequest(KafkaRequest request) {} - - void handleCreatePartitionsRequest(KafkaRequest request) {} - - void handleDescribeClusterRequest(KafkaRequest request) {} } diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/versions/ApiVersionsHandler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/versions/ApiVersionsHandler.java new file mode 100644 index 0000000000..c38d7bc6cb --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/versions/ApiVersionsHandler.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.api.versions; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.kafka.KafkaRequestContext; +import org.apache.fluss.kafka.dispatcher.KafkaApiHandler; +import org.apache.fluss.kafka.dispatcher.KafkaApiRegistry; +import org.apache.fluss.kafka.dispatcher.KafkaApiSpec; + +import org.apache.kafka.common.message.ApiVersionsResponseData; +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.requests.AbstractResponse; +import org.apache.kafka.common.requests.ApiVersionsRequest; +import org.apache.kafka.common.requests.ApiVersionsResponse; + +import java.util.concurrent.CompletableFuture; + +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Implements ApiVersions from the capabilities actually registered on this server. */ +@Internal +public final class ApiVersionsHandler implements KafkaApiHandler { + + private static final KafkaApiSpec API_SPEC = + new KafkaApiSpec( + ApiKeys.API_VERSIONS, + ApiKeys.API_VERSIONS.oldestVersion(), + ApiKeys.API_VERSIONS.latestVersion(), + true); + + private final KafkaApiRegistry registry; + + /** Creates an ApiVersions handler backed by the server capability registry. */ + public ApiVersionsHandler(KafkaApiRegistry registry) { + this.registry = checkNotNull(registry); + } + + @Override + public KafkaApiSpec apiSpec() { + return API_SPEC; + } + + @Override + public CompletableFuture handle( + KafkaRequestContext context, ApiVersionsRequest request) { + if (!request.isValid()) { + return CompletableFuture.completedFuture( + request.getErrorResponse(Errors.INVALID_REQUEST.exception())); + } + ApiVersionsResponseData data = new ApiVersionsResponseData(); + for (KafkaApiSpec spec : registry.advertisedApiSpecs()) { + data.apiKeys() + .add( + new ApiVersionsResponseData.ApiVersion() + .setApiKey(spec.apiKey().id) + .setMinVersion(spec.minVersion()) + .setMaxVersion(spec.maxVersion())); + } + return CompletableFuture.completedFuture(new ApiVersionsResponse(data)); + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java index bf8e23b435..b0f25149e4 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java @@ -17,24 +17,32 @@ package org.apache.fluss.kafka; -import org.apache.fluss.rpc.TestingTabletGatewayService; import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBuf; import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBufAllocator; import org.apache.fluss.shaded.netty4.io.netty.channel.ChannelHandlerContext; +import org.apache.kafka.common.message.ApiVersionsRequestData; +import org.apache.kafka.common.message.ApiVersionsResponseData.ApiVersion; +import org.apache.kafka.common.message.CreateTopicsRequestData; import org.apache.kafka.common.protocol.ApiKeys; import org.apache.kafka.common.protocol.Errors; import org.apache.kafka.common.requests.AbstractRequest; import org.apache.kafka.common.requests.AbstractResponse; import org.apache.kafka.common.requests.ApiVersionsRequest; import org.apache.kafka.common.requests.ApiVersionsResponse; +import org.apache.kafka.common.requests.CreateTopicsRequest; +import org.apache.kafka.common.requests.CreateTopicsResponse; import org.apache.kafka.common.requests.RequestHeader; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import java.util.Collections; import java.util.Map; import java.util.concurrent.CompletableFuture; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.tuple; /** Tests for {@link KafkaRequestHandler}. */ public class KafkaRequestHandlerTest { @@ -53,7 +61,7 @@ public void testKafkaApiVersionsNotSupported() { new RequestHeader(ApiKeys.API_VERSIONS, latestVersion, "client-id", 0), apiVersionsRequest, ctx); - handler.handleApiVersionsRequest(request); + handler.processRequest(request); ApiVersionsResponse response = (ApiVersionsResponse) parseResponse(request); Map errorCounts = response.errorCounts(); @@ -61,44 +69,111 @@ public void testKafkaApiVersionsNotSupported() { assertThat(1).isEqualTo(errorCounts.get(Errors.UNSUPPORTED_VERSION)); } + @ParameterizedTest + @ValueSource(shorts = {0, 1, 2, 3, 4}) + public void testKafkaApiVersionsRequest(short version) { + KafkaRequestHandler handler = createKafkaRequestHandler(); + ApiVersionsResponse response = requestApiVersions(handler, version); + + assertSuccessfulResponseDefaults(response); + assertBrokerCapabilities(response); + } + + private static ApiVersionsResponse requestApiVersions( + KafkaRequestHandler handler, short version) { + ApiVersionsRequest apiVersionsRequest = new ApiVersionsRequest.Builder().build(version); + ChannelHandlerContext ctx = new TestingChannelHandlerContext(); + KafkaRequest request = + newRequest( + ApiKeys.API_VERSIONS, + version, + new RequestHeader(ApiKeys.API_VERSIONS, version, "client-id", 0), + apiVersionsRequest, + ctx); + handler.processRequest(request); + + return parseApiVersionsResponse(request); + } + + private static ApiVersionsResponse parseApiVersionsResponse(KafkaRequest request) { + return (ApiVersionsResponse) parseResponse(request); + } + + private static void assertSuccessfulResponseDefaults(ApiVersionsResponse response) { + assertThat(response.errorCounts()) + .containsExactlyEntriesOf(Collections.singletonMap(Errors.NONE, 1)); + assertThat(response.data().throttleTimeMs()).isZero(); + assertThat(response.data().supportedFeatures()).isEmpty(); + assertThat(response.data().finalizedFeaturesEpoch()).isEqualTo(-1L); + assertThat(response.data().finalizedFeatures()).isEmpty(); + assertThat(response.data().zkMigrationReady()).isFalse(); + } + + private static void assertBrokerCapabilities(ApiVersionsResponse response) { + assertThat(response.data().apiKeys()) + .extracting(ApiVersion::apiKey, ApiVersion::minVersion, ApiVersion::maxVersion) + .containsExactly( + tuple( + ApiKeys.API_VERSIONS.id, + ApiKeys.API_VERSIONS.oldestVersion(), + ApiKeys.API_VERSIONS.latestVersion())); + } + @Test - public void testKafkaApiVersionsRequest() { + public void testInvalidApiVersionsRequest() { KafkaRequestHandler handler = createKafkaRequestHandler(); short latestVersion = ApiKeys.API_VERSIONS.latestVersion(); - ApiVersionsRequest apiVersionsRequest = - new ApiVersionsRequest.Builder().build(latestVersion); - ChannelHandlerContext ctx = new TestingChannelHandlerContext(); + ApiVersionsRequest requestBody = + new ApiVersionsRequest.Builder( + new ApiVersionsRequestData() + .setClientSoftwareName("invalid client name") + .setClientSoftwareVersion("1.0"), + latestVersion, + latestVersion) + .build(latestVersion); KafkaRequest request = newRequest( ApiKeys.API_VERSIONS, latestVersion, new RequestHeader(ApiKeys.API_VERSIONS, latestVersion, "client-id", 0), - apiVersionsRequest, - ctx); - handler.handleApiVersionsRequest(request); + requestBody, + new TestingChannelHandlerContext()); + + handler.processRequest(request); ApiVersionsResponse response = (ApiVersionsResponse) parseResponse(request); - Map errorCounts = response.errorCounts(); - assertThat(1).isEqualTo(errorCounts.size()); - assertThat(1).isEqualTo(errorCounts.get(Errors.NONE)); - response.data() - .apiKeys() - .forEach( - apiVersion -> { - if (ApiKeys.METADATA.id == apiVersion.apiKey()) { - assertThat((short) 11) - .isGreaterThanOrEqualTo(apiVersion.maxVersion()); - } else if (ApiKeys.FETCH.id == apiVersion.apiKey()) { - assertThat((short) 12) - .isGreaterThanOrEqualTo(apiVersion.maxVersion()); - } else { - ApiKeys apiKeys = ApiKeys.forId(apiVersion.apiKey()); - assertThat(apiVersion.minVersion()) - .isEqualTo(apiKeys.oldestVersion()); - assertThat(apiVersion.maxVersion()) - .isEqualTo(apiKeys.latestVersion()); - } - }); + assertThat(response.errorCounts()).containsEntry(Errors.INVALID_REQUEST, 1); + } + + @Test + public void testUnregisteredApiIsNotRouted() { + KafkaRequestHandler handler = createKafkaRequestHandler(); + short version = ApiKeys.CREATE_TOPICS.latestVersion(); + CreateTopicsRequestData requestData = + new CreateTopicsRequestData() + .setTimeoutMs(1000) + .setTopics( + new CreateTopicsRequestData.CreatableTopicCollection( + Collections.singletonList( + new CreateTopicsRequestData.CreatableTopic() + .setName("topic") + .setNumPartitions(1) + .setReplicationFactor((short) 1)) + .iterator())); + CreateTopicsRequest requestBody = + new CreateTopicsRequest.Builder(requestData).build(version); + KafkaRequest request = + newRequest( + ApiKeys.CREATE_TOPICS, + version, + new RequestHeader(ApiKeys.CREATE_TOPICS, version, "client-id", 0), + requestBody, + new TestingChannelHandlerContext()); + + handler.processRequest(request); + + CreateTopicsResponse response = (CreateTopicsResponse) parseResponse(request); + assertThat(response.errorCounts()).containsEntry(Errors.UNSUPPORTED_VERSION, 1); } private static KafkaRequest newRequest( @@ -134,6 +209,6 @@ private static AbstractResponse parseResponse(KafkaRequest request) { } private static KafkaRequestHandler createKafkaRequestHandler() { - return new KafkaRequestHandler(new TestingTabletGatewayService()); + return new KafkaRequestHandler(); } } From 4317d87922b826751d2a23283e8f3eb5e3044a0f Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Sat, 10 Oct 2026 17:28:33 +0800 Subject: [PATCH 2/5] [kafka] Cover ApiVersions wire negotiation and capability ranges Exercise future-version decoding, v0 error serialization, correlation IDs, unknown tagged fields, buffer release, and renegotiation on one connection. Verify registry-defined ranges, hidden APIs, ordering, and invalid client software fields across the supported ApiVersions schemas. Validated with Java 11 reactor clean verify: 37 unit tests and 2 integration tests passed, with Checkstyle, Spotless, RAT, and git diff checks. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 0/0 AI-Contributed/UT: 238/238 --- .../fluss/kafka/KafkaRequestHandlerTest.java | 149 ++++++++++++++---- .../api/versions/ApiVersionsHandlerTest.java | 89 +++++++++++ 2 files changed, 205 insertions(+), 33 deletions(-) create mode 100644 fluss-kafka/src/test/java/org/apache/fluss/kafka/api/versions/ApiVersionsHandlerTest.java diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java index b0f25149e4..40fdab04cc 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaRequestHandlerTest.java @@ -17,15 +17,21 @@ package org.apache.fluss.kafka; +import org.apache.fluss.rpc.netty.server.RequestChannel; import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBuf; import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBufAllocator; +import org.apache.fluss.shaded.netty4.io.netty.buffer.Unpooled; import org.apache.fluss.shaded.netty4.io.netty.channel.ChannelHandlerContext; +import org.apache.fluss.shaded.netty4.io.netty.channel.embedded.EmbeddedChannel; import org.apache.kafka.common.message.ApiVersionsRequestData; +import org.apache.kafka.common.message.ApiVersionsResponseData; import org.apache.kafka.common.message.ApiVersionsResponseData.ApiVersion; import org.apache.kafka.common.message.CreateTopicsRequestData; import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.ByteBufferAccessor; import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.protocol.types.RawTaggedField; import org.apache.kafka.common.requests.AbstractRequest; import org.apache.kafka.common.requests.AbstractResponse; import org.apache.kafka.common.requests.ApiVersionsRequest; @@ -33,12 +39,15 @@ import org.apache.kafka.common.requests.CreateTopicsRequest; import org.apache.kafka.common.requests.CreateTopicsResponse; import org.apache.kafka.common.requests.RequestHeader; +import org.apache.kafka.common.requests.RequestUtils; +import org.apache.kafka.common.requests.ResponseHeader; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; import org.junit.jupiter.params.provider.ValueSource; +import java.nio.ByteBuffer; import java.util.Collections; -import java.util.Map; import java.util.concurrent.CompletableFuture; import static org.assertj.core.api.Assertions.assertThat; @@ -48,25 +57,49 @@ public class KafkaRequestHandlerTest { @Test - public void testKafkaApiVersionsNotSupported() { + public void testUnsupportedWireVersionFallsBackAndAllowsRenegotiation() { KafkaRequestHandler handler = createKafkaRequestHandler(); - short latestVersion = ApiKeys.API_VERSIONS.latestVersion(); - ApiVersionsRequest apiVersionsRequest = - new ApiVersionsRequest.Builder().build(latestVersion); - ChannelHandlerContext ctx = new TestingChannelHandlerContext(); - KafkaRequest request = - newRequest( - ApiKeys.API_VERSIONS, - (short) (latestVersion + 1), // unsupported version - new RequestHeader(ApiKeys.API_VERSIONS, latestVersion, "client-id", 0), - apiVersionsRequest, - ctx); - handler.processRequest(request); + RequestChannel requests = new RequestChannel(100); + EmbeddedChannel channel = + new EmbeddedChannel( + new KafkaCommandDecoder(new RequestChannel[] {requests}, "KAFKA")); + short unsupportedVersion = (short) (ApiKeys.API_VERSIONS.latestVersion() + 1); + try { + // An unknown request version must be rejected without parsing its body schema. + processWireRequest( + handler, + requests, + channel, + unsupportedVersion, + new ApiVersionsRequest.Builder().build((short) 4), + 41); + ApiVersionsResponse unsupportedResponse = parseWireResponse(channel, (short) 0, 41); + assertThat(unsupportedResponse.data().errorCode()) + .isEqualTo(Errors.UNSUPPORTED_VERSION.code()); + assertBrokerCapabilities(unsupportedResponse); - ApiVersionsResponse response = (ApiVersionsResponse) parseResponse(request); - Map errorCounts = response.errorCounts(); - assertThat(1).isEqualTo(errorCounts.size()); - assertThat(1).isEqualTo(errorCounts.get(Errors.UNSUPPORTED_VERSION)); + ApiVersionsRequestData data = + new ApiVersionsRequestData() + .setClientSoftwareName("test-client") + .setClientSoftwareVersion("1.0"); + data.unknownTaggedFields().add(new RawTaggedField(42, new byte[] {1, 2, 3})); + processWireRequest( + handler, + requests, + channel, + (short) 4, + new ApiVersionsRequest(data, (short) 4), + 42); + ApiVersionsResponse response = parseWireResponse(channel, (short) 4, 42); + assertSuccessfulResponseDefaults(response); + assertBrokerCapabilities(response); + assertThat(channel.isActive()).isTrue(); + assertThat(requests.requestsCount()).isZero(); + Object extraResponse = channel.readOutbound(); + assertThat(extraResponse).isNull(); + } finally { + channel.finishAndReleaseAll(); + } } @ParameterizedTest @@ -112,30 +145,32 @@ private static void assertSuccessfulResponseDefaults(ApiVersionsResponse respons private static void assertBrokerCapabilities(ApiVersionsResponse response) { assertThat(response.data().apiKeys()) .extracting(ApiVersion::apiKey, ApiVersion::minVersion, ApiVersion::maxVersion) - .containsExactly( - tuple( - ApiKeys.API_VERSIONS.id, - ApiKeys.API_VERSIONS.oldestVersion(), - ApiKeys.API_VERSIONS.latestVersion())); + .containsExactly(tuple(ApiKeys.API_VERSIONS.id, (short) 0, (short) 4)); } - @Test - public void testInvalidApiVersionsRequest() { + @ParameterizedTest + @CsvSource({ + "3, invalid client name, 1.0", + "3, client, ''", + "4, invalid client name, 1.0", + "4, client, ''" + }) + public void testInvalidApiVersionsRequest( + short version, String softwareName, String softwareVersion) { KafkaRequestHandler handler = createKafkaRequestHandler(); - short latestVersion = ApiKeys.API_VERSIONS.latestVersion(); ApiVersionsRequest requestBody = new ApiVersionsRequest.Builder( new ApiVersionsRequestData() - .setClientSoftwareName("invalid client name") - .setClientSoftwareVersion("1.0"), - latestVersion, - latestVersion) - .build(latestVersion); + .setClientSoftwareName(softwareName) + .setClientSoftwareVersion(softwareVersion), + version, + version) + .build(version); KafkaRequest request = newRequest( ApiKeys.API_VERSIONS, - latestVersion, - new RequestHeader(ApiKeys.API_VERSIONS, latestVersion, "client-id", 0), + version, + new RequestHeader(ApiKeys.API_VERSIONS, version, "client-id", 0), requestBody, new TestingChannelHandlerContext()); @@ -143,6 +178,7 @@ public void testInvalidApiVersionsRequest() { ApiVersionsResponse response = (ApiVersionsResponse) parseResponse(request); assertThat(response.errorCounts()).containsEntry(Errors.INVALID_REQUEST, 1); + assertThat(response.data().apiKeys()).isEmpty(); } @Test @@ -176,6 +212,53 @@ public void testUnregisteredApiIsNotRouted() { assertThat(response.errorCounts()).containsEntry(Errors.UNSUPPORTED_VERSION, 1); } + private static void processWireRequest( + KafkaRequestHandler handler, + RequestChannel requests, + EmbeddedChannel channel, + short version, + ApiVersionsRequest body, + int correlationId) { + RequestHeader header = + new RequestHeader(ApiKeys.API_VERSIONS, version, "test-client", correlationId); + ByteBuf buffer = + Unpooled.wrappedBuffer( + RequestUtils.serialize( + header.data(), + header.headerVersion(), + body.data(), + body.version())); + channel.writeInbound(buffer); + KafkaRequest request = (KafkaRequest) requests.pollRequest(1000); + assertThat(request).isNotNull(); + try { + assertThat(request.apiVersion()).isEqualTo(version); + handler.processRequest(request); + assertThat(request.future()).isDone(); + channel.runPendingTasks(); + assertThat(buffer.refCnt()).isZero(); + } finally { + request.releaseBuffer(); + } + } + + private static ApiVersionsResponse parseWireResponse( + EmbeddedChannel channel, short version, int correlationId) { + ByteBuf buffer = channel.readOutbound(); + assertThat(buffer).isNotNull(); + try { + ByteBuffer bytes = buffer.nioBuffer(); + assertThat(ResponseHeader.parse(bytes, (short) 0).correlationId()) + .isEqualTo(correlationId); + ApiVersionsResponseData data = + new ApiVersionsResponseData(new ByteBufferAccessor(bytes), version); + assertThat(bytes.hasRemaining()).isFalse(); + return new ApiVersionsResponse(data); + } finally { + buffer.release(); + } + } + private static KafkaRequest newRequest( ApiKeys apiKey, short apiVersion, diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/api/versions/ApiVersionsHandlerTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/api/versions/ApiVersionsHandlerTest.java new file mode 100644 index 0000000000..31269af56c --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/api/versions/ApiVersionsHandlerTest.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.api.versions; + +import org.apache.fluss.kafka.KafkaRequestContext; +import org.apache.fluss.kafka.dispatcher.KafkaApiHandler; +import org.apache.fluss.kafka.dispatcher.KafkaApiRegistry; +import org.apache.fluss.kafka.dispatcher.KafkaApiSpec; + +import org.apache.kafka.common.message.ApiVersionsResponseData.ApiVersion; +import org.apache.kafka.common.protocol.ApiKeys; +import org.apache.kafka.common.protocol.Errors; +import org.apache.kafka.common.requests.AbstractRequest; +import org.apache.kafka.common.requests.AbstractResponse; +import org.apache.kafka.common.requests.ApiVersionsRequest; +import org.apache.kafka.common.requests.ApiVersionsResponse; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.util.concurrent.CompletableFuture; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.tuple; + +/** Tests registry-backed capability advertisement by {@link ApiVersionsHandler}. */ +class ApiVersionsHandlerTest { + + @ParameterizedTest + @ValueSource(shorts = {0, 1, 2, 3, 4}) + void testAdvertiseRegisteredRangesAndExcludeHiddenApis(short version) { + KafkaApiRegistry registry = new KafkaApiRegistry(); + ApiVersionsHandler handler = new ApiVersionsHandler(registry); + registry.register(handler); + registry.register( + new TestingApiHandler( + new KafkaApiSpec(ApiKeys.METADATA, (short) 2, (short) 7, true))); + registry.register( + new TestingApiHandler( + new KafkaApiSpec(ApiKeys.FETCH, (short) 0, (short) 3, false))); + registry.freeze(); + + ApiVersionsResponse response = + (ApiVersionsResponse) + handler.handle(null, new ApiVersionsRequest.Builder().build(version)) + .join(); + + assertThat(response.data().errorCode()).isEqualTo(Errors.NONE.code()); + assertThat(response.data().apiKeys()) + .extracting(ApiVersion::apiKey, ApiVersion::minVersion, ApiVersion::maxVersion) + .containsExactly( + tuple(ApiKeys.METADATA.id, (short) 2, (short) 7), + tuple(ApiKeys.API_VERSIONS.id, (short) 0, (short) 4)); + } + + private static final class TestingApiHandler implements KafkaApiHandler { + private final KafkaApiSpec spec; + + private TestingApiHandler(KafkaApiSpec spec) { + this.spec = spec; + } + + @Override + public KafkaApiSpec apiSpec() { + return spec; + } + + @Override + public CompletableFuture handle( + KafkaRequestContext context, AbstractRequest request) { + throw new AssertionError( + "Capability advertisement must not invoke other API handlers."); + } + } +} From 229fc9434f705b4df9a894a5743c72166679cc45 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Tue, 13 Oct 2026 09:43:06 +0800 Subject: [PATCH 3/5] [kafka] Adapt ApiVersions to immutable API registry Construct the registry once with all handlers and defer ApiVersions registry access until request handling. Advertised capabilities and dispatch continue to share the same immutable registry without a separate capability list. Update coverage to verify lazy access and handler identity across versions. Validated with Java 11 reactor clean verify: 44 unit and 2 integration cases passed in both Maven test executions, including inherited error-mapping and idle regressions. Kafka main and test sources also compile with --release 8. Spotless, Checkstyle, RAT, and diff checks passed. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 28/28 AI-Contributed/UT: 33/33 --- .../fluss/kafka/KafkaRequestHandler.java | 13 ++++++-- .../api/versions/ApiVersionsHandler.java | 15 ++++++--- .../api/versions/ApiVersionsHandlerTest.java | 33 ++++++++++++++----- 3 files changed, 44 insertions(+), 17 deletions(-) diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java index 2df16a8bd7..b3b555a8f9 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/KafkaRequestHandler.java @@ -24,16 +24,19 @@ import org.apache.fluss.rpc.netty.server.RequestHandler; import org.apache.fluss.rpc.protocol.RequestType; +import java.util.Collections; + /** Entry point that dispatches Kafka protocol requests to registered API handlers. */ public class KafkaRequestHandler implements RequestHandler { + private final KafkaApiRegistry registry; private final KafkaRequestDispatcher dispatcher; /** Creates a Kafka request handler with the implemented server capabilities. */ public KafkaRequestHandler() { - KafkaApiRegistry registry = new KafkaApiRegistry(); - registry.register(new ApiVersionsHandler(registry)); - registry.freeze(); + this.registry = + new KafkaApiRegistry( + Collections.singletonList(new ApiVersionsHandler(this::apiRegistry))); this.dispatcher = new KafkaRequestDispatcher(registry, new KafkaErrorMapper()); } @@ -55,4 +58,8 @@ public void processRequest(KafkaRequest request) { } }); } + + private KafkaApiRegistry apiRegistry() { + return registry; + } } diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/versions/ApiVersionsHandler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/versions/ApiVersionsHandler.java index c38d7bc6cb..82e3559c5f 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/versions/ApiVersionsHandler.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/versions/ApiVersionsHandler.java @@ -31,6 +31,7 @@ import org.apache.kafka.common.requests.ApiVersionsResponse; import java.util.concurrent.CompletableFuture; +import java.util.function.Supplier; import static org.apache.fluss.utils.Preconditions.checkNotNull; @@ -45,11 +46,15 @@ public final class ApiVersionsHandler implements KafkaApiHandler registrySupplier; - /** Creates an ApiVersions handler backed by the server capability registry. */ - public ApiVersionsHandler(KafkaApiRegistry registry) { - this.registry = checkNotNull(registry); + /** + * Creates an ApiVersions handler backed by the same immutable registry used for dispatch. The + * supplier is read only when handling requests, so this handler can be included in the + * registry's construction snapshot. + */ + public ApiVersionsHandler(Supplier registrySupplier) { + this.registrySupplier = checkNotNull(registrySupplier); } @Override @@ -65,7 +70,7 @@ public CompletableFuture handle( request.getErrorResponse(Errors.INVALID_REQUEST.exception())); } ApiVersionsResponseData data = new ApiVersionsResponseData(); - for (KafkaApiSpec spec : registry.advertisedApiSpecs()) { + for (KafkaApiSpec spec : registrySupplier.get().advertisedApiSpecs()) { data.apiKeys() .add( new ApiVersionsResponseData.ApiVersion() diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/api/versions/ApiVersionsHandlerTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/api/versions/ApiVersionsHandlerTest.java index 31269af56c..0492ee3572 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/api/versions/ApiVersionsHandlerTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/api/versions/ApiVersionsHandlerTest.java @@ -32,7 +32,9 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; +import java.util.Arrays; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.atomic.AtomicReference; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.tuple; @@ -43,16 +45,29 @@ class ApiVersionsHandlerTest { @ParameterizedTest @ValueSource(shorts = {0, 1, 2, 3, 4}) void testAdvertiseRegisteredRangesAndExcludeHiddenApis(short version) { - KafkaApiRegistry registry = new KafkaApiRegistry(); - ApiVersionsHandler handler = new ApiVersionsHandler(registry); - registry.register(handler); - registry.register( + AtomicReference registryReference = new AtomicReference<>(); + ApiVersionsHandler handler = + new ApiVersionsHandler( + () -> { + assertThat(registryReference.get()).isNotNull(); + return registryReference.get(); + }); + TestingApiHandler metadataHandler = new TestingApiHandler( - new KafkaApiSpec(ApiKeys.METADATA, (short) 2, (short) 7, true))); - registry.register( - new TestingApiHandler( - new KafkaApiSpec(ApiKeys.FETCH, (short) 0, (short) 3, false))); - registry.freeze(); + new KafkaApiSpec(ApiKeys.METADATA, (short) 2, (short) 7, true)); + KafkaApiRegistry registry = + new KafkaApiRegistry( + Arrays.asList( + handler, + metadataHandler, + new TestingApiHandler( + new KafkaApiSpec( + ApiKeys.FETCH, (short) 0, (short) 3, false)))); + registryReference.set(registry); + + assertThat(registry.lookup(ApiKeys.API_VERSIONS)).isSameAs(handler); + assertThat(registry.lookup(ApiKeys.METADATA)).isSameAs(metadataHandler); + assertThat(registry.lookup(ApiKeys.FETCH)).isNull(); ApiVersionsResponse response = (ApiVersionsResponse) From 10bf516b6a04c579079b6e69b3788ac633041525 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Tue, 13 Oct 2026 18:06:21 +0800 Subject: [PATCH 4/5] [kafka] Adapt version fallback regression to PR02 handler Use the dispatcher-backed handler constructor in the fallback test added by PR01. Preserve its unsupported-version, listener, response, and buffer checks. Validated with Java 11 reactor clean verify: 39 unit and 2 integration cases passed in both Maven test executions. Kafka main and test sources compile with --release 8; Spotless, Checkstyle, RAT, and diff checks passed. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 0/0 AI-Contributed/UT: 3/3 --- .../java/org/apache/fluss/kafka/KafkaCommandDecoderTest.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaCommandDecoderTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaCommandDecoderTest.java index f4a255b013..dda2e52bde 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaCommandDecoderTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaCommandDecoderTest.java @@ -17,7 +17,6 @@ package org.apache.fluss.kafka; -import org.apache.fluss.rpc.TestingTabletGatewayService; import org.apache.fluss.rpc.netty.server.RequestChannel; import org.apache.fluss.shaded.netty4.io.netty.buffer.ByteBuf; import org.apache.fluss.shaded.netty4.io.netty.buffer.Unpooled; @@ -71,7 +70,7 @@ public void testUnsupportedApiVersionsRequestReturnsVersionZeroErrorResponse() { assertThat(parsedRequest.hasUnsupportedRequestVersion()).isTrue(); assertThat(parsedRequest.version()).isZero(); - new KafkaRequestHandler(new TestingTabletGatewayService()).processRequest(request); + new KafkaRequestHandler().processRequest(request); channel.runPendingTasks(); responseBuffer = channel.readOutbound(); From 062047def41ff5fe2d89b6c43b0b355e3772de1f Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Tue, 13 Oct 2026 21:00:25 +0800 Subject: [PATCH 5/5] [kafka] Clarify uninitialized API registry failures Check the deferred registry before reading advertised capabilities so an initialization-order failure reports an explicit diagnostic message. Validated with Java 11 reactor tests: 16 ApiVersions and request-handler cases passed, along with Spotless, Checkstyle, RAT, and diff checks. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 4/4 AI-Contributed/UT: 0/0 --- .../apache/fluss/kafka/api/versions/ApiVersionsHandler.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/versions/ApiVersionsHandler.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/versions/ApiVersionsHandler.java index 82e3559c5f..5cac0c3cca 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/versions/ApiVersionsHandler.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/api/versions/ApiVersionsHandler.java @@ -69,8 +69,10 @@ public CompletableFuture handle( return CompletableFuture.completedFuture( request.getErrorResponse(Errors.INVALID_REQUEST.exception())); } + KafkaApiRegistry registry = + checkNotNull(registrySupplier.get(), "Kafka API registry is not initialized."); ApiVersionsResponseData data = new ApiVersionsResponseData(); - for (KafkaApiSpec spec : registrySupplier.get().advertisedApiSpecs()) { + for (KafkaApiSpec spec : registry.advertisedApiSpecs()) { data.apiKeys() .add( new ApiVersionsResponseData.ApiVersion()