From cc82655a5631bdce5d33a365f40546c2204fd04d Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Thu, 10 Sep 2026 15:28:16 +0800 Subject: [PATCH 01/15] [kafka] Define DDL table mapping for Kafka compatibility Extract topic identity and the raw/string table mapping contract before Metadata. Validate table kinds, field projections and metadata columns independently of request handling and record decoding. Validation: Java 11, mvn -o -pl fluss-kafka clean verify (47 unit tests and 2 integration tests); Checkstyle, Spotless and RAT passed. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 769/769 AI-Contributed/UT: 480/480 --- fluss-kafka/pom.xml | 7 + .../fluss/kafka/format/KafkaDataFormat.java | 73 ++++ .../fluss/kafka/mapping/KafkaTopicMapper.java | 73 ++++ .../kafka/schema/KafkaFieldProjection.java | 123 +++++++ .../fluss/kafka/schema/KafkaTopicSchema.java | 150 ++++++++ .../schema/KafkaTopicSchemaException.java | 30 ++ .../schema/KafkaTopicSchemaResolver.java | 313 +++++++++++++++++ .../kafka/mapping/KafkaTopicMapperTest.java | 62 ++++ .../kafka/schema/KafkaTableMappingITCase.java | 88 +++++ .../schema/KafkaTopicSchemaResolverTest.java | 330 ++++++++++++++++++ 10 files changed, 1249 insertions(+) create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaDataFormat.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/mapping/KafkaTopicMapper.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaFieldProjection.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchema.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaException.java create mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolver.java create mode 100644 fluss-kafka/src/test/java/org/apache/fluss/kafka/mapping/KafkaTopicMapperTest.java create mode 100644 fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java create mode 100644 fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolverTest.java diff --git a/fluss-kafka/pom.xml b/fluss-kafka/pom.xml index f48a94ab9e9..b77695848ca 100644 --- a/fluss-kafka/pom.xml +++ b/fluss-kafka/pom.xml @@ -64,6 +64,13 @@ + + org.apache.curator + curator-test + ${curator.version} + test + + org.apache.fluss fluss-test-utils diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaDataFormat.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaDataFormat.java new file mode 100644 index 00000000000..ad0039b7e78 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaDataFormat.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.format; + +import org.apache.fluss.annotation.Internal; + +import java.util.Locale; + +/** Supported interpretations of Kafka record key and value bytes. */ +@Internal +public enum KafkaDataFormat { + RAW("raw"), + STRING("string"); + + /** Fluss table custom property controlling the record key format. */ + public static final String KEY_FORMAT_CONFIG = "kafka.key.format"; + + /** Fluss table custom property controlling the record value format. */ + public static final String VALUE_FORMAT_CONFIG = "kafka.value.format"; + + /** Fluss fields populated from the Kafka record key. */ + public static final String KEY_FIELDS_CONFIG = "kafka.key.fields"; + + /** Strategy for deriving fields populated from the Kafka record value. */ + public static final String VALUE_FIELDS_INCLUDE_CONFIG = "kafka.value.fields-include"; + + /** Fluss column populated from the Kafka record timestamp. */ + public static final String TIMESTAMP_COLUMN_CONFIG = "kafka.metadata.timestamp.column"; + + /** Fluss column populated from the Kafka record headers. */ + public static final String HEADERS_COLUMN_CONFIG = "kafka.metadata.headers.column"; + + private final String value; + + KafkaDataFormat(String value) { + this.value = value; + } + + /** Parses a table custom property value. */ + public static KafkaDataFormat parse(String value) { + if (value == null) { + throw new IllegalArgumentException("Kafka data format must not be null."); + } + String normalized = value.trim().toLowerCase(Locale.ROOT); + for (KafkaDataFormat format : values()) { + if (format.value.equals(normalized)) { + return format; + } + } + throw new IllegalArgumentException( + "Unsupported Kafka data format '" + value + "'. Expected raw or string."); + } + + /** Returns the persisted table property value. */ + public String value() { + return value; + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/mapping/KafkaTopicMapper.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/mapping/KafkaTopicMapper.java new file mode 100644 index 00000000000..9f8ed63f543 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/mapping/KafkaTopicMapper.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.mapping; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.metadata.TablePath; + +import org.apache.kafka.common.Uuid; +import org.apache.kafka.common.internals.Topic; + +import static org.apache.fluss.utils.Preconditions.checkArgument; +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Maps Kafka topic identities to tables in the configured Fluss Kafka database. */ +@Internal +public final class KafkaTopicMapper { + + // ASCII "Fluss" followed by zero bytes. A dedicated namespace avoids Kafka-reserved UUIDs. + private static final long TOPIC_ID_NAMESPACE = 0x466c757373000000L; + + private final String databaseName; + + /** Creates a topic mapper for one Fluss database. */ + public KafkaTopicMapper(String databaseName) { + this.databaseName = checkNotNull(databaseName); + } + + /** Maps a Kafka topic name to its Fluss table path. */ + public TablePath toTablePath(String topicName) { + Topic.validate(topicName); + return TablePath.of(databaseName, topicName); + } + + /** Returns whether a table belongs to this database and has a valid Kafka topic name. */ + public boolean isMappedTable(TablePath tablePath) { + return databaseName.equals(tablePath.getDatabaseName()) + && Topic.isValid(tablePath.getTableName()); + } + + /** Maps a Fluss table ID to a stable Kafka topic ID. */ + public Uuid toTopicId(long tableId) { + checkArgument(tableId >= 0, "Table ID must be non-negative, but was %s.", tableId); + return new Uuid(TOPIC_ID_NAMESPACE, tableId); + } + + /** Returns whether a Kafka topic ID can represent a Fluss table ID. */ + public boolean isMappedTopicId(Uuid topicId) { + return topicId != null + && topicId.getMostSignificantBits() == TOPIC_ID_NAMESPACE + && topicId.getLeastSignificantBits() >= 0L; + } + + /** Extracts the Fluss table ID encoded in a Kafka topic ID. */ + public long toTableId(Uuid topicId) { + checkArgument(isMappedTopicId(topicId), "Topic ID %s is not a Fluss topic ID.", topicId); + return topicId.getLeastSignificantBits(); + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaFieldProjection.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaFieldProjection.java new file mode 100644 index 00000000000..b1715758a9d --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaFieldProjection.java @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.schema; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.types.DataType; +import org.apache.fluss.types.RowType; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Objects; + +import static org.apache.fluss.utils.Preconditions.checkArgument; +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Ordered physical Fluss fields populated by one Kafka record component. */ +@Internal +public final class KafkaFieldProjection { + + private final List positions; + private final List names; + private final List dataTypes; + + /** Creates a projection from physical row positions. */ + public KafkaFieldProjection(RowType rowType, List positions) { + checkNotNull(rowType); + checkNotNull(positions); + List positionCopy = new ArrayList<>(positions.size()); + List projectedNames = new ArrayList<>(positions.size()); + List projectedTypes = new ArrayList<>(positions.size()); + for (Integer position : positions) { + checkArgument( + position != null && position >= 0 && position < rowType.getFieldCount(), + "Invalid Kafka field projection position %s.", + position); + positionCopy.add(position); + projectedNames.add(rowType.getFieldNames().get(position)); + projectedTypes.add(rowType.getTypeAt(position)); + } + this.positions = Collections.unmodifiableList(positionCopy); + this.names = Collections.unmodifiableList(projectedNames); + this.dataTypes = Collections.unmodifiableList(projectedTypes); + } + + /** Returns the number of projected fields. */ + public int size() { + return positions.size(); + } + + /** Returns whether this projection owns no fields. */ + public boolean isEmpty() { + return positions.isEmpty(); + } + + /** Returns the physical row position at the projection position. */ + public int positionAt(int projectionPosition) { + return positions.get(projectionPosition); + } + + /** Returns the physical field name at the projection position. */ + public String nameAt(int projectionPosition) { + return names.get(projectionPosition); + } + + /** Returns the physical data type at the projection position. */ + public DataType dataTypeAt(int projectionPosition) { + return dataTypes.get(projectionPosition); + } + + /** Returns the projected physical positions. */ + public List positions() { + return positions; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof KafkaFieldProjection)) { + return false; + } + KafkaFieldProjection that = (KafkaFieldProjection) obj; + return Objects.equals(positions, that.positions) + && Objects.equals(names, that.names) + && Objects.equals(dataTypes, that.dataTypes); + } + + @Override + public int hashCode() { + return Objects.hash(positions, names, dataTypes); + } + + @Override + public String toString() { + return "KafkaFieldProjection{" + + "positions=" + + positions + + ", " + + "names=" + + names + + ", " + + "dataTypes=" + + dataTypes + + "}"; + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchema.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchema.java new file mode 100644 index 00000000000..64263e69286 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchema.java @@ -0,0 +1,150 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.schema; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.kafka.format.KafkaDataFormat; +import org.apache.fluss.types.RowType; + +import javax.annotation.Nullable; + +import java.util.Objects; + +import static org.apache.fluss.utils.Preconditions.checkNotNull; + +/** Resolved Kafka key, value, and metadata mapping for one Fluss table schema. */ +@Internal +public final class KafkaTopicSchema { + + private final RowType rowType; + private final @Nullable KafkaDataFormat keyFormat; + private final KafkaFieldProjection keyProjection; + private final KafkaDataFormat valueFormat; + private final KafkaFieldProjection valueProjection; + private final int timestampPosition; + private final int headersPosition; + + /** Creates a resolved Kafka topic schema. */ + public KafkaTopicSchema( + RowType rowType, + @Nullable KafkaDataFormat keyFormat, + KafkaFieldProjection keyProjection, + KafkaDataFormat valueFormat, + KafkaFieldProjection valueProjection, + int timestampPosition, + int headersPosition) { + this.rowType = checkNotNull(rowType); + this.keyFormat = keyFormat; + this.keyProjection = checkNotNull(keyProjection); + this.valueFormat = checkNotNull(valueFormat); + this.valueProjection = checkNotNull(valueProjection); + this.timestampPosition = timestampPosition; + this.headersPosition = headersPosition; + } + + /** Returns the physical Fluss row type. */ + public RowType rowType() { + return rowType; + } + + /** Returns the key format, or null when the Kafka key is not mapped. */ + public @Nullable KafkaDataFormat keyFormat() { + return keyFormat; + } + + /** Returns the key field projection. */ + public KafkaFieldProjection keyProjection() { + return keyProjection; + } + + /** Returns the value format. */ + public KafkaDataFormat valueFormat() { + return valueFormat; + } + + /** Returns the value field projection. */ + public KafkaFieldProjection valueProjection() { + return valueProjection; + } + + /** Returns the timestamp physical position, or -1 when it is not mapped. */ + public int timestampPosition() { + return timestampPosition; + } + + /** Returns the headers physical position, or -1 when they are not mapped. */ + public int headersPosition() { + return headersPosition; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof KafkaTopicSchema)) { + return false; + } + KafkaTopicSchema that = (KafkaTopicSchema) obj; + return Objects.equals(rowType, that.rowType) + && Objects.equals(keyFormat, that.keyFormat) + && Objects.equals(keyProjection, that.keyProjection) + && Objects.equals(valueFormat, that.valueFormat) + && Objects.equals(valueProjection, that.valueProjection) + && Objects.equals(timestampPosition, that.timestampPosition) + && Objects.equals(headersPosition, that.headersPosition); + } + + @Override + public int hashCode() { + return Objects.hash( + rowType, + keyFormat, + keyProjection, + valueFormat, + valueProjection, + timestampPosition, + headersPosition); + } + + @Override + public String toString() { + return "KafkaTopicSchema{" + + "rowType=" + + rowType + + ", " + + "keyFormat=" + + keyFormat + + ", " + + "keyProjection=" + + keyProjection + + ", " + + "valueFormat=" + + valueFormat + + ", " + + "valueProjection=" + + valueProjection + + ", " + + "timestampPosition=" + + timestampPosition + + ", " + + "headersPosition=" + + headersPosition + + "}"; + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaException.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaException.java new file mode 100644 index 00000000000..0e6a294c4d5 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaException.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.schema; + +import org.apache.fluss.annotation.Internal; + +/** Indicates that a Fluss table does not define a valid Kafka record mapping contract. */ +@Internal +public final class KafkaTopicSchemaException extends IllegalArgumentException { + + /** Creates a Kafka topic schema exception. */ + public KafkaTopicSchemaException(String message) { + super(message); + } +} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolver.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolver.java new file mode 100644 index 00000000000..840ce8a4f58 --- /dev/null +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolver.java @@ -0,0 +1,313 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.schema; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.config.Configuration; +import org.apache.fluss.config.TableConfig; +import org.apache.fluss.kafka.format.KafkaDataFormat; +import org.apache.fluss.metadata.LogFormat; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.types.ArrayType; +import org.apache.fluss.types.BytesType; +import org.apache.fluss.types.LocalZonedTimestampType; +import org.apache.fluss.types.RowType; +import org.apache.fluss.types.StringType; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; + +/** Resolves and validates the Kafka record mapping stored in Fluss table custom properties. */ +@Internal +public final class KafkaTopicSchemaResolver { + + private static final String INCLUDE_ALL = "ALL"; + private static final String INCLUDE_EXCEPT_KEY = "EXCEPT_KEY"; + + private static final Set SUPPORTED_PROPERTIES = + new HashSet<>( + Arrays.asList( + KafkaDataFormat.KEY_FORMAT_CONFIG, + KafkaDataFormat.KEY_FIELDS_CONFIG, + KafkaDataFormat.VALUE_FORMAT_CONFIG, + KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, + KafkaDataFormat.TIMESTAMP_COLUMN_CONFIG, + KafkaDataFormat.HEADERS_COLUMN_CONFIG)); + + /** Resolves one table's Kafka record mapping contract. */ + public KafkaTopicSchema resolve(TableDescriptor table) { + validateTableKind(table); + RowType rowType = table.getSchema().getRowType(); + Map properties = table.getCustomProperties(); + + for (String property : properties.keySet()) { + if (property.startsWith("kafka.") && !SUPPORTED_PROPERTIES.contains(property)) { + throw invalid("Unsupported Kafka table property '" + property + "'."); + } + } + + int timestampPosition = + resolveOptionalPosition( + rowType, properties.get(KafkaDataFormat.TIMESTAMP_COLUMN_CONFIG)); + int headersPosition = + resolveOptionalPosition( + rowType, properties.get(KafkaDataFormat.HEADERS_COLUMN_CONFIG)); + if (timestampPosition >= 0 && timestampPosition == headersPosition) { + throw invalid("Kafka timestamp and headers cannot map to the same Fluss column."); + } + validateMetadataColumns(rowType, timestampPosition, headersPosition); + + String keyFormatValue = properties.get(KafkaDataFormat.KEY_FORMAT_CONFIG); + KafkaDataFormat keyFormat = keyFormatValue == null ? null : parseFormat(keyFormatValue); + List keyPositions = + resolveKeyPositions( + rowType, keyFormat, properties.get(KafkaDataFormat.KEY_FIELDS_CONFIG)); + for (Integer keyPosition : keyPositions) { + if (keyPosition == timestampPosition || keyPosition == headersPosition) { + throw invalid( + "Kafka key field '" + + rowType.getFieldNames().get(keyPosition) + + "' cannot be a Kafka metadata column."); + } + } + KafkaFieldProjection keyProjection = new KafkaFieldProjection(rowType, keyPositions); + + String valueFormatValue = properties.get(KafkaDataFormat.VALUE_FORMAT_CONFIG); + if (valueFormatValue == null) { + throw invalid( + "Missing required table property '" + + KafkaDataFormat.VALUE_FORMAT_CONFIG + + "'."); + } + KafkaDataFormat valueFormat = parseFormat(valueFormatValue); + String fieldsInclude = + normalizeFieldsInclude(properties.get(KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG)); + if (!keyPositions.isEmpty() && INCLUDE_ALL.equals(fieldsInclude)) { + throw invalid( + "Mapping Kafka key fields requires " + + KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG + + "=EXCEPT_KEY."); + } + + Set metadataPositions = new HashSet<>(); + if (timestampPosition >= 0) { + metadataPositions.add(timestampPosition); + } + if (headersPosition >= 0) { + metadataPositions.add(headersPosition); + } + List valuePositions = + resolveValuePositions(rowType, fieldsInclude, keyPositions, metadataPositions); + KafkaFieldProjection valueProjection = new KafkaFieldProjection(rowType, valuePositions); + if (valueProjection.isEmpty()) { + throw invalid("Kafka value projection must contain at least one Fluss column."); + } + + validateSingleFieldFormat(keyFormat, keyProjection, "key"); + validateSingleFieldFormat(valueFormat, valueProjection, "value"); + return new KafkaTopicSchema( + rowType, + keyFormat, + keyProjection, + valueFormat, + valueProjection, + timestampPosition, + headersPosition); + } + + private static void validateTableKind(TableDescriptor table) { + if (table.hasPrimaryKey()) { + throw invalid("Kafka topic table must be a log table."); + } + if (table.isPartitioned()) { + throw invalid("Partitioned Fluss tables are not supported."); + } + if (new TableConfig(Configuration.fromMap(table.getProperties())).getLogFormat() + != LogFormat.ARROW) { + throw invalid("Kafka topic table must use the Arrow log format."); + } + } + + private static List resolveKeyPositions( + RowType rowType, KafkaDataFormat keyFormat, String keyFieldsValue) { + if (keyFormat == null) { + if (keyFieldsValue != null) { + throw invalid( + KafkaDataFormat.KEY_FIELDS_CONFIG + + " requires " + + KafkaDataFormat.KEY_FORMAT_CONFIG + + "."); + } + return Collections.emptyList(); + } + if (keyFieldsValue == null || keyFieldsValue.trim().isEmpty()) { + throw invalid( + "Missing required table property '" + KafkaDataFormat.KEY_FIELDS_CONFIG + "'."); + } + String[] fieldNames = keyFieldsValue.split(",", -1); + List positions = new ArrayList<>(fieldNames.length); + Set uniquePositions = new HashSet<>(); + for (String fieldNameValue : fieldNames) { + String fieldName = fieldNameValue.trim(); + if (fieldName.isEmpty()) { + throw invalid("Kafka key field names must not be empty."); + } + int position = rowType.getFieldIndex(fieldName); + if (position < 0) { + throw invalid("Kafka key field '" + fieldName + "' does not exist."); + } + if (!uniquePositions.add(position)) { + throw invalid("Duplicate Kafka key field '" + fieldName + "'."); + } + positions.add(position); + } + return positions; + } + + private static String normalizeFieldsInclude(String value) { + String normalized = value == null ? INCLUDE_ALL : value.trim().toUpperCase(Locale.ROOT); + if (!INCLUDE_ALL.equals(normalized) && !INCLUDE_EXCEPT_KEY.equals(normalized)) { + throw invalid( + "Invalid " + + KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG + + " '" + + value + + "'. Expected ALL or EXCEPT_KEY."); + } + return normalized; + } + + private static List resolveValuePositions( + RowType rowType, + String fieldsInclude, + List keyPositions, + Set metadataPositions) { + Set excludedKeyPositions = + INCLUDE_EXCEPT_KEY.equals(fieldsInclude) + ? new HashSet<>(keyPositions) + : Collections.emptySet(); + List positions = new ArrayList<>(); + for (int position = 0; position < rowType.getFieldCount(); position++) { + if (!metadataPositions.contains(position) && !excludedKeyPositions.contains(position)) { + positions.add(position); + } + } + return positions; + } + + private static int resolveOptionalPosition(RowType rowType, String fieldNameValue) { + if (fieldNameValue == null) { + return -1; + } + if (fieldNameValue.trim().isEmpty()) { + throw invalid("Kafka metadata column name must not be empty."); + } + String fieldName = fieldNameValue.trim(); + int position = rowType.getFieldIndex(fieldName); + if (position < 0) { + throw invalid("Kafka metadata column '" + fieldName + "' does not exist."); + } + return position; + } + + private static void validateMetadataColumns( + RowType rowType, int timestampPosition, int headersPosition) { + if (timestampPosition >= 0) { + if (!(rowType.getTypeAt(timestampPosition) instanceof LocalZonedTimestampType) + || rowType.getTypeAt(timestampPosition).isNullable() + || ((LocalZonedTimestampType) rowType.getTypeAt(timestampPosition)) + .getPrecision() + != 3) { + throw invalid("Kafka timestamp column must be TIMESTAMP_LTZ(3) NOT NULL."); + } + } + if (headersPosition >= 0) { + validateHeadersType(rowType.getTypeAt(headersPosition)); + } + } + + private static void validateHeadersType(org.apache.fluss.types.DataType dataType) { + if (!(dataType instanceof ArrayType) || !dataType.isNullable()) { + throw invalid( + "Kafka headers column must be nullable " + + "ARRAY>."); + } + ArrayType arrayType = (ArrayType) dataType; + if (!(arrayType.getElementType() instanceof RowType)) { + throw invalid("Kafka headers elements must be rows."); + } + RowType headerType = (RowType) arrayType.getElementType(); + if (!headerType.getFieldNames().equals(Arrays.asList("name", "value")) + || !(headerType.getTypeAt(0) instanceof StringType) + || !(headerType.getTypeAt(1) instanceof BytesType) + || !headerType.getTypeAt(1).isNullable()) { + throw invalid("Kafka headers elements must be ROW."); + } + } + + private static void validateSingleFieldFormat( + KafkaDataFormat format, KafkaFieldProjection projection, String component) { + if (format == null) { + return; + } + if (format == KafkaDataFormat.RAW || format == KafkaDataFormat.STRING) { + if (projection.size() != 1) { + throw invalid( + "Kafka " + + component + + " format " + + format.value() + + " requires exactly one Fluss field."); + } + boolean validType = + format == KafkaDataFormat.RAW + ? projection.dataTypeAt(0) instanceof BytesType + : projection.dataTypeAt(0) instanceof StringType; + if (!validType) { + throw invalid( + "Kafka " + + component + + " field '" + + projection.nameAt(0) + + "' must be " + + (format == KafkaDataFormat.RAW ? "BYTES" : "STRING") + + " for format " + + format.value() + + "."); + } + } + } + + private static KafkaDataFormat parseFormat(String value) { + try { + return KafkaDataFormat.parse(value); + } catch (IllegalArgumentException e) { + throw invalid(e.getMessage()); + } + } + + private static KafkaTopicSchemaException invalid(String message) { + return new KafkaTopicSchemaException(message); + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/mapping/KafkaTopicMapperTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/mapping/KafkaTopicMapperTest.java new file mode 100644 index 00000000000..20c32ded941 --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/mapping/KafkaTopicMapperTest.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.mapping; + +import org.apache.fluss.metadata.TablePath; + +import org.apache.kafka.common.Uuid; +import org.apache.kafka.common.errors.InvalidTopicException; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests for {@link KafkaTopicMapper}. */ +public class KafkaTopicMapperTest { + + @Test + public void testTopicNameAndIdMapping() { + KafkaTopicMapper mapper = new KafkaTopicMapper("kafka"); + + assertThat(mapper.toTablePath("topic").toString()).isEqualTo("kafka.topic"); + Uuid topicId = mapper.toTopicId(123L); + assertThat(topicId).isNotIn(Uuid.ZERO_UUID, Uuid.ONE_UUID, Uuid.METADATA_TOPIC_ID); + assertThat(mapper.isMappedTopicId(topicId)).isTrue(); + assertThat(mapper.toTableId(topicId)).isEqualTo(123L); + + Uuid firstTableTopicId = mapper.toTopicId(0L); + assertThat(firstTableTopicId).isNotEqualTo(Uuid.ZERO_UUID); + assertThat(mapper.isMappedTopicId(firstTableTopicId)).isTrue(); + assertThat(mapper.toTableId(firstTableTopicId)).isZero(); + } + + @Test + public void testOnlyValidTopicsInConfiguredDatabaseAreMapped() { + KafkaTopicMapper mapper = new KafkaTopicMapper("kafka"); + assertThat(mapper.isMappedTable(TablePath.of("kafka", "events"))).isTrue(); + assertThat(mapper.isMappedTable(TablePath.of("other", "events"))).isFalse(); + assertThat(mapper.isMappedTable(TablePath.of("kafka", "invalid topic"))).isFalse(); + assertThatThrownBy(() -> mapper.toTablePath("invalid topic")) + .isInstanceOf(InvalidTopicException.class); + assertThatThrownBy(() -> mapper.toTopicId(-1L)) + .isInstanceOf(IllegalArgumentException.class); + assertThat(mapper.isMappedTopicId(Uuid.ZERO_UUID)).isFalse(); + assertThatThrownBy(() -> mapper.toTableId(Uuid.ZERO_UUID)) + .isInstanceOf(IllegalArgumentException.class); + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java new file mode 100644 index 00000000000..0c92b8a7300 --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.schema; + +import org.apache.fluss.kafka.format.KafkaDataFormat; +import org.apache.fluss.kafka.mapping.KafkaTopicMapper; +import org.apache.fluss.metadata.LogFormat; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.rpc.messages.MetadataRequest; +import org.apache.fluss.rpc.messages.PbTableMetadata; +import org.apache.fluss.server.testutils.FlussClusterExtension; +import org.apache.fluss.types.DataTypes; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import static org.apache.fluss.server.testutils.RpcMessageTestUtils.createTable; +import static org.assertj.core.api.Assertions.assertThat; + +/** Verifies Kafka mapping properties survive the native Fluss table creation path. */ +public class KafkaTableMappingITCase { + + @RegisterExtension + public static final FlussClusterExtension FLUSS_CLUSTER_EXTENSION = + FlussClusterExtension.builder().setNumOfTabletServers(1).build(); + + @Test + public void testResolveMappingFromCreatedTableMetadata() throws Exception { + TableDescriptor descriptor = + TableDescriptor.builder() + .schema( + Schema.newBuilder() + .column("event_key", DataTypes.STRING()) + .column("event_body", DataTypes.BYTES()) + .build()) + .distributedBy(2) + .logFormat(LogFormat.ARROW) + .customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, "string") + .customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "event_key") + .customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, "raw") + .customProperty(KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, "EXCEPT_KEY") + .build(); + KafkaTopicMapper mapper = new KafkaTopicMapper("kafka_ddl"); + TablePath tablePath = mapper.toTablePath("events"); + long tableId = createTable(FLUSS_CLUSTER_EXTENSION, tablePath, descriptor); + FLUSS_CLUSTER_EXTENSION.waitUntilAllGatewayHasSameMetadata(); + MetadataRequest request = new MetadataRequest(); + request.addTablePath() + .setDatabaseName(tablePath.getDatabaseName()) + .setTableName(tablePath.getTableName()); + PbTableMetadata metadata = + FLUSS_CLUSTER_EXTENSION + .newCoordinatorClient() + .metadata(request) + .get() + .getTableMetadatasList() + .get(0); + TableDescriptor persisted = TableDescriptor.fromJsonBytes(metadata.getTableJson()); + KafkaTopicSchema mapping = new KafkaTopicSchemaResolver().resolve(persisted); + + assertThat(metadata.getTableId()).isEqualTo(tableId); + assertThat(mapper.toTableId(mapper.toTopicId(metadata.getTableId()))).isEqualTo(tableId); + assertThat(metadata.getBucketMetadatasList()).hasSize(2); + assertThat(persisted.getCustomProperties()) + .containsAllEntriesOf(descriptor.getCustomProperties()); + assertThat(mapping.keyProjection().positions()).containsExactly(0); + assertThat(mapping.keyFormat()).isEqualTo(KafkaDataFormat.STRING); + assertThat(mapping.valueProjection().positions()).containsExactly(1); + assertThat(mapping.valueFormat()).isEqualTo(KafkaDataFormat.RAW); + } +} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolverTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolverTest.java new file mode 100644 index 00000000000..4c698b5ad6b --- /dev/null +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolverTest.java @@ -0,0 +1,330 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.kafka.schema; + +import org.apache.fluss.kafka.format.KafkaDataFormat; +import org.apache.fluss.metadata.LogFormat; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.types.DataType; +import org.apache.fluss.types.DataTypes; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests the DDL contract independently of Kafka request handling and record decoding. */ +public class KafkaTopicSchemaResolverTest { + + private final KafkaTopicSchemaResolver resolver = new KafkaTopicSchemaResolver(); + + @Test + public void testRawMappingSurvivesTableMetadataSerialization() { + Schema schema = + Schema.newBuilder() + .column("message", DataTypes.BYTES()) + .column("received_at", DataTypes.TIMESTAMP_LTZ(3).copy(false)) + .column("attributes", headersType()) + .column("message_key", DataTypes.BYTES()) + .build(); + TableDescriptor table = + table(schema, "raw") + .customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, "raw") + .customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "message_key") + .customProperty(KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, "EXCEPT_KEY") + .customProperty(KafkaDataFormat.TIMESTAMP_COLUMN_CONFIG, "received_at") + .customProperty(KafkaDataFormat.HEADERS_COLUMN_CONFIG, "attributes") + .build(); + KafkaTopicSchema mapping = + resolver.resolve(TableDescriptor.fromJsonBytes(table.toJsonBytes())); + + assertThat(mapping.rowType()).isEqualTo(schema.getRowType()); + assertThat(mapping.keyFormat()).isEqualTo(KafkaDataFormat.RAW); + assertThat(mapping.keyProjection().positions()).containsExactly(3); + assertThat(mapping.keyProjection().nameAt(0)).isEqualTo("message_key"); + assertThat(mapping.valueProjection().positions()).containsExactly(0); + assertThat(mapping.valueProjection().dataTypeAt(0)).isEqualTo(DataTypes.BYTES()); + assertThat(mapping.timestampPosition()).isEqualTo(1); + assertThat(mapping.headersPosition()).isEqualTo(2); + assertThat(mapping).isEqualTo(resolver.resolve(table)); + assertThatThrownBy(() -> mapping.keyProjection().positions().add(1)) + .isInstanceOf(UnsupportedOperationException.class); + } + + @Test + public void testValueOnlyStringAndNullableColumns() { + for (boolean nullable : new boolean[] {true, false}) { + KafkaTopicSchema mapping = + resolver.resolve( + table( + Schema.newBuilder() + .column( + "body", + DataTypes.STRING().copy(nullable)) + .build(), + " STRING ") + .build()); + assertThat(mapping.keyFormat()).isNull(); + assertThat(mapping.keyProjection().isEmpty()).isTrue(); + assertThat(mapping.valueFormat()).isEqualTo(KafkaDataFormat.STRING); + assertThat(mapping.valueProjection().positions()).containsExactly(0); + assertThat(mapping.valueProjection().dataTypeAt(0).isNullable()).isEqualTo(nullable); + assertThat(mapping.timestampPosition()).isEqualTo(-1); + assertThat(mapping.headersPosition()).isEqualTo(-1); + } + } + + @Test + public void testMixedFormatsAndMetadataAreOptional() { + KafkaTopicSchema mapping = + resolver.resolve( + keyValueTable() + .customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, "string") + .customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "id") + .customProperty( + KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, " except_key ") + .build()); + assertThat(mapping.keyFormat()).isEqualTo(KafkaDataFormat.STRING); + assertThat(mapping.valueFormat()).isEqualTo(KafkaDataFormat.RAW); + assertThat(mapping.keyProjection().positions()).containsExactly(0); + assertThat(mapping.valueProjection().positions()).containsExactly(1); + } + + @Test + public void testRejectsUnsupportedTableKinds() { + Schema primaryKeySchema = + Schema.newBuilder() + .column("id", DataTypes.STRING().copy(false)) + .primaryKey("id") + .build(); + assertInvalid(table(primaryKeySchema, "string"), "must be a log table"); + assertInvalid(keyValueTable().partitionedBy("id"), "Partitioned Fluss tables"); + assertInvalid(valueTable().logFormat(LogFormat.INDEXED), "Arrow log format"); + } + + @Test + public void testRequiresExplicitValueFormat() { + assertInvalid( + TableDescriptor.builder() + .schema(Schema.newBuilder().column("body", DataTypes.BYTES()).build()) + .distributedBy(1) + .customProperty("fluss.value.format", "raw"), + KafkaDataFormat.VALUE_FORMAT_CONFIG); + } + + @ParameterizedTest + @ValueSource(strings = {"json", "avro", ""}) + public void testRejectsUnavailableFormats(String format) { + assertInvalid( + valueTable().customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, format), + "Unsupported Kafka data format"); + assertInvalid( + keyValueTable().customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, format), + "Unsupported Kafka data format"); + } + + @Test + public void testRejectsUnsupportedKafkaOptions() { + assertInvalid( + valueTable().customProperty("kafka.value.rescue-column", "body"), + "Unsupported Kafka table property"); + assertInvalid( + valueTable().customProperty("kafka.key.field", "body"), + "Unsupported Kafka table property"); + assertThat(resolver.resolve(valueTable().customProperty("owner", "team").build())) + .isNotNull(); + } + + @Test + public void testKeyFormatAndFieldMustBeSpecifiedTogether() { + assertInvalid( + keyValueTable().customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "id"), + "requires kafka.key.format"); + assertInvalid( + keyValueTable().customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, "string"), + "kafka.key.fields"); + } + + @ParameterizedTest + @ValueSource(strings = {"", " ", "missing", "id,id", "id,", ",id", "id,,body"}) + public void testRejectsInvalidKeyFields(String fields) { + assertThatThrownBy( + () -> + resolver.resolve( + keyValueTable() + .customProperty( + KafkaDataFormat.KEY_FORMAT_CONFIG, "string") + .customProperty( + KafkaDataFormat.KEY_FIELDS_CONFIG, fields) + .customProperty( + KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, + "EXCEPT_KEY") + .build())) + .isInstanceOf(KafkaTopicSchemaException.class); + } + + @Test + public void testRejectsAmbiguousAndEmptyValueProjections() { + assertInvalid( + keyValueTable() + .customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, "string") + .customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "id"), + "requires"); + assertInvalid( + valueTable().customProperty(KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, "bad"), + "Expected ALL or EXCEPT_KEY"); + assertInvalid( + valueTable() + .customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, "raw") + .customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "body") + .customProperty(KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, "EXCEPT_KEY"), + "at least one Fluss column"); + assertInvalid(keyValueTable(), "exactly one Fluss field"); + assertInvalid( + keyValueTable() + .customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, "string") + .customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "id,body") + .customProperty(KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, "EXCEPT_KEY"), + "at least one Fluss column"); + } + + @Test + public void testRejectsWrongPhysicalTypes() { + assertInvalid( + valueTable().customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, "string"), + "must be STRING"); + assertInvalid( + keyValueTable() + .customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, "raw") + .customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "id") + .customProperty(KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, "EXCEPT_KEY"), + "must be BYTES"); + } + + @Test + public void testRejectsWrongTimestampTypes() { + for (DataType type : + new DataType[] { + DataTypes.STRING(), + DataTypes.TIMESTAMP_LTZ(3), + DataTypes.TIMESTAMP_LTZ(6).copy(false) + }) { + assertInvalid( + table( + Schema.newBuilder() + .column("body", DataTypes.BYTES()) + .column("ts", type) + .build(), + "raw") + .customProperty(KafkaDataFormat.TIMESTAMP_COLUMN_CONFIG, "ts"), + "TIMESTAMP_LTZ(3) NOT NULL"); + } + } + + @Test + public void testRejectsWrongHeaderTypes() { + for (DataType type : + new DataType[] { + DataTypes.STRING(), + headersType().copy(false), + DataTypes.ARRAY(DataTypes.STRING()), + DataTypes.ARRAY( + DataTypes.ROW( + DataTypes.FIELD("key", DataTypes.STRING()), + DataTypes.FIELD("value", DataTypes.BYTES()))), + DataTypes.ARRAY( + DataTypes.ROW( + DataTypes.FIELD("name", DataTypes.STRING()), + DataTypes.FIELD("value", DataTypes.BYTES().copy(false)))) + }) { + assertInvalid( + table( + Schema.newBuilder() + .column("body", DataTypes.BYTES()) + .column("attributes", type) + .build(), + "raw") + .customProperty(KafkaDataFormat.HEADERS_COLUMN_CONFIG, "attributes"), + "headers"); + } + } + + @Test + public void testRejectsConflictingOrMissingMetadataColumns() { + assertInvalid( + valueTable().customProperty(KafkaDataFormat.HEADERS_COLUMN_CONFIG, "missing"), + "does not exist"); + assertInvalid( + valueTable().customProperty(KafkaDataFormat.TIMESTAMP_COLUMN_CONFIG, " "), + "must not be empty"); + TableDescriptor.Builder table = + table( + Schema.newBuilder() + .column("body", DataTypes.BYTES()) + .column("ts", DataTypes.TIMESTAMP_LTZ(3).copy(false)) + .build(), + "raw") + .customProperty(KafkaDataFormat.TIMESTAMP_COLUMN_CONFIG, "ts"); + assertInvalid( + TableDescriptor.builder(table.build()) + .customProperty(KafkaDataFormat.HEADERS_COLUMN_CONFIG, "ts"), + "same Fluss column"); + assertInvalid( + TableDescriptor.builder(table.build()) + .customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, "raw") + .customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "ts"), + "metadata column"); + } + + private void assertInvalid(TableDescriptor.Builder table, String message) { + assertThatThrownBy(() -> resolver.resolve(table.build())) + .isInstanceOf(KafkaTopicSchemaException.class) + .hasMessageContaining(message); + } + + private static TableDescriptor.Builder keyValueTable() { + return table( + Schema.newBuilder() + .column("id", DataTypes.STRING()) + .column("body", DataTypes.BYTES()) + .build(), + "raw"); + } + + private static TableDescriptor.Builder valueTable() { + return table(Schema.newBuilder().column("body", DataTypes.BYTES()).build(), "raw"); + } + + private static TableDescriptor.Builder table(Schema schema, String valueFormat) { + return TableDescriptor.builder() + .schema(schema) + .distributedBy(2) + .logFormat(LogFormat.ARROW) + .customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, valueFormat); + } + + private static DataType headersType() { + return DataTypes.ARRAY( + DataTypes.ROW( + DataTypes.FIELD("name", DataTypes.STRING()), + DataTypes.FIELD("value", DataTypes.BYTES()))); + } +} From e2e991cba41b73cde6e41c3bca64dbb1df7675bb Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Thu, 17 Sep 2026 11:39:47 +0800 Subject: [PATCH 02/15] [kafka] Route qualified topics and streamline field projections Resolve database.table topic names directly to Fluss table paths and return qualified names for metadata. Validate name boundaries and cover same-named tables in different databases through native metadata round trips. Read projected field names directly from DataField to avoid rebuilding the complete field-name list for every projected column. Validated on Java 11: 67 Kafka unit tests and 2 integration tests passed, along with Checkstyle, Spotless, RAT, and git diff --check. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 52/52 AI-Contributed/UT: 167/167 --- .../fluss/kafka/mapping/KafkaTopicMapper.java | 50 ++++++--- .../kafka/schema/KafkaFieldProjection.java | 2 +- .../kafka/mapping/KafkaTopicMapperTest.java | 101 ++++++++++++++++-- .../kafka/schema/KafkaTableMappingITCase.java | 66 ++++++++---- 4 files changed, 172 insertions(+), 47 deletions(-) diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/mapping/KafkaTopicMapper.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/mapping/KafkaTopicMapper.java index 9f8ed63f543..f8b5744f71d 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/mapping/KafkaTopicMapper.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/mapping/KafkaTopicMapper.java @@ -21,35 +21,57 @@ import org.apache.fluss.metadata.TablePath; import org.apache.kafka.common.Uuid; +import org.apache.kafka.common.errors.InvalidTopicException; import org.apache.kafka.common.internals.Topic; import static org.apache.fluss.utils.Preconditions.checkArgument; -import static org.apache.fluss.utils.Preconditions.checkNotNull; -/** Maps Kafka topic identities to tables in the configured Fluss Kafka database. */ +/** Maps fully qualified Kafka topic identities to Fluss tables. */ @Internal public final class KafkaTopicMapper { // ASCII "Fluss" followed by zero bytes. A dedicated namespace avoids Kafka-reserved UUIDs. private static final long TOPIC_ID_NAMESPACE = 0x466c757373000000L; - private final String databaseName; - - /** Creates a topic mapper for one Fluss database. */ - public KafkaTopicMapper(String databaseName) { - this.databaseName = checkNotNull(databaseName); + /** Maps a Kafka topic in database.table form to its Fluss table path. */ + public TablePath toTablePath(String topicName) { + if (!isValidTopic(topicName)) { + throw new InvalidTopicException( + "Kafka topic must be a valid database.table name: " + topicName); + } + int separator = topicName.indexOf('.'); + return TablePath.of(topicName.substring(0, separator), topicName.substring(separator + 1)); } - /** Maps a Kafka topic name to its Fluss table path. */ - public TablePath toTablePath(String topicName) { - Topic.validate(topicName); - return TablePath.of(databaseName, topicName); + /** Returns the fully qualified Kafka name of a representable Fluss table. */ + public String toTopicName(TablePath tablePath) { + if (!isMappedTable(tablePath)) { + throw new InvalidTopicException( + "Fluss table cannot be represented as a Kafka database.table name: " + + tablePath); + } + return tablePath.toString(); } - /** Returns whether a table belongs to this database and has a valid Kafka topic name. */ + /** Returns whether a Fluss user table has a valid, fully qualified Kafka topic name. */ public boolean isMappedTable(TablePath tablePath) { - return databaseName.equals(tablePath.getDatabaseName()) - && Topic.isValid(tablePath.getTableName()); + return tablePath != null && tablePath.isValid() && isValidTopic(tablePath.toString()); + } + + /** Returns whether a name uniquely represents a user table in a Fluss database. */ + public static boolean isValidTopic(String topicName) { + if (topicName == null || !Topic.isValid(topicName)) { + return false; + } + int separator = topicName.indexOf('.'); + if (separator <= 0 || separator != topicName.lastIndexOf('.')) { + return false; + } + String database = topicName.substring(0, separator); + String table = topicName.substring(separator + 1); + return TablePath.of(database, table).isValid() + && TablePath.validatePrefix(database) == null + && TablePath.validatePrefix(table) == null; } /** Maps a Fluss table ID to a stable Kafka topic ID. */ diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaFieldProjection.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaFieldProjection.java index b1715758a9d..b6833475a20 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaFieldProjection.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaFieldProjection.java @@ -50,7 +50,7 @@ public KafkaFieldProjection(RowType rowType, List positions) { "Invalid Kafka field projection position %s.", position); positionCopy.add(position); - projectedNames.add(rowType.getFieldNames().get(position)); + projectedNames.add(rowType.getFields().get(position).getName()); projectedTypes.add(rowType.getTypeAt(position)); } this.positions = Collections.unmodifiableList(positionCopy); diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/mapping/KafkaTopicMapperTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/mapping/KafkaTopicMapperTest.java index 20c32ded941..c5d4a582a88 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/mapping/KafkaTopicMapperTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/mapping/KafkaTopicMapperTest.java @@ -22,6 +22,12 @@ import org.apache.kafka.common.Uuid; import org.apache.kafka.common.errors.InvalidTopicException; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.NullAndEmptySource; +import org.junit.jupiter.params.provider.ValueSource; + +import java.util.Arrays; +import java.util.Collections; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; @@ -29,11 +35,88 @@ /** Tests for {@link KafkaTopicMapper}. */ public class KafkaTopicMapperTest { + private final KafkaTopicMapper mapper = new KafkaTopicMapper(); + + @Test + public void testQualifiedNamesAcrossDatabases() { + for (String database : Arrays.asList("sales", "archive", "sales_1-2026")) { + TablePath tablePath = TablePath.of(database, "orders_1-2026"); + String topicName = database + ".orders_1-2026"; + assertThat(mapper.toTablePath(topicName)).isEqualTo(tablePath); + assertThat(mapper.toTopicName(tablePath)).isEqualTo(topicName); + assertThat(mapper.isMappedTable(tablePath)).isTrue(); + assertThat(KafkaTopicMapper.isValidTopic(topicName)).isTrue(); + } + } + + @ParameterizedTest + @NullAndEmptySource + @ValueSource( + strings = { + "orders", + ".orders", + "sales.", + "sales.orders.extra", + "sales..orders", + "sales.bad name", + " sales.orders", + "sales.orders ", + "sales/orders", + "__system.orders", + "sales.__internal", + "数据库.orders", + "sales.订单" + }) + public void testRejectsInvalidTopicNames(String topicName) { + assertThat(KafkaTopicMapper.isValidTopic(topicName)).isFalse(); + assertThatThrownBy(() -> mapper.toTablePath(topicName)) + .isInstanceOf(InvalidTopicException.class); + } + @Test - public void testTopicNameAndIdMapping() { - KafkaTopicMapper mapper = new KafkaTopicMapper("kafka"); + public void testNameLengthLimits() { + for (TablePath valid : + Arrays.asList( + TablePath.of(repeat('a', 200), "orders"), + TablePath.of("sales", repeat('a', 200)), + TablePath.of(repeat('a', 124), repeat('b', 124)))) { + assertThat(mapper.toTablePath(mapper.toTopicName(valid))).isEqualTo(valid); + } + for (TablePath invalid : + Arrays.asList( + TablePath.of(repeat('a', 201), "orders"), + TablePath.of("sales", repeat('a', 201)), + TablePath.of(repeat('a', 124), repeat('b', 125)))) { + assertThat(mapper.isMappedTable(invalid)).isFalse(); + assertThatThrownBy(() -> mapper.toTablePath(invalid.toString())) + .isInstanceOf(InvalidTopicException.class); + assertThatThrownBy(() -> mapper.toTopicName(invalid)) + .isInstanceOf(InvalidTopicException.class); + } + } + + @Test + public void testRejectsUnrepresentableTablePaths() { + for (TablePath invalid : + Arrays.asList( + null, + TablePath.of(null, "orders"), + TablePath.of("sales", null), + TablePath.of("", "orders"), + TablePath.of("sales", ""), + TablePath.of("sales.region", "orders"), + TablePath.of("sales", "orders.v1"), + TablePath.of("__system", "orders"), + TablePath.of("sales", "__internal"), + TablePath.of("sales", "bad name"))) { + assertThat(mapper.isMappedTable(invalid)).isFalse(); + assertThatThrownBy(() -> mapper.toTopicName(invalid)) + .isInstanceOf(InvalidTopicException.class); + } + } - assertThat(mapper.toTablePath("topic").toString()).isEqualTo("kafka.topic"); + @Test + public void testTopicIdMapping() { Uuid topicId = mapper.toTopicId(123L); assertThat(topicId).isNotIn(Uuid.ZERO_UUID, Uuid.ONE_UUID, Uuid.METADATA_TOPIC_ID); assertThat(mapper.isMappedTopicId(topicId)).isTrue(); @@ -46,17 +129,15 @@ public void testTopicNameAndIdMapping() { } @Test - public void testOnlyValidTopicsInConfiguredDatabaseAreMapped() { - KafkaTopicMapper mapper = new KafkaTopicMapper("kafka"); - assertThat(mapper.isMappedTable(TablePath.of("kafka", "events"))).isTrue(); - assertThat(mapper.isMappedTable(TablePath.of("other", "events"))).isFalse(); - assertThat(mapper.isMappedTable(TablePath.of("kafka", "invalid topic"))).isFalse(); - assertThatThrownBy(() -> mapper.toTablePath("invalid topic")) - .isInstanceOf(InvalidTopicException.class); + public void testRejectsInvalidTopicIds() { assertThatThrownBy(() -> mapper.toTopicId(-1L)) .isInstanceOf(IllegalArgumentException.class); assertThat(mapper.isMappedTopicId(Uuid.ZERO_UUID)).isFalse(); assertThatThrownBy(() -> mapper.toTableId(Uuid.ZERO_UUID)) .isInstanceOf(IllegalArgumentException.class); } + + private static String repeat(char character, int count) { + return String.join("", Collections.nCopies(count, String.valueOf(character))); + } } diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java index 0c92b8a7300..2ded7bd17f6 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java @@ -31,6 +31,11 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + import static org.apache.fluss.server.testutils.RpcMessageTestUtils.createTable; import static org.assertj.core.api.Assertions.assertThat; @@ -42,7 +47,7 @@ public class KafkaTableMappingITCase { FlussClusterExtension.builder().setNumOfTabletServers(1).build(); @Test - public void testResolveMappingFromCreatedTableMetadata() throws Exception { + public void testResolveMappingsFromQualifiedTopicsAcrossDatabases() throws Exception { TableDescriptor descriptor = TableDescriptor.builder() .schema( @@ -57,32 +62,49 @@ public void testResolveMappingFromCreatedTableMetadata() throws Exception { .customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, "raw") .customProperty(KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, "EXCEPT_KEY") .build(); - KafkaTopicMapper mapper = new KafkaTopicMapper("kafka_ddl"); - TablePath tablePath = mapper.toTablePath("events"); - long tableId = createTable(FLUSS_CLUSTER_EXTENSION, tablePath, descriptor); - FLUSS_CLUSTER_EXTENSION.waitUntilAllGatewayHasSameMetadata(); + KafkaTopicMapper mapper = new KafkaTopicMapper(); + Map tableIds = new LinkedHashMap<>(); MetadataRequest request = new MetadataRequest(); - request.addTablePath() - .setDatabaseName(tablePath.getDatabaseName()) - .setTableName(tablePath.getTableName()); - PbTableMetadata metadata = + for (String database : Arrays.asList("kafka_ddl", "kafka_ddl_archive")) { + TablePath tablePath = mapper.toTablePath(database + ".events"); + assertThat(tablePath).isEqualTo(TablePath.of(database, "events")); + tableIds.put(tablePath, createTable(FLUSS_CLUSTER_EXTENSION, tablePath, descriptor)); + request.addTablePath() + .setDatabaseName(tablePath.getDatabaseName()) + .setTableName(tablePath.getTableName()); + } + FLUSS_CLUSTER_EXTENSION.waitUntilAllGatewayHasSameMetadata(); + List metadatas = FLUSS_CLUSTER_EXTENSION .newCoordinatorClient() .metadata(request) .get() - .getTableMetadatasList() - .get(0); - TableDescriptor persisted = TableDescriptor.fromJsonBytes(metadata.getTableJson()); - KafkaTopicSchema mapping = new KafkaTopicSchemaResolver().resolve(persisted); + .getTableMetadatasList(); + assertThat(metadatas).hasSize(2); + assertThat(metadatas) + .extracting(PbTableMetadata::getTableId) + .containsExactlyInAnyOrderElementsOf(tableIds.values()) + .doesNotHaveDuplicates(); + for (PbTableMetadata metadata : metadatas) { + TablePath tablePath = + TablePath.of( + metadata.getTablePath().getDatabaseName(), + metadata.getTablePath().getTableName()); + TableDescriptor persisted = TableDescriptor.fromJsonBytes(metadata.getTableJson()); + KafkaTopicSchema mapping = new KafkaTopicSchemaResolver().resolve(persisted); - assertThat(metadata.getTableId()).isEqualTo(tableId); - assertThat(mapper.toTableId(mapper.toTopicId(metadata.getTableId()))).isEqualTo(tableId); - assertThat(metadata.getBucketMetadatasList()).hasSize(2); - assertThat(persisted.getCustomProperties()) - .containsAllEntriesOf(descriptor.getCustomProperties()); - assertThat(mapping.keyProjection().positions()).containsExactly(0); - assertThat(mapping.keyFormat()).isEqualTo(KafkaDataFormat.STRING); - assertThat(mapping.valueProjection().positions()).containsExactly(1); - assertThat(mapping.valueFormat()).isEqualTo(KafkaDataFormat.RAW); + assertThat(metadata.getTableId()).isEqualTo(tableIds.get(tablePath)); + assertThat(mapper.toTopicName(tablePath)) + .isIn("kafka_ddl.events", "kafka_ddl_archive.events"); + assertThat(mapper.toTableId(mapper.toTopicId(metadata.getTableId()))) + .isEqualTo(tableIds.get(tablePath)); + assertThat(metadata.getBucketMetadatasList()).hasSize(2); + assertThat(persisted.getCustomProperties()) + .containsAllEntriesOf(descriptor.getCustomProperties()); + assertThat(mapping.keyProjection().positions()).containsExactly(0); + assertThat(mapping.keyFormat()).isEqualTo(KafkaDataFormat.STRING); + assertThat(mapping.valueProjection().positions()).containsExactly(1); + assertThat(mapping.valueFormat()).isEqualTo(KafkaDataFormat.RAW); + } } } From 1082ed63229a39adb25cf8918e9cebb05e12fcd7 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Mon, 21 Sep 2026 11:28:20 +0800 Subject: [PATCH 03/15] [kafka] Define Kafka timestamp metadata as TIMESTAMP(3) Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 16/16 AI-Contributed/UT: 33/33 --- .../fluss/kafka/format/KafkaDataFormat.java | 5 +++- .../schema/KafkaTopicSchemaResolver.java | 11 +++----- .../kafka/schema/KafkaTableMappingITCase.java | 5 ++++ .../schema/KafkaTopicSchemaResolverTest.java | 28 ++++++++++++++++--- 4 files changed, 37 insertions(+), 12 deletions(-) diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaDataFormat.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaDataFormat.java index ad0039b7e78..df5d871907c 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaDataFormat.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaDataFormat.java @@ -39,7 +39,10 @@ public enum KafkaDataFormat { /** Strategy for deriving fields populated from the Kafka record value. */ public static final String VALUE_FIELDS_INCLUDE_CONFIG = "kafka.value.fields-include"; - /** Fluss column populated from the Kafka record timestamp. */ + /** + * TIMESTAMP(3) column populated from the Kafka record epoch milliseconds as UTC calendar + * values. A missing Kafka timestamp (-1) maps to null and requires a nullable column. + */ public static final String TIMESTAMP_COLUMN_CONFIG = "kafka.metadata.timestamp.column"; /** Fluss column populated from the Kafka record headers. */ diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolver.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolver.java index 840ce8a4f58..8959c420d15 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolver.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolver.java @@ -25,9 +25,9 @@ import org.apache.fluss.metadata.TableDescriptor; import org.apache.fluss.types.ArrayType; import org.apache.fluss.types.BytesType; -import org.apache.fluss.types.LocalZonedTimestampType; import org.apache.fluss.types.RowType; import org.apache.fluss.types.StringType; +import org.apache.fluss.types.TimestampType; import java.util.ArrayList; import java.util.Arrays; @@ -234,12 +234,9 @@ private static int resolveOptionalPosition(RowType rowType, String fieldNameValu private static void validateMetadataColumns( RowType rowType, int timestampPosition, int headersPosition) { if (timestampPosition >= 0) { - if (!(rowType.getTypeAt(timestampPosition) instanceof LocalZonedTimestampType) - || rowType.getTypeAt(timestampPosition).isNullable() - || ((LocalZonedTimestampType) rowType.getTypeAt(timestampPosition)) - .getPrecision() - != 3) { - throw invalid("Kafka timestamp column must be TIMESTAMP_LTZ(3) NOT NULL."); + if (!(rowType.getTypeAt(timestampPosition) instanceof TimestampType) + || ((TimestampType) rowType.getTypeAt(timestampPosition)).getPrecision() != 3) { + throw invalid("Kafka timestamp column must be TIMESTAMP(3)."); } } if (headersPosition >= 0) { diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java index 2ded7bd17f6..2f3e75bdd3d 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java @@ -54,12 +54,14 @@ public void testResolveMappingsFromQualifiedTopicsAcrossDatabases() throws Excep Schema.newBuilder() .column("event_key", DataTypes.STRING()) .column("event_body", DataTypes.BYTES()) + .column("event_time", DataTypes.TIMESTAMP(3)) .build()) .distributedBy(2) .logFormat(LogFormat.ARROW) .customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, "string") .customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "event_key") .customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, "raw") + .customProperty(KafkaDataFormat.TIMESTAMP_COLUMN_CONFIG, "event_time") .customProperty(KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, "EXCEPT_KEY") .build(); KafkaTopicMapper mapper = new KafkaTopicMapper(); @@ -105,6 +107,9 @@ public void testResolveMappingsFromQualifiedTopicsAcrossDatabases() throws Excep assertThat(mapping.keyFormat()).isEqualTo(KafkaDataFormat.STRING); assertThat(mapping.valueProjection().positions()).containsExactly(1); assertThat(mapping.valueFormat()).isEqualTo(KafkaDataFormat.RAW); + assertThat(mapping.timestampPosition()).isEqualTo(2); + assertThat(persisted.getSchema().getRowType().getTypeAt(2)) + .isEqualTo(DataTypes.TIMESTAMP(3)); } } } diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolverTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolverTest.java index 4c698b5ad6b..66c13a4711a 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolverTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolverTest.java @@ -41,7 +41,7 @@ public void testRawMappingSurvivesTableMetadataSerialization() { Schema schema = Schema.newBuilder() .column("message", DataTypes.BYTES()) - .column("received_at", DataTypes.TIMESTAMP_LTZ(3).copy(false)) + .column("received_at", DataTypes.TIMESTAMP(3).copy(false)) .column("attributes", headersType()) .column("message_key", DataTypes.BYTES()) .build(); @@ -219,13 +219,33 @@ public void testRejectsWrongPhysicalTypes() { "must be BYTES"); } + @Test + public void testAcceptsTimestampWithEitherNullability() { + for (boolean nullable : new boolean[] {true, false}) { + TableDescriptor descriptor = + table( + Schema.newBuilder() + .column("body", DataTypes.BYTES()) + .column("ts", DataTypes.TIMESTAMP(3).copy(nullable)) + .build(), + "raw") + .customProperty(KafkaDataFormat.TIMESTAMP_COLUMN_CONFIG, "ts") + .build(); + KafkaTopicSchema mapping = new KafkaTopicSchemaResolver().resolve(descriptor); + assertThat(mapping.timestampPosition()).isEqualTo(1); + assertThat(mapping.valueProjection().positions()).containsExactly(0); + } + } + @Test public void testRejectsWrongTimestampTypes() { for (DataType type : new DataType[] { DataTypes.STRING(), DataTypes.TIMESTAMP_LTZ(3), - DataTypes.TIMESTAMP_LTZ(6).copy(false) + DataTypes.TIMESTAMP_LTZ(3).copy(false), + DataTypes.TIMESTAMP(2), + DataTypes.TIMESTAMP(6).copy(false) }) { assertInvalid( table( @@ -235,7 +255,7 @@ public void testRejectsWrongTimestampTypes() { .build(), "raw") .customProperty(KafkaDataFormat.TIMESTAMP_COLUMN_CONFIG, "ts"), - "TIMESTAMP_LTZ(3) NOT NULL"); + "TIMESTAMP(3)"); } } @@ -279,7 +299,7 @@ public void testRejectsConflictingOrMissingMetadataColumns() { table( Schema.newBuilder() .column("body", DataTypes.BYTES()) - .column("ts", DataTypes.TIMESTAMP_LTZ(3).copy(false)) + .column("ts", DataTypes.TIMESTAMP(3).copy(false)) .build(), "raw") .customProperty(KafkaDataFormat.TIMESTAMP_COLUMN_CONFIG, "ts"); From 9e8b68012b48f0a9814c531b86ff74e9a3ddde63 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Sun, 11 Oct 2026 23:16:34 +0800 Subject: [PATCH 04/15] [test] Wait for table readiness in snapshot lease validation Wait for the newly created log table to become ready before acquiring or releasing snapshot leases. This prevents the coordinator's asynchronous table-ID registration from masking the expected NonPrimaryKeyTableException. Validated on Java 11 with the same-worktree reactor: FlussAdminITCase#testKvSnapshotLeaseForLogTable passed, along with Checkstyle, Spotless, RAT, and git diff --check. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 0/0 AI-Contributed/UT: 1/1 --- .../java/org/apache/fluss/client/admin/FlussAdminITCase.java | 1 + 1 file changed, 1 insertion(+) diff --git a/fluss-client/src/test/java/org/apache/fluss/client/admin/FlussAdminITCase.java b/fluss-client/src/test/java/org/apache/fluss/client/admin/FlussAdminITCase.java index e67faf84b24..b39739c6b01 100644 --- a/fluss-client/src/test/java/org/apache/fluss/client/admin/FlussAdminITCase.java +++ b/fluss-client/src/test/java/org/apache/fluss/client/admin/FlussAdminITCase.java @@ -2831,6 +2831,7 @@ void testKvSnapshotLeaseForLogTable() throws Exception { TableDescriptor logTableDescriptor = TableDescriptor.builder().schema(logSchema).distributedBy(3).build(); long tableId = createTable(logTablePath, logTableDescriptor, true); + FLUSS_CLUSTER_EXTENSION.waitUntilTableReady(tableId); // Create a KvSnapshotLease KvSnapshotLease lease = admin.createKvSnapshotLease("test-lease-log", 60000); From 68194786bf648686ddfe73ee618b2dbc7109bb32 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Mon, 12 Oct 2026 15:55:33 +0800 Subject: [PATCH 05/15] [kafka] Align DDL table mapping with the FIP contract Register table.kafka options and validate the declared fixed-column mapping through a shared schema contract. Support binary, string and flat/nested JSON values with optional key, headers and rescue columns. Validate CREATE and ALTER before persistence, and fence schema publication with the table version and coordinator epoch so concurrent schema and property changes cannot bypass the contract. Resolve bare topic aliases through kafka.default.database and stop automatically creating that database. Keep real offset persistence deferred and reject Kafka enablement for tables that declare offset metadata until that capability is available. Validation: 146 targeted tests passed on Java 11; Checkstyle, Spotless, RAT and git diff checks passed. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 1131/1131 AI-Contributed/UT: 1451/1451 --- .../apache/fluss/config/ConfigOptions.java | 39 +- .../apache/fluss/config/FlussConfigUtils.java | 4 + .../org/apache/fluss/config/TableConfig.java | 5 + .../fluss/metadata}/KafkaFieldProjection.java | 2 +- .../fluss/metadata/KafkaTableSchema.java | 391 ++++++++++++++++ .../fluss/metadata/KafkaTableSchemaTest.java | 437 ++++++++++++++++++ .../flink/utils/FlinkConversionsTest.java | 45 ++ .../fluss/kafka/format/KafkaDataFormat.java | 76 --- .../fluss/kafka/mapping/KafkaTopicMapper.java | 49 +- .../fluss/kafka/schema/KafkaTopicSchema.java | 150 ------ .../schema/KafkaTopicSchemaResolver.java | 292 +----------- .../apache/fluss/kafka/KafkaConfigsTest.java | 8 +- .../kafka/mapping/KafkaTopicMapperTest.java | 34 +- .../kafka/schema/KafkaTableMappingITCase.java | 410 +++++++++++++--- .../schema/KafkaTopicSchemaResolverTest.java | 335 ++------------ .../server/coordinator/CoordinatorServer.java | 8 - .../coordinator/CoordinatorService.java | 5 +- .../server/coordinator/MetadataManager.java | 36 +- .../utils/TableDescriptorValidation.java | 7 + .../fluss/server/zk/ZooKeeperClient.java | 30 ++ .../KafkaTableSchemaAlterTest.java | 182 ++++++++ website/docs/maintenance/configuration.md | 37 +- 22 files changed, 1672 insertions(+), 910 deletions(-) rename {fluss-kafka/src/main/java/org/apache/fluss/kafka/schema => fluss-common/src/main/java/org/apache/fluss/metadata}/KafkaFieldProjection.java (99%) create mode 100644 fluss-common/src/main/java/org/apache/fluss/metadata/KafkaTableSchema.java create mode 100644 fluss-common/src/test/java/org/apache/fluss/metadata/KafkaTableSchemaTest.java delete mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaDataFormat.java delete mode 100644 fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchema.java create mode 100644 fluss-server/src/test/java/org/apache/fluss/server/coordinator/KafkaTableSchemaAlterTest.java diff --git a/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java b/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java index f02fdbe0066..8e34ddb8bb6 100644 --- a/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java +++ b/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java @@ -2269,6 +2269,38 @@ public class ConfigOptions { + "Downstream consumers must be upgraded to Fluss v1.0+ before enabling this option, " + "as older versions cannot parse the extended batch format."); + public static final ConfigOption TABLE_KAFKA_ENABLED = + key("table.kafka.enabled") + .booleanType() + .defaultValue(false) + .withDescription( + "Whether the table is exposed as a Kafka topic. Enabling validates the " + + "existing schema and conversion options without adding columns or defaults."); + + public static final ConfigOption TABLE_KAFKA_CONVERT_VALUE_TYPE = + key("table.kafka.convert.value.type") + .stringType() + .noDefaultValue() + .withDescription( + "Required when Kafka compatibility is enabled: binary, string, or parse_json. " + + "Binary and string target the declared _kafka_value column."); + + public static final ConfigOption TABLE_KAFKA_CONVERT_VALUE_LAYOUT = + key("table.kafka.convert.value.layout") + .stringType() + .noDefaultValue() + .withDescription( + "JSON layout: flat (the read-time default for parse_json) or nested. " + + "Nested targets _kafka_value ROW. Not allowed for binary or string."); + + public static final ConfigOption TABLE_KAFKA_CONVERT_KEY_TYPE = + key("table.kafka.convert.key.type") + .stringType() + .noDefaultValue() + .withDescription( + "Optional Kafka key conversion type. Only binary is supported and requires " + + "_kafka_key BYTES. Declaring that column implies binary without persisting a default."); + // ------------------------------------------------------------------------ // ConfigOptions for Consumer Offsets // ------------------------------------------------------------------------ @@ -2947,12 +2979,13 @@ public class ConfigOptions { .withDescription( "The listener names for Kafka wire protocol communication. Support multiple listener names, separated by comma."); - public static final ConfigOption KAFKA_DATABASE = - key("kafka.database") + public static final ConfigOption KAFKA_DEFAULT_DATABASE = + key("kafka.default.database") .stringType() .defaultValue("kafka") .withDescription( - "The database for fluss kafka. The default database is `kafka`."); + "Default database for unqualified Kafka topic names. Fully qualified names may " + + "access other databases. Databases must already exist; this option does not create them."); /** * Compaction style for Fluss's kv, which is same to rocksdb's, but help use avoid including diff --git a/fluss-common/src/main/java/org/apache/fluss/config/FlussConfigUtils.java b/fluss-common/src/main/java/org/apache/fluss/config/FlussConfigUtils.java index 37d5cc9f88e..415103c46ea 100644 --- a/fluss-common/src/main/java/org/apache/fluss/config/FlussConfigUtils.java +++ b/fluss-common/src/main/java/org/apache/fluss/config/FlussConfigUtils.java @@ -47,6 +47,10 @@ public class FlussConfigUtils { CLIENT_OPTIONS = extractConfigOptions("client."); ALTERABLE_TABLE_OPTIONS = Arrays.asList( + ConfigOptions.TABLE_KAFKA_ENABLED.key(), + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE.key(), + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT.key(), + ConfigOptions.TABLE_KAFKA_CONVERT_KEY_TYPE.key(), ConfigOptions.TABLE_DATALAKE_ENABLED.key(), ConfigOptions.TABLE_DATALAKE_HISTORICAL_PARTITION_ENABLED.key(), ConfigOptions.TABLE_DATALAKE_HISTORICAL_PARTITION_LOOKUP_MODE.key(), diff --git a/fluss-common/src/main/java/org/apache/fluss/config/TableConfig.java b/fluss-common/src/main/java/org/apache/fluss/config/TableConfig.java index ed2c111c335..b1bc64dfc83 100644 --- a/fluss-common/src/main/java/org/apache/fluss/config/TableConfig.java +++ b/fluss-common/src/main/java/org/apache/fluss/config/TableConfig.java @@ -54,6 +54,11 @@ public TableConfig(Configuration config) { this.config = config; } + /** Whether the table explicitly enables Kafka compatibility. */ + public boolean isKafkaEnabled() { + return config.get(ConfigOptions.TABLE_KAFKA_ENABLED); + } + /** Gets the replication factor of the table. */ public int getReplicationFactor() { return config.get(ConfigOptions.TABLE_REPLICATION_FACTOR); diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaFieldProjection.java b/fluss-common/src/main/java/org/apache/fluss/metadata/KafkaFieldProjection.java similarity index 99% rename from fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaFieldProjection.java rename to fluss-common/src/main/java/org/apache/fluss/metadata/KafkaFieldProjection.java index b6833475a20..f8946107020 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaFieldProjection.java +++ b/fluss-common/src/main/java/org/apache/fluss/metadata/KafkaFieldProjection.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.fluss.kafka.schema; +package org.apache.fluss.metadata; import org.apache.fluss.annotation.Internal; import org.apache.fluss.types.DataType; diff --git a/fluss-common/src/main/java/org/apache/fluss/metadata/KafkaTableSchema.java b/fluss-common/src/main/java/org/apache/fluss/metadata/KafkaTableSchema.java new file mode 100644 index 00000000000..7c040f15871 --- /dev/null +++ b/fluss-common/src/main/java/org/apache/fluss/metadata/KafkaTableSchema.java @@ -0,0 +1,391 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.metadata; + +import org.apache.fluss.annotation.Internal; +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.config.Configuration; +import org.apache.fluss.config.TableConfig; +import org.apache.fluss.exception.InvalidTableException; +import org.apache.fluss.types.BytesType; +import org.apache.fluss.types.DataType; +import org.apache.fluss.types.DataTypes; +import org.apache.fluss.types.MapType; +import org.apache.fluss.types.RowType; +import org.apache.fluss.types.StringType; + +import javax.annotation.Nullable; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Locale; +import java.util.Objects; +import java.util.Set; + +/** + * Immutable Kafka record mapping, shared by native DDL validation and Kafka request handling. + * Resolving a contract never changes the schema or materializes configuration defaults. + */ +@Internal +public final class KafkaTableSchema { + + public static final String KEY_COLUMN = "_kafka_key"; + public static final String VALUE_COLUMN = "_kafka_value"; + public static final String HEADERS_COLUMN = "_kafka_headers"; + public static final String METADATA_COLUMN = "_kafka_metadata"; + public static final String RESCUE_COLUMN = "_kafka_rescue"; + + private static final Set RESERVED_NAMES = + Collections.unmodifiableSet( + new HashSet<>( + Arrays.asList( + KEY_COLUMN, + VALUE_COLUMN, + HEADERS_COLUMN, + METADATA_COLUMN, + RESCUE_COLUMN))); + private static final Set OPTIONS = + new HashSet<>( + Arrays.asList( + ConfigOptions.TABLE_KAFKA_ENABLED.key(), + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE.key(), + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT.key(), + ConfigOptions.TABLE_KAFKA_CONVERT_KEY_TYPE.key())); + + /** Supported value conversion contracts; decoders are provided by the Kafka module. */ + public enum ValueType { + BINARY, + STRING, + PARSE_JSON + } + + /** JSON business fields either occupy the table row or the fixed value ROW column. */ + public enum ValueLayout { + FLAT, + NESTED + } + + private final RowType rowType; + private final ValueType valueType; + private final @Nullable ValueLayout valueLayout; + private final KafkaFieldProjection valueProjection; + private final int keyPosition; + private final int headersPosition; + private final int rescuePosition; + + private KafkaTableSchema( + RowType rowType, + ValueType valueType, + @Nullable ValueLayout valueLayout, + List valuePositions, + int keyPosition, + int headersPosition, + int rescuePosition) { + this.rowType = rowType; + this.valueType = valueType; + this.valueLayout = valueLayout; + this.valueProjection = new KafkaFieldProjection(rowType, valuePositions); + this.keyPosition = keyPosition; + this.headersPosition = headersPosition; + this.rescuePosition = rescuePosition; + } + + /** Returns only the persisted opt-in flag; schema compatibility does not imply enablement. */ + public static boolean isEnabled(TableDescriptor table) { + return new TableConfig(Configuration.fromMap(table.getProperties())).isKafkaEnabled(); + } + + /** Validates the Kafka contract only when the final table state enables it. */ + public static void validate(TableDescriptor table) { + if (isEnabled(table)) { + resolve(table); + } + } + + /** Checks whether the canonical topic name satisfies Fluss and Kafka name restrictions. */ + public static boolean isValidTablePath(TablePath path) { + // Fluss identifiers are already restricted to Kafka's ASCII character set, without dots. + return path != null + && path.isValid() + && TablePath.validatePrefix(path.getDatabaseName()) == null + && TablePath.validatePrefix(path.getTableName()) == null + && path.toString().length() <= 249; + } + + /** Validates the canonical topic name before persisting an enabled table. */ + public static void validateTablePath(TablePath path, TableDescriptor table) { + if (isEnabled(table) && !isValidTablePath(path)) { + throw invalid("Table path cannot be represented as a Kafka topic: " + path); + } + } + + /** Returns whether a JSON member at the value mapping level has a reserved name. */ + public static boolean isReservedName(String name) { + return RESERVED_NAMES.contains(name); + } + + /** Resolves an explicitly enabled table or rejects its invalid contract. */ + public static KafkaTableSchema resolve(TableDescriptor table) { + if (!isEnabled(table)) { + throw invalid("Kafka compatibility is disabled; table.kafka.enabled must be true."); + } + validateOptions(table); + if (table.hasPrimaryKey() + || table.isPartitioned() + || new TableConfig(Configuration.fromMap(table.getProperties())).getLogFormat() + != LogFormat.ARROW) { + throw invalid( + "Kafka requires a non-partitioned Arrow log table without a primary key."); + } + Configuration options = Configuration.fromMap(table.getProperties()); + String value = options.get(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE); + if (value == null) { + throw invalid("Missing required table property 'table.kafka.convert.value.type'."); + } + ValueType valueType; + try { + valueType = ValueType.valueOf(value.trim().toUpperCase(Locale.ROOT)); + } catch (IllegalArgumentException e) { + throw invalid( + "Unsupported table.kafka.convert.value.type '" + + value + + "'. Expected binary, string or parse_json."); + } + String layout = options.get(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT); + ValueLayout valueLayout = null; + if (valueType == ValueType.PARSE_JSON) { + try { + valueLayout = + layout == null + ? ValueLayout.FLAT + : ValueLayout.valueOf(layout.trim().toUpperCase(Locale.ROOT)); + } catch (IllegalArgumentException e) { + throw invalid("Unsupported table.kafka.convert.value.layout '" + layout + "'."); + } + } else if (layout != null) { + throw invalid("table.kafka.convert.value.layout is only supported for parse_json."); + } + + RowType rowType = table.getSchema().getRowType(); + int keyPosition = rowType.getFieldIndex(KEY_COLUMN); + if (keyPosition >= 0 && !(rowType.getTypeAt(keyPosition) instanceof BytesType)) { + throw invalid("_kafka_key must be BYTES."); + } + String keyType = options.get(ConfigOptions.TABLE_KAFKA_CONVERT_KEY_TYPE); + if (keyType != null && (!"binary".equalsIgnoreCase(keyType.trim()) || keyPosition < 0)) { + throw invalid( + "table.kafka.convert.key.type only supports binary and requires _kafka_key BYTES."); + } + int headersPosition = rowType.getFieldIndex(HEADERS_COLUMN); + if (headersPosition >= 0) { + DataType headers = rowType.getTypeAt(headersPosition); + if (!(headers instanceof MapType) + || !(((MapType) headers).getKeyType() instanceof StringType) + || !(((MapType) headers).getValueType() instanceof BytesType)) { + throw invalid("_kafka_headers must be MAP."); + } + } + int rescuePosition = rowType.getFieldIndex(RESCUE_COLUMN); + if (rescuePosition >= 0 + && (!(rowType.getTypeAt(rescuePosition) instanceof StringType) + || !rowType.getTypeAt(rescuePosition).isNullable())) { + throw invalid("_kafka_rescue must be nullable STRING."); + } + int metadataPosition = rowType.getFieldIndex(METADATA_COLUMN); + if (metadataPosition >= 0) { + validateMetadataType(rowType.getTypeAt(metadataPosition)); + throw invalid( + "_kafka_metadata requires native append support for persisting actual per-row offsets; " + + "this capability is not available yet."); + } + + List valuePositions = new ArrayList<>(); + if (valueLayout == ValueLayout.FLAT) { + if (rowType.getFieldIndex(VALUE_COLUMN) >= 0) { + throw invalid( + "_kafka_value is reserved for binary, string or nested JSON; " + + "it cannot be a flat JSON business field."); + } + for (int i = 0; i < rowType.getFieldCount(); i++) { + if (!isReservedName(rowType.getFieldNames().get(i))) { + valuePositions.add(i); + } + } + } else { + int valuePosition = rowType.getFieldIndex(VALUE_COLUMN); + if (valuePosition < 0) { + throw invalid("Missing required value column _kafka_value."); + } + DataType type = rowType.getTypeAt(valuePosition); + boolean valid = + valueType == ValueType.BINARY + ? type instanceof BytesType + : valueType == ValueType.STRING + ? type instanceof StringType + : type instanceof RowType; + if (!valid) { + throw invalid( + "_kafka_value must be " + + (valueType == ValueType.BINARY + ? "BYTES" + : valueType == ValueType.STRING ? "STRING" : "ROW") + + " for " + + valueType + + "."); + } + for (String name : rowType.getFieldNames()) { + if (!isReservedName(name)) { + throw invalid( + "Column '" + + name + + "' has no value mapping; this conversion targets _kafka_value only."); + } + } + if (type instanceof RowType) { + for (String name : ((RowType) type).getFieldNames()) { + if (isReservedName(name)) { + throw invalid( + "Reserved name '" + + name + + "' cannot be a nested JSON business field."); + } + } + } + valuePositions.add(valuePosition); + } + return new KafkaTableSchema( + rowType, + valueType, + valueLayout, + valuePositions, + keyPosition, + headersPosition, + rescuePosition); + } + + private static void validateOptions(TableDescriptor table) { + for (String name : table.getProperties().keySet()) { + if (name.startsWith("table.kafka.") && !OPTIONS.contains(name)) { + throw invalid("Unsupported Kafka table property '" + name + "'."); + } + } + for (String name : table.getCustomProperties().keySet()) { + if (name.startsWith("table.kafka.") + || name.startsWith("kafka.") + || name.startsWith("fluss.")) { + throw invalid( + "Unsupported Kafka custom property '" + + name + + "'. Use registered table.kafka.* properties and fixed system columns."); + } + } + } + + private static void validateMetadataType(DataType type) { + String message = + "_kafka_metadata must be ROW."; + if (!(type instanceof RowType)) { + throw invalid(message); + } + RowType row = (RowType) type; + if (!row.getFieldNames() + .equals(Arrays.asList("timestamp", "timestamp_type", "partition", "offset")) + || !row.getTypeAt(0).copy(true).equals(DataTypes.TIMESTAMP_LTZ(3)) + || !row.getTypeAt(1).copy(true).equals(DataTypes.INT()) + || !row.getTypeAt(2).copy(true).equals(DataTypes.INT()) + || !row.getTypeAt(3).copy(true).equals(DataTypes.BIGINT())) { + throw invalid(message); + } + } + + /** Returns the unchanged physical schema, including declared nullability. */ + public RowType rowType() { + return rowType; + } + + /** Returns the value conversion type. */ + public ValueType valueType() { + return valueType; + } + + /** Returns the JSON layout, or null for binary/string. */ + public @Nullable ValueLayout valueLayout() { + return valueLayout; + } + + /** Returns physical value targets; nested JSON projects its single ROW container. */ + public KafkaFieldProjection valueProjection() { + return valueProjection; + } + + /** Returns the key column position, or -1 when keys are not stored. */ + public int keyPosition() { + return keyPosition; + } + + /** + * Returns the headers column position, or -1 when headers are not stored. Decoding maps names + * to binary values, with the last occurrence winning for duplicate header names. + */ + public int headersPosition() { + return headersPosition; + } + + /** + * Returns the rescue column position, or -1 when absent. JSON decoders collect unknown and + * reserved members here; binary/string decoders write null to this optional column. + */ + public int rescuePosition() { + return rescuePosition; + } + + @Override + public boolean equals(Object other) { + if (!(other instanceof KafkaTableSchema)) { + return false; + } + KafkaTableSchema that = (KafkaTableSchema) other; + return rowType.equals(that.rowType) + && valueType == that.valueType + && valueLayout == that.valueLayout + && valueProjection.equals(that.valueProjection) + && keyPosition == that.keyPosition + && headersPosition == that.headersPosition + && rescuePosition == that.rescuePosition; + } + + @Override + public int hashCode() { + return Objects.hash( + rowType, + valueType, + valueLayout, + valueProjection, + keyPosition, + headersPosition, + rescuePosition); + } + + private static InvalidTableException invalid(String message) { + return new InvalidTableException(message); + } +} diff --git a/fluss-common/src/test/java/org/apache/fluss/metadata/KafkaTableSchemaTest.java b/fluss-common/src/test/java/org/apache/fluss/metadata/KafkaTableSchemaTest.java new file mode 100644 index 00000000000..e2b9c5be187 --- /dev/null +++ b/fluss-common/src/test/java/org/apache/fluss/metadata/KafkaTableSchemaTest.java @@ -0,0 +1,437 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.metadata; + +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.config.Configuration; +import org.apache.fluss.config.FlussConfigUtils; +import org.apache.fluss.config.TableConfig; +import org.apache.fluss.exception.InvalidTableException; +import org.apache.fluss.metadata.KafkaTableSchema.ValueLayout; +import org.apache.fluss.metadata.KafkaTableSchema.ValueType; +import org.apache.fluss.types.DataType; +import org.apache.fluss.types.DataTypes; +import org.apache.fluss.types.RowType; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.util.Arrays; +import java.util.Collections; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatCode; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests the shared DDL and Kafka conversion contract. */ +class KafkaTableSchemaTest { + + @Test + void testRegisteredAlterableOptionsAndReadTimeDefaults() { + for (String key : + Arrays.asList( + "table.kafka.enabled", + "table.kafka.convert.value.type", + "table.kafka.convert.value.layout", + "table.kafka.convert.key.type")) { + assertThat(FlussConfigUtils.TABLE_OPTIONS).containsKey(key); + assertThat(FlussConfigUtils.isAlterableTableOption(key)).isTrue(); + } + Configuration config = new Configuration(); + assertThat(new TableConfig(config).isKafkaEnabled()).isFalse(); + assertThat(config.toMap()).isEmpty(); + } + + @Test + void testExplicitOptInAndDisabledTables() { + TableDescriptor disabled = + TableDescriptor.builder() + .schema( + Schema.newBuilder() + .column("_kafka_value", DataTypes.BYTES()) + .build()) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "binary") + .build(); + assertThat(KafkaTableSchema.isEnabled(disabled)).isFalse(); + assertThatCode(() -> KafkaTableSchema.validate(disabled)).doesNotThrowAnyException(); + assertThatThrownBy(() -> KafkaTableSchema.resolve(disabled)) + .hasMessageContaining("disabled"); + TableDescriptor arbitrary = + TableDescriptor.builder(disabled) + .schema( + Schema.newBuilder() + .column("id", DataTypes.INT().copy(false)) + .primaryKey("id") + .build()) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "future-format") + .property(ConfigOptions.TABLE_KAFKA_ENABLED, false) + .build(); + assertThatCode(() -> KafkaTableSchema.validate(arbitrary)).doesNotThrowAnyException(); + } + + @ParameterizedTest + @ValueSource(strings = {"binary", "string"}) + void testFixedValueColumnWithOptionalSystemFields(String format) { + DataType valueType = format.equals("binary") ? DataTypes.BYTES() : DataTypes.STRING(); + for (int mask = 0; mask < 8; mask++) { + Schema.Builder schema = + Schema.newBuilder().column("_kafka_value", valueType.copy(false)); + if ((mask & 1) != 0) { + schema.column("_kafka_key", DataTypes.BYTES()); + } + if ((mask & 2) != 0) { + schema.column( + "_kafka_headers", DataTypes.MAP(DataTypes.STRING(), DataTypes.BYTES())); + } + if ((mask & 4) != 0) { + schema.column("_kafka_rescue", DataTypes.STRING()); + } + TableDescriptor table = enabled(schema.build(), format).build(); + byte[] before = table.toJsonBytes(); + KafkaTableSchema resolved = KafkaTableSchema.resolve(table); + assertThat(resolved.valueProjection().positions()).containsExactly(0); + assertThat(resolved.valueProjection().dataTypeAt(0).isNullable()).isFalse(); + assertThat(resolved.valueLayout()).isNull(); + assertThat(resolved.keyPosition() >= 0).isEqualTo((mask & 1) != 0); + assertThat(resolved.headersPosition() >= 0).isEqualTo((mask & 2) != 0); + assertThat(resolved.rescuePosition() >= 0).isEqualTo((mask & 4) != 0); + assertThat(table.toJsonBytes()).isEqualTo(before); + assertThat(table.getProperties()) + .doesNotContainKeys( + "table.kafka.convert.key.type", "table.kafka.convert.value.layout"); + assertThat(KafkaTableSchema.resolve(TableDescriptor.fromJsonBytes(before))) + .isEqualTo(resolved); + } + } + + @Test + void testJsonFlatPreservesNestedTypesAndExcludesSystemColumns() { + RowType nested = + DataTypes.ROW( + DataTypes.FIELD("_kafka_key", DataTypes.STRING()), + DataTypes.FIELD( + "items", + DataTypes.ARRAY( + DataTypes.MAP(DataTypes.STRING(), DataTypes.INT())))); + TableDescriptor table = + enabled( + Schema.newBuilder() + .column("id", DataTypes.BIGINT().copy(false)) + .column("_kafka_rescue", DataTypes.STRING()) + .column("payload", nested) + .column("_kafka_key", DataTypes.BYTES()) + .column( + "_kafka_headers", + DataTypes.MAP( + DataTypes.STRING(), DataTypes.BYTES())) + .column("custom_rescue", DataTypes.STRING()) + .build(), + "parse_json") + .build(); + KafkaTableSchema resolved = KafkaTableSchema.resolve(table); + assertThat(resolved.valueType()).isEqualTo(ValueType.PARSE_JSON); + assertThat(resolved.valueLayout()).isEqualTo(ValueLayout.FLAT); + assertThat(resolved.valueProjection().positions()).containsExactly(0, 2, 5); + assertThat(resolved.valueProjection().dataTypeAt(1)).isEqualTo(nested); + assertThat(resolved.rescuePosition()).isEqualTo(1); + assertThat(table.getProperties()).doesNotContainKey("table.kafka.convert.value.layout"); + TableDescriptor noRescue = + enabled( + Schema.newBuilder() + .column("custom_rescue", DataTypes.STRING()) + .build(), + "parse_json") + .build(); + assertThat(KafkaTableSchema.resolve(noRescue).rescuePosition()).isEqualTo(-1); + } + + @Test + void testNestedJsonTargetsOneRow() { + RowType row = + DataTypes.ROW( + DataTypes.FIELD("id", DataTypes.STRING()), + DataTypes.FIELD( + "details", + DataTypes.ROW( + DataTypes.FIELD("_kafka_rescue", DataTypes.STRING())))); + TableDescriptor table = + enabled( + Schema.newBuilder() + .column("_kafka_rescue", DataTypes.STRING()) + .column("_kafka_value", row) + .build(), + "parse_json") + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT, "nested") + .build(); + KafkaTableSchema resolved = KafkaTableSchema.resolve(table); + assertThat(resolved.valueLayout()).isEqualTo(ValueLayout.NESTED); + assertThat(resolved.valueProjection().positions()).containsExactly(1); + assertThat(resolved.valueProjection().dataTypeAt(0)).isEqualTo(row); + assertThat(resolved.rescuePosition()).isZero(); + } + + @ParameterizedTest + @ValueSource( + strings = { + "_kafka_key", + "_kafka_value", + "_kafka_headers", + "_kafka_metadata", + "_kafka_rescue" + }) + void testReservedNamesAtValueMappingLevel(String name) { + assertThat(KafkaTableSchema.isReservedName(name)).isTrue(); + TableDescriptor nested = + enabled( + Schema.newBuilder() + .column( + "_kafka_value", + DataTypes.ROW( + DataTypes.FIELD(name, DataTypes.STRING()))) + .build(), + "parse_json") + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT, "nested") + .build(); + assertInvalid(nested, "Reserved name"); + } + + @ParameterizedTest + @ValueSource(strings = {"raw", "json", "schema_id_prefix", "schema_latest", "avro", ""}) + void testRejectsUnsupportedValueTypes(String type) { + assertInvalid(enabled(valueSchema(), type).build(), "table.kafka.convert.value.type"); + } + + @Test + void testRejectsMissingValueTypeAndTargetConflicts() { + assertInvalid( + TableDescriptor.builder() + .schema(valueSchema()) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) + .build(), + "Missing required"); + assertInvalid( + enabled(Schema.newBuilder().column("body", DataTypes.BYTES()).build(), "binary") + .build(), + "Missing required value column"); + assertInvalid(enabled(valueSchema(), "string").build(), "must be STRING"); + assertInvalid( + enabled(valueSchema(), "parse_json") + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT, "nested") + .build(), + "must be ROW"); + assertInvalid(enabled(valueSchema(), "parse_json").build(), "flat JSON"); + assertInvalid( + enabled( + Schema.newBuilder() + .column("_kafka_value", DataTypes.BYTES()) + .column("unused", DataTypes.STRING()) + .build(), + "binary") + .build(), + "no value mapping"); + assertInvalid( + enabled( + Schema.newBuilder() + .column( + "_kafka_value", + DataTypes.ROW( + DataTypes.FIELD("id", DataTypes.STRING()))) + .column("unused", DataTypes.STRING()) + .build(), + "parse_json") + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT, "nested") + .build(), + "no value mapping"); + } + + @ParameterizedTest + @ValueSource(strings = {"binary", "string"}) + void testNonJsonRejectsExplicitLayout(String type) { + assertInvalid( + enabled(valueSchema(), type) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT, "flat") + .build(), + "only supported for parse_json"); + } + + @Test + void testInvalidLayoutAndKeyConversion() { + assertInvalid( + enabled(valueSchema(), "parse_json") + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT, "unknown") + .build(), + "value.layout"); + assertInvalid( + enabled(valueSchema(), "binary") + .property(ConfigOptions.TABLE_KAFKA_CONVERT_KEY_TYPE, "binary") + .build(), + "requires _kafka_key"); + Schema schema = + Schema.newBuilder() + .column("_kafka_value", DataTypes.BYTES()) + .column("_kafka_key", DataTypes.BYTES()) + .build(); + assertInvalid( + enabled(schema, "binary") + .property(ConfigOptions.TABLE_KAFKA_CONVERT_KEY_TYPE, "string") + .build(), + "only supports binary"); + assertThat( + KafkaTableSchema.resolve( + enabled(schema, "binary") + .property( + ConfigOptions.TABLE_KAFKA_CONVERT_KEY_TYPE, + "binary") + .build()) + .keyPosition()) + .isEqualTo(1); + } + + @Test + void testSystemColumnTypes() { + for (DataType invalid : Arrays.asList(DataTypes.STRING(), DataTypes.BINARY(8))) { + assertInvalid(withSystem("_kafka_key", invalid), "_kafka_key must be BYTES"); + } + for (DataType invalid : + Arrays.asList( + DataTypes.BYTES(), + DataTypes.ARRAY(DataTypes.STRING()), + DataTypes.MAP(DataTypes.INT(), DataTypes.BYTES()), + DataTypes.MAP(DataTypes.STRING(), DataTypes.STRING()))) { + assertInvalid(withSystem("_kafka_headers", invalid), "MAP"); + } + assertInvalid(withSystem("_kafka_rescue", DataTypes.INT()), "nullable STRING"); + assertInvalid( + withSystem("_kafka_rescue", DataTypes.STRING().copy(false)), "nullable STRING"); + } + + @Test + void testMetadataStructureAndUnavailableOffsetPersistence() { + RowType metadata = + DataTypes.ROW( + DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP_LTZ(3)), + DataTypes.FIELD("timestamp_type", DataTypes.INT()), + DataTypes.FIELD("partition", DataTypes.INT()), + DataTypes.FIELD("offset", DataTypes.BIGINT())); + assertInvalid(withSystem("_kafka_metadata", metadata), "persisting actual per-row offsets"); + assertInvalid(withSystem("_kafka_metadata", DataTypes.STRING()), "must be ROW"); + assertInvalid( + withSystem( + "_kafka_metadata", + DataTypes.ROW( + DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP(3)), + DataTypes.FIELD("timestamp_type", DataTypes.INT()), + DataTypes.FIELD("partition", DataTypes.INT()), + DataTypes.FIELD("offset", DataTypes.BIGINT()))), + "TIMESTAMP_LTZ(3)"); + assertInvalid( + withSystem( + "_kafka_metadata", + DataTypes.ROW( + DataTypes.FIELD("offset", DataTypes.BIGINT()), + DataTypes.FIELD("timestamp_type", DataTypes.INT()), + DataTypes.FIELD("partition", DataTypes.INT()), + DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP_LTZ(3)))), + "must be ROW"); + } + + @Test + void testUnknownAndPrototypeOptions() { + for (String key : + Arrays.asList("table.kafka.unknown", "table.kafka.convert.rescue-column")) { + assertInvalid( + enabled(valueSchema(), "binary").property(key, "anything").build(), + "Unsupported Kafka table property"); + } + for (String key : + Arrays.asList( + "kafka.value.format", + "kafka.key.fields", + "kafka.metadata.headers.column", + "fluss.value.format", + "table.kafka.convert.value.type")) { + assertInvalid( + enabled(valueSchema(), "binary").customProperty(key, "raw").build(), + "Unsupported Kafka custom property"); + } + assertThat( + KafkaTableSchema.resolve( + enabled(valueSchema(), "binary") + .customProperty("owner", "team") + .build())) + .isNotNull(); + } + + @Test + void testTableKindsAndCanonicalNameLimits() { + Schema keyed = + Schema.newBuilder() + .column("_kafka_value", DataTypes.BYTES().copy(false)) + .primaryKey("_kafka_value") + .build(); + assertInvalid(enabled(keyed, "binary").build(), "without a primary key"); + assertInvalid( + enabled(valueSchema(), "binary").partitionedBy("_kafka_value").build(), + "non-partitioned"); + assertInvalid( + enabled(valueSchema(), "binary").logFormat(LogFormat.INDEXED).build(), "Arrow"); + String database = String.join("", Collections.nCopies(124, "d")); + String tableName = String.join("", Collections.nCopies(124, "t")); + TableDescriptor table = enabled(valueSchema(), "binary").build(); + assertThatCode( + () -> + KafkaTableSchema.validateTablePath( + TablePath.of(database, tableName), table)) + .doesNotThrowAnyException(); + assertThatThrownBy( + () -> + KafkaTableSchema.validateTablePath( + TablePath.of(database, tableName + "t"), table)) + .isInstanceOf(InvalidTableException.class) + .hasMessageContaining("Kafka topic"); + } + + private static TableDescriptor withSystem(String name, DataType type) { + return enabled( + Schema.newBuilder() + .column("_kafka_value", DataTypes.BYTES()) + .column(name, type) + .build(), + "binary") + .build(); + } + + private static Schema valueSchema() { + return Schema.newBuilder().column("_kafka_value", DataTypes.BYTES()).build(); + } + + private static TableDescriptor.Builder enabled(Schema schema, String type) { + return TableDescriptor.builder() + .schema(schema) + .logFormat(LogFormat.ARROW) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, type); + } + + private static void assertInvalid(TableDescriptor table, String message) { + assertThatThrownBy(() -> KafkaTableSchema.resolve(table)) + .isInstanceOf(InvalidTableException.class) + .hasMessageContaining(message); + } +} diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/utils/FlinkConversionsTest.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/utils/FlinkConversionsTest.java index 67f843a7276..56e990d8cd5 100644 --- a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/utils/FlinkConversionsTest.java +++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/utils/FlinkConversionsTest.java @@ -19,6 +19,7 @@ import org.apache.fluss.config.ConfigOptions; import org.apache.fluss.flink.catalog.TestSchemaResolver; +import org.apache.fluss.metadata.KafkaTableSchema; import org.apache.fluss.metadata.KvFormat; import org.apache.fluss.metadata.TableChange; import org.apache.fluss.metadata.TableDescriptor; @@ -75,6 +76,50 @@ /** Test for {@link FlinkConversions}. */ public class FlinkConversionsTest { + @Test + void testKafkaDdlUsesRegisteredPropertiesAndFixedColumns() { + ResolvedSchema schema = + new ResolvedSchema( + Arrays.asList( + Column.physical( + "_kafka_value", + org.apache.flink.table.api.DataTypes.BYTES()), + Column.physical( + "_kafka_key", org.apache.flink.table.api.DataTypes.BYTES()), + Column.physical( + "_kafka_headers", + org.apache.flink.table.api.DataTypes.MAP( + org.apache.flink.table.api.DataTypes.STRING(), + org.apache.flink.table.api.DataTypes.BYTES())), + Column.physical( + "_kafka_rescue", + org.apache.flink.table.api.DataTypes.STRING())), + Collections.emptyList(), + null); + Map options = new HashMap<>(); + options.put("table.kafka.enabled", "true"); + options.put("table.kafka.convert.value.type", "binary"); + CatalogTable table = + CatalogTable.of( + Schema.newBuilder().fromResolvedSchema(schema).build(), + "Kafka DDL", + Collections.emptyList(), + options); + TableDescriptor descriptor = + FlinkConversions.toFlussTable(new ResolvedCatalogTable(table, schema)); + assertThat(descriptor.getProperties()) + .containsAllEntriesOf(options) + .doesNotContainKeys( + "table.kafka.convert.key.type", "table.kafka.convert.value.layout"); + assertThat(descriptor.getCustomProperties()) + .doesNotContainKeys("table.kafka.enabled", "table.kafka.convert.value.type"); + KafkaTableSchema mapping = KafkaTableSchema.resolve(descriptor); + assertThat(mapping.valueProjection().positions()).containsExactly(0); + assertThat(mapping.keyPosition()).isEqualTo(1); + assertThat(mapping.headersPosition()).isEqualTo(2); + assertThat(mapping.rescuePosition()).isEqualTo(3); + } + @Test void testConvertBucketCountChange() { assertThat( diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaDataFormat.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaDataFormat.java deleted file mode 100644 index df5d871907c..00000000000 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/format/KafkaDataFormat.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.fluss.kafka.format; - -import org.apache.fluss.annotation.Internal; - -import java.util.Locale; - -/** Supported interpretations of Kafka record key and value bytes. */ -@Internal -public enum KafkaDataFormat { - RAW("raw"), - STRING("string"); - - /** Fluss table custom property controlling the record key format. */ - public static final String KEY_FORMAT_CONFIG = "kafka.key.format"; - - /** Fluss table custom property controlling the record value format. */ - public static final String VALUE_FORMAT_CONFIG = "kafka.value.format"; - - /** Fluss fields populated from the Kafka record key. */ - public static final String KEY_FIELDS_CONFIG = "kafka.key.fields"; - - /** Strategy for deriving fields populated from the Kafka record value. */ - public static final String VALUE_FIELDS_INCLUDE_CONFIG = "kafka.value.fields-include"; - - /** - * TIMESTAMP(3) column populated from the Kafka record epoch milliseconds as UTC calendar - * values. A missing Kafka timestamp (-1) maps to null and requires a nullable column. - */ - public static final String TIMESTAMP_COLUMN_CONFIG = "kafka.metadata.timestamp.column"; - - /** Fluss column populated from the Kafka record headers. */ - public static final String HEADERS_COLUMN_CONFIG = "kafka.metadata.headers.column"; - - private final String value; - - KafkaDataFormat(String value) { - this.value = value; - } - - /** Parses a table custom property value. */ - public static KafkaDataFormat parse(String value) { - if (value == null) { - throw new IllegalArgumentException("Kafka data format must not be null."); - } - String normalized = value.trim().toLowerCase(Locale.ROOT); - for (KafkaDataFormat format : values()) { - if (format.value.equals(normalized)) { - return format; - } - } - throw new IllegalArgumentException( - "Unsupported Kafka data format '" + value + "'. Expected raw or string."); - } - - /** Returns the persisted table property value. */ - public String value() { - return value; - } -} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/mapping/KafkaTopicMapper.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/mapping/KafkaTopicMapper.java index f8b5744f71d..704d322f2e0 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/mapping/KafkaTopicMapper.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/mapping/KafkaTopicMapper.java @@ -18,6 +18,9 @@ package org.apache.fluss.kafka.mapping; import org.apache.fluss.annotation.Internal; +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.config.Configuration; +import org.apache.fluss.metadata.KafkaTableSchema; import org.apache.fluss.metadata.TablePath; import org.apache.kafka.common.Uuid; @@ -26,21 +29,43 @@ import static org.apache.fluss.utils.Preconditions.checkArgument; -/** Maps fully qualified Kafka topic identities to Fluss tables. */ +/** Maps canonical Kafka topic names and default-database aliases to Fluss tables. */ @Internal public final class KafkaTopicMapper { // ASCII "Fluss" followed by zero bytes. A dedicated namespace avoids Kafka-reserved UUIDs. private static final long TOPIC_ID_NAMESPACE = 0x466c757373000000L; - /** Maps a Kafka topic in database.table form to its Fluss table path. */ + private final String defaultDatabase; + + /** Creates a mapper using the default database configuration. */ + public KafkaTopicMapper() { + this(new Configuration()); + } + + /** Creates a mapper using the server's default database; it never creates databases. */ + public KafkaTopicMapper(Configuration config) { + this.defaultDatabase = config.get(ConfigOptions.KAFKA_DEFAULT_DATABASE); + TablePath.validateDatabaseName(defaultDatabase); + } + + /** Maps a canonical topic or an unqualified alias to a table path, without fallback lookup. */ public TablePath toTablePath(String topicName) { if (!isValidTopic(topicName)) { throw new InvalidTopicException( - "Kafka topic must be a valid database.table name: " + topicName); + "Kafka topic must be a valid table or database.table name: " + topicName); } int separator = topicName.indexOf('.'); - return TablePath.of(topicName.substring(0, separator), topicName.substring(separator + 1)); + TablePath path = + separator < 0 + ? TablePath.of(defaultDatabase, topicName) + : TablePath.of( + topicName.substring(0, separator), + topicName.substring(separator + 1)); + if (!isMappedTable(path)) { + throw new InvalidTopicException("Canonical Kafka topic name is invalid: " + path); + } + return path; } /** Returns the fully qualified Kafka name of a representable Fluss table. */ @@ -55,7 +80,7 @@ public String toTopicName(TablePath tablePath) { /** Returns whether a Fluss user table has a valid, fully qualified Kafka topic name. */ public boolean isMappedTable(TablePath tablePath) { - return tablePath != null && tablePath.isValid() && isValidTopic(tablePath.toString()); + return KafkaTableSchema.isValidTablePath(tablePath); } /** Returns whether a name uniquely represents a user table in a Fluss database. */ @@ -64,14 +89,16 @@ public static boolean isValidTopic(String topicName) { return false; } int separator = topicName.indexOf('.'); - if (separator <= 0 || separator != topicName.lastIndexOf('.')) { + if (separator < 0) { + return TablePath.detectInvalidName(topicName) == null + && TablePath.validatePrefix(topicName) == null; + } + if (separator == 0 || separator != topicName.lastIndexOf('.')) { return false; } - String database = topicName.substring(0, separator); - String table = topicName.substring(separator + 1); - return TablePath.of(database, table).isValid() - && TablePath.validatePrefix(database) == null - && TablePath.validatePrefix(table) == null; + return KafkaTableSchema.isValidTablePath( + TablePath.of( + topicName.substring(0, separator), topicName.substring(separator + 1))); } /** Maps a Fluss table ID to a stable Kafka topic ID. */ diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchema.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchema.java deleted file mode 100644 index 64263e69286..00000000000 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchema.java +++ /dev/null @@ -1,150 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.fluss.kafka.schema; - -import org.apache.fluss.annotation.Internal; -import org.apache.fluss.kafka.format.KafkaDataFormat; -import org.apache.fluss.types.RowType; - -import javax.annotation.Nullable; - -import java.util.Objects; - -import static org.apache.fluss.utils.Preconditions.checkNotNull; - -/** Resolved Kafka key, value, and metadata mapping for one Fluss table schema. */ -@Internal -public final class KafkaTopicSchema { - - private final RowType rowType; - private final @Nullable KafkaDataFormat keyFormat; - private final KafkaFieldProjection keyProjection; - private final KafkaDataFormat valueFormat; - private final KafkaFieldProjection valueProjection; - private final int timestampPosition; - private final int headersPosition; - - /** Creates a resolved Kafka topic schema. */ - public KafkaTopicSchema( - RowType rowType, - @Nullable KafkaDataFormat keyFormat, - KafkaFieldProjection keyProjection, - KafkaDataFormat valueFormat, - KafkaFieldProjection valueProjection, - int timestampPosition, - int headersPosition) { - this.rowType = checkNotNull(rowType); - this.keyFormat = keyFormat; - this.keyProjection = checkNotNull(keyProjection); - this.valueFormat = checkNotNull(valueFormat); - this.valueProjection = checkNotNull(valueProjection); - this.timestampPosition = timestampPosition; - this.headersPosition = headersPosition; - } - - /** Returns the physical Fluss row type. */ - public RowType rowType() { - return rowType; - } - - /** Returns the key format, or null when the Kafka key is not mapped. */ - public @Nullable KafkaDataFormat keyFormat() { - return keyFormat; - } - - /** Returns the key field projection. */ - public KafkaFieldProjection keyProjection() { - return keyProjection; - } - - /** Returns the value format. */ - public KafkaDataFormat valueFormat() { - return valueFormat; - } - - /** Returns the value field projection. */ - public KafkaFieldProjection valueProjection() { - return valueProjection; - } - - /** Returns the timestamp physical position, or -1 when it is not mapped. */ - public int timestampPosition() { - return timestampPosition; - } - - /** Returns the headers physical position, or -1 when they are not mapped. */ - public int headersPosition() { - return headersPosition; - } - - @Override - public boolean equals(Object obj) { - if (this == obj) { - return true; - } - if (!(obj instanceof KafkaTopicSchema)) { - return false; - } - KafkaTopicSchema that = (KafkaTopicSchema) obj; - return Objects.equals(rowType, that.rowType) - && Objects.equals(keyFormat, that.keyFormat) - && Objects.equals(keyProjection, that.keyProjection) - && Objects.equals(valueFormat, that.valueFormat) - && Objects.equals(valueProjection, that.valueProjection) - && Objects.equals(timestampPosition, that.timestampPosition) - && Objects.equals(headersPosition, that.headersPosition); - } - - @Override - public int hashCode() { - return Objects.hash( - rowType, - keyFormat, - keyProjection, - valueFormat, - valueProjection, - timestampPosition, - headersPosition); - } - - @Override - public String toString() { - return "KafkaTopicSchema{" - + "rowType=" - + rowType - + ", " - + "keyFormat=" - + keyFormat - + ", " - + "keyProjection=" - + keyProjection - + ", " - + "valueFormat=" - + valueFormat - + ", " - + "valueProjection=" - + valueProjection - + ", " - + "timestampPosition=" - + timestampPosition - + ", " - + "headersPosition=" - + headersPosition - + "}"; - } -} diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolver.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolver.java index 8959c420d15..c02879e922a 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolver.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolver.java @@ -18,293 +18,29 @@ package org.apache.fluss.kafka.schema; import org.apache.fluss.annotation.Internal; -import org.apache.fluss.config.Configuration; -import org.apache.fluss.config.TableConfig; -import org.apache.fluss.kafka.format.KafkaDataFormat; -import org.apache.fluss.metadata.LogFormat; +import org.apache.fluss.exception.InvalidTableException; +import org.apache.fluss.metadata.KafkaTableSchema; import org.apache.fluss.metadata.TableDescriptor; -import org.apache.fluss.types.ArrayType; -import org.apache.fluss.types.BytesType; -import org.apache.fluss.types.RowType; -import org.apache.fluss.types.StringType; -import org.apache.fluss.types.TimestampType; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Set; - -/** Resolves and validates the Kafka record mapping stored in Fluss table custom properties. */ +/** Resolves the same persisted Kafka contract that native DDL validates. */ @Internal public final class KafkaTopicSchemaResolver { - private static final String INCLUDE_ALL = "ALL"; - private static final String INCLUDE_EXCEPT_KEY = "EXCEPT_KEY"; - - private static final Set SUPPORTED_PROPERTIES = - new HashSet<>( - Arrays.asList( - KafkaDataFormat.KEY_FORMAT_CONFIG, - KafkaDataFormat.KEY_FIELDS_CONFIG, - KafkaDataFormat.VALUE_FORMAT_CONFIG, - KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, - KafkaDataFormat.TIMESTAMP_COLUMN_CONFIG, - KafkaDataFormat.HEADERS_COLUMN_CONFIG)); - - /** Resolves one table's Kafka record mapping contract. */ - public KafkaTopicSchema resolve(TableDescriptor table) { - validateTableKind(table); - RowType rowType = table.getSchema().getRowType(); - Map properties = table.getCustomProperties(); - - for (String property : properties.keySet()) { - if (property.startsWith("kafka.") && !SUPPORTED_PROPERTIES.contains(property)) { - throw invalid("Unsupported Kafka table property '" + property + "'."); - } - } - - int timestampPosition = - resolveOptionalPosition( - rowType, properties.get(KafkaDataFormat.TIMESTAMP_COLUMN_CONFIG)); - int headersPosition = - resolveOptionalPosition( - rowType, properties.get(KafkaDataFormat.HEADERS_COLUMN_CONFIG)); - if (timestampPosition >= 0 && timestampPosition == headersPosition) { - throw invalid("Kafka timestamp and headers cannot map to the same Fluss column."); - } - validateMetadataColumns(rowType, timestampPosition, headersPosition); - - String keyFormatValue = properties.get(KafkaDataFormat.KEY_FORMAT_CONFIG); - KafkaDataFormat keyFormat = keyFormatValue == null ? null : parseFormat(keyFormatValue); - List keyPositions = - resolveKeyPositions( - rowType, keyFormat, properties.get(KafkaDataFormat.KEY_FIELDS_CONFIG)); - for (Integer keyPosition : keyPositions) { - if (keyPosition == timestampPosition || keyPosition == headersPosition) { - throw invalid( - "Kafka key field '" - + rowType.getFieldNames().get(keyPosition) - + "' cannot be a Kafka metadata column."); - } - } - KafkaFieldProjection keyProjection = new KafkaFieldProjection(rowType, keyPositions); - - String valueFormatValue = properties.get(KafkaDataFormat.VALUE_FORMAT_CONFIG); - if (valueFormatValue == null) { - throw invalid( - "Missing required table property '" - + KafkaDataFormat.VALUE_FORMAT_CONFIG - + "'."); - } - KafkaDataFormat valueFormat = parseFormat(valueFormatValue); - String fieldsInclude = - normalizeFieldsInclude(properties.get(KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG)); - if (!keyPositions.isEmpty() && INCLUDE_ALL.equals(fieldsInclude)) { - throw invalid( - "Mapping Kafka key fields requires " - + KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG - + "=EXCEPT_KEY."); - } - - Set metadataPositions = new HashSet<>(); - if (timestampPosition >= 0) { - metadataPositions.add(timestampPosition); - } - if (headersPosition >= 0) { - metadataPositions.add(headersPosition); - } - List valuePositions = - resolveValuePositions(rowType, fieldsInclude, keyPositions, metadataPositions); - KafkaFieldProjection valueProjection = new KafkaFieldProjection(rowType, valuePositions); - if (valueProjection.isEmpty()) { - throw invalid("Kafka value projection must contain at least one Fluss column."); - } - - validateSingleFieldFormat(keyFormat, keyProjection, "key"); - validateSingleFieldFormat(valueFormat, valueProjection, "value"); - return new KafkaTopicSchema( - rowType, - keyFormat, - keyProjection, - valueFormat, - valueProjection, - timestampPosition, - headersPosition); - } - - private static void validateTableKind(TableDescriptor table) { - if (table.hasPrimaryKey()) { - throw invalid("Kafka topic table must be a log table."); - } - if (table.isPartitioned()) { - throw invalid("Partitioned Fluss tables are not supported."); - } - if (new TableConfig(Configuration.fromMap(table.getProperties())).getLogFormat() - != LogFormat.ARROW) { - throw invalid("Kafka topic table must use the Arrow log format."); - } - } - - private static List resolveKeyPositions( - RowType rowType, KafkaDataFormat keyFormat, String keyFieldsValue) { - if (keyFormat == null) { - if (keyFieldsValue != null) { - throw invalid( - KafkaDataFormat.KEY_FIELDS_CONFIG - + " requires " - + KafkaDataFormat.KEY_FORMAT_CONFIG - + "."); - } - return Collections.emptyList(); - } - if (keyFieldsValue == null || keyFieldsValue.trim().isEmpty()) { - throw invalid( - "Missing required table property '" + KafkaDataFormat.KEY_FIELDS_CONFIG + "'."); - } - String[] fieldNames = keyFieldsValue.split(",", -1); - List positions = new ArrayList<>(fieldNames.length); - Set uniquePositions = new HashSet<>(); - for (String fieldNameValue : fieldNames) { - String fieldName = fieldNameValue.trim(); - if (fieldName.isEmpty()) { - throw invalid("Kafka key field names must not be empty."); - } - int position = rowType.getFieldIndex(fieldName); - if (position < 0) { - throw invalid("Kafka key field '" + fieldName + "' does not exist."); - } - if (!uniquePositions.add(position)) { - throw invalid("Duplicate Kafka key field '" + fieldName + "'."); - } - positions.add(position); - } - return positions; - } - - private static String normalizeFieldsInclude(String value) { - String normalized = value == null ? INCLUDE_ALL : value.trim().toUpperCase(Locale.ROOT); - if (!INCLUDE_ALL.equals(normalized) && !INCLUDE_EXCEPT_KEY.equals(normalized)) { - throw invalid( - "Invalid " - + KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG - + " '" - + value - + "'. Expected ALL or EXCEPT_KEY."); - } - return normalized; - } - - private static List resolveValuePositions( - RowType rowType, - String fieldsInclude, - List keyPositions, - Set metadataPositions) { - Set excludedKeyPositions = - INCLUDE_EXCEPT_KEY.equals(fieldsInclude) - ? new HashSet<>(keyPositions) - : Collections.emptySet(); - List positions = new ArrayList<>(); - for (int position = 0; position < rowType.getFieldCount(); position++) { - if (!metadataPositions.contains(position) && !excludedKeyPositions.contains(position)) { - positions.add(position); - } - } - return positions; - } - - private static int resolveOptionalPosition(RowType rowType, String fieldNameValue) { - if (fieldNameValue == null) { - return -1; - } - if (fieldNameValue.trim().isEmpty()) { - throw invalid("Kafka metadata column name must not be empty."); - } - String fieldName = fieldNameValue.trim(); - int position = rowType.getFieldIndex(fieldName); - if (position < 0) { - throw invalid("Kafka metadata column '" + fieldName + "' does not exist."); - } - return position; - } - - private static void validateMetadataColumns( - RowType rowType, int timestampPosition, int headersPosition) { - if (timestampPosition >= 0) { - if (!(rowType.getTypeAt(timestampPosition) instanceof TimestampType) - || ((TimestampType) rowType.getTypeAt(timestampPosition)).getPrecision() != 3) { - throw invalid("Kafka timestamp column must be TIMESTAMP(3)."); - } - } - if (headersPosition >= 0) { - validateHeadersType(rowType.getTypeAt(headersPosition)); - } - } - - private static void validateHeadersType(org.apache.fluss.types.DataType dataType) { - if (!(dataType instanceof ArrayType) || !dataType.isNullable()) { - throw invalid( - "Kafka headers column must be nullable " - + "ARRAY>."); - } - ArrayType arrayType = (ArrayType) dataType; - if (!(arrayType.getElementType() instanceof RowType)) { - throw invalid("Kafka headers elements must be rows."); - } - RowType headerType = (RowType) arrayType.getElementType(); - if (!headerType.getFieldNames().equals(Arrays.asList("name", "value")) - || !(headerType.getTypeAt(0) instanceof StringType) - || !(headerType.getTypeAt(1) instanceof BytesType) - || !headerType.getTypeAt(1).isNullable()) { - throw invalid("Kafka headers elements must be ROW."); - } - } - - private static void validateSingleFieldFormat( - KafkaDataFormat format, KafkaFieldProjection projection, String component) { - if (format == null) { - return; - } - if (format == KafkaDataFormat.RAW || format == KafkaDataFormat.STRING) { - if (projection.size() != 1) { - throw invalid( - "Kafka " - + component - + " format " - + format.value() - + " requires exactly one Fluss field."); - } - boolean validType = - format == KafkaDataFormat.RAW - ? projection.dataTypeAt(0) instanceof BytesType - : projection.dataTypeAt(0) instanceof StringType; - if (!validType) { - throw invalid( - "Kafka " - + component - + " field '" - + projection.nameAt(0) - + "' must be " - + (format == KafkaDataFormat.RAW ? "BYTES" : "STRING") - + " for format " - + format.value() - + "."); - } - } - } - - private static KafkaDataFormat parseFormat(String value) { + /** Returns the table's explicit opt-in flag without inferring it from schema compatibility. */ + public boolean isEnabled(TableDescriptor table) { try { - return KafkaDataFormat.parse(value); + return KafkaTableSchema.isEnabled(table); } catch (IllegalArgumentException e) { - throw invalid(e.getMessage()); + throw new KafkaTopicSchemaException(e.getMessage()); } } - private static KafkaTopicSchemaException invalid(String message) { - return new KafkaTopicSchemaException(message); + /** Resolves one enabled table's contract; invalid enabled tables remain visible as errors. */ + public KafkaTableSchema resolve(TableDescriptor table) { + try { + return KafkaTableSchema.resolve(table); + } catch (InvalidTableException | IllegalArgumentException e) { + throw new KafkaTopicSchemaException(e.getMessage()); + } } } diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaConfigsTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaConfigsTest.java index 7f502494f2e..55e8ba6fc57 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaConfigsTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaConfigsTest.java @@ -36,13 +36,14 @@ public void testFromMap() throws Exception { Map map = new HashMap<>(); map.put(ConfigOptions.KAFKA_ENABLED.key(), "true"); map.put(ConfigOptions.KAFKA_LISTENER_NAMES.key(), "kafka,kafka_sasl"); - map.put(ConfigOptions.KAFKA_DATABASE.key(), "fluss"); + map.put(ConfigOptions.KAFKA_DEFAULT_DATABASE.key(), "fluss"); Configuration configuration = Configuration.fromMap(map); assertThat(configuration.getBoolean(ConfigOptions.KAFKA_ENABLED)).isTrue(); assertThat(configuration.get(ConfigOptions.KAFKA_LISTENER_NAMES)) .isEqualTo(Arrays.asList("kafka", "kafka_sasl")); - assertThat(configuration.getString(ConfigOptions.KAFKA_DATABASE)).isEqualTo("fluss"); + assertThat(configuration.getString(ConfigOptions.KAFKA_DEFAULT_DATABASE)) + .isEqualTo("fluss"); } @Test @@ -51,6 +52,7 @@ public void testFromDefault() throws Exception { assertThat(configuration.getBoolean(ConfigOptions.KAFKA_ENABLED)).isFalse(); assertThat(configuration.get(ConfigOptions.KAFKA_LISTENER_NAMES)) .isEqualTo(Collections.singletonList("KAFKA")); - assertThat(configuration.getString(ConfigOptions.KAFKA_DATABASE)).isEqualTo("kafka"); + assertThat(configuration.getString(ConfigOptions.KAFKA_DEFAULT_DATABASE)) + .isEqualTo("kafka"); } } diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/mapping/KafkaTopicMapperTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/mapping/KafkaTopicMapperTest.java index c5d4a582a88..7ef7e281244 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/mapping/KafkaTopicMapperTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/mapping/KafkaTopicMapperTest.java @@ -17,6 +17,8 @@ package org.apache.fluss.kafka.mapping; +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.config.Configuration; import org.apache.fluss.metadata.TablePath; import org.apache.kafka.common.Uuid; @@ -49,11 +51,41 @@ public void testQualifiedNamesAcrossDatabases() { } } + @Test + void testDefaultDatabaseAliases() { + assertThat(mapper.toTablePath("orders")).isEqualTo(mapper.toTablePath("kafka.orders")); + Configuration config = new Configuration(); + config.set(ConfigOptions.KAFKA_DEFAULT_DATABASE, "sales"); + KafkaTopicMapper custom = new KafkaTopicMapper(config); + assertThat(custom.toTablePath("orders")).isEqualTo(TablePath.of("sales", "orders")); + assertThat(custom.toTablePath("warehouse.orders")) + .isEqualTo(TablePath.of("warehouse", "orders")); + assertThat(custom.toTopicName(custom.toTablePath("orders"))).isEqualTo("sales.orders"); + config.setString("kafka.database", "legacy"); + assertThat(new KafkaTopicMapper(config).toTablePath("orders")) + .isEqualTo(TablePath.of("sales", "orders")); + } + + @Test + void testInvalidDefaultDatabaseAndCanonicalAliasLength() { + Configuration config = new Configuration(); + config.set(ConfigOptions.KAFKA_DEFAULT_DATABASE, "bad.name"); + assertThatThrownBy(() -> new KafkaTopicMapper(config)).hasMessageContaining("invalid"); + config.set( + ConfigOptions.KAFKA_DEFAULT_DATABASE, + String.join("", Collections.nCopies(128, "d"))); + KafkaTopicMapper longDatabase = new KafkaTopicMapper(config); + assertThatThrownBy( + () -> + longDatabase.toTablePath( + String.join("", Collections.nCopies(128, "t")))) + .isInstanceOf(InvalidTopicException.class); + } + @ParameterizedTest @NullAndEmptySource @ValueSource( strings = { - "orders", ".orders", "sales.", "sales.orders.extra", diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java index 2f3e75bdd3d..45744dbc87a 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java @@ -4,7 +4,7 @@ * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -17,99 +17,371 @@ package org.apache.fluss.kafka.schema; -import org.apache.fluss.kafka.format.KafkaDataFormat; +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.config.Configuration; +import org.apache.fluss.exception.DatabaseNotExistException; +import org.apache.fluss.exception.InvalidConfigException; +import org.apache.fluss.exception.InvalidTableException; import org.apache.fluss.kafka.mapping.KafkaTopicMapper; +import org.apache.fluss.metadata.KafkaTableSchema; import org.apache.fluss.metadata.LogFormat; import org.apache.fluss.metadata.Schema; import org.apache.fluss.metadata.TableDescriptor; import org.apache.fluss.metadata.TablePath; -import org.apache.fluss.rpc.messages.MetadataRequest; -import org.apache.fluss.rpc.messages.PbTableMetadata; +import org.apache.fluss.rpc.gateway.CoordinatorGateway; +import org.apache.fluss.rpc.messages.AlterTableRequest; +import org.apache.fluss.rpc.messages.GetTableInfoResponse; +import org.apache.fluss.rpc.messages.PbAddColumn; import org.apache.fluss.server.testutils.FlussClusterExtension; +import org.apache.fluss.types.DataType; import org.apache.fluss.types.DataTypes; +import org.apache.fluss.utils.json.DataTypeJsonSerde; +import org.apache.fluss.utils.json.JsonSerdeUtils; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; import java.util.Arrays; -import java.util.LinkedHashMap; -import java.util.List; +import java.util.Collections; +import java.util.HashMap; import java.util.Map; +import static org.apache.fluss.server.coordinator.CoordinatorServer.DEFAULT_DATABASE; import static org.apache.fluss.server.testutils.RpcMessageTestUtils.createTable; +import static org.apache.fluss.server.testutils.RpcMessageTestUtils.newAlterTableRequest; +import static org.apache.fluss.server.testutils.RpcMessageTestUtils.newCreateDatabaseRequest; +import static org.apache.fluss.server.testutils.RpcMessageTestUtils.newCreateTableRequest; +import static org.apache.fluss.server.testutils.RpcMessageTestUtils.newDatabaseExistsRequest; +import static org.apache.fluss.server.testutils.RpcMessageTestUtils.newGetTableInfoRequest; +import static org.apache.fluss.server.testutils.RpcMessageTestUtils.newTableExistsRequest; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; -/** Verifies Kafka mapping properties survive the native Fluss table creation path. */ -public class KafkaTableMappingITCase { +/** Verifies Kafka DDL contracts through real coordinator RPC and persisted metadata. */ +class KafkaTableMappingITCase { + + private static final Configuration CONFIG = new Configuration(); + + static { + CONFIG.set(ConfigOptions.KAFKA_ENABLED, true); + CONFIG.set(ConfigOptions.KAFKA_DEFAULT_DATABASE, "missing_default"); + CONFIG.setString("kafka.database", "legacy_database"); + } @RegisterExtension - public static final FlussClusterExtension FLUSS_CLUSTER_EXTENSION = - FlussClusterExtension.builder().setNumOfTabletServers(1).build(); + public static final FlussClusterExtension CLUSTER = + FlussClusterExtension.builder().setNumOfTabletServers(1).setClusterConf(CONFIG).build(); + + private CoordinatorGateway gateway; + + @BeforeEach + void setup() { + gateway = CLUSTER.newCoordinatorClient(); + } @Test - public void testResolveMappingsFromQualifiedTopicsAcrossDatabases() throws Exception { + void testNoAutomaticKafkaDatabaseAndQualifiedAccessWithoutDefaultDatabase() throws Exception { + for (String database : Arrays.asList("missing_default", "kafka", "legacy_database")) { + assertThat(gateway.databaseExists(newDatabaseExistsRequest(database)).get().isExists()) + .isFalse(); + } + assertThat( + gateway.databaseExists(newDatabaseExistsRequest(DEFAULT_DATABASE)) + .get() + .isExists()) + .isTrue(); + KafkaTopicMapper mapper = new KafkaTopicMapper(CONFIG); + TablePath sales = mapper.toTablePath("sales.orders"); + TablePath warehouse = mapper.toTablePath("warehouse.orders"); + long salesId = createTable(CLUSTER, sales, enabled().build()); + long warehouseId = createTable(CLUSTER, warehouse, enabled().build()); + assertThat(salesId).isNotEqualTo(warehouseId); + assertThat(mapper.toTablePath("orders")) + .isEqualTo(TablePath.of("missing_default", "orders")); + assertThatThrownBy( + () -> + gateway.createTable( + newCreateTableRequest( + mapper.toTablePath("orders"), + enabled().build(), + false)) + .get()) + .hasCauseInstanceOf(DatabaseNotExistException.class); + assertThat( + gateway.databaseExists(newDatabaseExistsRequest("missing_default")) + .get() + .isExists()) + .isFalse(); + Configuration config = new Configuration(); + config.set(ConfigOptions.KAFKA_DEFAULT_DATABASE, "sales"); + KafkaTopicMapper aliasMapper = new KafkaTopicMapper(config); + assertThat(info(aliasMapper.toTablePath("orders")).getTableId()) + .isEqualTo(info(sales).getTableId()); + assertThat(aliasMapper.toTopicName(sales)).isEqualTo("sales.orders"); + } + + @Test + void testOnlyRequestedKafkaPropertiesArePersisted() throws Exception { + TablePath path = TablePath.of("ddl", "roundtrip"); + Schema schema = + Schema.newBuilder() + .column("_kafka_key", DataTypes.BYTES()) + .column( + "payload", + DataTypes.ROW(DataTypes.FIELD("nested", DataTypes.STRING()))) + .column( + "_kafka_headers", + DataTypes.MAP(DataTypes.STRING(), DataTypes.BYTES())) + .column("_kafka_rescue", DataTypes.STRING()) + .build(); TableDescriptor descriptor = - TableDescriptor.builder() + enabled() + .schema(schema) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "parse_json") + .build(); + createTable(CLUSTER, path, descriptor); + TableDescriptor stored = stored(path); + assertThat(stored.getSchema()).isEqualTo(schema); + assertThat(kafkaProperties(stored)).isEqualTo(kafkaProperties(descriptor)); + assertThat(new KafkaTopicSchemaResolver().resolve(stored).valueProjection().positions()) + .containsExactly(1); + } + + @Test + void testEnableDisableAndFailedPropertyAlterAreAtomic() throws Exception { + TablePath path = TablePath.of("ddl", "enable_disable"); + TableDescriptor disabled = + enabled().property(ConfigOptions.TABLE_KAFKA_ENABLED, false).build(); + createTable(CLUSTER, path, disabled); + TableDescriptor before = stored(path); + assertThat(KafkaTableSchema.isEnabled(before)).isFalse(); + set(path, Collections.singletonMap("table.kafka.enabled", "true")); + TableDescriptor enabled = stored(path); + assertThat(KafkaTableSchema.isEnabled(enabled)).isTrue(); + Map expected = new HashMap<>(before.getProperties()); + expected.put("table.kafka.enabled", "true"); + assertThat(enabled.getProperties()).isEqualTo(expected); + assertThat(enabled.getSchema()).isEqualTo(before.getSchema()); + Map invalid = new HashMap<>(); + invalid.put("table.kafka.convert.value.type", "string"); + invalid.put("table.kafka.convert.key.type", "binary"); + assertThatThrownBy(() -> set(path, invalid)) + .hasCauseInstanceOf(InvalidTableException.class); + assertThat(stored(path)).isEqualTo(enabled); + assertThatThrownBy(() -> reset(path, "table.kafka.convert.value.type")) + .hasCauseInstanceOf(InvalidTableException.class); + assertThat(stored(path)).isEqualTo(enabled); + reset(path, "table.kafka.enabled"); + assertThat(KafkaTableSchema.isEnabled(stored(path))).isFalse(); + assertThat(stored(path).getProperties()).doesNotContainKey("table.kafka.enabled"); + // Disabled tables can stage incompatible Kafka conversion properties without Kafka + // validation. + set(path, Collections.singletonMap("table.kafka.convert.value.type", "string")); + TableDescriptor staged = stored(path); + assertThatThrownBy(() -> set(path, Collections.singletonMap("table.kafka.enabled", "true"))) + .hasCauseInstanceOf(InvalidTableException.class); + assertThat(stored(path)).isEqualTo(staged); + } + + @Test + void testSchemaAlterValidatesBeforePersistenceAndCanBeDisabled() throws Exception { + TablePath path = TablePath.of("ddl", "schema_change"); + createTable(CLUSTER, path, enabled().build()); + GetTableInfoResponse before = info(path); + assertThatThrownBy(() -> addColumn(path, "_kafka_headers", DataTypes.STRING())) + .hasCauseInstanceOf(InvalidTableException.class); + assertThat(info(path).getTableJson()).isEqualTo(before.getTableJson()); + assertThat(info(path).getSchemaId()).isEqualTo(before.getSchemaId()); + addColumn(path, "_kafka_headers", DataTypes.MAP(DataTypes.STRING(), DataTypes.BYTES())); + assertThat(KafkaTableSchema.resolve(stored(path)).headersPosition()).isEqualTo(1); + assertThat(info(path).getSchemaId()).isGreaterThan(before.getSchemaId()); + TableDescriptor withHeaders = stored(path); + assertThatThrownBy(() -> addColumn(path, "_kafka_rescue", DataTypes.INT())) + .hasCauseInstanceOf(InvalidTableException.class); + assertThat(stored(path)).isEqualTo(withHeaders); + set(path, Collections.singletonMap("table.kafka.enabled", "false")); + addColumn(path, "unmapped_business_field", DataTypes.INT()); + assertThat(stored(path).getSchema().getRowType().getFieldNames()) + .contains("unmapped_business_field"); + assertThatThrownBy(() -> set(path, Collections.singletonMap("table.kafka.enabled", "true"))) + .hasCauseInstanceOf(InvalidTableException.class); + assertThat(KafkaTableSchema.isEnabled(stored(path))).isFalse(); + } + + @Test + void testCreateRejectsInvalidContractWithoutPublishingMetadata() throws Exception { + gateway.createDatabase(newCreateDatabaseRequest("ddl_invalid", true)).get(); + TablePath path = TablePath.of("ddl_invalid", "bad"); + assertThatThrownBy( + () -> + gateway.createTable( + newCreateTableRequest( + path, + enabled() + .property( + ConfigOptions + .TABLE_KAFKA_CONVERT_VALUE_TYPE, + "string") + .build(), + false)) + .get()) + .hasCauseInstanceOf(InvalidTableException.class); + assertThat(gateway.tableExists(newTableExistsRequest(path)).get().isExists()).isFalse(); + assertThat(CLUSTER.getZooKeeperClient().getCurrentSchemaId(path)).isZero(); + // Non-null added columns are rejected by generic schema evolution first, so exercise + // Kafka-specific rescue nullability through CREATE, where such columns are allowed. + TableDescriptor nonNullableRescue = + enabled() .schema( Schema.newBuilder() - .column("event_key", DataTypes.STRING()) - .column("event_body", DataTypes.BYTES()) - .column("event_time", DataTypes.TIMESTAMP(3)) + .column("_kafka_value", DataTypes.BYTES()) + .column("_kafka_rescue", DataTypes.STRING().copy(false)) .build()) - .distributedBy(2) - .logFormat(LogFormat.ARROW) - .customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, "string") - .customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "event_key") - .customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, "raw") - .customProperty(KafkaDataFormat.TIMESTAMP_COLUMN_CONFIG, "event_time") - .customProperty(KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, "EXCEPT_KEY") .build(); - KafkaTopicMapper mapper = new KafkaTopicMapper(); - Map tableIds = new LinkedHashMap<>(); - MetadataRequest request = new MetadataRequest(); - for (String database : Arrays.asList("kafka_ddl", "kafka_ddl_archive")) { - TablePath tablePath = mapper.toTablePath(database + ".events"); - assertThat(tablePath).isEqualTo(TablePath.of(database, "events")); - tableIds.put(tablePath, createTable(FLUSS_CLUSTER_EXTENSION, tablePath, descriptor)); - request.addTablePath() - .setDatabaseName(tablePath.getDatabaseName()) - .setTableName(tablePath.getTableName()); - } - FLUSS_CLUSTER_EXTENSION.waitUntilAllGatewayHasSameMetadata(); - List metadatas = - FLUSS_CLUSTER_EXTENSION - .newCoordinatorClient() - .metadata(request) - .get() - .getTableMetadatasList(); - assertThat(metadatas).hasSize(2); - assertThat(metadatas) - .extracting(PbTableMetadata::getTableId) - .containsExactlyInAnyOrderElementsOf(tableIds.values()) - .doesNotHaveDuplicates(); - for (PbTableMetadata metadata : metadatas) { - TablePath tablePath = - TablePath.of( - metadata.getTablePath().getDatabaseName(), - metadata.getTablePath().getTableName()); - TableDescriptor persisted = TableDescriptor.fromJsonBytes(metadata.getTableJson()); - KafkaTopicSchema mapping = new KafkaTopicSchemaResolver().resolve(persisted); - - assertThat(metadata.getTableId()).isEqualTo(tableIds.get(tablePath)); - assertThat(mapper.toTopicName(tablePath)) - .isIn("kafka_ddl.events", "kafka_ddl_archive.events"); - assertThat(mapper.toTableId(mapper.toTopicId(metadata.getTableId()))) - .isEqualTo(tableIds.get(tablePath)); - assertThat(metadata.getBucketMetadatasList()).hasSize(2); - assertThat(persisted.getCustomProperties()) - .containsAllEntriesOf(descriptor.getCustomProperties()); - assertThat(mapping.keyProjection().positions()).containsExactly(0); - assertThat(mapping.keyFormat()).isEqualTo(KafkaDataFormat.STRING); - assertThat(mapping.valueProjection().positions()).containsExactly(1); - assertThat(mapping.valueFormat()).isEqualTo(KafkaDataFormat.RAW); - assertThat(mapping.timestampPosition()).isEqualTo(2); - assertThat(persisted.getSchema().getRowType().getTypeAt(2)) - .isEqualTo(DataTypes.TIMESTAMP(3)); - } + assertThatThrownBy( + () -> + gateway.createTable( + newCreateTableRequest( + path, nonNullableRescue, false)) + .get()) + .hasCauseInstanceOf(InvalidTableException.class) + .hasStackTraceContaining("nullable STRING"); + assertThat(CLUSTER.getZooKeeperClient().getCurrentSchemaId(path)).isZero(); + assertThatThrownBy( + () -> + gateway.createTable( + newCreateTableRequest( + path, + enabled() + .property( + "table.kafka.convert.rescue-column", + "rescue") + .build(), + false)) + .get()) + .hasCauseInstanceOf(InvalidConfigException.class); + assertThat(gateway.tableExists(newTableExistsRequest(path)).get().isExists()).isFalse(); + createTable(CLUSTER, path, enabled().build()); + assertThat(KafkaTableSchema.isEnabled(stored(path))).isTrue(); + } + + @Test + void testUnsupportedMetadataOffsetCannotBeEnabled() throws Exception { + TablePath path = TablePath.of("ddl", "offset_capability"); + Schema schema = + Schema.newBuilder() + .column("_kafka_value", DataTypes.BYTES()) + .column( + "_kafka_metadata", + DataTypes.ROW( + DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP_LTZ(3)), + DataTypes.FIELD("timestamp_type", DataTypes.INT()), + DataTypes.FIELD("partition", DataTypes.INT()), + DataTypes.FIELD("offset", DataTypes.BIGINT()))) + .build(); + createTable( + CLUSTER, + path, + enabled() + .schema(schema) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, false) + .build()); + TableDescriptor before = stored(path); + assertThatThrownBy(() -> set(path, Collections.singletonMap("table.kafka.enabled", "true"))) + .hasCauseInstanceOf(InvalidTableException.class) + .hasStackTraceContaining("actual per-row offsets"); + assertThat(stored(path)).isEqualTo(before); + } + + @Test + void testCanonicalNameCheckedOnCreateAndEnable() throws Exception { + String database = String.join("", Collections.nCopies(128, "d")); + TablePath path = TablePath.of(database, String.join("", Collections.nCopies(128, "t"))); + gateway.createDatabase(newCreateDatabaseRequest(database, true)).get(); + assertThatThrownBy( + () -> + gateway.createTable( + newCreateTableRequest( + path, enabled().build(), false)) + .get()) + .hasCauseInstanceOf(InvalidTableException.class) + .hasStackTraceContaining("Kafka topic"); + createTable( + CLUSTER, + path, + enabled().property(ConfigOptions.TABLE_KAFKA_ENABLED, false).build()); + TableDescriptor before = stored(path); + assertThatThrownBy(() -> set(path, Collections.singletonMap("table.kafka.enabled", "true"))) + .hasCauseInstanceOf(InvalidTableException.class); + assertThat(stored(path)).isEqualTo(before); + } + + private GetTableInfoResponse info(TablePath path) throws Exception { + return gateway.getTableInfo(newGetTableInfoRequest(path)).get(); + } + + private TableDescriptor stored(TablePath path) throws Exception { + return TableDescriptor.fromJsonBytes(info(path).getTableJson()); + } + + private void set(TablePath path, Map properties) throws Exception { + gateway.alterTable( + newAlterTableRequest( + path, + properties, + Collections.emptyList(), + Collections.emptyList(), + false)) + .get(); + } + + private void reset(TablePath path, String property) throws Exception { + gateway.alterTable( + newAlterTableRequest( + path, + Collections.emptyMap(), + Collections.singletonList(property), + Collections.emptyList(), + false)) + .get(); + } + + private void addColumn(TablePath path, String name, DataType type) throws Exception { + PbAddColumn column = + new PbAddColumn() + .setColumnName(name) + .setDataTypeJson( + JsonSerdeUtils.writeValueAsBytes(type, DataTypeJsonSerde.INSTANCE)) + .setColumnPositionType(0); + AlterTableRequest request = + newAlterTableRequest( + path, + Collections.emptyMap(), + Collections.emptyList(), + Collections.singletonList(column), + false); + gateway.alterTable(request).get(); + } + + private static Map kafkaProperties(TableDescriptor table) { + Map properties = new HashMap<>(); + table.getProperties() + .forEach( + (key, value) -> { + if (key.startsWith("table.kafka.")) { + properties.put(key, value); + } + }); + return properties; + } + + private static TableDescriptor.Builder enabled() { + return TableDescriptor.builder() + .schema(Schema.newBuilder().column("_kafka_value", DataTypes.BYTES()).build()) + .distributedBy(1) + .logFormat(LogFormat.ARROW) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "binary"); } } diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolverTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolverTest.java index 66c13a4711a..b7e4174421c 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolverTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolverTest.java @@ -4,7 +4,7 @@ * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -17,334 +17,61 @@ package org.apache.fluss.kafka.schema; -import org.apache.fluss.kafka.format.KafkaDataFormat; -import org.apache.fluss.metadata.LogFormat; +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.metadata.KafkaTableSchema; import org.apache.fluss.metadata.Schema; import org.apache.fluss.metadata.TableDescriptor; -import org.apache.fluss.types.DataType; import org.apache.fluss.types.DataTypes; import org.junit.jupiter.api.Test; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.ValueSource; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; -/** Tests the DDL contract independently of Kafka request handling and record decoding. */ -public class KafkaTopicSchemaResolverTest { - +/** Tests the Kafka adapter's use of the shared DDL contract. */ +class KafkaTopicSchemaResolverTest { private final KafkaTopicSchemaResolver resolver = new KafkaTopicSchemaResolver(); @Test - public void testRawMappingSurvivesTableMetadataSerialization() { - Schema schema = - Schema.newBuilder() - .column("message", DataTypes.BYTES()) - .column("received_at", DataTypes.TIMESTAMP(3).copy(false)) - .column("attributes", headersType()) - .column("message_key", DataTypes.BYTES()) - .build(); - TableDescriptor table = - table(schema, "raw") - .customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, "raw") - .customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "message_key") - .customProperty(KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, "EXCEPT_KEY") - .customProperty(KafkaDataFormat.TIMESTAMP_COLUMN_CONFIG, "received_at") - .customProperty(KafkaDataFormat.HEADERS_COLUMN_CONFIG, "attributes") - .build(); - KafkaTopicSchema mapping = - resolver.resolve(TableDescriptor.fromJsonBytes(table.toJsonBytes())); - - assertThat(mapping.rowType()).isEqualTo(schema.getRowType()); - assertThat(mapping.keyFormat()).isEqualTo(KafkaDataFormat.RAW); - assertThat(mapping.keyProjection().positions()).containsExactly(3); - assertThat(mapping.keyProjection().nameAt(0)).isEqualTo("message_key"); - assertThat(mapping.valueProjection().positions()).containsExactly(0); - assertThat(mapping.valueProjection().dataTypeAt(0)).isEqualTo(DataTypes.BYTES()); - assertThat(mapping.timestampPosition()).isEqualTo(1); - assertThat(mapping.headersPosition()).isEqualTo(2); - assertThat(mapping).isEqualTo(resolver.resolve(table)); - assertThatThrownBy(() -> mapping.keyProjection().positions().add(1)) - .isInstanceOf(UnsupportedOperationException.class); - } - - @Test - public void testValueOnlyStringAndNullableColumns() { - for (boolean nullable : new boolean[] {true, false}) { - KafkaTopicSchema mapping = - resolver.resolve( - table( - Schema.newBuilder() - .column( - "body", - DataTypes.STRING().copy(nullable)) - .build(), - " STRING ") - .build()); - assertThat(mapping.keyFormat()).isNull(); - assertThat(mapping.keyProjection().isEmpty()).isTrue(); - assertThat(mapping.valueFormat()).isEqualTo(KafkaDataFormat.STRING); - assertThat(mapping.valueProjection().positions()).containsExactly(0); - assertThat(mapping.valueProjection().dataTypeAt(0).isNullable()).isEqualTo(nullable); - assertThat(mapping.timestampPosition()).isEqualTo(-1); - assertThat(mapping.headersPosition()).isEqualTo(-1); - } + void testEnabledContractUsesSharedValidation() { + TableDescriptor table = table().property(ConfigOptions.TABLE_KAFKA_ENABLED, true).build(); + assertThat(resolver.isEnabled(table)).isTrue(); + assertThat(resolver.resolve(table)).isEqualTo(KafkaTableSchema.resolve(table)); + assertThat(resolver.resolve(TableDescriptor.fromJsonBytes(table.toJsonBytes()))) + .isEqualTo(resolver.resolve(table)); } @Test - public void testMixedFormatsAndMetadataAreOptional() { - KafkaTopicSchema mapping = - resolver.resolve( - keyValueTable() - .customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, "string") - .customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "id") - .customProperty( - KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, " except_key ") - .build()); - assertThat(mapping.keyFormat()).isEqualTo(KafkaDataFormat.STRING); - assertThat(mapping.valueFormat()).isEqualTo(KafkaDataFormat.RAW); - assertThat(mapping.keyProjection().positions()).containsExactly(0); - assertThat(mapping.valueProjection().positions()).containsExactly(1); + void testCompatibleSchemaDoesNotEnableTable() { + assertThat(resolver.isEnabled(table().build())).isFalse(); + assertThatThrownBy(() -> resolver.resolve(table().build())) + .isInstanceOf(KafkaTopicSchemaException.class) + .hasMessageContaining("disabled"); } @Test - public void testRejectsUnsupportedTableKinds() { - Schema primaryKeySchema = - Schema.newBuilder() - .column("id", DataTypes.STRING().copy(false)) - .primaryKey("id") + void testInvalidEnabledContractIsNotTreatedAsDisabled() { + TableDescriptor table = + table().property(ConfigOptions.TABLE_KAFKA_ENABLED, true) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "schema_latest") .build(); - assertInvalid(table(primaryKeySchema, "string"), "must be a log table"); - assertInvalid(keyValueTable().partitionedBy("id"), "Partitioned Fluss tables"); - assertInvalid(valueTable().logFormat(LogFormat.INDEXED), "Arrow log format"); - } - - @Test - public void testRequiresExplicitValueFormat() { - assertInvalid( - TableDescriptor.builder() - .schema(Schema.newBuilder().column("body", DataTypes.BYTES()).build()) - .distributedBy(1) - .customProperty("fluss.value.format", "raw"), - KafkaDataFormat.VALUE_FORMAT_CONFIG); - } - - @ParameterizedTest - @ValueSource(strings = {"json", "avro", ""}) - public void testRejectsUnavailableFormats(String format) { - assertInvalid( - valueTable().customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, format), - "Unsupported Kafka data format"); - assertInvalid( - keyValueTable().customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, format), - "Unsupported Kafka data format"); - } - - @Test - public void testRejectsUnsupportedKafkaOptions() { - assertInvalid( - valueTable().customProperty("kafka.value.rescue-column", "body"), - "Unsupported Kafka table property"); - assertInvalid( - valueTable().customProperty("kafka.key.field", "body"), - "Unsupported Kafka table property"); - assertThat(resolver.resolve(valueTable().customProperty("owner", "team").build())) - .isNotNull(); - } - - @Test - public void testKeyFormatAndFieldMustBeSpecifiedTogether() { - assertInvalid( - keyValueTable().customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "id"), - "requires kafka.key.format"); - assertInvalid( - keyValueTable().customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, "string"), - "kafka.key.fields"); - } - - @ParameterizedTest - @ValueSource(strings = {"", " ", "missing", "id,id", "id,", ",id", "id,,body"}) - public void testRejectsInvalidKeyFields(String fields) { - assertThatThrownBy( - () -> - resolver.resolve( - keyValueTable() - .customProperty( - KafkaDataFormat.KEY_FORMAT_CONFIG, "string") - .customProperty( - KafkaDataFormat.KEY_FIELDS_CONFIG, fields) - .customProperty( - KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, - "EXCEPT_KEY") - .build())) - .isInstanceOf(KafkaTopicSchemaException.class); - } - - @Test - public void testRejectsAmbiguousAndEmptyValueProjections() { - assertInvalid( - keyValueTable() - .customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, "string") - .customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "id"), - "requires"); - assertInvalid( - valueTable().customProperty(KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, "bad"), - "Expected ALL or EXCEPT_KEY"); - assertInvalid( - valueTable() - .customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, "raw") - .customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "body") - .customProperty(KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, "EXCEPT_KEY"), - "at least one Fluss column"); - assertInvalid(keyValueTable(), "exactly one Fluss field"); - assertInvalid( - keyValueTable() - .customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, "string") - .customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "id,body") - .customProperty(KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, "EXCEPT_KEY"), - "at least one Fluss column"); - } - - @Test - public void testRejectsWrongPhysicalTypes() { - assertInvalid( - valueTable().customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, "string"), - "must be STRING"); - assertInvalid( - keyValueTable() - .customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, "raw") - .customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "id") - .customProperty(KafkaDataFormat.VALUE_FIELDS_INCLUDE_CONFIG, "EXCEPT_KEY"), - "must be BYTES"); - } - - @Test - public void testAcceptsTimestampWithEitherNullability() { - for (boolean nullable : new boolean[] {true, false}) { - TableDescriptor descriptor = - table( - Schema.newBuilder() - .column("body", DataTypes.BYTES()) - .column("ts", DataTypes.TIMESTAMP(3).copy(nullable)) - .build(), - "raw") - .customProperty(KafkaDataFormat.TIMESTAMP_COLUMN_CONFIG, "ts") - .build(); - KafkaTopicSchema mapping = new KafkaTopicSchemaResolver().resolve(descriptor); - assertThat(mapping.timestampPosition()).isEqualTo(1); - assertThat(mapping.valueProjection().positions()).containsExactly(0); - } - } - - @Test - public void testRejectsWrongTimestampTypes() { - for (DataType type : - new DataType[] { - DataTypes.STRING(), - DataTypes.TIMESTAMP_LTZ(3), - DataTypes.TIMESTAMP_LTZ(3).copy(false), - DataTypes.TIMESTAMP(2), - DataTypes.TIMESTAMP(6).copy(false) - }) { - assertInvalid( - table( - Schema.newBuilder() - .column("body", DataTypes.BYTES()) - .column("ts", type) - .build(), - "raw") - .customProperty(KafkaDataFormat.TIMESTAMP_COLUMN_CONFIG, "ts"), - "TIMESTAMP(3)"); - } - } - - @Test - public void testRejectsWrongHeaderTypes() { - for (DataType type : - new DataType[] { - DataTypes.STRING(), - headersType().copy(false), - DataTypes.ARRAY(DataTypes.STRING()), - DataTypes.ARRAY( - DataTypes.ROW( - DataTypes.FIELD("key", DataTypes.STRING()), - DataTypes.FIELD("value", DataTypes.BYTES()))), - DataTypes.ARRAY( - DataTypes.ROW( - DataTypes.FIELD("name", DataTypes.STRING()), - DataTypes.FIELD("value", DataTypes.BYTES().copy(false)))) - }) { - assertInvalid( - table( - Schema.newBuilder() - .column("body", DataTypes.BYTES()) - .column("attributes", type) - .build(), - "raw") - .customProperty(KafkaDataFormat.HEADERS_COLUMN_CONFIG, "attributes"), - "headers"); - } + assertThat(resolver.isEnabled(table)).isTrue(); + assertThatThrownBy(() -> resolver.resolve(table)) + .isInstanceOf(KafkaTopicSchemaException.class) + .hasMessageContaining("schema_latest"); } @Test - public void testRejectsConflictingOrMissingMetadataColumns() { - assertInvalid( - valueTable().customProperty(KafkaDataFormat.HEADERS_COLUMN_CONFIG, "missing"), - "does not exist"); - assertInvalid( - valueTable().customProperty(KafkaDataFormat.TIMESTAMP_COLUMN_CONFIG, " "), - "must not be empty"); - TableDescriptor.Builder table = - table( - Schema.newBuilder() - .column("body", DataTypes.BYTES()) - .column("ts", DataTypes.TIMESTAMP(3).copy(false)) - .build(), - "raw") - .customProperty(KafkaDataFormat.TIMESTAMP_COLUMN_CONFIG, "ts"); - assertInvalid( - TableDescriptor.builder(table.build()) - .customProperty(KafkaDataFormat.HEADERS_COLUMN_CONFIG, "ts"), - "same Fluss column"); - assertInvalid( - TableDescriptor.builder(table.build()) - .customProperty(KafkaDataFormat.KEY_FORMAT_CONFIG, "raw") - .customProperty(KafkaDataFormat.KEY_FIELDS_CONFIG, "ts"), - "metadata column"); - } - - private void assertInvalid(TableDescriptor.Builder table, String message) { - assertThatThrownBy(() -> resolver.resolve(table.build())) + void testMalformedPersistedEnableFlagIsAnExplicitContractError() { + TableDescriptor table = table().property("table.kafka.enabled", "maybe").build(); + assertThatThrownBy(() -> resolver.isEnabled(table)) .isInstanceOf(KafkaTopicSchemaException.class) - .hasMessageContaining(message); + .hasMessageContaining("table.kafka.enabled"); } - private static TableDescriptor.Builder keyValueTable() { - return table( - Schema.newBuilder() - .column("id", DataTypes.STRING()) - .column("body", DataTypes.BYTES()) - .build(), - "raw"); - } - - private static TableDescriptor.Builder valueTable() { - return table(Schema.newBuilder().column("body", DataTypes.BYTES()).build(), "raw"); - } - - private static TableDescriptor.Builder table(Schema schema, String valueFormat) { + private static TableDescriptor.Builder table() { return TableDescriptor.builder() - .schema(schema) - .distributedBy(2) - .logFormat(LogFormat.ARROW) - .customProperty(KafkaDataFormat.VALUE_FORMAT_CONFIG, valueFormat); - } - - private static DataType headersType() { - return DataTypes.ARRAY( - DataTypes.ROW( - DataTypes.FIELD("name", DataTypes.STRING()), - DataTypes.FIELD("value", DataTypes.BYTES()))); + .schema(Schema.newBuilder().column("_kafka_value", DataTypes.BYTES()).build()) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "binary"); } } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorServer.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorServer.java index 2cdd2c5bf97..aff8bb78761 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorServer.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorServer.java @@ -553,14 +553,6 @@ private void createDefaultDatabase() { metadataManager.createDatabase(DEFAULT_DATABASE, DatabaseDescriptor.EMPTY, true); LOG.info("Created default database '{}' because no database exists.", DEFAULT_DATABASE); } - // create Kafka default database if Kafka is enabled. - if (conf.get(ConfigOptions.KAFKA_ENABLED)) { - String kafkaDB = conf.get(ConfigOptions.KAFKA_DATABASE); - if (!databases.contains(kafkaDB)) { - metadataManager.createDatabase(kafkaDB, DatabaseDescriptor.EMPTY, true); - LOG.info("Created default database '{}' for Kafka protocol.", kafkaDB); - } - } } /** diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorService.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorService.java index 6bed0ecd3f0..e8e09dab26d 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorService.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorService.java @@ -507,7 +507,7 @@ public CompletableFuture createTable(CreateTableRequest req // validate table descriptor before creating table in lake or fluss metadata, // to avoid orphaned lake tables when validation fails - metadataManager.validateTableDescriptor(tableDescriptor); + metadataManager.validateTableDescriptor(tablePath, tableDescriptor); // the distribution and bucket count must be set now //noinspection OptionalGetWithoutIsPresent @@ -615,7 +615,8 @@ public CompletableFuture alterTable(AlterTableRequest reques tablePath, alterSchemaChanges, request.isIgnoreIfNotExists(), - currentSession().getPrincipal()); + currentSession().getPrincipal(), + coordinatorZkVersionSupplier.get()); } if (hasConfigChanges) { diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/MetadataManager.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/MetadataManager.java index 14b97c927eb..b1a5fae3d04 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/MetadataManager.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/MetadataManager.java @@ -39,6 +39,7 @@ import org.apache.fluss.metadata.DatabaseDescriptor; import org.apache.fluss.metadata.DatabaseInfo; import org.apache.fluss.metadata.DatabaseSummary; +import org.apache.fluss.metadata.KafkaTableSchema; import org.apache.fluss.metadata.LakeTableUtil; import org.apache.fluss.metadata.MergeEngineType; import org.apache.fluss.metadata.ResolvedPartitionSpec; @@ -58,6 +59,7 @@ import org.apache.fluss.server.zk.data.PartitionRegistration; import org.apache.fluss.server.zk.data.TableAssignment; import org.apache.fluss.server.zk.data.TableRegistration; +import org.apache.fluss.server.zk.data.ZkVersion; import org.apache.fluss.shaded.zookeeper3.org.apache.zookeeper.KeeperException; import org.apache.fluss.utils.function.RunnableWithException; import org.apache.fluss.utils.function.ThrowingRunnable; @@ -131,6 +133,12 @@ public void validateTableDescriptor(TableDescriptor tableDescriptor) { lakeCatalogDynamicLoader.getLakeCatalogContainer().getDataLakeFormat()); } + /** Validates a descriptor and its Kafka topic identity before persistence. */ + public void validateTableDescriptor(TablePath tablePath, TableDescriptor tableDescriptor) { + validateTableDescriptor(tableDescriptor); + KafkaTableSchema.validateTablePath(tablePath, tableDescriptor); + } + public void createDatabase( String databaseName, DatabaseDescriptor databaseDescriptor, boolean ignoreIfExists) throws DatabaseAlreadyExistException { @@ -458,8 +466,25 @@ public void alterTableSchema( boolean ignoreIfNotExists, FlussPrincipal flussPrincipal) throws TableNotExistException, TableNotPartitionedException { - try { + alterTableSchema( + tablePath, + schemaChanges, + ignoreIfNotExists, + flussPrincipal, + ZkVersion.MATCH_ANY_VERSION.getVersion()); + } + /** Alters schema with a table-version and coordinator-epoch fence against concurrent ALTER. */ + public void alterTableSchema( + TablePath tablePath, + List schemaChanges, + boolean ignoreIfNotExists, + FlussPrincipal flussPrincipal, + int coordinatorZkVersion) + throws TableNotExistException, TableNotPartitionedException { + try { + ZooKeeperClient.VersionedData versionedTable = + getTableRegistrationWithVersion(tablePath); TableInfo table = getTable(tablePath); TableDescriptor tableDescriptor = table.toTableDescriptor(); @@ -480,7 +505,12 @@ public void alterTableSchema( // Update Fluss schema (ZK) after Lake sync succeeds if (!newSchema.equals(table.getSchema())) { - zookeeperClient.registerSchema(tablePath, newSchema, table.getSchemaId() + 1); + zookeeperClient.registerSchemaAndBumpTableVersion( + tablePath, + newSchema, + table.getSchemaId() + 1, + versionedTable, + coordinatorZkVersion); } else { LOG.info( "Skipping schema evolution for table {} because the column(s) to add {} already exist.", @@ -852,7 +882,7 @@ private void doAlterTablePropertiesOnce( } // reuse the same validate logic with the createTable() method - validateTableDescriptor(newDescriptor); + validateTableDescriptor(tablePath, newDescriptor); beforeUpdate.accept(tableInfo, newDescriptor); diff --git a/fluss-server/src/main/java/org/apache/fluss/server/utils/TableDescriptorValidation.java b/fluss-server/src/main/java/org/apache/fluss/server/utils/TableDescriptorValidation.java index de40fbf01a8..fafbbf9a6c1 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/utils/TableDescriptorValidation.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/utils/TableDescriptorValidation.java @@ -32,6 +32,7 @@ import org.apache.fluss.metadata.ChangelogImage; import org.apache.fluss.metadata.DataLakeFormat; import org.apache.fluss.metadata.DeleteBehavior; +import org.apache.fluss.metadata.KafkaTableSchema; import org.apache.fluss.metadata.KvFormat; import org.apache.fluss.metadata.LogFormat; import org.apache.fluss.metadata.MergeEngineType; @@ -122,6 +123,8 @@ public static void validateTableDescriptor( } } + KafkaTableSchema.validate(tableDescriptor); + // check distribution checkDistribution(tableDescriptor, maxBucketNum); checkPrimaryKey(tableDescriptor); @@ -146,6 +149,10 @@ public static void validateTableDescriptor( /** Validates the schema after altering table columns. */ @Internal public static void validateAlterTableSchema(TableInfo table, Schema newSchema) { + TableDescriptor updated = + TableDescriptor.builder(table.toTableDescriptor()).schema(newSchema).build(); + KafkaTableSchema.validate(updated); + KafkaTableSchema.validateTablePath(table.getTablePath(), updated); checkSystemColumns(newSchema.getRowType()); if (table.getTableConfig() .getMergeEngineType() diff --git a/fluss-server/src/main/java/org/apache/fluss/server/zk/ZooKeeperClient.java b/fluss-server/src/main/java/org/apache/fluss/server/zk/ZooKeeperClient.java index 01f897cff29..a7a706c214e 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/zk/ZooKeeperClient.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/zk/ZooKeeperClient.java @@ -1315,6 +1315,36 @@ public int registerSchema(TablePath tablePath, Schema schema, int schemaId) thro return schemaId; } + /** + * Publishes a schema and advances the table registration version in one fenced transaction. + * Property ALTER validates against a schema snapshot and compares the table version, so schema + * ALTER must participate in the same CAS to prevent enabling an incompatible Kafka contract. + */ + public void registerSchemaAndBumpTableVersion( + TablePath tablePath, + Schema schema, + int schemaId, + VersionedData expectedTable, + int expectedCoordinatorEpochZkVersion) + throws Exception { + List ops = new ArrayList<>(); + ops.add( + zkClient.transactionOp() + .setData() + .withVersion(expectedTable.zkVersion()) + .forPath( + TableZNode.path(tablePath), + TableZNode.encode(expectedTable.data()))); + ops.add( + zkClient.transactionOp() + .create() + .withMode(CreateMode.PERSISTENT) + .forPath( + SchemaZNode.path(tablePath, schemaId), SchemaZNode.encode(schema))); + zkClient.transaction() + .forOperations(wrapRequestsWithEpochCheck(ops, expectedCoordinatorEpochZkVersion)); + } + /** Get the specific schema by schema id in ZK metadata. */ public Optional getSchemaById(TablePath tablePath, int schemaId) throws Exception { Optional bytes = getOrEmpty(SchemaZNode.path(tablePath, schemaId)); diff --git a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/KafkaTableSchemaAlterTest.java b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/KafkaTableSchemaAlterTest.java new file mode 100644 index 00000000000..e76dbda662d --- /dev/null +++ b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/KafkaTableSchemaAlterTest.java @@ -0,0 +1,182 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.server.coordinator; + +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.config.Configuration; +import org.apache.fluss.exception.FlussRuntimeException; +import org.apache.fluss.metadata.DatabaseDescriptor; +import org.apache.fluss.metadata.KafkaTableSchema; +import org.apache.fluss.metadata.Schema; +import org.apache.fluss.metadata.TableChange; +import org.apache.fluss.metadata.TableDescriptor; +import org.apache.fluss.metadata.TablePath; +import org.apache.fluss.server.entity.TablePropertyChanges; +import org.apache.fluss.server.zk.NOPErrorHandler; +import org.apache.fluss.server.zk.ZooKeeperClient; +import org.apache.fluss.server.zk.ZooKeeperExtension; +import org.apache.fluss.server.zk.data.TableRegistration; +import org.apache.fluss.server.zk.data.ZkVersion; +import org.apache.fluss.shaded.zookeeper3.org.apache.zookeeper.KeeperException; +import org.apache.fluss.testutils.common.AllCallbackWrapper; +import org.apache.fluss.types.DataTypes; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import java.util.Collections; +import java.util.function.BiConsumer; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Deterministic interleavings of schema ALTER and Kafka enablement using real ZooKeeper. */ +class KafkaTableSchemaAlterTest { + + @RegisterExtension + public static final AllCallbackWrapper ZK = + new AllCallbackWrapper<>(new ZooKeeperExtension()); + + private ZooKeeperClient zk; + private MetadataManager manager; + private int epoch; + + @BeforeEach + void setup() throws Exception { + zk = ZK.getCustomExtension().getZooKeeperClient(NOPErrorHandler.INSTANCE); + epoch = zk.fenceBecomeCoordinatorLeader("kafka-ddl").getCoordinatorEpochZkVersion(); + manager = + new MetadataManager( + zk, + new Configuration(), + new LakeCatalogDynamicLoader(new Configuration(), null, true)); + manager.createDatabase("ddl", DatabaseDescriptor.EMPTY, true); + } + + @Test + void testConcurrentSchemaChangePreventsStaleEnablement() throws Exception { + TablePath path = create("schema_wins"); + assertThatThrownBy( + () -> + enable( + path, + (oldTable, updated) -> + manager.alterTableSchema( + path, + Collections.singletonList( + TableChange.addColumn( + "unmapped", + DataTypes.STRING(), + null, + TableChange.ColumnPosition + .last())), + false, + null, + epoch))) + .isInstanceOf(FlussRuntimeException.class) + .hasMessageContaining("Concurrent metadata change"); + assertThat(manager.getTable(path).getTableConfig().isKafkaEnabled()).isFalse(); + assertThat(manager.getTable(path).getSchema().getRowType().getFieldNames()) + .contains("unmapped"); + } + + @Test + void testConcurrentEnablementPreventsStaleSchemaPublication() throws Exception { + TablePath path = create("enable_wins"); + ZooKeeperClient.VersionedData snapshot = + zk.getTableWithVersion(path).get(); + int schemaId = zk.getCurrentSchemaId(path); + Schema incompatible = + Schema.newBuilder() + .column("_kafka_value", DataTypes.BYTES()) + .column("unmapped", DataTypes.STRING()) + .build(); + // This candidate is valid for the disabled snapshot, but not after the competing enable. + KafkaTableSchema.validate( + TableDescriptor.builder(manager.getTable(path).toTableDescriptor()) + .schema(incompatible) + .build()); + enable(path, (oldTable, updated) -> {}); + assertThatThrownBy( + () -> + zk.registerSchemaAndBumpTableVersion( + path, incompatible, schemaId + 1, snapshot, epoch)) + .isInstanceOf(KeeperException.BadVersionException.class); + assertThat(zk.getCurrentSchemaId(path)).isEqualTo(schemaId); + assertThat(manager.getTable(path).getTableConfig().isKafkaEnabled()).isTrue(); + assertThat(manager.getTable(path).getSchema().getRowType().getFieldNames()) + .containsExactly("_kafka_value"); + } + + @Test + void testSchemaFailureDoesNotAdvanceTableVersion() throws Exception { + TablePath path = create("atomic_failure"); + ZooKeeperClient.VersionedData snapshot = + zk.getTableWithVersion(path).get(); + int schemaId = zk.getCurrentSchemaId(path); + Schema schema = manager.getTable(path).getSchema(); + assertThatThrownBy( + () -> + zk.registerSchemaAndBumpTableVersion( + path, schema, schemaId, snapshot, epoch)) + .isInstanceOf(KeeperException.NodeExistsException.class); + assertThat(zk.getTableWithVersion(path).get().zkVersion()).isEqualTo(snapshot.zkVersion()); + assertThat(zk.getCurrentSchemaId(path)).isEqualTo(schemaId); + zk.fenceBecomeCoordinatorLeader("next-coordinator"); + assertThatThrownBy( + () -> + zk.registerSchemaAndBumpTableVersion( + path, schema, schemaId + 1, snapshot, epoch)) + .isInstanceOf(KeeperException.BadVersionException.class); + assertThat(zk.getTableWithVersion(path).get().zkVersion()).isEqualTo(snapshot.zkVersion()); + assertThat(zk.getCurrentSchemaId(path)).isEqualTo(schemaId); + } + + private TablePath create(String name) { + TablePath path = TablePath.of("ddl", name); + TableDescriptor table = + TableDescriptor.builder() + .schema( + Schema.newBuilder() + .column("_kafka_value", DataTypes.BYTES()) + .build()) + .distributedBy(1) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "binary") + .build() + .withReplicationFactor(1); + manager.createTable(path, zk.getDefaultRemoteDataDir(), table, null, false); + return path; + } + + private void enable( + TablePath path, + BiConsumer beforeUpdate) { + TablePropertyChanges.Builder changes = new TablePropertyChanges.Builder(); + changes.setTableProperty("table.kafka.enabled", "true"); + manager.alterTableProperties( + path, + Collections.singletonList(TableChange.set("table.kafka.enabled", "true")), + changes.build(), + false, + null, + beforeUpdate, + (oldTable, updated) -> {}, + ZkVersion.MATCH_ANY_VERSION.getVersion()); + } +} diff --git a/website/docs/maintenance/configuration.md b/website/docs/maintenance/configuration.md index c88f83ccf08..04efa02f6c3 100644 --- a/website/docs/maintenance/configuration.md +++ b/website/docs/maintenance/configuration.md @@ -259,7 +259,42 @@ Kafka protocol compatibility is still in development. |--------------------------------|----------|---------|--------------------------------------------------------------------------------------------------------------------| | kafka.enabled | Boolean | false | Whether enable Fluss Kafka. Disabled by default. When this option is set to true, the Fluss Kafka will be enabled. | | kafka.listener.names | String | KAFKA | The listener names for Kafka wire protocol communication. Support multiple listener names, separated by comma. | -| kafka.database | String | kafka | The database for Fluss Kafka. The default database is `kafka`. | +| kafka.default.database | String | kafka | Database used for unqualified Kafka topic names. Fully qualified names can access other databases. This option does not create a database. | Kafka listeners share `netty.connection.max-idle-time` with native Fluss RPC listeners (see [Netty](#netty)). The default is `10min`; connections close when neither reads nor writes occur for the configured duration. + +Kafka tables must be created through Fluss or the Flink Fluss catalog. A topic named +`sales.orders` maps to that table directly; `orders` maps to the same-named table in +`kafka.default.database`. Databases and tables must already exist. Reverse mapping always uses +the fully qualified name. All TabletServers must use the same default database. Changing it +redirects unqualified names, so migrate clients to fully qualified names before changing the +configuration. The prototype `kafka.database` option is no longer used. + +### Kafka table properties + +| Option | Type | Default | Description | +|---|---|---|---| +| table.kafka.enabled | Boolean | false | Explicitly expose a non-partitioned Arrow log table as a Kafka topic. CREATE and ALTER validate the contract before persisting it. | +| table.kafka.convert.value.type | String | (None) | Required when enabled: `binary`, `string`, or `parse_json`. | +| table.kafka.convert.value.layout | String | (None) | JSON uses `flat` when omitted; `nested` targets `_kafka_value ROW<...>`. Setting this property for binary/string is invalid. | +| table.kafka.convert.key.type | String | (None) | Only `binary` is supported, and requires `_kafka_key BYTES`. Declaring that column implies binary without adding a property. | + +Binary and string values require a user-declared `_kafka_value BYTES` or `_kafka_value STRING` +column respectively. Flat JSON targets business columns at the table level; nested JSON targets +the fields of `_kafka_value ROW<...>`. Both exclude reserved system names from business matching. +Optional `_kafka_key BYTES` and `_kafka_headers MAP` columns retain keys and headers. +Duplicate header names use the last value. Optional `_kafka_rescue STRING` must be nullable and +collects unknown or reserved JSON members; it requires no mapping property. Binary/string +conversion writes null to a declared rescue column. + +Enabling Kafka compatibility does not add columns or persist default Kafka properties. When +disabled, ordinary Fluss tables are not subject to Kafka-specific schema validation. Prototype +`kafka.key.*`, `kafka.value.*`, `kafka.metadata.*`, and `fluss.*` conversion properties are not +accepted on enabled tables. + +The `_kafka_metadata` contract is +`ROW`. +Enabling tables with this column is currently rejected until native append can assign and +persist each row's actual offset. Other optional system columns can be used independently. +These DDL contracts do not by themselves advertise Kafka Metadata or Produce API support. From 0aee8ee0d85bc43bd2b559536feca4da47bc3fe2 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Mon, 12 Oct 2026 17:15:35 +0800 Subject: [PATCH 06/15] [kafka] Validate recursive JSON schemas and default to fluss Default bare topic names to the native fluss database while preserving explicit database overrides, canonical names and no automatic Kafka database creation. Validate JSON business types recursively before CREATE, ALTER and enablement. Reject non-STRING map keys at every nesting level and schemas deeper than the supported limit. Expose the validation entry point and depth constant for subsequent decoders. Count business fields equally in flat and nested layouts, without charging a level for the physical _kafka_value ROW container. Keep disabled tables outside Kafka validation and preserve user schemas and explicit properties. Cover complex scalar/container types, depth boundaries, failed DDL state preservation and successful enablement with real RPC tests. Validation: Java 11 clean reactor verify passed 161 targeted cases, including snapshot lease and concurrent ALTER regressions. Checkstyle, Spotless, RAT and git diff checks passed. The five prior PR03 commits were rebased onto PR02 97d20b18549ec34f49e4e608392cdf1ec3a1b68a and checked with range-diff. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 92/92 AI-Contributed/UT: 292/292 --- .../apache/fluss/config/ConfigOptions.java | 2 +- .../fluss/metadata/KafkaTableSchema.java | 80 ++++++++ .../fluss/metadata/KafkaTableSchemaTest.java | 103 ++++++++++ .../apache/fluss/kafka/KafkaConfigsTest.java | 7 +- .../kafka/mapping/KafkaTopicMapperTest.java | 4 +- .../kafka/schema/KafkaTableMappingITCase.java | 178 ++++++++++++++++++ website/docs/maintenance/configuration.md | 10 +- 7 files changed, 377 insertions(+), 7 deletions(-) diff --git a/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java b/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java index 8e34ddb8bb6..43630778e8e 100644 --- a/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java +++ b/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java @@ -2982,7 +2982,7 @@ public class ConfigOptions { public static final ConfigOption KAFKA_DEFAULT_DATABASE = key("kafka.default.database") .stringType() - .defaultValue("kafka") + .defaultValue("fluss") .withDescription( "Default database for unqualified Kafka topic names. Fully qualified names may " + "access other databases. Databases must already exist; this option does not create them."); diff --git a/fluss-common/src/main/java/org/apache/fluss/metadata/KafkaTableSchema.java b/fluss-common/src/main/java/org/apache/fluss/metadata/KafkaTableSchema.java index 7c040f15871..990cec50416 100644 --- a/fluss-common/src/main/java/org/apache/fluss/metadata/KafkaTableSchema.java +++ b/fluss-common/src/main/java/org/apache/fluss/metadata/KafkaTableSchema.java @@ -22,6 +22,7 @@ import org.apache.fluss.config.Configuration; import org.apache.fluss.config.TableConfig; import org.apache.fluss.exception.InvalidTableException; +import org.apache.fluss.types.ArrayType; import org.apache.fluss.types.BytesType; import org.apache.fluss.types.DataType; import org.apache.fluss.types.DataTypes; @@ -53,6 +54,9 @@ public final class KafkaTableSchema { public static final String METADATA_COLUMN = "_kafka_metadata"; public static final String RESCUE_COLUMN = "_kafka_rescue"; + /** Maximum recursive depth below each top-level JSON business field. */ + public static final int MAX_JSON_SCHEMA_NESTING_DEPTH = 64; + private static final Set RESERVED_NAMES = Collections.unmodifiableSet( new HashSet<>( @@ -271,6 +275,20 @@ public static KafkaTableSchema resolve(TableDescriptor table) { } valuePositions.add(valuePosition); } + if (valueType == ValueType.PARSE_JSON) { + for (int position : valuePositions) { + DataType type = rowType.getTypeAt(position); + if (valueLayout == ValueLayout.NESTED) { + // The physical value container represents the JSON object root, just like + // the table row in flat layout; it must not add a business nesting level. + for (DataType field : ((RowType) type).getChildren()) { + validateJsonType(field); + } + } else { + validateJsonType(type); + } + } + } return new KafkaTableSchema( rowType, valueType, @@ -281,6 +299,68 @@ public static KafkaTableSchema resolve(TableDescriptor table) { rescuePosition); } + /** + * Validates a top-level JSON business field without parser or protocol dependencies. DDL and + * decoder construction share this contract. Each business field starts at depth zero; ROW + * fields, ARRAY elements and MAP values each advance the depth by one. Nullability and scalar + * value constraints are enforced when decoding individual records. The JSON object root + * (including the physical nested _kafka_value ROW) is not counted. + */ + public static void validateJsonType(DataType type) { + validateJsonType(type, 0); + } + + private static void validateJsonType(DataType type, int depth) { + if (depth > MAX_JSON_SCHEMA_NESTING_DEPTH) { + throw invalid( + "Kafka JSON schema exceeds the maximum nesting depth of " + + MAX_JSON_SCHEMA_NESTING_DEPTH + + "."); + } + switch (type.getTypeRoot()) { + case CHAR: + case STRING: + case BOOLEAN: + case BINARY: + case BYTES: + case DECIMAL: + case TINYINT: + case SMALLINT: + case INTEGER: + case BIGINT: + case FLOAT: + case DOUBLE: + case DATE: + case TIME_WITHOUT_TIME_ZONE: + case TIMESTAMP_WITHOUT_TIME_ZONE: + case TIMESTAMP_WITH_LOCAL_TIME_ZONE: + return; + case ARRAY: + validateJsonType(((ArrayType) type).getElementType(), depth + 1); + return; + case MAP: + MapType map = (MapType) type; + if (!(map.getKeyType() instanceof StringType)) { + throw invalid( + "Kafka JSON format only supports STRING map keys, but found " + + type.asSummaryString() + + "."); + } + validateJsonType(map.getValueType(), depth + 1); + return; + case ROW: + for (DataType field : ((RowType) type).getChildren()) { + validateJsonType(field, depth + 1); + } + return; + default: + throw invalid( + "Kafka JSON format does not support Fluss type " + + type.asSummaryString() + + "."); + } + } + private static void validateOptions(TableDescriptor table) { for (String name : table.getProperties().keySet()) { if (name.startsWith("table.kafka.") && !OPTIONS.contains(name)) { diff --git a/fluss-common/src/test/java/org/apache/fluss/metadata/KafkaTableSchemaTest.java b/fluss-common/src/test/java/org/apache/fluss/metadata/KafkaTableSchemaTest.java index e2b9c5be187..809f13ac553 100644 --- a/fluss-common/src/test/java/org/apache/fluss/metadata/KafkaTableSchemaTest.java +++ b/fluss-common/src/test/java/org/apache/fluss/metadata/KafkaTableSchemaTest.java @@ -186,6 +186,109 @@ void testNestedJsonTargetsOneRow() { assertThat(resolved.rescuePosition()).isZero(); } + @ParameterizedTest + @ValueSource(strings = {"flat", "nested"}) + void testJsonRejectsNonStringMapKeysRecursively(String layout) { + for (DataType key : Arrays.asList(DataTypes.INT(), DataTypes.CHAR(10), DataTypes.BYTES())) { + DataType invalidMap = DataTypes.MAP(key, DataTypes.STRING()); + for (DataType type : + Arrays.asList( + invalidMap, + DataTypes.ARRAY(invalidMap), + DataTypes.ROW(DataTypes.FIELD("attributes", invalidMap)), + DataTypes.MAP(DataTypes.STRING(), DataTypes.ARRAY(invalidMap)))) { + TableDescriptor table = jsonTable(layout, type); + assertInvalid(table, "only supports STRING map keys"); + assertThatCode( + () -> + KafkaTableSchema.validate( + TableDescriptor.builder(table) + .property( + ConfigOptions.TABLE_KAFKA_ENABLED, + false) + .build())) + .doesNotThrowAnyException(); + } + } + } + + @ParameterizedTest + @ValueSource(strings = {"flat", "nested"}) + void testJsonAcceptsAllScalarTypesInNestedContainers(String layout) { + for (DataType scalar : + Arrays.asList( + DataTypes.CHAR(10), + DataTypes.STRING(), + DataTypes.BOOLEAN(), + DataTypes.BINARY(4), + DataTypes.BYTES(), + DataTypes.DECIMAL(18, 2), + DataTypes.TINYINT(), + DataTypes.SMALLINT(), + DataTypes.INT(), + DataTypes.BIGINT(), + DataTypes.FLOAT(), + DataTypes.DOUBLE(), + DataTypes.DATE(), + DataTypes.TIME(3), + DataTypes.TIMESTAMP(9), + DataTypes.TIMESTAMP_LTZ(9))) { + DataType type = + DataTypes.ARRAY( + DataTypes.MAP( + DataTypes.STRING().copy(false), + DataTypes.ROW( + DataTypes.FIELD("_kafka_key", scalar.copy(false))))); + TableDescriptor table = jsonTable(layout, type); + byte[] before = table.toJsonBytes(); + assertThatCode(() -> KafkaTableSchema.validate(table)).doesNotThrowAnyException(); + assertThat(table.toJsonBytes()).isEqualTo(before); + assertThat(KafkaTableSchema.resolve(TableDescriptor.fromJsonBytes(before))) + .isEqualTo(KafkaTableSchema.resolve(table)); + } + } + + @ParameterizedTest + @ValueSource(strings = {"flat", "nested"}) + void testJsonBusinessNestingDepthBoundary(String layout) { + // Exercise every recursive edge, including map values (keys do not add a level). + DataType boundary = DataTypes.STRING(); + for (int depth = 0; depth < KafkaTableSchema.MAX_JSON_SCHEMA_NESTING_DEPTH; depth++) { + if (depth % 3 == 0) { + boundary = DataTypes.ARRAY(boundary); + } else if (depth % 3 == 1) { + boundary = DataTypes.MAP(DataTypes.STRING(), boundary); + } else { + boundary = DataTypes.ROW(DataTypes.FIELD("child", boundary)); + } + } + TableDescriptor valid = jsonTable(layout, boundary); + assertThatCode(() -> KafkaTableSchema.validate(valid)).doesNotThrowAnyException(); + TableDescriptor tooDeep = jsonTable(layout, DataTypes.ARRAY(boundary)); + assertInvalid(tooDeep, "maximum nesting depth of 64"); + assertThatCode( + () -> + KafkaTableSchema.validate( + TableDescriptor.builder(tooDeep) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, false) + .build())) + .doesNotThrowAnyException(); + } + + private static TableDescriptor jsonTable(String layout, DataType businessType) { + Schema schema = + "flat".equals(layout) + ? Schema.newBuilder().column("payload", businessType).build() + : Schema.newBuilder() + .column( + "_kafka_value", + DataTypes.ROW(DataTypes.FIELD("payload", businessType))) + .build(); + return enabled(schema, "parse_json") + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT, layout) + .build(); + } + @ParameterizedTest @ValueSource( strings = { diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaConfigsTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaConfigsTest.java index 55e8ba6fc57..17a8c6c0f9c 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaConfigsTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaConfigsTest.java @@ -36,14 +36,14 @@ public void testFromMap() throws Exception { Map map = new HashMap<>(); map.put(ConfigOptions.KAFKA_ENABLED.key(), "true"); map.put(ConfigOptions.KAFKA_LISTENER_NAMES.key(), "kafka,kafka_sasl"); - map.put(ConfigOptions.KAFKA_DEFAULT_DATABASE.key(), "fluss"); + map.put(ConfigOptions.KAFKA_DEFAULT_DATABASE.key(), "sales"); Configuration configuration = Configuration.fromMap(map); assertThat(configuration.getBoolean(ConfigOptions.KAFKA_ENABLED)).isTrue(); assertThat(configuration.get(ConfigOptions.KAFKA_LISTENER_NAMES)) .isEqualTo(Arrays.asList("kafka", "kafka_sasl")); assertThat(configuration.getString(ConfigOptions.KAFKA_DEFAULT_DATABASE)) - .isEqualTo("fluss"); + .isEqualTo("sales"); } @Test @@ -53,6 +53,7 @@ public void testFromDefault() throws Exception { assertThat(configuration.get(ConfigOptions.KAFKA_LISTENER_NAMES)) .isEqualTo(Collections.singletonList("KAFKA")); assertThat(configuration.getString(ConfigOptions.KAFKA_DEFAULT_DATABASE)) - .isEqualTo("kafka"); + .isEqualTo("fluss"); + assertThat(configuration.toMap()).doesNotContainKey("kafka.default.database"); } } diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/mapping/KafkaTopicMapperTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/mapping/KafkaTopicMapperTest.java index 7ef7e281244..f70c7ffca22 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/mapping/KafkaTopicMapperTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/mapping/KafkaTopicMapperTest.java @@ -53,7 +53,9 @@ public void testQualifiedNamesAcrossDatabases() { @Test void testDefaultDatabaseAliases() { - assertThat(mapper.toTablePath("orders")).isEqualTo(mapper.toTablePath("kafka.orders")); + assertThat(mapper.toTablePath("orders")).isEqualTo(TablePath.of("fluss", "orders")); + assertThat(mapper.toTopicName(mapper.toTablePath("orders"))).isEqualTo("fluss.orders"); + assertThat(mapper.toTablePath("kafka.orders")).isEqualTo(TablePath.of("kafka", "orders")); Configuration config = new Configuration(); config.set(ConfigOptions.KAFKA_DEFAULT_DATABASE, "sales"); KafkaTopicMapper custom = new KafkaTopicMapper(config); diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java index 45744dbc87a..76b9c0f773d 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java @@ -41,6 +41,8 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; import java.util.Arrays; import java.util.Collections; @@ -121,6 +123,182 @@ void testNoAutomaticKafkaDatabaseAndQualifiedAccessWithoutDefaultDatabase() thro assertThat(aliasMapper.toTopicName(sales)).isEqualTo("sales.orders"); } + @Test + void testDefaultAliasUsesExistingFlussDatabase() throws Exception { + KafkaTopicMapper mapper = new KafkaTopicMapper(); + TablePath path = mapper.toTablePath("default_alias"); + assertThat(path).isEqualTo(TablePath.of(DEFAULT_DATABASE, "default_alias")); + createTable(CLUSTER, path, enabled().build()); + assertThat(info(mapper.toTablePath("fluss.default_alias")).getTableId()) + .isEqualTo(info(path).getTableId()); + assertThat(mapper.toTopicName(path)).isEqualTo("fluss.default_alias"); + } + + @ParameterizedTest + @ValueSource(strings = {"flat", "nested"}) + void testJsonCreateAndEnableRejectUnsupportedBusinessSchema(String layout) throws Exception { + gateway.createDatabase(newCreateDatabaseRequest("ddl_json", true)).get(); + DataType invalidMap = DataTypes.MAP(DataTypes.INT(), DataTypes.STRING()); + int index = 0; + for (DataType type : + Arrays.asList( + invalidMap, + DataTypes.ARRAY(invalidMap), + DataTypes.ROW(DataTypes.FIELD("attributes", invalidMap)), + DataTypes.MAP(DataTypes.STRING(), DataTypes.ARRAY(invalidMap)))) { + TablePath path = TablePath.of("ddl_json", layout + "_invalid_" + index++); + TableDescriptor descriptor = jsonTable(layout, type).build(); + assertThatThrownBy( + () -> + gateway.createTable( + newCreateTableRequest(path, descriptor, false)) + .get()) + .hasCauseInstanceOf(InvalidTableException.class) + .hasStackTraceContaining("only supports STRING map keys"); + assertThat(gateway.tableExists(newTableExistsRequest(path)).get().isExists()).isFalse(); + assertThat(CLUSTER.getZooKeeperClient().getCurrentSchemaId(path)).isZero(); + + // The identical schema is legal for an ordinary, explicitly disabled Fluss table. + createTable( + CLUSTER, + path, + TableDescriptor.builder(descriptor) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, false) + .build()); + GetTableInfoResponse before = info(path); + assertThatThrownBy( + () -> + set( + path, + Collections.singletonMap( + "table.kafka.enabled", "true"))) + .hasCauseInstanceOf(InvalidTableException.class) + .hasStackTraceContaining("only supports STRING map keys"); + assertUnchanged(path, before); + assertThat(KafkaTableSchema.isEnabled(stored(path))).isFalse(); + } + } + + @ParameterizedTest + @ValueSource(strings = {"flat", "nested"}) + void testJsonComplexSchemaRoundtripAndEnable(String layout) throws Exception { + TablePath path = TablePath.of("ddl_json", layout + "_complex"); + DataType type = + DataTypes.ARRAY( + DataTypes.MAP( + DataTypes.STRING(), + DataTypes.ROW( + DataTypes.FIELD("amount", DataTypes.DECIMAL(18, 2)), + DataTypes.FIELD("time", DataTypes.TIMESTAMP_LTZ(3)), + DataTypes.FIELD("_kafka_key", DataTypes.BYTES())))); + TableDescriptor descriptor = + jsonTable(layout, type).property(ConfigOptions.TABLE_KAFKA_ENABLED, false).build(); + createTable(CLUSTER, path, descriptor); + GetTableInfoResponse before = info(path); + set(path, Collections.singletonMap("table.kafka.enabled", "true")); + TableDescriptor actual = stored(path); + assertThat(actual.getSchema()).isEqualTo(descriptor.getSchema()); + assertThat(info(path).getSchemaId()).isEqualTo(before.getSchemaId()); + Map expected = new HashMap<>(kafkaProperties(descriptor)); + expected.put("table.kafka.enabled", "true"); + assertThat(kafkaProperties(actual)).isEqualTo(expected); + addColumn(path, "_kafka_rescue", DataTypes.STRING()); + assertThat(info(path).getSchemaId()).isGreaterThan(before.getSchemaId()); + assertThat(stored(path).getSchema().getRowType().getTypeAt(0)) + .isEqualTo(descriptor.getSchema().getRowType().getTypeAt(0)); + assertThat(KafkaTableSchema.resolve(stored(path)).rescuePosition()).isEqualTo(1); + assertThat(kafkaProperties(stored(path))).isEqualTo(expected); + } + + @Test + void testJsonSchemaAlterRejectsUnsupportedNestedMapAtomically() throws Exception { + TablePath path = TablePath.of("ddl_json", "alter_business_type"); + createTable(CLUSTER, path, jsonTable("flat", DataTypes.STRING()).build()); + DataType invalid = + DataTypes.ARRAY( + DataTypes.ROW( + DataTypes.FIELD( + "attributes", + DataTypes.MAP(DataTypes.INT(), DataTypes.STRING())))); + GetTableInfoResponse before = info(path); + assertThatThrownBy(() -> addColumn(path, "bad", invalid)) + .hasCauseInstanceOf(InvalidTableException.class) + .hasStackTraceContaining("only supports STRING map keys"); + assertUnchanged(path, before); + addColumn( + path, "good", DataTypes.MAP(DataTypes.STRING(), DataTypes.ARRAY(DataTypes.INT()))); + assertThat(info(path).getSchemaId()).isGreaterThan(before.getSchemaId()); + set(path, Collections.singletonMap("table.kafka.enabled", "false")); + addColumn(path, "ordinary", invalid); + GetTableInfoResponse disabled = info(path); + assertThat(stored(path).getSchema().getRowType().getFieldNames()).contains("ordinary"); + assertThatThrownBy(() -> set(path, Collections.singletonMap("table.kafka.enabled", "true"))) + .hasCauseInstanceOf(InvalidTableException.class) + .hasStackTraceContaining("only supports STRING map keys"); + assertUnchanged(path, disabled); + } + + @ParameterizedTest + @ValueSource(strings = {"flat", "nested"}) + void testJsonDepthValidationThroughDdl(String layout) throws Exception { + DataType boundary = DataTypes.STRING(); + for (int i = 0; i < KafkaTableSchema.MAX_JSON_SCHEMA_NESTING_DEPTH; i++) { + boundary = DataTypes.ARRAY(boundary); + } + TablePath valid = TablePath.of("ddl_json", layout + "_depth_boundary"); + TableDescriptor descriptor = jsonTable(layout, boundary).build(); + createTable(CLUSTER, valid, descriptor); + assertThat(stored(valid).getSchema()).isEqualTo(descriptor.getSchema()); + TablePath invalid = TablePath.of("ddl_json", layout + "_too_deep"); + TableDescriptor tooDeep = jsonTable(layout, DataTypes.ARRAY(boundary)).build(); + assertThatThrownBy( + () -> + gateway.createTable(newCreateTableRequest(invalid, tooDeep, false)) + .get()) + .hasCauseInstanceOf(InvalidTableException.class) + .hasStackTraceContaining("maximum nesting depth of 64"); + assertThat(gateway.tableExists(newTableExistsRequest(invalid)).get().isExists()).isFalse(); + assertThat(CLUSTER.getZooKeeperClient().getCurrentSchemaId(invalid)).isZero(); + createTable( + CLUSTER, + invalid, + TableDescriptor.builder(tooDeep) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, false) + .build()); + GetTableInfoResponse before = info(invalid); + assertThatThrownBy( + () -> set(invalid, Collections.singletonMap("table.kafka.enabled", "true"))) + .hasCauseInstanceOf(InvalidTableException.class) + .hasStackTraceContaining("maximum nesting depth of 64"); + assertUnchanged(invalid, before); + } + + private void assertUnchanged(TablePath path, GetTableInfoResponse before) throws Exception { + GetTableInfoResponse after = info(path); + assertThat(after.getTableJson()).isEqualTo(before.getTableJson()); + assertThat(after.getSchemaId()).isEqualTo(before.getSchemaId()); + } + + private static TableDescriptor.Builder jsonTable(String layout, DataType businessType) { + Schema schema = + "flat".equals(layout) + ? Schema.newBuilder().column("payload", businessType).build() + : Schema.newBuilder() + .column( + "_kafka_value", + DataTypes.ROW(DataTypes.FIELD("payload", businessType))) + .build(); + TableDescriptor.Builder builder = + enabled() + .schema(schema) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "parse_json"); + // Omitted flat layout must remain omitted through CREATE and ALTER. + if ("nested".equals(layout)) { + builder.property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT, layout); + } + return builder; + } + @Test void testOnlyRequestedKafkaPropertiesArePersisted() throws Exception { TablePath path = TablePath.of("ddl", "roundtrip"); diff --git a/website/docs/maintenance/configuration.md b/website/docs/maintenance/configuration.md index 04efa02f6c3..45cc108f29b 100644 --- a/website/docs/maintenance/configuration.md +++ b/website/docs/maintenance/configuration.md @@ -259,14 +259,14 @@ Kafka protocol compatibility is still in development. |--------------------------------|----------|---------|--------------------------------------------------------------------------------------------------------------------| | kafka.enabled | Boolean | false | Whether enable Fluss Kafka. Disabled by default. When this option is set to true, the Fluss Kafka will be enabled. | | kafka.listener.names | String | KAFKA | The listener names for Kafka wire protocol communication. Support multiple listener names, separated by comma. | -| kafka.default.database | String | kafka | Database used for unqualified Kafka topic names. Fully qualified names can access other databases. This option does not create a database. | +| kafka.default.database | String | fluss | Database used for unqualified Kafka topic names. Fully qualified names can access other databases. This option does not create a database. | Kafka listeners share `netty.connection.max-idle-time` with native Fluss RPC listeners (see [Netty](#netty)). The default is `10min`; connections close when neither reads nor writes occur for the configured duration. Kafka tables must be created through Fluss or the Flink Fluss catalog. A topic named `sales.orders` maps to that table directly; `orders` maps to the same-named table in -`kafka.default.database`. Databases and tables must already exist. Reverse mapping always uses +`kafka.default.database` (by default, `fluss.orders`). Databases and tables must already exist. Reverse mapping always uses the fully qualified name. All TabletServers must use the same default database. Changing it redirects unqualified names, so migrate clients to fully qualified names before changing the configuration. The prototype `kafka.database` option is no longer used. @@ -283,6 +283,12 @@ configuration. The prototype `kafka.database` option is no longer used. Binary and string values require a user-declared `_kafka_value BYTES` or `_kafka_value STRING` column respectively. Flat JSON targets business columns at the table level; nested JSON targets the fields of `_kafka_value ROW<...>`. Both exclude reserved system names from business matching. +JSON business schemas support nested ROW, ARRAY and MAP values. Every MAP key must be STRING, +including maps inside other containers. CREATE and ALTER reject unsupported types or a nesting +depth above 64, counted from each top-level JSON business field at depth zero; ROW fields, +ARRAY elements and MAP values each add one level. The JSON root and the physical nested +`_kafka_value ROW` container add no level, so equivalent flat/nested schemas have the same limit. +Scalar value and nullability checks apply when decoding records. Optional `_kafka_key BYTES` and `_kafka_headers MAP` columns retain keys and headers. Duplicate header names use the last value. Optional `_kafka_rescue STRING` must be nullable and collects unknown or reserved JSON members; it requires no mapping property. Binary/string From 61535a6d179bf251a0085b740f0b5afc20ae5b71 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Wed, 14 Oct 2026 10:54:36 +0800 Subject: [PATCH 07/15] [kafka] Enable metadata mapping and fix table change notifications Allow optional _kafka_metadata with timestamp, timestamp_type and partition, without a physical offset field. Cover optional fixed-column combinations, metadata validation and CREATE/ALTER behavior. Ignore unchanged table registrations when schema ALTER advances the ZK version. Preserve atomic concurrency fencing and notifications for real property and bucket changes, with deterministic regression coverage. Validation: Java 11 reactor verify passed 111 targeted test cases; Checkstyle, Spotless and git diff --check passed. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 57/57 AI-Contributed/UT: 265/265 --- .../fluss/metadata/KafkaTableSchema.java | 25 +++-- .../fluss/metadata/KafkaTableSchemaTest.java | 93 ++++++++++++++----- .../kafka/schema/KafkaTableMappingITCase.java | 79 +++++++++++++++- .../schema/KafkaTopicSchemaResolverTest.java | 20 +++- .../event/watcher/TableChangeWatcher.java | 21 +++-- .../event/watcher/TableChangeWatcherTest.java | 73 ++++++++++++++- website/docs/maintenance/configuration.md | 11 ++- 7 files changed, 277 insertions(+), 45 deletions(-) diff --git a/fluss-common/src/main/java/org/apache/fluss/metadata/KafkaTableSchema.java b/fluss-common/src/main/java/org/apache/fluss/metadata/KafkaTableSchema.java index 990cec50416..9ab16195fbd 100644 --- a/fluss-common/src/main/java/org/apache/fluss/metadata/KafkaTableSchema.java +++ b/fluss-common/src/main/java/org/apache/fluss/metadata/KafkaTableSchema.java @@ -93,6 +93,7 @@ public enum ValueLayout { private final KafkaFieldProjection valueProjection; private final int keyPosition; private final int headersPosition; + private final int metadataPosition; private final int rescuePosition; private KafkaTableSchema( @@ -102,6 +103,7 @@ private KafkaTableSchema( List valuePositions, int keyPosition, int headersPosition, + int metadataPosition, int rescuePosition) { this.rowType = rowType; this.valueType = valueType; @@ -109,6 +111,7 @@ private KafkaTableSchema( this.valueProjection = new KafkaFieldProjection(rowType, valuePositions); this.keyPosition = keyPosition; this.headersPosition = headersPosition; + this.metadataPosition = metadataPosition; this.rescuePosition = rescuePosition; } @@ -216,9 +219,6 @@ public static KafkaTableSchema resolve(TableDescriptor table) { int metadataPosition = rowType.getFieldIndex(METADATA_COLUMN); if (metadataPosition >= 0) { validateMetadataType(rowType.getTypeAt(metadataPosition)); - throw invalid( - "_kafka_metadata requires native append support for persisting actual per-row offsets; " - + "this capability is not available yet."); } List valuePositions = new ArrayList<>(); @@ -296,6 +296,7 @@ public static KafkaTableSchema resolve(TableDescriptor table) { valuePositions, keyPosition, headersPosition, + metadataPosition, rescuePosition); } @@ -382,17 +383,15 @@ private static void validateOptions(TableDescriptor table) { private static void validateMetadataType(DataType type) { String message = "_kafka_metadata must be ROW."; + + "timestamp_type INT, partition INT>."; if (!(type instanceof RowType)) { throw invalid(message); } RowType row = (RowType) type; - if (!row.getFieldNames() - .equals(Arrays.asList("timestamp", "timestamp_type", "partition", "offset")) + if (!row.getFieldNames().equals(Arrays.asList("timestamp", "timestamp_type", "partition")) || !row.getTypeAt(0).copy(true).equals(DataTypes.TIMESTAMP_LTZ(3)) || !row.getTypeAt(1).copy(true).equals(DataTypes.INT()) - || !row.getTypeAt(2).copy(true).equals(DataTypes.INT()) - || !row.getTypeAt(3).copy(true).equals(DataTypes.BIGINT())) { + || !row.getTypeAt(2).copy(true).equals(DataTypes.INT())) { throw invalid(message); } } @@ -430,6 +429,14 @@ public int headersPosition() { return headersPosition; } + /** + * Returns the metadata column position, or -1 when absent. The ROW stores the input record + * timestamp, batch timestamp type and requested partition; log offsets are not row fields. + */ + public int metadataPosition() { + return metadataPosition; + } + /** * Returns the rescue column position, or -1 when absent. JSON decoders collect unknown and * reserved members here; binary/string decoders write null to this optional column. @@ -450,6 +457,7 @@ public boolean equals(Object other) { && valueProjection.equals(that.valueProjection) && keyPosition == that.keyPosition && headersPosition == that.headersPosition + && metadataPosition == that.metadataPosition && rescuePosition == that.rescuePosition; } @@ -462,6 +470,7 @@ public int hashCode() { valueProjection, keyPosition, headersPosition, + metadataPosition, rescuePosition); } diff --git a/fluss-common/src/test/java/org/apache/fluss/metadata/KafkaTableSchemaTest.java b/fluss-common/src/test/java/org/apache/fluss/metadata/KafkaTableSchemaTest.java index 809f13ac553..00aacbd51e5 100644 --- a/fluss-common/src/test/java/org/apache/fluss/metadata/KafkaTableSchemaTest.java +++ b/fluss-common/src/test/java/org/apache/fluss/metadata/KafkaTableSchemaTest.java @@ -89,7 +89,7 @@ void testExplicitOptInAndDisabledTables() { @ValueSource(strings = {"binary", "string"}) void testFixedValueColumnWithOptionalSystemFields(String format) { DataType valueType = format.equals("binary") ? DataTypes.BYTES() : DataTypes.STRING(); - for (int mask = 0; mask < 8; mask++) { + for (int mask = 0; mask < 16; mask++) { Schema.Builder schema = Schema.newBuilder().column("_kafka_value", valueType.copy(false)); if ((mask & 1) != 0) { @@ -102,6 +102,9 @@ void testFixedValueColumnWithOptionalSystemFields(String format) { if ((mask & 4) != 0) { schema.column("_kafka_rescue", DataTypes.STRING()); } + if ((mask & 8) != 0) { + schema.column("_kafka_metadata", metadataType()); + } TableDescriptor table = enabled(schema.build(), format).build(); byte[] before = table.toJsonBytes(); KafkaTableSchema resolved = KafkaTableSchema.resolve(table); @@ -111,6 +114,9 @@ void testFixedValueColumnWithOptionalSystemFields(String format) { assertThat(resolved.keyPosition() >= 0).isEqualTo((mask & 1) != 0); assertThat(resolved.headersPosition() >= 0).isEqualTo((mask & 2) != 0); assertThat(resolved.rescuePosition() >= 0).isEqualTo((mask & 4) != 0); + assertThat(resolved.metadataPosition() >= 0).isEqualTo((mask & 8) != 0); + assertThat(resolved.metadataPosition()) + .isEqualTo(table.getSchema().getRowType().getFieldIndex("_kafka_metadata")); assertThat(table.toJsonBytes()).isEqualTo(before); assertThat(table.getProperties()) .doesNotContainKeys( @@ -134,6 +140,7 @@ void testJsonFlatPreservesNestedTypesAndExcludesSystemColumns() { Schema.newBuilder() .column("id", DataTypes.BIGINT().copy(false)) .column("_kafka_rescue", DataTypes.STRING()) + .column("_kafka_metadata", metadataType()) .column("payload", nested) .column("_kafka_key", DataTypes.BYTES()) .column( @@ -147,9 +154,10 @@ void testJsonFlatPreservesNestedTypesAndExcludesSystemColumns() { KafkaTableSchema resolved = KafkaTableSchema.resolve(table); assertThat(resolved.valueType()).isEqualTo(ValueType.PARSE_JSON); assertThat(resolved.valueLayout()).isEqualTo(ValueLayout.FLAT); - assertThat(resolved.valueProjection().positions()).containsExactly(0, 2, 5); + assertThat(resolved.valueProjection().positions()).containsExactly(0, 3, 6); assertThat(resolved.valueProjection().dataTypeAt(1)).isEqualTo(nested); assertThat(resolved.rescuePosition()).isEqualTo(1); + assertThat(resolved.metadataPosition()).isEqualTo(2); assertThat(table.getProperties()).doesNotContainKey("table.kafka.convert.value.layout"); TableDescriptor noRescue = enabled( @@ -174,6 +182,7 @@ void testNestedJsonTargetsOneRow() { enabled( Schema.newBuilder() .column("_kafka_rescue", DataTypes.STRING()) + .column("_kafka_metadata", metadataType()) .column("_kafka_value", row) .build(), "parse_json") @@ -181,9 +190,10 @@ void testNestedJsonTargetsOneRow() { .build(); KafkaTableSchema resolved = KafkaTableSchema.resolve(table); assertThat(resolved.valueLayout()).isEqualTo(ValueLayout.NESTED); - assertThat(resolved.valueProjection().positions()).containsExactly(1); + assertThat(resolved.valueProjection().positions()).containsExactly(2); assertThat(resolved.valueProjection().dataTypeAt(0)).isEqualTo(row); assertThat(resolved.rescuePosition()).isZero(); + assertThat(resolved.metadataPosition()).isEqualTo(1); } @ParameterizedTest @@ -425,33 +435,63 @@ void testSystemColumnTypes() { } @Test - void testMetadataStructureAndUnavailableOffsetPersistence() { + void testMetadataPreservesDeclaredNullability() { RowType metadata = DataTypes.ROW( DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP_LTZ(3)), - DataTypes.FIELD("timestamp_type", DataTypes.INT()), - DataTypes.FIELD("partition", DataTypes.INT()), - DataTypes.FIELD("offset", DataTypes.BIGINT())); - assertInvalid(withSystem("_kafka_metadata", metadata), "persisting actual per-row offsets"); - assertInvalid(withSystem("_kafka_metadata", DataTypes.STRING()), "must be ROW"); - assertInvalid( - withSystem( - "_kafka_metadata", + DataTypes.FIELD("timestamp_type", DataTypes.INT().copy(false)), + DataTypes.FIELD("partition", DataTypes.INT().copy(false))); + for (boolean nullable : new boolean[] {true, false}) { + DataType type = metadata.copy(nullable); + TableDescriptor table = withSystem("_kafka_metadata", type); + KafkaTableSchema resolved = KafkaTableSchema.resolve(table); + assertThat(resolved.rowType().getTypeAt(resolved.metadataPosition())).isEqualTo(type); + assertThat(KafkaTableSchema.resolve(TableDescriptor.fromJsonBytes(table.toJsonBytes()))) + .isEqualTo(resolved); + } + } + + @Test + void testMetadataRejectsInvalidStructureAndOffsetField() { + for (DataType invalid : + Arrays.asList( + DataTypes.STRING(), DataTypes.ROW( - DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP(3)), - DataTypes.FIELD("timestamp_type", DataTypes.INT()), + DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP_LTZ(3)), + DataTypes.FIELD("timestamp_type", DataTypes.INT())), + DataTypes.ROW( + DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP_LTZ(3)), + DataTypes.FIELD("timestamp_type", DataTypes.INT()), DataTypes.FIELD("partition", DataTypes.INT()), - DataTypes.FIELD("offset", DataTypes.BIGINT()))), - "TIMESTAMP_LTZ(3)"); - assertInvalid( - withSystem( - "_kafka_metadata", + DataTypes.FIELD("offset", DataTypes.BIGINT())), + DataTypes.ROW( + DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP(3)), + DataTypes.FIELD("timestamp_type", DataTypes.INT()), + DataTypes.FIELD("partition", DataTypes.INT())), + DataTypes.ROW( + DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP_LTZ(6)), + DataTypes.FIELD("timestamp_type", DataTypes.INT()), + DataTypes.FIELD("partition", DataTypes.INT())), + DataTypes.ROW( + DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP_LTZ(3)), + DataTypes.FIELD("timestamp_type", DataTypes.STRING()), + DataTypes.FIELD("partition", DataTypes.INT())), + DataTypes.ROW( + DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP_LTZ(3)), + DataTypes.FIELD("timestamp_type", DataTypes.INT()), + DataTypes.FIELD("partition", DataTypes.BIGINT())), DataTypes.ROW( - DataTypes.FIELD("offset", DataTypes.BIGINT()), - DataTypes.FIELD("timestamp_type", DataTypes.INT()), + DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP_LTZ(3)), DataTypes.FIELD("partition", DataTypes.INT()), - DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP_LTZ(3)))), - "must be ROW"); + DataTypes.FIELD("timestamp_type", DataTypes.INT())), + DataTypes.ROW( + DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP_LTZ(3)), + DataTypes.FIELD("timestamp_type", DataTypes.INT()), + DataTypes.FIELD("bucket", DataTypes.INT())))) { + assertInvalid( + withSystem("_kafka_metadata", invalid), + "ROW"); + } } @Test @@ -510,6 +550,13 @@ void testTableKindsAndCanonicalNameLimits() { .hasMessageContaining("Kafka topic"); } + private static RowType metadataType() { + return DataTypes.ROW( + DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP_LTZ(3)), + DataTypes.FIELD("timestamp_type", DataTypes.INT()), + DataTypes.FIELD("partition", DataTypes.INT())); + } + private static TableDescriptor withSystem(String name, DataType type) { return enabled( Schema.newBuilder() diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java index 76b9c0f773d..97e75987f89 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java @@ -444,9 +444,62 @@ void testCreateRejectsInvalidContractWithoutPublishingMetadata() throws Exceptio assertThat(KafkaTableSchema.isEnabled(stored(path))).isTrue(); } + @ParameterizedTest + @ValueSource(strings = {"binary", "string"}) + void testMetadataCanBeCreatedAndEnabledWithoutOffset(String format) throws Exception { + Schema schema = + Schema.newBuilder() + .column("_kafka_metadata", metadataType()) + .column( + "_kafka_value", + "binary".equals(format) ? DataTypes.BYTES() : DataTypes.STRING()) + .build(); + TableDescriptor descriptor = + enabled() + .schema(schema) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, format) + .build(); + TablePath created = TablePath.of("ddl", "metadata_create_" + format); + createTable(CLUSTER, created, descriptor); + TableDescriptor persisted = stored(created); + assertThat(persisted.getSchema()).isEqualTo(schema); + assertThat(kafkaProperties(persisted)).isEqualTo(kafkaProperties(descriptor)); + assertThat(KafkaTableSchema.resolve(persisted).metadataPosition()).isZero(); + + TablePath altered = TablePath.of("ddl", "metadata_enable_" + format); + createTable( + CLUSTER, + altered, + TableDescriptor.builder(descriptor) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, false) + .build()); + int schemaId = info(altered).getSchemaId(); + set(altered, Collections.singletonMap("table.kafka.enabled", "true")); + persisted = stored(altered); + assertThat(info(altered).getSchemaId()).isEqualTo(schemaId); + assertThat(persisted.getSchema()).isEqualTo(schema); + assertThat(kafkaProperties(persisted)).isEqualTo(kafkaProperties(descriptor)); + assertThat(KafkaTableSchema.resolve(persisted).metadataPosition()).isZero(); + } + @Test - void testUnsupportedMetadataOffsetCannotBeEnabled() throws Exception { - TablePath path = TablePath.of("ddl", "offset_capability"); + void testMetadataCanBeAddedToEnabledTable() throws Exception { + TablePath path = TablePath.of("ddl", "metadata_add"); + createTable(CLUSTER, path, enabled().build()); + TableDescriptor before = stored(path); + addColumn(path, "_kafka_metadata", metadataType()); + TableDescriptor after = stored(path); + KafkaTableSchema resolved = KafkaTableSchema.resolve(after); + assertThat(resolved.metadataPosition()).isGreaterThanOrEqualTo(0); + assertThat(resolved.rowType().getTypeAt(resolved.metadataPosition())) + .isEqualTo(metadataType()); + assertThat(kafkaProperties(after)).isEqualTo(kafkaProperties(before)); + } + + @Test + void testMetadataOffsetFieldRejectedOnCreateAndEnable() throws Exception { + gateway.createDatabase(newCreateDatabaseRequest("ddl", true)).get(); + TablePath path = TablePath.of("ddl", "metadata_with_offset"); Schema schema = Schema.newBuilder() .column("_kafka_value", DataTypes.BYTES()) @@ -458,6 +511,18 @@ void testUnsupportedMetadataOffsetCannotBeEnabled() throws Exception { DataTypes.FIELD("partition", DataTypes.INT()), DataTypes.FIELD("offset", DataTypes.BIGINT()))) .build(); + assertThatThrownBy( + () -> + gateway.createTable( + newCreateTableRequest( + path, + enabled().schema(schema).build(), + false)) + .get()) + .hasCauseInstanceOf(InvalidTableException.class) + .hasStackTraceContaining( + "ROW"); + assertThat(gateway.tableExists(newTableExistsRequest(path)).get().isExists()).isFalse(); createTable( CLUSTER, path, @@ -468,7 +533,8 @@ void testUnsupportedMetadataOffsetCannotBeEnabled() throws Exception { TableDescriptor before = stored(path); assertThatThrownBy(() -> set(path, Collections.singletonMap("table.kafka.enabled", "true"))) .hasCauseInstanceOf(InvalidTableException.class) - .hasStackTraceContaining("actual per-row offsets"); + .hasStackTraceContaining( + "ROW"); assertThat(stored(path)).isEqualTo(before); } @@ -495,6 +561,13 @@ path, enabled().build(), false)) assertThat(stored(path)).isEqualTo(before); } + private static DataType metadataType() { + return DataTypes.ROW( + DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP_LTZ(3)), + DataTypes.FIELD("timestamp_type", DataTypes.INT()), + DataTypes.FIELD("partition", DataTypes.INT())); + } + private GetTableInfoResponse info(TablePath path) throws Exception { return gateway.getTableInfo(newGetTableInfoRequest(path)).get(); } diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolverTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolverTest.java index b7e4174421c..8327c6fb60e 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolverTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolverTest.java @@ -34,9 +34,27 @@ class KafkaTopicSchemaResolverTest { @Test void testEnabledContractUsesSharedValidation() { - TableDescriptor table = table().property(ConfigOptions.TABLE_KAFKA_ENABLED, true).build(); + TableDescriptor table = + table().schema( + Schema.newBuilder() + .column( + "_kafka_metadata", + DataTypes.ROW( + DataTypes.FIELD( + "timestamp", + DataTypes.TIMESTAMP_LTZ(3)), + DataTypes.FIELD( + "timestamp_type", DataTypes.INT()), + DataTypes.FIELD( + "partition", DataTypes.INT()))) + .column("_kafka_value", DataTypes.BYTES()) + .build()) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) + .build(); assertThat(resolver.isEnabled(table)).isTrue(); assertThat(resolver.resolve(table)).isEqualTo(KafkaTableSchema.resolve(table)); + assertThat(resolver.resolve(table).metadataPosition()).isZero(); + assertThat(resolver.resolve(table).valueProjection().positions()).containsExactly(1); assertThat(resolver.resolve(TableDescriptor.fromJsonBytes(table.toJsonBytes()))) .isEqualTo(resolver.resolve(table)); } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/event/watcher/TableChangeWatcher.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/event/watcher/TableChangeWatcher.java index 5e05877b4be..f38a7f0afe4 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/event/watcher/TableChangeWatcher.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/event/watcher/TableChangeWatcher.java @@ -17,6 +17,7 @@ package org.apache.fluss.server.coordinator.event.watcher; +import org.apache.fluss.annotation.VisibleForTesting; import org.apache.fluss.config.Configuration; import org.apache.fluss.config.TableConfig; import org.apache.fluss.metadata.PhysicalTablePath; @@ -87,7 +88,8 @@ public void stop() { } /** A listener to monitor the changes of table nodes in zookeeper. */ - private final class TablePathChangeListener implements CuratorCacheListener { + @VisibleForTesting + final class TablePathChangeListener implements CuratorCacheListener { @Override public void event(Type type, ChildData oldData, ChildData newData) { @@ -150,13 +152,13 @@ public void event(Type type, ChildData oldData, ChildData newData) { if (tablePath == null) { break; } - // Distinguish between table creation and properties change. + // Distinguish between table creation and registration updates. // If oldData exists and contains valid table registration data, - // it's a properties change; otherwise, it's a table creation. + // compare its contents; otherwise, it's a table creation. if (oldData != null && oldData.getData() != null && oldData.getData().length > 0) { - processTableRegistrationChange(tablePath, newData); + processTableRegistrationChange(tablePath, oldData, newData); } else { processCreateTable(tablePath, newData); } @@ -276,9 +278,16 @@ private void processCreatePartition( tablePath, tableId, partitionId, partitionName, partitionAssignment)); } - private void processTableRegistrationChange(TablePath tablePath, ChildData newData) { + private void processTableRegistrationChange( + TablePath tablePath, ChildData oldData, ChildData newData) { + TableRegistration oldTable = TableZNode.decode(oldData.getData()); TableRegistration newTable = TableZNode.decode(newData.getData()); - eventManager.put(new TableRegistrationChangeEvent(tablePath, newTable)); + // Schema ALTER bumps the table znode version to fence concurrent registration updates, + // even when its contents are unchanged. Compare decoded registrations so differences + // in JSON serialization do not trigger a registration change either. + if (!oldTable.equals(newTable)) { + eventManager.put(new TableRegistrationChangeEvent(tablePath, newTable)); + } } } diff --git a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/event/watcher/TableChangeWatcherTest.java b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/event/watcher/TableChangeWatcherTest.java index 28a4131ecd8..9a36280d891 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/event/watcher/TableChangeWatcherTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/event/watcher/TableChangeWatcherTest.java @@ -42,11 +42,16 @@ import org.apache.fluss.server.entity.TablePropertyChanges; import org.apache.fluss.server.zk.NOPErrorHandler; import org.apache.fluss.server.zk.ZooKeeperClient; +import org.apache.fluss.server.zk.ZooKeeperClient.VersionedData; import org.apache.fluss.server.zk.ZooKeeperExtension; import org.apache.fluss.server.zk.data.PartitionAssignment; import org.apache.fluss.server.zk.data.TableAssignment; import org.apache.fluss.server.zk.data.TableRegistration; +import org.apache.fluss.server.zk.data.ZkData.TableZNode; import org.apache.fluss.server.zk.data.ZkVersion; +import org.apache.fluss.shaded.curator5.org.apache.curator.framework.recipes.cache.ChildData; +import org.apache.fluss.shaded.curator5.org.apache.curator.framework.recipes.cache.CuratorCacheListener; +import org.apache.fluss.shaded.zookeeper3.org.apache.zookeeper.data.Stat; import org.apache.fluss.testutils.common.AllCallbackWrapper; import org.apache.fluss.types.DataTypes; import org.apache.fluss.utils.clock.SystemClock; @@ -56,9 +61,13 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import java.nio.charset.StandardCharsets; import java.time.Duration; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -302,7 +311,7 @@ void testPartitionedTable() throws Exception { } @Test - void testSchemaChanges() { + void testSchemaChanges() throws Exception { // create tables, collect create table events List expectedEvents = new ArrayList<>(); for (int i = 0; i < 10; i++) { @@ -349,6 +358,8 @@ void testSchemaChanges() { } CreateTableEvent createTableEvent = (CreateTableEvent) coordinatorEvent; TableInfo tableInfo = createTableEvent.getTableInfo(); + VersionedData beforeAlter = + zookeeperClient.getTableWithVersion(tableInfo.getTablePath()).get(); metadataManager.alterTableSchema( tableInfo.getTablePath(), Collections.singletonList( @@ -359,6 +370,10 @@ void testSchemaChanges() { TableChange.ColumnPosition.last())), false, null); + VersionedData afterAlter = + zookeeperClient.getTableWithVersion(tableInfo.getTablePath()).get(); + assertThat(afterAlter.zkVersion()).isEqualTo(beforeAlter.zkVersion() + 1); + assertThat(afterAlter.data()).isEqualTo(beforeAlter.data()); Schema newSchema = Schema.newBuilder() .fromSchema(tableInfo.getSchema()) @@ -378,6 +393,62 @@ void testSchemaChanges() { .containsExactlyInAnyOrderElementsOf(allEvents)); } + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testUnchangedTableRegistration(boolean differentJsonFormatting) { + tableChangeWatcher.stop(); + TablePath tablePath = TablePath.of(DEFAULT_DB, "unchanged_table_registration"); + TableRegistration table = TableRegistration.newTable(1L, remoteDataDir, TEST_TABLE); + byte[] oldData = TableZNode.encode(table); + byte[] newData = + differentJsonFormatting + ? ("\n" + new String(oldData, StandardCharsets.UTF_8)) + .getBytes(StandardCharsets.UTF_8) + : oldData; + + // Deliver the callback synchronously so the absence of an event cannot pass before + // CuratorCache has observed the version bump. + CuratorCacheListener listener = tableChangeWatcher.new TablePathChangeListener(); + listener.event( + CuratorCacheListener.Type.NODE_CHANGED, + tableNodeData(tablePath, oldData, 1), + tableNodeData(tablePath, newData, 2)); + + assertThat(eventManager.getEvents()).isEmpty(); + } + + @Test + void testChangedTableRegistrationContents() { + tableChangeWatcher.stop(); + TablePath tablePath = TablePath.of(DEFAULT_DB, "changed_table_registration"); + TableRegistration table = TableRegistration.newTable(1L, remoteDataDir, TEST_TABLE); + CuratorCacheListener listener = tableChangeWatcher.new TablePathChangeListener(); + List expectedEvents = new ArrayList<>(); + for (TableRegistration updatedTable : + Arrays.asList( + table.newProperties( + Collections.singletonMap( + ConfigOptions.TABLE_KAFKA_ENABLED.key(), "true"), + table.customProperties), + table.newProperties( + table.properties, + Collections.singletonMap("custom.key", "custom.value")), + table.newBucketCount(4))) { + listener.event( + CuratorCacheListener.Type.NODE_CHANGED, + tableNodeData(tablePath, TableZNode.encode(table), 1), + tableNodeData(tablePath, TableZNode.encode(updatedTable), 2)); + expectedEvents.add(new TableRegistrationChangeEvent(tablePath, updatedTable)); + assertThat(eventManager.getEvents()).containsExactlyElementsOf(expectedEvents); + } + } + + private static ChildData tableNodeData(TablePath tablePath, byte[] data, int version) { + Stat stat = new Stat(); + stat.setVersion(version); + return new ChildData(TableZNode.path(tablePath), stat, data); + } + @Test void testTableRegistrationChange() { // create a table diff --git a/website/docs/maintenance/configuration.md b/website/docs/maintenance/configuration.md index 45cc108f29b..2f656004c43 100644 --- a/website/docs/maintenance/configuration.md +++ b/website/docs/maintenance/configuration.md @@ -300,7 +300,12 @@ disabled, ordinary Fluss tables are not subject to Kafka-specific schema validat accepted on enabled tables. The `_kafka_metadata` contract is -`ROW`. -Enabling tables with this column is currently rejected until native append can assign and -persist each row's actual offset. Other optional system columns can be used independently. +`ROW`. +This optional physical column stores the input Kafka record timestamp, the batch timestamp type +(`-1` for no timestamp, `0` for CreateTime, `1` for LogAppendTime), and the requested partition. +Without a timestamp, the timestamp field is null; declared NOT NULL constraints still apply +when converting records. These values are available before Arrow encoding. Log offsets remain +native log metadata and are not part of this ROW, so enabling the column requires no append-time +offset backfill. CREATE and ALTER reject extra fields, including the former offset field, and +incorrect field names, ordering or types. These DDL contracts do not by themselves advertise Kafka Metadata or Produce API support. From 2ce83f190510dea12ede39fcb6e6ba287a66f0f7 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Wed, 14 Oct 2026 17:17:18 +0800 Subject: [PATCH 08/15] [kafka] Focus DDL mapping changes and streamline tests Create the configured Kafka default database when missing and preserve existing metadata. Fence schema and property ALTER with schema-parent version checks without rewriting table registrations, restoring the original table watcher. Remove redundant configuration, Flink conversion and mapping test coverage while retaining DDL validation, optional system columns and concurrent ALTER checks. Validation: targeted clean verify passed 173 tests with Checkstyle and Spotless. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 120/120 AI-Contributed/UT: 683/683 --- .../apache/fluss/config/ConfigOptions.java | 3 +- .../fluss/metadata/KafkaTableSchemaTest.java | 83 +----- .../flink/utils/FlinkConversionsTest.java | 45 ---- .../apache/fluss/kafka/KafkaConfigsTest.java | 59 ---- .../kafka/schema/KafkaTableMappingITCase.java | 255 ++++-------------- .../schema/KafkaTopicSchemaResolverTest.java | 28 -- .../server/coordinator/CoordinatorServer.java | 16 ++ .../server/coordinator/MetadataManager.java | 20 +- .../event/watcher/TableChangeWatcher.java | 21 +- .../fluss/server/zk/ZooKeeperClient.java | 54 +++- .../coordinator/KafkaDefaultDatabaseTest.java | 94 +++++++ .../KafkaTableSchemaAlterTest.java | 46 +++- .../event/watcher/TableChangeWatcherTest.java | 73 +---- website/docs/maintenance/configuration.md | 6 +- 14 files changed, 275 insertions(+), 528 deletions(-) delete mode 100644 fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaConfigsTest.java create mode 100644 fluss-server/src/test/java/org/apache/fluss/server/coordinator/KafkaDefaultDatabaseTest.java diff --git a/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java b/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java index 43630778e8e..ac75410f7f9 100644 --- a/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java +++ b/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java @@ -2985,7 +2985,8 @@ public class ConfigOptions { .defaultValue("fluss") .withDescription( "Default database for unqualified Kafka topic names. Fully qualified names may " - + "access other databases. Databases must already exist; this option does not create them."); + + "access other databases. When Kafka is enabled, the coordinator creates this " + + "database if it does not exist."); /** * Compaction style for Fluss's kv, which is same to rocksdb's, but help use avoid including diff --git a/fluss-common/src/test/java/org/apache/fluss/metadata/KafkaTableSchemaTest.java b/fluss-common/src/test/java/org/apache/fluss/metadata/KafkaTableSchemaTest.java index 00aacbd51e5..458242feadd 100644 --- a/fluss-common/src/test/java/org/apache/fluss/metadata/KafkaTableSchemaTest.java +++ b/fluss-common/src/test/java/org/apache/fluss/metadata/KafkaTableSchemaTest.java @@ -18,9 +18,6 @@ package org.apache.fluss.metadata; import org.apache.fluss.config.ConfigOptions; -import org.apache.fluss.config.Configuration; -import org.apache.fluss.config.FlussConfigUtils; -import org.apache.fluss.config.TableConfig; import org.apache.fluss.exception.InvalidTableException; import org.apache.fluss.metadata.KafkaTableSchema.ValueLayout; import org.apache.fluss.metadata.KafkaTableSchema.ValueType; @@ -33,7 +30,6 @@ import org.junit.jupiter.params.provider.ValueSource; import java.util.Arrays; -import java.util.Collections; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatCode; @@ -42,22 +38,6 @@ /** Tests the shared DDL and Kafka conversion contract. */ class KafkaTableSchemaTest { - @Test - void testRegisteredAlterableOptionsAndReadTimeDefaults() { - for (String key : - Arrays.asList( - "table.kafka.enabled", - "table.kafka.convert.value.type", - "table.kafka.convert.value.layout", - "table.kafka.convert.key.type")) { - assertThat(FlussConfigUtils.TABLE_OPTIONS).containsKey(key); - assertThat(FlussConfigUtils.isAlterableTableOption(key)).isTrue(); - } - Configuration config = new Configuration(); - assertThat(new TableConfig(config).isKafkaEnabled()).isFalse(); - assertThat(config.toMap()).isEmpty(); - } - @Test void testExplicitOptInAndDisabledTables() { TableDescriptor disabled = @@ -106,35 +86,22 @@ void testFixedValueColumnWithOptionalSystemFields(String format) { schema.column("_kafka_metadata", metadataType()); } TableDescriptor table = enabled(schema.build(), format).build(); - byte[] before = table.toJsonBytes(); KafkaTableSchema resolved = KafkaTableSchema.resolve(table); assertThat(resolved.valueProjection().positions()).containsExactly(0); assertThat(resolved.valueProjection().dataTypeAt(0).isNullable()).isFalse(); - assertThat(resolved.valueLayout()).isNull(); assertThat(resolved.keyPosition() >= 0).isEqualTo((mask & 1) != 0); assertThat(resolved.headersPosition() >= 0).isEqualTo((mask & 2) != 0); assertThat(resolved.rescuePosition() >= 0).isEqualTo((mask & 4) != 0); assertThat(resolved.metadataPosition() >= 0).isEqualTo((mask & 8) != 0); - assertThat(resolved.metadataPosition()) - .isEqualTo(table.getSchema().getRowType().getFieldIndex("_kafka_metadata")); - assertThat(table.toJsonBytes()).isEqualTo(before); assertThat(table.getProperties()) .doesNotContainKeys( "table.kafka.convert.key.type", "table.kafka.convert.value.layout"); - assertThat(KafkaTableSchema.resolve(TableDescriptor.fromJsonBytes(before))) - .isEqualTo(resolved); } } @Test void testJsonFlatPreservesNestedTypesAndExcludesSystemColumns() { - RowType nested = - DataTypes.ROW( - DataTypes.FIELD("_kafka_key", DataTypes.STRING()), - DataTypes.FIELD( - "items", - DataTypes.ARRAY( - DataTypes.MAP(DataTypes.STRING(), DataTypes.INT())))); + RowType nested = DataTypes.ROW(DataTypes.FIELD("_kafka_key", DataTypes.STRING())); TableDescriptor table = enabled( Schema.newBuilder() @@ -159,14 +126,6 @@ void testJsonFlatPreservesNestedTypesAndExcludesSystemColumns() { assertThat(resolved.rescuePosition()).isEqualTo(1); assertThat(resolved.metadataPosition()).isEqualTo(2); assertThat(table.getProperties()).doesNotContainKey("table.kafka.convert.value.layout"); - TableDescriptor noRescue = - enabled( - Schema.newBuilder() - .column("custom_rescue", DataTypes.STRING()) - .build(), - "parse_json") - .build(); - assertThat(KafkaTableSchema.resolve(noRescue).rescuePosition()).isEqualTo(-1); } @Test @@ -209,15 +168,6 @@ void testJsonRejectsNonStringMapKeysRecursively(String layout) { DataTypes.MAP(DataTypes.STRING(), DataTypes.ARRAY(invalidMap)))) { TableDescriptor table = jsonTable(layout, type); assertInvalid(table, "only supports STRING map keys"); - assertThatCode( - () -> - KafkaTableSchema.validate( - TableDescriptor.builder(table) - .property( - ConfigOptions.TABLE_KAFKA_ENABLED, - false) - .build())) - .doesNotThrowAnyException(); } } } @@ -250,11 +200,7 @@ void testJsonAcceptsAllScalarTypesInNestedContainers(String layout) { DataTypes.ROW( DataTypes.FIELD("_kafka_key", scalar.copy(false))))); TableDescriptor table = jsonTable(layout, type); - byte[] before = table.toJsonBytes(); assertThatCode(() -> KafkaTableSchema.validate(table)).doesNotThrowAnyException(); - assertThat(table.toJsonBytes()).isEqualTo(before); - assertThat(KafkaTableSchema.resolve(TableDescriptor.fromJsonBytes(before))) - .isEqualTo(KafkaTableSchema.resolve(table)); } } @@ -276,13 +222,6 @@ void testJsonBusinessNestingDepthBoundary(String layout) { assertThatCode(() -> KafkaTableSchema.validate(valid)).doesNotThrowAnyException(); TableDescriptor tooDeep = jsonTable(layout, DataTypes.ARRAY(boundary)); assertInvalid(tooDeep, "maximum nesting depth of 64"); - assertThatCode( - () -> - KafkaTableSchema.validate( - TableDescriptor.builder(tooDeep) - .property(ConfigOptions.TABLE_KAFKA_ENABLED, false) - .build())) - .doesNotThrowAnyException(); } private static TableDescriptor jsonTable(String layout, DataType businessType) { @@ -325,7 +264,7 @@ void testReservedNamesAtValueMappingLevel(String name) { } @ParameterizedTest - @ValueSource(strings = {"raw", "json", "schema_id_prefix", "schema_latest", "avro", ""}) + @ValueSource(strings = {"raw", "avro", ""}) void testRejectsUnsupportedValueTypes(String type) { assertInvalid(enabled(valueSchema(), type).build(), "table.kafka.convert.value.type"); } @@ -446,8 +385,6 @@ void testMetadataPreservesDeclaredNullability() { TableDescriptor table = withSystem("_kafka_metadata", type); KafkaTableSchema resolved = KafkaTableSchema.resolve(table); assertThat(resolved.rowType().getTypeAt(resolved.metadataPosition())).isEqualTo(type); - assertThat(KafkaTableSchema.resolve(TableDescriptor.fromJsonBytes(table.toJsonBytes()))) - .isEqualTo(resolved); } } @@ -522,7 +459,7 @@ void testUnknownAndPrototypeOptions() { } @Test - void testTableKindsAndCanonicalNameLimits() { + void testUnsupportedTableKinds() { Schema keyed = Schema.newBuilder() .column("_kafka_value", DataTypes.BYTES().copy(false)) @@ -534,20 +471,6 @@ void testTableKindsAndCanonicalNameLimits() { "non-partitioned"); assertInvalid( enabled(valueSchema(), "binary").logFormat(LogFormat.INDEXED).build(), "Arrow"); - String database = String.join("", Collections.nCopies(124, "d")); - String tableName = String.join("", Collections.nCopies(124, "t")); - TableDescriptor table = enabled(valueSchema(), "binary").build(); - assertThatCode( - () -> - KafkaTableSchema.validateTablePath( - TablePath.of(database, tableName), table)) - .doesNotThrowAnyException(); - assertThatThrownBy( - () -> - KafkaTableSchema.validateTablePath( - TablePath.of(database, tableName + "t"), table)) - .isInstanceOf(InvalidTableException.class) - .hasMessageContaining("Kafka topic"); } private static RowType metadataType() { diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/utils/FlinkConversionsTest.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/utils/FlinkConversionsTest.java index 56e990d8cd5..67f843a7276 100644 --- a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/utils/FlinkConversionsTest.java +++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/utils/FlinkConversionsTest.java @@ -19,7 +19,6 @@ import org.apache.fluss.config.ConfigOptions; import org.apache.fluss.flink.catalog.TestSchemaResolver; -import org.apache.fluss.metadata.KafkaTableSchema; import org.apache.fluss.metadata.KvFormat; import org.apache.fluss.metadata.TableChange; import org.apache.fluss.metadata.TableDescriptor; @@ -76,50 +75,6 @@ /** Test for {@link FlinkConversions}. */ public class FlinkConversionsTest { - @Test - void testKafkaDdlUsesRegisteredPropertiesAndFixedColumns() { - ResolvedSchema schema = - new ResolvedSchema( - Arrays.asList( - Column.physical( - "_kafka_value", - org.apache.flink.table.api.DataTypes.BYTES()), - Column.physical( - "_kafka_key", org.apache.flink.table.api.DataTypes.BYTES()), - Column.physical( - "_kafka_headers", - org.apache.flink.table.api.DataTypes.MAP( - org.apache.flink.table.api.DataTypes.STRING(), - org.apache.flink.table.api.DataTypes.BYTES())), - Column.physical( - "_kafka_rescue", - org.apache.flink.table.api.DataTypes.STRING())), - Collections.emptyList(), - null); - Map options = new HashMap<>(); - options.put("table.kafka.enabled", "true"); - options.put("table.kafka.convert.value.type", "binary"); - CatalogTable table = - CatalogTable.of( - Schema.newBuilder().fromResolvedSchema(schema).build(), - "Kafka DDL", - Collections.emptyList(), - options); - TableDescriptor descriptor = - FlinkConversions.toFlussTable(new ResolvedCatalogTable(table, schema)); - assertThat(descriptor.getProperties()) - .containsAllEntriesOf(options) - .doesNotContainKeys( - "table.kafka.convert.key.type", "table.kafka.convert.value.layout"); - assertThat(descriptor.getCustomProperties()) - .doesNotContainKeys("table.kafka.enabled", "table.kafka.convert.value.type"); - KafkaTableSchema mapping = KafkaTableSchema.resolve(descriptor); - assertThat(mapping.valueProjection().positions()).containsExactly(0); - assertThat(mapping.keyPosition()).isEqualTo(1); - assertThat(mapping.headersPosition()).isEqualTo(2); - assertThat(mapping.rescuePosition()).isEqualTo(3); - } - @Test void testConvertBucketCountChange() { assertThat( diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaConfigsTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaConfigsTest.java deleted file mode 100644 index 17a8c6c0f9c..00000000000 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/KafkaConfigsTest.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.fluss.kafka; - -import org.apache.fluss.config.ConfigOptions; -import org.apache.fluss.config.Configuration; - -import org.junit.jupiter.api.Test; - -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; - -import static org.assertj.core.api.Assertions.assertThat; - -/** Tests for Kafka configuration. */ -public class KafkaConfigsTest { - @Test - public void testFromMap() throws Exception { - Map map = new HashMap<>(); - map.put(ConfigOptions.KAFKA_ENABLED.key(), "true"); - map.put(ConfigOptions.KAFKA_LISTENER_NAMES.key(), "kafka,kafka_sasl"); - map.put(ConfigOptions.KAFKA_DEFAULT_DATABASE.key(), "sales"); - Configuration configuration = Configuration.fromMap(map); - - assertThat(configuration.getBoolean(ConfigOptions.KAFKA_ENABLED)).isTrue(); - assertThat(configuration.get(ConfigOptions.KAFKA_LISTENER_NAMES)) - .isEqualTo(Arrays.asList("kafka", "kafka_sasl")); - assertThat(configuration.getString(ConfigOptions.KAFKA_DEFAULT_DATABASE)) - .isEqualTo("sales"); - } - - @Test - public void testFromDefault() throws Exception { - Configuration configuration = Configuration.fromMap(new HashMap<>()); - assertThat(configuration.getBoolean(ConfigOptions.KAFKA_ENABLED)).isFalse(); - assertThat(configuration.get(ConfigOptions.KAFKA_LISTENER_NAMES)) - .isEqualTo(Collections.singletonList("KAFKA")); - assertThat(configuration.getString(ConfigOptions.KAFKA_DEFAULT_DATABASE)) - .isEqualTo("fluss"); - assertThat(configuration.toMap()).doesNotContainKey("kafka.default.database"); - } -} diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java index 97e75987f89..a327df3d5a6 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java @@ -19,8 +19,6 @@ import org.apache.fluss.config.ConfigOptions; import org.apache.fluss.config.Configuration; -import org.apache.fluss.exception.DatabaseNotExistException; -import org.apache.fluss.exception.InvalidConfigException; import org.apache.fluss.exception.InvalidTableException; import org.apache.fluss.kafka.mapping.KafkaTopicMapper; import org.apache.fluss.metadata.KafkaTableSchema; @@ -38,6 +36,7 @@ import org.apache.fluss.utils.json.DataTypeJsonSerde; import org.apache.fluss.utils.json.JsonSerdeUtils; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; @@ -67,7 +66,7 @@ class KafkaTableMappingITCase { static { CONFIG.set(ConfigOptions.KAFKA_ENABLED, true); - CONFIG.set(ConfigOptions.KAFKA_DEFAULT_DATABASE, "missing_default"); + CONFIG.set(ConfigOptions.KAFKA_DEFAULT_DATABASE, "kafka_default"); CONFIG.setString("kafka.database", "legacy_database"); } @@ -77,14 +76,24 @@ class KafkaTableMappingITCase { private CoordinatorGateway gateway; + @BeforeAll + static void verifyConfiguredDatabaseCreatedOnStartup() throws Exception { + assertThat( + CLUSTER.newCoordinatorClient() + .databaseExists(newDatabaseExistsRequest("kafka_default")) + .get() + .isExists()) + .isTrue(); + } + @BeforeEach void setup() { gateway = CLUSTER.newCoordinatorClient(); } @Test - void testNoAutomaticKafkaDatabaseAndQualifiedAccessWithoutDefaultDatabase() throws Exception { - for (String database : Arrays.asList("missing_default", "kafka", "legacy_database")) { + void testConfiguredKafkaDatabaseAndQualifiedAccess() throws Exception { + for (String database : Arrays.asList("kafka", "legacy_database")) { assertThat(gateway.databaseExists(newDatabaseExistsRequest(database)).get().isExists()) .isFalse(); } @@ -99,22 +108,16 @@ void testNoAutomaticKafkaDatabaseAndQualifiedAccessWithoutDefaultDatabase() thro long salesId = createTable(CLUSTER, sales, enabled().build()); long warehouseId = createTable(CLUSTER, warehouse, enabled().build()); assertThat(salesId).isNotEqualTo(warehouseId); - assertThat(mapper.toTablePath("orders")) - .isEqualTo(TablePath.of("missing_default", "orders")); - assertThatThrownBy( - () -> - gateway.createTable( - newCreateTableRequest( - mapper.toTablePath("orders"), - enabled().build(), - false)) - .get()) - .hasCauseInstanceOf(DatabaseNotExistException.class); + assertThat(mapper.toTablePath("orders")).isEqualTo(TablePath.of("kafka_default", "orders")); + // The cluster fixture removes custom databases between tests; recreate this table's + // database through the usual DDL helper after checking startup creation in @BeforeAll. + createTable(CLUSTER, mapper.toTablePath("orders"), enabled().build()); assertThat( - gateway.databaseExists(newDatabaseExistsRequest("missing_default")) + gateway.databaseExists(newDatabaseExistsRequest("kafka_default")) .get() .isExists()) - .isFalse(); + .isTrue(); + assertThat(KafkaTableSchema.isEnabled(stored(mapper.toTablePath("orders")))).isTrue(); Configuration config = new Configuration(); config.set(ConfigOptions.KAFKA_DEFAULT_DATABASE, "sales"); KafkaTopicMapper aliasMapper = new KafkaTopicMapper(config); @@ -123,60 +126,36 @@ void testNoAutomaticKafkaDatabaseAndQualifiedAccessWithoutDefaultDatabase() thro assertThat(aliasMapper.toTopicName(sales)).isEqualTo("sales.orders"); } - @Test - void testDefaultAliasUsesExistingFlussDatabase() throws Exception { - KafkaTopicMapper mapper = new KafkaTopicMapper(); - TablePath path = mapper.toTablePath("default_alias"); - assertThat(path).isEqualTo(TablePath.of(DEFAULT_DATABASE, "default_alias")); - createTable(CLUSTER, path, enabled().build()); - assertThat(info(mapper.toTablePath("fluss.default_alias")).getTableId()) - .isEqualTo(info(path).getTableId()); - assertThat(mapper.toTopicName(path)).isEqualTo("fluss.default_alias"); - } - @ParameterizedTest @ValueSource(strings = {"flat", "nested"}) void testJsonCreateAndEnableRejectUnsupportedBusinessSchema(String layout) throws Exception { gateway.createDatabase(newCreateDatabaseRequest("ddl_json", true)).get(); - DataType invalidMap = DataTypes.MAP(DataTypes.INT(), DataTypes.STRING()); - int index = 0; - for (DataType type : - Arrays.asList( - invalidMap, - DataTypes.ARRAY(invalidMap), - DataTypes.ROW(DataTypes.FIELD("attributes", invalidMap)), - DataTypes.MAP(DataTypes.STRING(), DataTypes.ARRAY(invalidMap)))) { - TablePath path = TablePath.of("ddl_json", layout + "_invalid_" + index++); - TableDescriptor descriptor = jsonTable(layout, type).build(); - assertThatThrownBy( - () -> - gateway.createTable( - newCreateTableRequest(path, descriptor, false)) - .get()) - .hasCauseInstanceOf(InvalidTableException.class) - .hasStackTraceContaining("only supports STRING map keys"); - assertThat(gateway.tableExists(newTableExistsRequest(path)).get().isExists()).isFalse(); - assertThat(CLUSTER.getZooKeeperClient().getCurrentSchemaId(path)).isZero(); - - // The identical schema is legal for an ordinary, explicitly disabled Fluss table. - createTable( - CLUSTER, - path, - TableDescriptor.builder(descriptor) - .property(ConfigOptions.TABLE_KAFKA_ENABLED, false) - .build()); - GetTableInfoResponse before = info(path); - assertThatThrownBy( - () -> - set( - path, - Collections.singletonMap( - "table.kafka.enabled", "true"))) - .hasCauseInstanceOf(InvalidTableException.class) - .hasStackTraceContaining("only supports STRING map keys"); - assertUnchanged(path, before); - assertThat(KafkaTableSchema.isEnabled(stored(path))).isFalse(); - } + TablePath path = TablePath.of("ddl_json", layout + "_invalid"); + DataType invalid = + DataTypes.ARRAY( + DataTypes.ROW( + DataTypes.FIELD( + "attributes", + DataTypes.MAP(DataTypes.INT(), DataTypes.STRING())))); + TableDescriptor descriptor = jsonTable(layout, invalid).build(); + assertThatThrownBy( + () -> + gateway.createTable(newCreateTableRequest(path, descriptor, false)) + .get()) + .hasCauseInstanceOf(InvalidTableException.class); + assertThat(gateway.tableExists(newTableExistsRequest(path)).get().isExists()).isFalse(); + assertThat(CLUSTER.getZooKeeperClient().getCurrentSchemaId(path)).isZero(); + + createTable( + CLUSTER, + path, + TableDescriptor.builder(descriptor) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, false) + .build()); + GetTableInfoResponse before = info(path); + assertThatThrownBy(() -> set(path, Collections.singletonMap("table.kafka.enabled", "true"))) + .hasCauseInstanceOf(InvalidTableException.class); + assertUnchanged(path, before); } @ParameterizedTest @@ -238,41 +217,6 @@ void testJsonSchemaAlterRejectsUnsupportedNestedMapAtomically() throws Exception assertUnchanged(path, disabled); } - @ParameterizedTest - @ValueSource(strings = {"flat", "nested"}) - void testJsonDepthValidationThroughDdl(String layout) throws Exception { - DataType boundary = DataTypes.STRING(); - for (int i = 0; i < KafkaTableSchema.MAX_JSON_SCHEMA_NESTING_DEPTH; i++) { - boundary = DataTypes.ARRAY(boundary); - } - TablePath valid = TablePath.of("ddl_json", layout + "_depth_boundary"); - TableDescriptor descriptor = jsonTable(layout, boundary).build(); - createTable(CLUSTER, valid, descriptor); - assertThat(stored(valid).getSchema()).isEqualTo(descriptor.getSchema()); - TablePath invalid = TablePath.of("ddl_json", layout + "_too_deep"); - TableDescriptor tooDeep = jsonTable(layout, DataTypes.ARRAY(boundary)).build(); - assertThatThrownBy( - () -> - gateway.createTable(newCreateTableRequest(invalid, tooDeep, false)) - .get()) - .hasCauseInstanceOf(InvalidTableException.class) - .hasStackTraceContaining("maximum nesting depth of 64"); - assertThat(gateway.tableExists(newTableExistsRequest(invalid)).get().isExists()).isFalse(); - assertThat(CLUSTER.getZooKeeperClient().getCurrentSchemaId(invalid)).isZero(); - createTable( - CLUSTER, - invalid, - TableDescriptor.builder(tooDeep) - .property(ConfigOptions.TABLE_KAFKA_ENABLED, false) - .build()); - GetTableInfoResponse before = info(invalid); - assertThatThrownBy( - () -> set(invalid, Collections.singletonMap("table.kafka.enabled", "true"))) - .hasCauseInstanceOf(InvalidTableException.class) - .hasStackTraceContaining("maximum nesting depth of 64"); - assertUnchanged(invalid, before); - } - private void assertUnchanged(TablePath path, GetTableInfoResponse before) throws Exception { GetTableInfoResponse after = info(path); assertThat(after.getTableJson()).isEqualTo(before.getTableJson()); @@ -374,10 +318,12 @@ void testSchemaAlterValidatesBeforePersistenceAndCanBeDisabled() throws Exceptio addColumn(path, "_kafka_headers", DataTypes.MAP(DataTypes.STRING(), DataTypes.BYTES())); assertThat(KafkaTableSchema.resolve(stored(path)).headersPosition()).isEqualTo(1); assertThat(info(path).getSchemaId()).isGreaterThan(before.getSchemaId()); - TableDescriptor withHeaders = stored(path); + addColumn(path, "_kafka_metadata", metadataType()); + assertThat(KafkaTableSchema.resolve(stored(path)).metadataPosition()).isEqualTo(2); + TableDescriptor beforeFailedRescue = stored(path); assertThatThrownBy(() -> addColumn(path, "_kafka_rescue", DataTypes.INT())) .hasCauseInstanceOf(InvalidTableException.class); - assertThat(stored(path)).isEqualTo(withHeaders); + assertThat(stored(path)).isEqualTo(beforeFailedRescue); set(path, Collections.singletonMap("table.kafka.enabled", "false")); addColumn(path, "unmapped_business_field", DataTypes.INT()); assertThat(stored(path).getSchema().getRowType().getFieldNames()) @@ -388,85 +334,18 @@ void testSchemaAlterValidatesBeforePersistenceAndCanBeDisabled() throws Exceptio } @Test - void testCreateRejectsInvalidContractWithoutPublishingMetadata() throws Exception { - gateway.createDatabase(newCreateDatabaseRequest("ddl_invalid", true)).get(); - TablePath path = TablePath.of("ddl_invalid", "bad"); - assertThatThrownBy( - () -> - gateway.createTable( - newCreateTableRequest( - path, - enabled() - .property( - ConfigOptions - .TABLE_KAFKA_CONVERT_VALUE_TYPE, - "string") - .build(), - false)) - .get()) - .hasCauseInstanceOf(InvalidTableException.class); - assertThat(gateway.tableExists(newTableExistsRequest(path)).get().isExists()).isFalse(); - assertThat(CLUSTER.getZooKeeperClient().getCurrentSchemaId(path)).isZero(); - // Non-null added columns are rejected by generic schema evolution first, so exercise - // Kafka-specific rescue nullability through CREATE, where such columns are allowed. - TableDescriptor nonNullableRescue = - enabled() - .schema( - Schema.newBuilder() - .column("_kafka_value", DataTypes.BYTES()) - .column("_kafka_rescue", DataTypes.STRING().copy(false)) - .build()) - .build(); - assertThatThrownBy( - () -> - gateway.createTable( - newCreateTableRequest( - path, nonNullableRescue, false)) - .get()) - .hasCauseInstanceOf(InvalidTableException.class) - .hasStackTraceContaining("nullable STRING"); - assertThat(CLUSTER.getZooKeeperClient().getCurrentSchemaId(path)).isZero(); - assertThatThrownBy( - () -> - gateway.createTable( - newCreateTableRequest( - path, - enabled() - .property( - "table.kafka.convert.rescue-column", - "rescue") - .build(), - false)) - .get()) - .hasCauseInstanceOf(InvalidConfigException.class); - assertThat(gateway.tableExists(newTableExistsRequest(path)).get().isExists()).isFalse(); - createTable(CLUSTER, path, enabled().build()); - assertThat(KafkaTableSchema.isEnabled(stored(path))).isTrue(); - } - - @ParameterizedTest - @ValueSource(strings = {"binary", "string"}) - void testMetadataCanBeCreatedAndEnabledWithoutOffset(String format) throws Exception { + void testMetadataCanBeCreatedAndEnabledWithoutOffset() throws Exception { Schema schema = Schema.newBuilder() .column("_kafka_metadata", metadataType()) - .column( - "_kafka_value", - "binary".equals(format) ? DataTypes.BYTES() : DataTypes.STRING()) - .build(); - TableDescriptor descriptor = - enabled() - .schema(schema) - .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, format) + .column("_kafka_value", DataTypes.BYTES()) .build(); - TablePath created = TablePath.of("ddl", "metadata_create_" + format); + TableDescriptor descriptor = enabled().schema(schema).build(); + TablePath created = TablePath.of("ddl", "metadata_create"); createTable(CLUSTER, created, descriptor); - TableDescriptor persisted = stored(created); - assertThat(persisted.getSchema()).isEqualTo(schema); - assertThat(kafkaProperties(persisted)).isEqualTo(kafkaProperties(descriptor)); - assertThat(KafkaTableSchema.resolve(persisted).metadataPosition()).isZero(); + assertThat(stored(created).getSchema()).isEqualTo(schema); - TablePath altered = TablePath.of("ddl", "metadata_enable_" + format); + TablePath altered = TablePath.of("ddl", "metadata_enable"); createTable( CLUSTER, altered, @@ -475,25 +354,9 @@ void testMetadataCanBeCreatedAndEnabledWithoutOffset(String format) throws Excep .build()); int schemaId = info(altered).getSchemaId(); set(altered, Collections.singletonMap("table.kafka.enabled", "true")); - persisted = stored(altered); assertThat(info(altered).getSchemaId()).isEqualTo(schemaId); - assertThat(persisted.getSchema()).isEqualTo(schema); - assertThat(kafkaProperties(persisted)).isEqualTo(kafkaProperties(descriptor)); - assertThat(KafkaTableSchema.resolve(persisted).metadataPosition()).isZero(); - } - - @Test - void testMetadataCanBeAddedToEnabledTable() throws Exception { - TablePath path = TablePath.of("ddl", "metadata_add"); - createTable(CLUSTER, path, enabled().build()); - TableDescriptor before = stored(path); - addColumn(path, "_kafka_metadata", metadataType()); - TableDescriptor after = stored(path); - KafkaTableSchema resolved = KafkaTableSchema.resolve(after); - assertThat(resolved.metadataPosition()).isGreaterThanOrEqualTo(0); - assertThat(resolved.rowType().getTypeAt(resolved.metadataPosition())) - .isEqualTo(metadataType()); - assertThat(kafkaProperties(after)).isEqualTo(kafkaProperties(before)); + assertThat(stored(altered).getSchema()).isEqualTo(schema); + assertThat(KafkaTableSchema.resolve(stored(altered)).metadataPosition()).isZero(); } @Test diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolverTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolverTest.java index 8327c6fb60e..119e8aff38f 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolverTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolverTest.java @@ -18,7 +18,6 @@ package org.apache.fluss.kafka.schema; import org.apache.fluss.config.ConfigOptions; -import org.apache.fluss.metadata.KafkaTableSchema; import org.apache.fluss.metadata.Schema; import org.apache.fluss.metadata.TableDescriptor; import org.apache.fluss.types.DataTypes; @@ -32,33 +31,6 @@ class KafkaTopicSchemaResolverTest { private final KafkaTopicSchemaResolver resolver = new KafkaTopicSchemaResolver(); - @Test - void testEnabledContractUsesSharedValidation() { - TableDescriptor table = - table().schema( - Schema.newBuilder() - .column( - "_kafka_metadata", - DataTypes.ROW( - DataTypes.FIELD( - "timestamp", - DataTypes.TIMESTAMP_LTZ(3)), - DataTypes.FIELD( - "timestamp_type", DataTypes.INT()), - DataTypes.FIELD( - "partition", DataTypes.INT()))) - .column("_kafka_value", DataTypes.BYTES()) - .build()) - .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) - .build(); - assertThat(resolver.isEnabled(table)).isTrue(); - assertThat(resolver.resolve(table)).isEqualTo(KafkaTableSchema.resolve(table)); - assertThat(resolver.resolve(table).metadataPosition()).isZero(); - assertThat(resolver.resolve(table).valueProjection().positions()).containsExactly(1); - assertThat(resolver.resolve(TableDescriptor.fromJsonBytes(table.toJsonBytes()))) - .isEqualTo(resolver.resolve(table)); - } - @Test void testCompatibleSchemaDoesNotEnableTable() { assertThat(resolver.isEnabled(table().build())).isFalse(); diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorServer.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorServer.java index aff8bb78761..ac161728107 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorServer.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorServer.java @@ -548,11 +548,27 @@ private interface ThrowingRunnable { private void createDefaultDatabase() { MetadataManager metadataManager = new MetadataManager(zkClient, conf, lakeCatalogDynamicLoader); + createDefaultDatabases(metadataManager, conf); + } + + @VisibleForTesting + static void createDefaultDatabases(MetadataManager metadataManager, Configuration conf) { List databases = metadataManager.listDatabases(); if (databases.isEmpty()) { metadataManager.createDatabase(DEFAULT_DATABASE, DatabaseDescriptor.EMPTY, true); LOG.info("Created default database '{}' because no database exists.", DEFAULT_DATABASE); } + if (conf.get(ConfigOptions.KAFKA_ENABLED)) { + String kafkaDatabase = conf.get(ConfigOptions.KAFKA_DEFAULT_DATABASE); + // An empty cluster has already created the native default database above. + if (DEFAULT_DATABASE.equals(kafkaDatabase) && databases.isEmpty()) { + return; + } + if (!databases.contains(kafkaDatabase)) { + metadataManager.createDatabase(kafkaDatabase, DatabaseDescriptor.EMPTY, true); + LOG.info("Created default database '{}' for Kafka protocol.", kafkaDatabase); + } + } } /** diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/MetadataManager.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/MetadataManager.java index b1a5fae3d04..3354b869eb6 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/MetadataManager.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/MetadataManager.java @@ -474,7 +474,7 @@ public void alterTableSchema( ZkVersion.MATCH_ANY_VERSION.getVersion()); } - /** Alters schema with a table-version and coordinator-epoch fence against concurrent ALTER. */ + /** Alters schema with table, schema and coordinator version checks against concurrent ALTER. */ public void alterTableSchema( TablePath tablePath, List schemaChanges, @@ -485,6 +485,7 @@ public void alterTableSchema( try { ZooKeeperClient.VersionedData versionedTable = getTableRegistrationWithVersion(tablePath); + int schemaZkVersion = zookeeperClient.getSchemaZkVersion(tablePath); TableInfo table = getTable(tablePath); TableDescriptor tableDescriptor = table.toTableDescriptor(); @@ -505,11 +506,12 @@ public void alterTableSchema( // Update Fluss schema (ZK) after Lake sync succeeds if (!newSchema.equals(table.getSchema())) { - zookeeperClient.registerSchemaAndBumpTableVersion( + zookeeperClient.registerSchemaWithVersionCheck( tablePath, newSchema, table.getSchemaId() + 1, - versionedTable, + versionedTable.zkVersion(), + schemaZkVersion, coordinatorZkVersion); } else { LOG.info( @@ -813,10 +815,10 @@ private boolean isTablePresent(TablePath tablePath) { } /** - * One attempt of the property ALTER read-modify-write. The ZK write is CAS-guarded by the table - * version read here plus the coordinator epoch version. Since {@link #preAlterTableProperties} - * may apply external lake side effects, a version conflict is surfaced instead of being retried - * automatically. + * One attempt of the property ALTER read-modify-write. The ZK write is guarded by the table and + * schema versions read here plus the coordinator epoch version. Since {@link + * #preAlterTableProperties} may apply external lake side effects, a version conflict is + * surfaced instead of being retried automatically. */ private void doAlterTablePropertiesOnce( TablePath tablePath, @@ -832,6 +834,7 @@ private void doAlterTablePropertiesOnce( getTableRegistrationWithVersion(tablePath); TableRegistration tableReg = versionedTableReg.data(); int tableZkVersion = versionedTableReg.zkVersion(); + int schemaZkVersion = zookeeperClient.getSchemaZkVersion(tablePath); SchemaInfo schemaInfo = getLatestSchema(tablePath); // we can't use MetadataManager#getTable here, because it will add the default // lake options to the table properties, which may cause the validation failure @@ -901,7 +904,8 @@ private void doAlterTablePropertiesOnce( updatedTableRegistration, tableZkVersion, Collections.emptyMap(), - coordinatorZkVersion); + coordinatorZkVersion, + schemaZkVersion); } catch (KeeperException.BadVersionException e) { throw new FlussRuntimeException( String.format( diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/event/watcher/TableChangeWatcher.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/event/watcher/TableChangeWatcher.java index f38a7f0afe4..5e05877b4be 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/event/watcher/TableChangeWatcher.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/event/watcher/TableChangeWatcher.java @@ -17,7 +17,6 @@ package org.apache.fluss.server.coordinator.event.watcher; -import org.apache.fluss.annotation.VisibleForTesting; import org.apache.fluss.config.Configuration; import org.apache.fluss.config.TableConfig; import org.apache.fluss.metadata.PhysicalTablePath; @@ -88,8 +87,7 @@ public void stop() { } /** A listener to monitor the changes of table nodes in zookeeper. */ - @VisibleForTesting - final class TablePathChangeListener implements CuratorCacheListener { + private final class TablePathChangeListener implements CuratorCacheListener { @Override public void event(Type type, ChildData oldData, ChildData newData) { @@ -152,13 +150,13 @@ public void event(Type type, ChildData oldData, ChildData newData) { if (tablePath == null) { break; } - // Distinguish between table creation and registration updates. + // Distinguish between table creation and properties change. // If oldData exists and contains valid table registration data, - // compare its contents; otherwise, it's a table creation. + // it's a properties change; otherwise, it's a table creation. if (oldData != null && oldData.getData() != null && oldData.getData().length > 0) { - processTableRegistrationChange(tablePath, oldData, newData); + processTableRegistrationChange(tablePath, newData); } else { processCreateTable(tablePath, newData); } @@ -278,16 +276,9 @@ private void processCreatePartition( tablePath, tableId, partitionId, partitionName, partitionAssignment)); } - private void processTableRegistrationChange( - TablePath tablePath, ChildData oldData, ChildData newData) { - TableRegistration oldTable = TableZNode.decode(oldData.getData()); + private void processTableRegistrationChange(TablePath tablePath, ChildData newData) { TableRegistration newTable = TableZNode.decode(newData.getData()); - // Schema ALTER bumps the table znode version to fence concurrent registration updates, - // even when its contents are unchanged. Compare decoded registrations so differences - // in JSON serialization do not trigger a registration change either. - if (!oldTable.equals(newTable)) { - eventManager.put(new TableRegistrationChangeEvent(tablePath, newTable)); - } + eventManager.put(new TableRegistrationChangeEvent(tablePath, newTable)); } } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/zk/ZooKeeperClient.java b/fluss-server/src/main/java/org/apache/fluss/server/zk/ZooKeeperClient.java index a7a706c214e..99d2c6c01ae 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/zk/ZooKeeperClient.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/zk/ZooKeeperClient.java @@ -1159,7 +1159,32 @@ public void updateTableWithPartitionBucketCountBackfill( Map> partitionBackfills, int expectedCoordinatorEpochZkVersion) throws Exception { + updateTableWithPartitionBucketCountBackfill( + tablePath, + tableRegistration, + expectedTableZkVersion, + partitionBackfills, + expectedCoordinatorEpochZkVersion, + ZkVersion.MATCH_ANY_VERSION.getVersion()); + } + + /** Updates table properties with an additional fence on the schema used for validation. */ + public void updateTableWithPartitionBucketCountBackfill( + TablePath tablePath, + TableRegistration tableRegistration, + int expectedTableZkVersion, + Map> partitionBackfills, + int expectedCoordinatorEpochZkVersion, + int expectedSchemaZkVersion) + throws Exception { List ops = new ArrayList<>(partitionBackfills.size() + 1); + if (expectedSchemaZkVersion != ZkVersion.MATCH_ANY_VERSION.getVersion()) { + ops.add( + zkClient.transactionOp() + .check() + .withVersion(expectedSchemaZkVersion) + .forPath(SchemasZNode.path(tablePath))); + } for (Map.Entry> entry : partitionBackfills.entrySet()) { String partitionPath = PartitionZNode.path(tablePath, entry.getKey()); @@ -1316,25 +1341,29 @@ public int registerSchema(TablePath tablePath, Schema schema, int schemaId) thro } /** - * Publishes a schema and advances the table registration version in one fenced transaction. - * Property ALTER validates against a schema snapshot and compares the table version, so schema - * ALTER must participate in the same CAS to prevent enabling an incompatible Kafka contract. + * Publishes a schema and advances the schema version in one fenced transaction. The table + * registration is checked without being rewritten; property ALTER checks the schema version to + * prevent enabling a Kafka contract against a stale schema. */ - public void registerSchemaAndBumpTableVersion( + public void registerSchemaWithVersionCheck( TablePath tablePath, Schema schema, int schemaId, - VersionedData expectedTable, + int expectedTableZkVersion, + int expectedSchemaZkVersion, int expectedCoordinatorEpochZkVersion) throws Exception { List ops = new ArrayList<>(); + ops.add( + zkClient.transactionOp() + .check() + .withVersion(expectedTableZkVersion) + .forPath(TableZNode.path(tablePath))); ops.add( zkClient.transactionOp() .setData() - .withVersion(expectedTable.zkVersion()) - .forPath( - TableZNode.path(tablePath), - TableZNode.encode(expectedTable.data()))); + .withVersion(expectedSchemaZkVersion) + .forPath(SchemasZNode.path(tablePath), new byte[0])); ops.add( zkClient.transactionOp() .create() @@ -1345,6 +1374,13 @@ public void registerSchemaAndBumpTableVersion( .forOperations(wrapRequestsWithEpochCheck(ops, expectedCoordinatorEpochZkVersion)); } + /** Returns the schema parent znode version, read before loading a schema for DDL validation. */ + public int getSchemaZkVersion(TablePath tablePath) throws Exception { + Stat stat = new Stat(); + zkClient.getData().storingStatIn(stat).forPath(SchemasZNode.path(tablePath)); + return stat.getVersion(); + } + /** Get the specific schema by schema id in ZK metadata. */ public Optional getSchemaById(TablePath tablePath, int schemaId) throws Exception { Optional bytes = getOrEmpty(SchemaZNode.path(tablePath, schemaId)); diff --git a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/KafkaDefaultDatabaseTest.java b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/KafkaDefaultDatabaseTest.java new file mode 100644 index 00000000000..d846c95f3e6 --- /dev/null +++ b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/KafkaDefaultDatabaseTest.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.server.coordinator; + +import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.config.Configuration; +import org.apache.fluss.metadata.DatabaseDescriptor; +import org.apache.fluss.server.zk.NOPErrorHandler; +import org.apache.fluss.server.zk.ZooKeeperClient; +import org.apache.fluss.server.zk.ZooKeeperExtension; +import org.apache.fluss.server.zk.data.ZkData.DatabasesZNode; +import org.apache.fluss.testutils.common.AllCallbackWrapper; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; + +import static org.apache.fluss.server.coordinator.CoordinatorServer.DEFAULT_DATABASE; +import static org.assertj.core.api.Assertions.assertThat; + +/** Tests startup database creation for native and Kafka default database configurations. */ +class KafkaDefaultDatabaseTest { + + @RegisterExtension + public static final AllCallbackWrapper ZK = + new AllCallbackWrapper<>(new ZooKeeperExtension()); + + private MetadataManager manager; + + @BeforeEach + void setup() throws Exception { + ZooKeeperClient zk = ZK.getCustomExtension().getZooKeeperClient(NOPErrorHandler.INSTANCE); + if (zk.getStat(DatabasesZNode.path()).isPresent()) { + zk.getCuratorClient() + .delete() + .deletingChildrenIfNeeded() + .forPath(DatabasesZNode.path()); + } + manager = + new MetadataManager( + zk, + new Configuration(), + new LakeCatalogDynamicLoader(new Configuration(), null, true)); + } + + @ParameterizedTest + @CsvSource({"fluss,false", "kafka_default,false", "fluss,true", "kafka_default,true"}) + void testCreateMissingDefaultAndPreserveExisting(String database, boolean exists) { + manager.createDatabase("existing", DatabaseDescriptor.EMPTY, false); + DatabaseDescriptor descriptor = + DatabaseDescriptor.builder().comment("existing default").build(); + if (exists) { + manager.createDatabase(database, descriptor, false); + } + Configuration config = kafkaConfiguration(database); + CoordinatorServer.createDefaultDatabases(manager, config); + CoordinatorServer.createDefaultDatabases(manager, config); + assertThat(manager.listDatabases()).containsExactlyInAnyOrder("existing", database); + assertThat(manager.getDatabase(database).getDatabaseDescriptor().toJsonBytes()) + .isEqualTo((exists ? descriptor : DatabaseDescriptor.EMPTY).toJsonBytes()); + } + + @Test + void testDisabledKafkaDoesNotCreateConfiguredDatabase() { + Configuration config = kafkaConfiguration("kafka_default"); + config.set(ConfigOptions.KAFKA_ENABLED, false); + CoordinatorServer.createDefaultDatabases(manager, config); + assertThat(manager.listDatabases()).containsExactly(DEFAULT_DATABASE); + } + + private static Configuration kafkaConfiguration(String database) { + Configuration config = new Configuration(); + config.set(ConfigOptions.KAFKA_ENABLED, true); + config.set(ConfigOptions.KAFKA_DEFAULT_DATABASE, database); + return config; + } +} diff --git a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/KafkaTableSchemaAlterTest.java b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/KafkaTableSchemaAlterTest.java index e76dbda662d..dc08ed10385 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/KafkaTableSchemaAlterTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/KafkaTableSchemaAlterTest.java @@ -21,7 +21,6 @@ import org.apache.fluss.config.Configuration; import org.apache.fluss.exception.FlussRuntimeException; import org.apache.fluss.metadata.DatabaseDescriptor; -import org.apache.fluss.metadata.KafkaTableSchema; import org.apache.fluss.metadata.Schema; import org.apache.fluss.metadata.TableChange; import org.apache.fluss.metadata.TableDescriptor; @@ -72,6 +71,9 @@ void setup() throws Exception { @Test void testConcurrentSchemaChangePreventsStaleEnablement() throws Exception { TablePath path = create("schema_wins"); + ZooKeeperClient.VersionedData before = + zk.getTableWithVersion(path).get(); + int schemaZkVersion = zk.getSchemaZkVersion(path); assertThatThrownBy( () -> enable( @@ -91,6 +93,10 @@ void testConcurrentSchemaChangePreventsStaleEnablement() throws Exception { epoch))) .isInstanceOf(FlussRuntimeException.class) .hasMessageContaining("Concurrent metadata change"); + ZooKeeperClient.VersionedData after = zk.getTableWithVersion(path).get(); + assertThat(after.zkVersion()).isEqualTo(before.zkVersion()); + assertThat(after.data()).isEqualTo(before.data()); + assertThat(zk.getSchemaZkVersion(path)).isEqualTo(schemaZkVersion + 1); assertThat(manager.getTable(path).getTableConfig().isKafkaEnabled()).isFalse(); assertThat(manager.getTable(path).getSchema().getRowType().getFieldNames()) .contains("unmapped"); @@ -101,22 +107,23 @@ void testConcurrentEnablementPreventsStaleSchemaPublication() throws Exception { TablePath path = create("enable_wins"); ZooKeeperClient.VersionedData snapshot = zk.getTableWithVersion(path).get(); + int schemaZkVersion = zk.getSchemaZkVersion(path); int schemaId = zk.getCurrentSchemaId(path); Schema incompatible = Schema.newBuilder() .column("_kafka_value", DataTypes.BYTES()) .column("unmapped", DataTypes.STRING()) .build(); - // This candidate is valid for the disabled snapshot, but not after the competing enable. - KafkaTableSchema.validate( - TableDescriptor.builder(manager.getTable(path).toTableDescriptor()) - .schema(incompatible) - .build()); enable(path, (oldTable, updated) -> {}); assertThatThrownBy( () -> - zk.registerSchemaAndBumpTableVersion( - path, incompatible, schemaId + 1, snapshot, epoch)) + zk.registerSchemaWithVersionCheck( + path, + incompatible, + schemaId + 1, + snapshot.zkVersion(), + schemaZkVersion, + epoch)) .isInstanceOf(KeeperException.BadVersionException.class); assertThat(zk.getCurrentSchemaId(path)).isEqualTo(schemaId); assertThat(manager.getTable(path).getTableConfig().isKafkaEnabled()).isTrue(); @@ -125,26 +132,39 @@ void testConcurrentEnablementPreventsStaleSchemaPublication() throws Exception { } @Test - void testSchemaFailureDoesNotAdvanceTableVersion() throws Exception { + void testSchemaFailureDoesNotAdvanceSchemaVersion() throws Exception { TablePath path = create("atomic_failure"); ZooKeeperClient.VersionedData snapshot = zk.getTableWithVersion(path).get(); + int schemaZkVersion = zk.getSchemaZkVersion(path); int schemaId = zk.getCurrentSchemaId(path); Schema schema = manager.getTable(path).getSchema(); assertThatThrownBy( () -> - zk.registerSchemaAndBumpTableVersion( - path, schema, schemaId, snapshot, epoch)) + zk.registerSchemaWithVersionCheck( + path, + schema, + schemaId, + snapshot.zkVersion(), + schemaZkVersion, + epoch)) .isInstanceOf(KeeperException.NodeExistsException.class); assertThat(zk.getTableWithVersion(path).get().zkVersion()).isEqualTo(snapshot.zkVersion()); + assertThat(zk.getSchemaZkVersion(path)).isEqualTo(schemaZkVersion); assertThat(zk.getCurrentSchemaId(path)).isEqualTo(schemaId); zk.fenceBecomeCoordinatorLeader("next-coordinator"); assertThatThrownBy( () -> - zk.registerSchemaAndBumpTableVersion( - path, schema, schemaId + 1, snapshot, epoch)) + zk.registerSchemaWithVersionCheck( + path, + schema, + schemaId + 1, + snapshot.zkVersion(), + schemaZkVersion, + epoch)) .isInstanceOf(KeeperException.BadVersionException.class); assertThat(zk.getTableWithVersion(path).get().zkVersion()).isEqualTo(snapshot.zkVersion()); + assertThat(zk.getSchemaZkVersion(path)).isEqualTo(schemaZkVersion); assertThat(zk.getCurrentSchemaId(path)).isEqualTo(schemaId); } diff --git a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/event/watcher/TableChangeWatcherTest.java b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/event/watcher/TableChangeWatcherTest.java index 9a36280d891..28a4131ecd8 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/event/watcher/TableChangeWatcherTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/event/watcher/TableChangeWatcherTest.java @@ -42,16 +42,11 @@ import org.apache.fluss.server.entity.TablePropertyChanges; import org.apache.fluss.server.zk.NOPErrorHandler; import org.apache.fluss.server.zk.ZooKeeperClient; -import org.apache.fluss.server.zk.ZooKeeperClient.VersionedData; import org.apache.fluss.server.zk.ZooKeeperExtension; import org.apache.fluss.server.zk.data.PartitionAssignment; import org.apache.fluss.server.zk.data.TableAssignment; import org.apache.fluss.server.zk.data.TableRegistration; -import org.apache.fluss.server.zk.data.ZkData.TableZNode; import org.apache.fluss.server.zk.data.ZkVersion; -import org.apache.fluss.shaded.curator5.org.apache.curator.framework.recipes.cache.ChildData; -import org.apache.fluss.shaded.curator5.org.apache.curator.framework.recipes.cache.CuratorCacheListener; -import org.apache.fluss.shaded.zookeeper3.org.apache.zookeeper.data.Stat; import org.apache.fluss.testutils.common.AllCallbackWrapper; import org.apache.fluss.types.DataTypes; import org.apache.fluss.utils.clock.SystemClock; @@ -61,13 +56,9 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.ValueSource; -import java.nio.charset.StandardCharsets; import java.time.Duration; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -311,7 +302,7 @@ void testPartitionedTable() throws Exception { } @Test - void testSchemaChanges() throws Exception { + void testSchemaChanges() { // create tables, collect create table events List expectedEvents = new ArrayList<>(); for (int i = 0; i < 10; i++) { @@ -358,8 +349,6 @@ void testSchemaChanges() throws Exception { } CreateTableEvent createTableEvent = (CreateTableEvent) coordinatorEvent; TableInfo tableInfo = createTableEvent.getTableInfo(); - VersionedData beforeAlter = - zookeeperClient.getTableWithVersion(tableInfo.getTablePath()).get(); metadataManager.alterTableSchema( tableInfo.getTablePath(), Collections.singletonList( @@ -370,10 +359,6 @@ void testSchemaChanges() throws Exception { TableChange.ColumnPosition.last())), false, null); - VersionedData afterAlter = - zookeeperClient.getTableWithVersion(tableInfo.getTablePath()).get(); - assertThat(afterAlter.zkVersion()).isEqualTo(beforeAlter.zkVersion() + 1); - assertThat(afterAlter.data()).isEqualTo(beforeAlter.data()); Schema newSchema = Schema.newBuilder() .fromSchema(tableInfo.getSchema()) @@ -393,62 +378,6 @@ void testSchemaChanges() throws Exception { .containsExactlyInAnyOrderElementsOf(allEvents)); } - @ParameterizedTest - @ValueSource(booleans = {false, true}) - void testUnchangedTableRegistration(boolean differentJsonFormatting) { - tableChangeWatcher.stop(); - TablePath tablePath = TablePath.of(DEFAULT_DB, "unchanged_table_registration"); - TableRegistration table = TableRegistration.newTable(1L, remoteDataDir, TEST_TABLE); - byte[] oldData = TableZNode.encode(table); - byte[] newData = - differentJsonFormatting - ? ("\n" + new String(oldData, StandardCharsets.UTF_8)) - .getBytes(StandardCharsets.UTF_8) - : oldData; - - // Deliver the callback synchronously so the absence of an event cannot pass before - // CuratorCache has observed the version bump. - CuratorCacheListener listener = tableChangeWatcher.new TablePathChangeListener(); - listener.event( - CuratorCacheListener.Type.NODE_CHANGED, - tableNodeData(tablePath, oldData, 1), - tableNodeData(tablePath, newData, 2)); - - assertThat(eventManager.getEvents()).isEmpty(); - } - - @Test - void testChangedTableRegistrationContents() { - tableChangeWatcher.stop(); - TablePath tablePath = TablePath.of(DEFAULT_DB, "changed_table_registration"); - TableRegistration table = TableRegistration.newTable(1L, remoteDataDir, TEST_TABLE); - CuratorCacheListener listener = tableChangeWatcher.new TablePathChangeListener(); - List expectedEvents = new ArrayList<>(); - for (TableRegistration updatedTable : - Arrays.asList( - table.newProperties( - Collections.singletonMap( - ConfigOptions.TABLE_KAFKA_ENABLED.key(), "true"), - table.customProperties), - table.newProperties( - table.properties, - Collections.singletonMap("custom.key", "custom.value")), - table.newBucketCount(4))) { - listener.event( - CuratorCacheListener.Type.NODE_CHANGED, - tableNodeData(tablePath, TableZNode.encode(table), 1), - tableNodeData(tablePath, TableZNode.encode(updatedTable), 2)); - expectedEvents.add(new TableRegistrationChangeEvent(tablePath, updatedTable)); - assertThat(eventManager.getEvents()).containsExactlyElementsOf(expectedEvents); - } - } - - private static ChildData tableNodeData(TablePath tablePath, byte[] data, int version) { - Stat stat = new Stat(); - stat.setVersion(version); - return new ChildData(TableZNode.path(tablePath), stat, data); - } - @Test void testTableRegistrationChange() { // create a table diff --git a/website/docs/maintenance/configuration.md b/website/docs/maintenance/configuration.md index 2f656004c43..3a360df5319 100644 --- a/website/docs/maintenance/configuration.md +++ b/website/docs/maintenance/configuration.md @@ -259,14 +259,16 @@ Kafka protocol compatibility is still in development. |--------------------------------|----------|---------|--------------------------------------------------------------------------------------------------------------------| | kafka.enabled | Boolean | false | Whether enable Fluss Kafka. Disabled by default. When this option is set to true, the Fluss Kafka will be enabled. | | kafka.listener.names | String | KAFKA | The listener names for Kafka wire protocol communication. Support multiple listener names, separated by comma. | -| kafka.default.database | String | fluss | Database used for unqualified Kafka topic names. Fully qualified names can access other databases. This option does not create a database. | +| kafka.default.database | String | fluss | Database used for unqualified Kafka topic names. Fully qualified names can access other databases. When Kafka is enabled, the coordinator creates this database if it is missing. | Kafka listeners share `netty.connection.max-idle-time` with native Fluss RPC listeners (see [Netty](#netty)). The default is `10min`; connections close when neither reads nor writes occur for the configured duration. Kafka tables must be created through Fluss or the Flink Fluss catalog. A topic named `sales.orders` maps to that table directly; `orders` maps to the same-named table in -`kafka.default.database` (by default, `fluss.orders`). Databases and tables must already exist. Reverse mapping always uses +`kafka.default.database` (by default, `fluss.orders`). The coordinator creates the configured default +database when Kafka is enabled and the database is missing. Existing databases are preserved; +other databases and all tables must be created explicitly. Reverse mapping always uses the fully qualified name. All TabletServers must use the same default database. Changing it redirects unqualified names, so migrate clients to fully qualified names before changing the configuration. The prototype `kafka.database` option is no longer used. From 7badab020d60b214d9637afc0d01ae788befa48d Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Wed, 14 Oct 2026 20:11:19 +0800 Subject: [PATCH 09/15] [kafka] Simplify DDL validation and default database setup Reuse native initialization for the fluss database and create a custom Kafka default database only when it is missing. Remove PR03-only schema version and Coordinator epoch checks, preserving PR02 metadata writes and existing protections. Drop the corresponding concurrency tests while retaining Kafka DDL mapping validation. Validation: 121 focused tests passed; Maven verify checks passed. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 132/132 AI-Contributed/UT: 224/224 --- .../apache/fluss/config/ConfigOptions.java | 5 +- .../kafka/schema/KafkaTableMappingITCase.java | 2 +- .../server/coordinator/CoordinatorServer.java | 7 +- .../coordinator/CoordinatorService.java | 3 +- .../server/coordinator/MetadataManager.java | 41 +--- .../fluss/server/zk/ZooKeeperClient.java | 66 ------ .../coordinator/KafkaDefaultDatabaseTest.java | 20 +- .../KafkaTableSchemaAlterTest.java | 202 ------------------ website/docs/maintenance/configuration.md | 10 +- 9 files changed, 37 insertions(+), 319 deletions(-) delete mode 100644 fluss-server/src/test/java/org/apache/fluss/server/coordinator/KafkaTableSchemaAlterTest.java diff --git a/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java b/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java index ac75410f7f9..e2e1f4cbf25 100644 --- a/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java +++ b/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java @@ -2985,8 +2985,9 @@ public class ConfigOptions { .defaultValue("fluss") .withDescription( "Default database for unqualified Kafka topic names. Fully qualified names may " - + "access other databases. When Kafka is enabled, the coordinator creates this " - + "database if it does not exist."); + + "access other databases. When Kafka is enabled, a configured database other " + + "than 'fluss' is created if missing. The native 'fluss' database is created " + + "only when the cluster has no databases."); /** * Compaction style for Fluss's kv, which is same to rocksdb's, but help use avoid including diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java index a327df3d5a6..0c7ff17eda3 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java @@ -271,7 +271,7 @@ void testOnlyRequestedKafkaPropertiesArePersisted() throws Exception { } @Test - void testEnableDisableAndFailedPropertyAlterAreAtomic() throws Exception { + void testEnableDisableAndRejectInvalidProperties() throws Exception { TablePath path = TablePath.of("ddl", "enable_disable"); TableDescriptor disabled = enabled().property(ConfigOptions.TABLE_KAFKA_ENABLED, false).build(); diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorServer.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorServer.java index ac161728107..3b93e2c8c50 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorServer.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorServer.java @@ -560,11 +560,8 @@ static void createDefaultDatabases(MetadataManager metadataManager, Configuratio } if (conf.get(ConfigOptions.KAFKA_ENABLED)) { String kafkaDatabase = conf.get(ConfigOptions.KAFKA_DEFAULT_DATABASE); - // An empty cluster has already created the native default database above. - if (DEFAULT_DATABASE.equals(kafkaDatabase) && databases.isEmpty()) { - return; - } - if (!databases.contains(kafkaDatabase)) { + // The native default database follows the cluster initialization logic above. + if (!DEFAULT_DATABASE.equals(kafkaDatabase) && !databases.contains(kafkaDatabase)) { metadataManager.createDatabase(kafkaDatabase, DatabaseDescriptor.EMPTY, true); LOG.info("Created default database '{}' for Kafka protocol.", kafkaDatabase); } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorService.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorService.java index e8e09dab26d..df8713338a8 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorService.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorService.java @@ -615,8 +615,7 @@ public CompletableFuture alterTable(AlterTableRequest reques tablePath, alterSchemaChanges, request.isIgnoreIfNotExists(), - currentSession().getPrincipal(), - coordinatorZkVersionSupplier.get()); + currentSession().getPrincipal()); } if (hasConfigChanges) { diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/MetadataManager.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/MetadataManager.java index 3354b869eb6..41259eed034 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/MetadataManager.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/MetadataManager.java @@ -59,7 +59,6 @@ import org.apache.fluss.server.zk.data.PartitionRegistration; import org.apache.fluss.server.zk.data.TableAssignment; import org.apache.fluss.server.zk.data.TableRegistration; -import org.apache.fluss.server.zk.data.ZkVersion; import org.apache.fluss.shaded.zookeeper3.org.apache.zookeeper.KeeperException; import org.apache.fluss.utils.function.RunnableWithException; import org.apache.fluss.utils.function.ThrowingRunnable; @@ -466,26 +465,8 @@ public void alterTableSchema( boolean ignoreIfNotExists, FlussPrincipal flussPrincipal) throws TableNotExistException, TableNotPartitionedException { - alterTableSchema( - tablePath, - schemaChanges, - ignoreIfNotExists, - flussPrincipal, - ZkVersion.MATCH_ANY_VERSION.getVersion()); - } - - /** Alters schema with table, schema and coordinator version checks against concurrent ALTER. */ - public void alterTableSchema( - TablePath tablePath, - List schemaChanges, - boolean ignoreIfNotExists, - FlussPrincipal flussPrincipal, - int coordinatorZkVersion) - throws TableNotExistException, TableNotPartitionedException { try { - ZooKeeperClient.VersionedData versionedTable = - getTableRegistrationWithVersion(tablePath); - int schemaZkVersion = zookeeperClient.getSchemaZkVersion(tablePath); + TableInfo table = getTable(tablePath); TableDescriptor tableDescriptor = table.toTableDescriptor(); @@ -506,13 +487,7 @@ public void alterTableSchema( // Update Fluss schema (ZK) after Lake sync succeeds if (!newSchema.equals(table.getSchema())) { - zookeeperClient.registerSchemaWithVersionCheck( - tablePath, - newSchema, - table.getSchemaId() + 1, - versionedTable.zkVersion(), - schemaZkVersion, - coordinatorZkVersion); + zookeeperClient.registerSchema(tablePath, newSchema, table.getSchemaId() + 1); } else { LOG.info( "Skipping schema evolution for table {} because the column(s) to add {} already exist.", @@ -815,10 +790,10 @@ private boolean isTablePresent(TablePath tablePath) { } /** - * One attempt of the property ALTER read-modify-write. The ZK write is guarded by the table and - * schema versions read here plus the coordinator epoch version. Since {@link - * #preAlterTableProperties} may apply external lake side effects, a version conflict is - * surfaced instead of being retried automatically. + * One attempt of the property ALTER read-modify-write. The ZK write is CAS-guarded by the table + * version read here plus the coordinator epoch version. Since {@link #preAlterTableProperties} + * may apply external lake side effects, a version conflict is surfaced instead of being retried + * automatically. */ private void doAlterTablePropertiesOnce( TablePath tablePath, @@ -834,7 +809,6 @@ private void doAlterTablePropertiesOnce( getTableRegistrationWithVersion(tablePath); TableRegistration tableReg = versionedTableReg.data(); int tableZkVersion = versionedTableReg.zkVersion(); - int schemaZkVersion = zookeeperClient.getSchemaZkVersion(tablePath); SchemaInfo schemaInfo = getLatestSchema(tablePath); // we can't use MetadataManager#getTable here, because it will add the default // lake options to the table properties, which may cause the validation failure @@ -904,8 +878,7 @@ private void doAlterTablePropertiesOnce( updatedTableRegistration, tableZkVersion, Collections.emptyMap(), - coordinatorZkVersion, - schemaZkVersion); + coordinatorZkVersion); } catch (KeeperException.BadVersionException e) { throw new FlussRuntimeException( String.format( diff --git a/fluss-server/src/main/java/org/apache/fluss/server/zk/ZooKeeperClient.java b/fluss-server/src/main/java/org/apache/fluss/server/zk/ZooKeeperClient.java index 99d2c6c01ae..01f897cff29 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/zk/ZooKeeperClient.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/zk/ZooKeeperClient.java @@ -1159,32 +1159,7 @@ public void updateTableWithPartitionBucketCountBackfill( Map> partitionBackfills, int expectedCoordinatorEpochZkVersion) throws Exception { - updateTableWithPartitionBucketCountBackfill( - tablePath, - tableRegistration, - expectedTableZkVersion, - partitionBackfills, - expectedCoordinatorEpochZkVersion, - ZkVersion.MATCH_ANY_VERSION.getVersion()); - } - - /** Updates table properties with an additional fence on the schema used for validation. */ - public void updateTableWithPartitionBucketCountBackfill( - TablePath tablePath, - TableRegistration tableRegistration, - int expectedTableZkVersion, - Map> partitionBackfills, - int expectedCoordinatorEpochZkVersion, - int expectedSchemaZkVersion) - throws Exception { List ops = new ArrayList<>(partitionBackfills.size() + 1); - if (expectedSchemaZkVersion != ZkVersion.MATCH_ANY_VERSION.getVersion()) { - ops.add( - zkClient.transactionOp() - .check() - .withVersion(expectedSchemaZkVersion) - .forPath(SchemasZNode.path(tablePath))); - } for (Map.Entry> entry : partitionBackfills.entrySet()) { String partitionPath = PartitionZNode.path(tablePath, entry.getKey()); @@ -1340,47 +1315,6 @@ public int registerSchema(TablePath tablePath, Schema schema, int schemaId) thro return schemaId; } - /** - * Publishes a schema and advances the schema version in one fenced transaction. The table - * registration is checked without being rewritten; property ALTER checks the schema version to - * prevent enabling a Kafka contract against a stale schema. - */ - public void registerSchemaWithVersionCheck( - TablePath tablePath, - Schema schema, - int schemaId, - int expectedTableZkVersion, - int expectedSchemaZkVersion, - int expectedCoordinatorEpochZkVersion) - throws Exception { - List ops = new ArrayList<>(); - ops.add( - zkClient.transactionOp() - .check() - .withVersion(expectedTableZkVersion) - .forPath(TableZNode.path(tablePath))); - ops.add( - zkClient.transactionOp() - .setData() - .withVersion(expectedSchemaZkVersion) - .forPath(SchemasZNode.path(tablePath), new byte[0])); - ops.add( - zkClient.transactionOp() - .create() - .withMode(CreateMode.PERSISTENT) - .forPath( - SchemaZNode.path(tablePath, schemaId), SchemaZNode.encode(schema))); - zkClient.transaction() - .forOperations(wrapRequestsWithEpochCheck(ops, expectedCoordinatorEpochZkVersion)); - } - - /** Returns the schema parent znode version, read before loading a schema for DDL validation. */ - public int getSchemaZkVersion(TablePath tablePath) throws Exception { - Stat stat = new Stat(); - zkClient.getData().storingStatIn(stat).forPath(SchemasZNode.path(tablePath)); - return stat.getVersion(); - } - /** Get the specific schema by schema id in ZK metadata. */ public Optional getSchemaById(TablePath tablePath, int schemaId) throws Exception { Optional bytes = getOrEmpty(SchemaZNode.path(tablePath, schemaId)); diff --git a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/KafkaDefaultDatabaseTest.java b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/KafkaDefaultDatabaseTest.java index d846c95f3e6..5e4cc8bc3a5 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/KafkaDefaultDatabaseTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/KafkaDefaultDatabaseTest.java @@ -30,7 +30,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.ValueSource; import static org.apache.fluss.server.coordinator.CoordinatorServer.DEFAULT_DATABASE; import static org.assertj.core.api.Assertions.assertThat; @@ -61,8 +61,22 @@ void setup() throws Exception { } @ParameterizedTest - @CsvSource({"fluss,false", "kafka_default,false", "fluss,true", "kafka_default,true"}) - void testCreateMissingDefaultAndPreserveExisting(String database, boolean exists) { + @ValueSource(booleans = {false, true}) + void testNativeDefaultFollowsClusterInitialization(boolean hasDatabase) { + if (hasDatabase) { + manager.createDatabase("existing", DatabaseDescriptor.EMPTY, false); + } + Configuration config = kafkaConfiguration(DEFAULT_DATABASE); + CoordinatorServer.createDefaultDatabases(manager, config); + CoordinatorServer.createDefaultDatabases(manager, config); + assertThat(manager.listDatabases()) + .containsExactly(hasDatabase ? "existing" : DEFAULT_DATABASE); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + void testCreateMissingCustomDefaultAndPreserveExisting(boolean exists) { + String database = "kafka_default"; manager.createDatabase("existing", DatabaseDescriptor.EMPTY, false); DatabaseDescriptor descriptor = DatabaseDescriptor.builder().comment("existing default").build(); diff --git a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/KafkaTableSchemaAlterTest.java b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/KafkaTableSchemaAlterTest.java deleted file mode 100644 index dc08ed10385..00000000000 --- a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/KafkaTableSchemaAlterTest.java +++ /dev/null @@ -1,202 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.fluss.server.coordinator; - -import org.apache.fluss.config.ConfigOptions; -import org.apache.fluss.config.Configuration; -import org.apache.fluss.exception.FlussRuntimeException; -import org.apache.fluss.metadata.DatabaseDescriptor; -import org.apache.fluss.metadata.Schema; -import org.apache.fluss.metadata.TableChange; -import org.apache.fluss.metadata.TableDescriptor; -import org.apache.fluss.metadata.TablePath; -import org.apache.fluss.server.entity.TablePropertyChanges; -import org.apache.fluss.server.zk.NOPErrorHandler; -import org.apache.fluss.server.zk.ZooKeeperClient; -import org.apache.fluss.server.zk.ZooKeeperExtension; -import org.apache.fluss.server.zk.data.TableRegistration; -import org.apache.fluss.server.zk.data.ZkVersion; -import org.apache.fluss.shaded.zookeeper3.org.apache.zookeeper.KeeperException; -import org.apache.fluss.testutils.common.AllCallbackWrapper; -import org.apache.fluss.types.DataTypes; - -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.RegisterExtension; - -import java.util.Collections; -import java.util.function.BiConsumer; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatThrownBy; - -/** Deterministic interleavings of schema ALTER and Kafka enablement using real ZooKeeper. */ -class KafkaTableSchemaAlterTest { - - @RegisterExtension - public static final AllCallbackWrapper ZK = - new AllCallbackWrapper<>(new ZooKeeperExtension()); - - private ZooKeeperClient zk; - private MetadataManager manager; - private int epoch; - - @BeforeEach - void setup() throws Exception { - zk = ZK.getCustomExtension().getZooKeeperClient(NOPErrorHandler.INSTANCE); - epoch = zk.fenceBecomeCoordinatorLeader("kafka-ddl").getCoordinatorEpochZkVersion(); - manager = - new MetadataManager( - zk, - new Configuration(), - new LakeCatalogDynamicLoader(new Configuration(), null, true)); - manager.createDatabase("ddl", DatabaseDescriptor.EMPTY, true); - } - - @Test - void testConcurrentSchemaChangePreventsStaleEnablement() throws Exception { - TablePath path = create("schema_wins"); - ZooKeeperClient.VersionedData before = - zk.getTableWithVersion(path).get(); - int schemaZkVersion = zk.getSchemaZkVersion(path); - assertThatThrownBy( - () -> - enable( - path, - (oldTable, updated) -> - manager.alterTableSchema( - path, - Collections.singletonList( - TableChange.addColumn( - "unmapped", - DataTypes.STRING(), - null, - TableChange.ColumnPosition - .last())), - false, - null, - epoch))) - .isInstanceOf(FlussRuntimeException.class) - .hasMessageContaining("Concurrent metadata change"); - ZooKeeperClient.VersionedData after = zk.getTableWithVersion(path).get(); - assertThat(after.zkVersion()).isEqualTo(before.zkVersion()); - assertThat(after.data()).isEqualTo(before.data()); - assertThat(zk.getSchemaZkVersion(path)).isEqualTo(schemaZkVersion + 1); - assertThat(manager.getTable(path).getTableConfig().isKafkaEnabled()).isFalse(); - assertThat(manager.getTable(path).getSchema().getRowType().getFieldNames()) - .contains("unmapped"); - } - - @Test - void testConcurrentEnablementPreventsStaleSchemaPublication() throws Exception { - TablePath path = create("enable_wins"); - ZooKeeperClient.VersionedData snapshot = - zk.getTableWithVersion(path).get(); - int schemaZkVersion = zk.getSchemaZkVersion(path); - int schemaId = zk.getCurrentSchemaId(path); - Schema incompatible = - Schema.newBuilder() - .column("_kafka_value", DataTypes.BYTES()) - .column("unmapped", DataTypes.STRING()) - .build(); - enable(path, (oldTable, updated) -> {}); - assertThatThrownBy( - () -> - zk.registerSchemaWithVersionCheck( - path, - incompatible, - schemaId + 1, - snapshot.zkVersion(), - schemaZkVersion, - epoch)) - .isInstanceOf(KeeperException.BadVersionException.class); - assertThat(zk.getCurrentSchemaId(path)).isEqualTo(schemaId); - assertThat(manager.getTable(path).getTableConfig().isKafkaEnabled()).isTrue(); - assertThat(manager.getTable(path).getSchema().getRowType().getFieldNames()) - .containsExactly("_kafka_value"); - } - - @Test - void testSchemaFailureDoesNotAdvanceSchemaVersion() throws Exception { - TablePath path = create("atomic_failure"); - ZooKeeperClient.VersionedData snapshot = - zk.getTableWithVersion(path).get(); - int schemaZkVersion = zk.getSchemaZkVersion(path); - int schemaId = zk.getCurrentSchemaId(path); - Schema schema = manager.getTable(path).getSchema(); - assertThatThrownBy( - () -> - zk.registerSchemaWithVersionCheck( - path, - schema, - schemaId, - snapshot.zkVersion(), - schemaZkVersion, - epoch)) - .isInstanceOf(KeeperException.NodeExistsException.class); - assertThat(zk.getTableWithVersion(path).get().zkVersion()).isEqualTo(snapshot.zkVersion()); - assertThat(zk.getSchemaZkVersion(path)).isEqualTo(schemaZkVersion); - assertThat(zk.getCurrentSchemaId(path)).isEqualTo(schemaId); - zk.fenceBecomeCoordinatorLeader("next-coordinator"); - assertThatThrownBy( - () -> - zk.registerSchemaWithVersionCheck( - path, - schema, - schemaId + 1, - snapshot.zkVersion(), - schemaZkVersion, - epoch)) - .isInstanceOf(KeeperException.BadVersionException.class); - assertThat(zk.getTableWithVersion(path).get().zkVersion()).isEqualTo(snapshot.zkVersion()); - assertThat(zk.getSchemaZkVersion(path)).isEqualTo(schemaZkVersion); - assertThat(zk.getCurrentSchemaId(path)).isEqualTo(schemaId); - } - - private TablePath create(String name) { - TablePath path = TablePath.of("ddl", name); - TableDescriptor table = - TableDescriptor.builder() - .schema( - Schema.newBuilder() - .column("_kafka_value", DataTypes.BYTES()) - .build()) - .distributedBy(1) - .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "binary") - .build() - .withReplicationFactor(1); - manager.createTable(path, zk.getDefaultRemoteDataDir(), table, null, false); - return path; - } - - private void enable( - TablePath path, - BiConsumer beforeUpdate) { - TablePropertyChanges.Builder changes = new TablePropertyChanges.Builder(); - changes.setTableProperty("table.kafka.enabled", "true"); - manager.alterTableProperties( - path, - Collections.singletonList(TableChange.set("table.kafka.enabled", "true")), - changes.build(), - false, - null, - beforeUpdate, - (oldTable, updated) -> {}, - ZkVersion.MATCH_ANY_VERSION.getVersion()); - } -} diff --git a/website/docs/maintenance/configuration.md b/website/docs/maintenance/configuration.md index 3a360df5319..d62ecef66b1 100644 --- a/website/docs/maintenance/configuration.md +++ b/website/docs/maintenance/configuration.md @@ -259,16 +259,18 @@ Kafka protocol compatibility is still in development. |--------------------------------|----------|---------|--------------------------------------------------------------------------------------------------------------------| | kafka.enabled | Boolean | false | Whether enable Fluss Kafka. Disabled by default. When this option is set to true, the Fluss Kafka will be enabled. | | kafka.listener.names | String | KAFKA | The listener names for Kafka wire protocol communication. Support multiple listener names, separated by comma. | -| kafka.default.database | String | fluss | Database used for unqualified Kafka topic names. Fully qualified names can access other databases. When Kafka is enabled, the coordinator creates this database if it is missing. | +| kafka.default.database | String | fluss | Database used for unqualified Kafka topic names. Fully qualified names can access other databases. When Kafka is enabled, a configured database other than `fluss` is created if missing. The native `fluss` database is created only when the cluster has no databases. | Kafka listeners share `netty.connection.max-idle-time` with native Fluss RPC listeners (see [Netty](#netty)). The default is `10min`; connections close when neither reads nor writes occur for the configured duration. Kafka tables must be created through Fluss or the Flink Fluss catalog. A topic named `sales.orders` maps to that table directly; `orders` maps to the same-named table in -`kafka.default.database` (by default, `fluss.orders`). The coordinator creates the configured default -database when Kafka is enabled and the database is missing. Existing databases are preserved; -other databases and all tables must be created explicitly. Reverse mapping always uses +`kafka.default.database` (by default, `fluss.orders`). The native `fluss` database is created only +when the cluster has no databases; Kafka reuses this database when its default is `fluss`. +If Kafka is enabled and a different default database is configured, the coordinator creates it +if missing. Existing databases are preserved; other databases and all tables must be created +explicitly. Reverse mapping always uses the fully qualified name. All TabletServers must use the same default database. Changing it redirects unqualified names, so migrate clients to fully qualified names before changing the configuration. The prototype `kafka.database` option is no longer used. From 8cd649f2eb5349012f23abac32dddbcbf7879b25 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Sat, 17 Oct 2026 15:28:16 +0800 Subject: [PATCH 10/15] [docs] Clarify Kafka topic table mapping State that Kafka access requires an existing Fluss table and summarize qualified topic mapping and the default database fallback. Validation: git diff --check. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 13/13 AI-Contributed/UT: 0/0 --- website/docs/maintenance/configuration.md | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/website/docs/maintenance/configuration.md b/website/docs/maintenance/configuration.md index d62ecef66b1..cd5af868048 100644 --- a/website/docs/maintenance/configuration.md +++ b/website/docs/maintenance/configuration.md @@ -264,16 +264,9 @@ Kafka protocol compatibility is still in development. Kafka listeners share `netty.connection.max-idle-time` with native Fluss RPC listeners (see [Netty](#netty)). The default is `10min`; connections close when neither reads nor writes occur for the configured duration. -Kafka tables must be created through Fluss or the Flink Fluss catalog. A topic named -`sales.orders` maps to that table directly; `orders` maps to the same-named table in -`kafka.default.database` (by default, `fluss.orders`). The native `fluss` database is created only -when the cluster has no databases; Kafka reuses this database when its default is `fluss`. -If Kafka is enabled and a different default database is configured, the coordinator creates it -if missing. Existing databases are preserved; other databases and all tables must be created -explicitly. Reverse mapping always uses -the fully qualified name. All TabletServers must use the same default database. Changing it -redirects unqualified names, so migrate clients to fully qualified names before changing the -configuration. The prototype `kafka.database` option is no longer used. +Kafka clients can access a topic only after its corresponding table has been created in Fluss. +Kafka topics map to Fluss tables using `.`. Topic names without a database prefix +map to the same-named table in the database configured by `kafka.default.database` (default: `fluss`). ### Kafka table properties From 92522a9b2a4804e1309894851fca6ead96ac5a24 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Sat, 17 Oct 2026 15:49:14 +0800 Subject: [PATCH 11/15] [docs] Reorder Kafka topic mapping explanation Describe topic mapping before the requirement to create its Fluss table. Validation: git diff --check. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 2/2 AI-Contributed/UT: 0/0 --- website/docs/maintenance/configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/docs/maintenance/configuration.md b/website/docs/maintenance/configuration.md index cd5af868048..e451be59f4a 100644 --- a/website/docs/maintenance/configuration.md +++ b/website/docs/maintenance/configuration.md @@ -264,9 +264,9 @@ Kafka protocol compatibility is still in development. Kafka listeners share `netty.connection.max-idle-time` with native Fluss RPC listeners (see [Netty](#netty)). The default is `10min`; connections close when neither reads nor writes occur for the configured duration. -Kafka clients can access a topic only after its corresponding table has been created in Fluss. Kafka topics map to Fluss tables using `.
`. Topic names without a database prefix map to the same-named table in the database configured by `kafka.default.database` (default: `fluss`). +Kafka clients can access a topic only after its corresponding table has been created in Fluss. ### Kafka table properties From 6049dd895a33dde6f1c79f3a4f9956b8e8632fb5 Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Sun, 18 Oct 2026 18:11:10 +0800 Subject: [PATCH 12/15] [kafka] Refine DDL mapping configuration and validation Use typed conversion options with read-time defaults and keep all four system columns optional. Validate nullable fields recursively while preserving Fluss's non-null MAP key requirement and allow nested business fields to use system names. Ensure the configured Kafka default database exists, consolidate table validation entry points, and document the mapping contract in a dedicated Kafka compatibility guide with regression coverage. Validation: 102 targeted tests passed; after the enum rename, 52 related tests and the affected Maven reactor verification passed. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 648/648 AI-Contributed/UT: 492/492 --- .../apache/fluss/config/ConfigOptions.java | 38 ++- .../fluss/config/KafkaKeyConversionType.java | 27 ++ .../org/apache/fluss/config/TableConfig.java | 5 - .../fluss/config/ValueConversionLayout.java | 30 ++ .../fluss/config/ValueConversionType.java | 33 ++ .../fluss/metadata/KafkaTableSchema.java | 181 +++++------ .../fluss/metadata/KafkaTableSchemaTest.java | 298 ++++++++++++------ .../fluss/kafka/mapping/KafkaTopicMapper.java | 13 +- .../kafka/mapping/KafkaTopicMapperTest.java | 6 +- .../kafka/schema/KafkaTableMappingITCase.java | 159 +++++++++- .../schema/KafkaTopicSchemaResolverTest.java | 5 +- .../server/coordinator/CoordinatorServer.java | 8 +- .../coordinator/CoordinatorService.java | 2 +- .../server/coordinator/MetadataManager.java | 13 +- .../utils/TableDescriptorValidation.java | 8 +- .../coordinator/KafkaDefaultDatabaseTest.java | 24 +- website/docs/kafka-compatibility.md | 244 ++++++++++++++ website/docs/maintenance/configuration.md | 46 +-- 18 files changed, 827 insertions(+), 313 deletions(-) create mode 100644 fluss-common/src/main/java/org/apache/fluss/config/KafkaKeyConversionType.java create mode 100644 fluss-common/src/main/java/org/apache/fluss/config/ValueConversionLayout.java create mode 100644 fluss-common/src/main/java/org/apache/fluss/config/ValueConversionType.java create mode 100644 website/docs/kafka-compatibility.md diff --git a/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java b/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java index e2e1f4cbf25..8d1cbe0319e 100644 --- a/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java +++ b/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java @@ -2277,29 +2277,33 @@ public class ConfigOptions { "Whether the table is exposed as a Kafka topic. Enabling validates the " + "existing schema and conversion options without adding columns or defaults."); - public static final ConfigOption TABLE_KAFKA_CONVERT_VALUE_TYPE = + public static final ConfigOption TABLE_KAFKA_CONVERT_VALUE_TYPE = key("table.kafka.convert.value.type") - .stringType() - .noDefaultValue() + .enumType(ValueConversionType.class) + .defaultValue(ValueConversionType.PARSE_JSON) .withDescription( - "Required when Kafka compatibility is enabled: binary, string, or parse_json. " - + "Binary and string target the declared _kafka_value column."); + "Kafka value conversion: binary, string, or parse_json. " + + "Binary and string target the declared _kafka_value column. " + + "The default is used at read time without adding a table property."); - public static final ConfigOption TABLE_KAFKA_CONVERT_VALUE_LAYOUT = + public static final ConfigOption TABLE_KAFKA_CONVERT_VALUE_LAYOUT = key("table.kafka.convert.value.layout") - .stringType() - .noDefaultValue() + .enumType(ValueConversionLayout.class) + .defaultValue(ValueConversionLayout.FLAT) .withDescription( - "JSON layout: flat (the read-time default for parse_json) or nested. " - + "Nested targets _kafka_value ROW. Not allowed for binary or string."); + "JSON layout for parse_json: flat maps top-level business columns; " + + "nested targets _kafka_value ROW. Ignored for binary and string, " + + "whether defaulted or explicitly configured. The default is used " + + "at read time without adding a table property."); - public static final ConfigOption TABLE_KAFKA_CONVERT_KEY_TYPE = + public static final ConfigOption TABLE_KAFKA_CONVERT_KEY_TYPE = key("table.kafka.convert.key.type") - .stringType() + .enumType(KafkaKeyConversionType.class) .noDefaultValue() .withDescription( - "Optional Kafka key conversion type. Only binary is supported and requires " - + "_kafka_key BYTES. Declaring that column implies binary without persisting a default."); + "Optional Kafka key conversion. Only binary is supported and requires " + + "_kafka_key BYTES. Declaring that column implies binary without " + + "adding a table property."); // ------------------------------------------------------------------------ // ConfigOptions for Consumer Offsets @@ -2985,9 +2989,9 @@ public class ConfigOptions { .defaultValue("fluss") .withDescription( "Default database for unqualified Kafka topic names. Fully qualified names may " - + "access other databases. When Kafka is enabled, a configured database other " - + "than 'fluss' is created if missing. The native 'fluss' database is created " - + "only when the cluster has no databases."); + + "access other databases. When Kafka is enabled, the coordinator creates " + + "the configured database if missing, including 'fluss'. With Kafka disabled, " + + "native initialization creates 'fluss' only when the cluster has no databases."); /** * Compaction style for Fluss's kv, which is same to rocksdb's, but help use avoid including diff --git a/fluss-common/src/main/java/org/apache/fluss/config/KafkaKeyConversionType.java b/fluss-common/src/main/java/org/apache/fluss/config/KafkaKeyConversionType.java new file mode 100644 index 00000000000..5c0ebbdc05d --- /dev/null +++ b/fluss-common/src/main/java/org/apache/fluss/config/KafkaKeyConversionType.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.config; + +import org.apache.fluss.annotation.PublicEvolving; + +/** Conversion applied to Kafka record keys. */ +@PublicEvolving +public enum KafkaKeyConversionType { + /** Store the key as bytes in the fixed key column. */ + BINARY +} diff --git a/fluss-common/src/main/java/org/apache/fluss/config/TableConfig.java b/fluss-common/src/main/java/org/apache/fluss/config/TableConfig.java index b1bc64dfc83..ed2c111c335 100644 --- a/fluss-common/src/main/java/org/apache/fluss/config/TableConfig.java +++ b/fluss-common/src/main/java/org/apache/fluss/config/TableConfig.java @@ -54,11 +54,6 @@ public TableConfig(Configuration config) { this.config = config; } - /** Whether the table explicitly enables Kafka compatibility. */ - public boolean isKafkaEnabled() { - return config.get(ConfigOptions.TABLE_KAFKA_ENABLED); - } - /** Gets the replication factor of the table. */ public int getReplicationFactor() { return config.get(ConfigOptions.TABLE_REPLICATION_FACTOR); diff --git a/fluss-common/src/main/java/org/apache/fluss/config/ValueConversionLayout.java b/fluss-common/src/main/java/org/apache/fluss/config/ValueConversionLayout.java new file mode 100644 index 00000000000..58e12e85f2f --- /dev/null +++ b/fluss-common/src/main/java/org/apache/fluss/config/ValueConversionLayout.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.config; + +import org.apache.fluss.annotation.PublicEvolving; + +/** Mapping of JSON object members to Fluss table columns. */ +@PublicEvolving +public enum ValueConversionLayout { + /** Map members to top-level business columns. */ + FLAT, + + /** Map members to fields inside the fixed value ROW column. */ + NESTED +} diff --git a/fluss-common/src/main/java/org/apache/fluss/config/ValueConversionType.java b/fluss-common/src/main/java/org/apache/fluss/config/ValueConversionType.java new file mode 100644 index 00000000000..9d3683d5372 --- /dev/null +++ b/fluss-common/src/main/java/org/apache/fluss/config/ValueConversionType.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.fluss.config; + +import org.apache.fluss.annotation.PublicEvolving; + +/** Conversion applied to Kafka record values. */ +@PublicEvolving +public enum ValueConversionType { + /** Store the value as bytes in the fixed value column. */ + BINARY, + + /** Decode the value as a UTF-8 string in the fixed value column. */ + STRING, + + /** Parse the JSON object using the table schema and configured value layout. */ + PARSE_JSON +} diff --git a/fluss-common/src/main/java/org/apache/fluss/metadata/KafkaTableSchema.java b/fluss-common/src/main/java/org/apache/fluss/metadata/KafkaTableSchema.java index 9ab16195fbd..e02e5daa2d1 100644 --- a/fluss-common/src/main/java/org/apache/fluss/metadata/KafkaTableSchema.java +++ b/fluss-common/src/main/java/org/apache/fluss/metadata/KafkaTableSchema.java @@ -21,9 +21,12 @@ import org.apache.fluss.config.ConfigOptions; import org.apache.fluss.config.Configuration; import org.apache.fluss.config.TableConfig; +import org.apache.fluss.config.ValueConversionLayout; +import org.apache.fluss.config.ValueConversionType; import org.apache.fluss.exception.InvalidTableException; import org.apache.fluss.types.ArrayType; import org.apache.fluss.types.BytesType; +import org.apache.fluss.types.DataField; import org.apache.fluss.types.DataType; import org.apache.fluss.types.DataTypes; import org.apache.fluss.types.MapType; @@ -37,7 +40,6 @@ import java.util.Collections; import java.util.HashSet; import java.util.List; -import java.util.Locale; import java.util.Objects; import java.util.Set; @@ -66,30 +68,10 @@ public final class KafkaTableSchema { HEADERS_COLUMN, METADATA_COLUMN, RESCUE_COLUMN))); - private static final Set OPTIONS = - new HashSet<>( - Arrays.asList( - ConfigOptions.TABLE_KAFKA_ENABLED.key(), - ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE.key(), - ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT.key(), - ConfigOptions.TABLE_KAFKA_CONVERT_KEY_TYPE.key())); - - /** Supported value conversion contracts; decoders are provided by the Kafka module. */ - public enum ValueType { - BINARY, - STRING, - PARSE_JSON - } - - /** JSON business fields either occupy the table row or the fixed value ROW column. */ - public enum ValueLayout { - FLAT, - NESTED - } private final RowType rowType; - private final ValueType valueType; - private final @Nullable ValueLayout valueLayout; + private final ValueConversionType valueConversionType; + private final @Nullable ValueConversionLayout valueConversionLayout; private final KafkaFieldProjection valueProjection; private final int keyPosition; private final int headersPosition; @@ -98,16 +80,16 @@ public enum ValueLayout { private KafkaTableSchema( RowType rowType, - ValueType valueType, - @Nullable ValueLayout valueLayout, + ValueConversionType valueConversionType, + @Nullable ValueConversionLayout valueConversionLayout, List valuePositions, int keyPosition, int headersPosition, int metadataPosition, int rescuePosition) { this.rowType = rowType; - this.valueType = valueType; - this.valueLayout = valueLayout; + this.valueConversionType = valueConversionType; + this.valueConversionLayout = valueConversionLayout; this.valueProjection = new KafkaFieldProjection(rowType, valuePositions); this.keyPosition = keyPosition; this.headersPosition = headersPosition; @@ -117,7 +99,20 @@ private KafkaTableSchema( /** Returns only the persisted opt-in flag; schema compatibility does not imply enablement. */ public static boolean isEnabled(TableDescriptor table) { - return new TableConfig(Configuration.fromMap(table.getProperties())).isKafkaEnabled(); + String value = table.getProperties().get(ConfigOptions.TABLE_KAFKA_ENABLED.key()); + if (value == null) { + return ConfigOptions.TABLE_KAFKA_ENABLED.defaultValue(); + } + if ("true".equalsIgnoreCase(value)) { + return true; + } + if ("false".equalsIgnoreCase(value)) { + return false; + } + throw new IllegalArgumentException( + "Invalid value for table.kafka.enabled: '" + + value + + "'. Expected true or false (case insensitive)."); } /** Validates the Kafka contract only when the final table state enables it. */ @@ -144,7 +139,7 @@ public static void validateTablePath(TablePath path, TableDescriptor table) { } } - /** Returns whether a JSON member at the value mapping level has a reserved name. */ + /** Returns whether a name is reserved at the physical table's top level. */ public static boolean isReservedName(String name) { return RESERVED_NAMES.contains(name); } @@ -154,7 +149,8 @@ public static KafkaTableSchema resolve(TableDescriptor table) { if (!isEnabled(table)) { throw invalid("Kafka compatibility is disabled; table.kafka.enabled must be true."); } - validateOptions(table); + // The initial Kafka write path encodes Arrow batches and appends them to log tables. + // Other log formats require matching encoders before they can be enabled here. if (table.hasPrimaryKey() || table.isPartitioned() || new TableConfig(Configuration.fromMap(table.getProperties())).getLogFormat() @@ -163,43 +159,21 @@ public static KafkaTableSchema resolve(TableDescriptor table) { "Kafka requires a non-partitioned Arrow log table without a primary key."); } Configuration options = Configuration.fromMap(table.getProperties()); - String value = options.get(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE); - if (value == null) { - throw invalid("Missing required table property 'table.kafka.convert.value.type'."); - } - ValueType valueType; - try { - valueType = ValueType.valueOf(value.trim().toUpperCase(Locale.ROOT)); - } catch (IllegalArgumentException e) { - throw invalid( - "Unsupported table.kafka.convert.value.type '" - + value - + "'. Expected binary, string or parse_json."); - } - String layout = options.get(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT); - ValueLayout valueLayout = null; - if (valueType == ValueType.PARSE_JSON) { - try { - valueLayout = - layout == null - ? ValueLayout.FLAT - : ValueLayout.valueOf(layout.trim().toUpperCase(Locale.ROOT)); - } catch (IllegalArgumentException e) { - throw invalid("Unsupported table.kafka.convert.value.layout '" + layout + "'."); - } - } else if (layout != null) { - throw invalid("table.kafka.convert.value.layout is only supported for parse_json."); - } + ValueConversionType valueConversionType = + options.get(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE); + ValueConversionLayout valueConversionLayout = + valueConversionType == ValueConversionType.PARSE_JSON + ? options.get(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT) + : null; RowType rowType = table.getSchema().getRowType(); int keyPosition = rowType.getFieldIndex(KEY_COLUMN); if (keyPosition >= 0 && !(rowType.getTypeAt(keyPosition) instanceof BytesType)) { throw invalid("_kafka_key must be BYTES."); } - String keyType = options.get(ConfigOptions.TABLE_KAFKA_CONVERT_KEY_TYPE); - if (keyType != null && (!"binary".equalsIgnoreCase(keyType.trim()) || keyPosition < 0)) { - throw invalid( - "table.kafka.convert.key.type only supports binary and requires _kafka_key BYTES."); + if (options.getOptional(ConfigOptions.TABLE_KAFKA_CONVERT_KEY_TYPE).isPresent() + && keyPosition < 0) { + throw invalid("table.kafka.convert.key.type requires _kafka_key BYTES."); } int headersPosition = rowType.getFieldIndex(HEADERS_COLUMN); if (headersPosition >= 0) { @@ -222,14 +196,14 @@ public static KafkaTableSchema resolve(TableDescriptor table) { } List valuePositions = new ArrayList<>(); - if (valueLayout == ValueLayout.FLAT) { + if (valueConversionLayout == ValueConversionLayout.FLAT) { if (rowType.getFieldIndex(VALUE_COLUMN) >= 0) { throw invalid( "_kafka_value is reserved for binary, string or nested JSON; " + "it cannot be a flat JSON business field."); } for (int i = 0; i < rowType.getFieldCount(); i++) { - if (!isReservedName(rowType.getFieldNames().get(i))) { + if (!isReservedName(rowType.getFields().get(i).getName())) { valuePositions.add(i); } } @@ -240,19 +214,21 @@ public static KafkaTableSchema resolve(TableDescriptor table) { } DataType type = rowType.getTypeAt(valuePosition); boolean valid = - valueType == ValueType.BINARY + valueConversionType == ValueConversionType.BINARY ? type instanceof BytesType - : valueType == ValueType.STRING + : valueConversionType == ValueConversionType.STRING ? type instanceof StringType : type instanceof RowType; if (!valid) { throw invalid( "_kafka_value must be " - + (valueType == ValueType.BINARY + + (valueConversionType == ValueConversionType.BINARY ? "BYTES" - : valueType == ValueType.STRING ? "STRING" : "ROW") + : valueConversionType == ValueConversionType.STRING + ? "STRING" + : "ROW") + " for " - + valueType + + valueConversionType + "."); } for (String name : rowType.getFieldNames()) { @@ -263,22 +239,12 @@ public static KafkaTableSchema resolve(TableDescriptor table) { + "' has no value mapping; this conversion targets _kafka_value only."); } } - if (type instanceof RowType) { - for (String name : ((RowType) type).getFieldNames()) { - if (isReservedName(name)) { - throw invalid( - "Reserved name '" - + name - + "' cannot be a nested JSON business field."); - } - } - } valuePositions.add(valuePosition); } - if (valueType == ValueType.PARSE_JSON) { + if (valueConversionType == ValueConversionType.PARSE_JSON) { for (int position : valuePositions) { DataType type = rowType.getTypeAt(position); - if (valueLayout == ValueLayout.NESTED) { + if (valueConversionLayout == ValueConversionLayout.NESTED) { // The physical value container represents the JSON object root, just like // the table row in flat layout; it must not add a business nesting level. for (DataType field : ((RowType) type).getChildren()) { @@ -289,10 +255,13 @@ public static KafkaTableSchema resolve(TableDescriptor table) { } } } + for (DataField field : rowType.getFields()) { + validateNullable(field.getType(), field.getName()); + } return new KafkaTableSchema( rowType, - valueType, - valueLayout, + valueConversionType, + valueConversionLayout, valuePositions, keyPosition, headersPosition, @@ -302,9 +271,11 @@ public static KafkaTableSchema resolve(TableDescriptor table) { /** * Validates a top-level JSON business field without parser or protocol dependencies. DDL and - * decoder construction share this contract. Each business field starts at depth zero; ROW - * fields, ARRAY elements and MAP values each advance the depth by one. Nullability and scalar - * value constraints are enforced when decoding individual records. The JSON object root + * decoder construction share this contract. All currently defined Fluss types are supported + * except MAP types with non-STRING keys (including nested maps) and schemas nested deeper than + * 64 levels. Each business field starts at depth zero; ROW fields, ARRAY elements and MAP + * values each advance the depth by one. Table resolution separately requires nullable fields; + * scalar value constraints are enforced when decoding individual records. The JSON object root * (including the physical nested _kafka_value ROW) is not counted. */ public static void validateJsonType(DataType type) { @@ -362,21 +333,19 @@ private static void validateJsonType(DataType type, int depth) { } } - private static void validateOptions(TableDescriptor table) { - for (String name : table.getProperties().keySet()) { - if (name.startsWith("table.kafka.") && !OPTIONS.contains(name)) { - throw invalid("Unsupported Kafka table property '" + name + "'."); - } + private static void validateNullable(DataType type, String path) { + if (!type.isNullable()) { + throw invalid("Kafka requires nullable fields, but '" + path + "' is NOT NULL."); } - for (String name : table.getCustomProperties().keySet()) { - if (name.startsWith("table.kafka.") - || name.startsWith("kafka.") - || name.startsWith("fluss.")) { - throw invalid( - "Unsupported Kafka custom property '" - + name - + "'. Use registered table.kafka.* properties and fixed system columns."); + if (type instanceof RowType) { + for (DataField field : ((RowType) type).getFields()) { + validateNullable(field.getType(), path + "." + field.getName()); } + } else if (type instanceof ArrayType) { + validateNullable(((ArrayType) type).getElementType(), path + "[]"); + } else if (type instanceof MapType) { + // Fluss requires non-null MAP keys. Only values can be nulled by field-level rescue. + validateNullable(((MapType) type).getValueType(), path + ""); } } @@ -402,13 +371,13 @@ public RowType rowType() { } /** Returns the value conversion type. */ - public ValueType valueType() { - return valueType; + public ValueConversionType valueConversionType() { + return valueConversionType; } /** Returns the JSON layout, or null for binary/string. */ - public @Nullable ValueLayout valueLayout() { - return valueLayout; + public @Nullable ValueConversionLayout valueConversionLayout() { + return valueConversionLayout; } /** Returns physical value targets; nested JSON projects its single ROW container. */ @@ -452,8 +421,8 @@ public boolean equals(Object other) { } KafkaTableSchema that = (KafkaTableSchema) other; return rowType.equals(that.rowType) - && valueType == that.valueType - && valueLayout == that.valueLayout + && valueConversionType == that.valueConversionType + && valueConversionLayout == that.valueConversionLayout && valueProjection.equals(that.valueProjection) && keyPosition == that.keyPosition && headersPosition == that.headersPosition @@ -465,8 +434,8 @@ public boolean equals(Object other) { public int hashCode() { return Objects.hash( rowType, - valueType, - valueLayout, + valueConversionType, + valueConversionLayout, valueProjection, keyPosition, headersPosition, diff --git a/fluss-common/src/test/java/org/apache/fluss/metadata/KafkaTableSchemaTest.java b/fluss-common/src/test/java/org/apache/fluss/metadata/KafkaTableSchemaTest.java index 458242feadd..4224845b1f4 100644 --- a/fluss-common/src/test/java/org/apache/fluss/metadata/KafkaTableSchemaTest.java +++ b/fluss-common/src/test/java/org/apache/fluss/metadata/KafkaTableSchemaTest.java @@ -18,9 +18,10 @@ package org.apache.fluss.metadata; import org.apache.fluss.config.ConfigOptions; +import org.apache.fluss.config.KafkaKeyConversionType; +import org.apache.fluss.config.ValueConversionLayout; +import org.apache.fluss.config.ValueConversionType; import org.apache.fluss.exception.InvalidTableException; -import org.apache.fluss.metadata.KafkaTableSchema.ValueLayout; -import org.apache.fluss.metadata.KafkaTableSchema.ValueType; import org.apache.fluss.types.DataType; import org.apache.fluss.types.DataTypes; import org.apache.fluss.types.RowType; @@ -46,7 +47,7 @@ void testExplicitOptInAndDisabledTables() { Schema.newBuilder() .column("_kafka_value", DataTypes.BYTES()) .build()) - .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "binary") + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE.key(), "binary") .build(); assertThat(KafkaTableSchema.isEnabled(disabled)).isFalse(); assertThatCode(() -> KafkaTableSchema.validate(disabled)).doesNotThrowAnyException(); @@ -59,19 +60,50 @@ void testExplicitOptInAndDisabledTables() { .column("id", DataTypes.INT().copy(false)) .primaryKey("id") .build()) - .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "future-format") + .property( + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE.key(), "future-format") .property(ConfigOptions.TABLE_KAFKA_ENABLED, false) .build(); assertThatCode(() -> KafkaTableSchema.validate(arbitrary)).doesNotThrowAnyException(); } + @Test + void testEnabledFlagParsing() { + TableDescriptor.Builder builder = enabled(valueSchema(), "binary"); + for (String value : Arrays.asList("true", "TRUE", "TrUe")) { + assertThat( + KafkaTableSchema.isEnabled( + builder.property("table.kafka.enabled", value).build())) + .isTrue(); + } + for (String value : Arrays.asList("false", "FALSE", "FaLsE")) { + assertThat( + KafkaTableSchema.isEnabled( + builder.property("table.kafka.enabled", value).build())) + .isFalse(); + } + for (String value : Arrays.asList("", "yes", "1", " true", "false ")) { + TableDescriptor table = builder.property("table.kafka.enabled", value).build(); + assertThatThrownBy(() -> KafkaTableSchema.isEnabled(table)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("table.kafka.enabled"); + } + } + @ParameterizedTest @ValueSource(strings = {"binary", "string"}) void testFixedValueColumnWithOptionalSystemFields(String format) { DataType valueType = format.equals("binary") ? DataTypes.BYTES() : DataTypes.STRING(); + assertInvalid( + enabled( + Schema.newBuilder() + .column("_kafka_value", valueType.copy(false)) + .build(), + format) + .build(), + "nullable fields"); for (int mask = 0; mask < 16; mask++) { - Schema.Builder schema = - Schema.newBuilder().column("_kafka_value", valueType.copy(false)); + Schema.Builder schema = Schema.newBuilder().column("_kafka_value", valueType); if ((mask & 1) != 0) { schema.column("_kafka_key", DataTypes.BYTES()); } @@ -88,7 +120,7 @@ void testFixedValueColumnWithOptionalSystemFields(String format) { TableDescriptor table = enabled(schema.build(), format).build(); KafkaTableSchema resolved = KafkaTableSchema.resolve(table); assertThat(resolved.valueProjection().positions()).containsExactly(0); - assertThat(resolved.valueProjection().dataTypeAt(0).isNullable()).isFalse(); + assertThat(resolved.valueProjection().dataTypeAt(0).isNullable()).isTrue(); assertThat(resolved.keyPosition() >= 0).isEqualTo((mask & 1) != 0); assertThat(resolved.headersPosition() >= 0).isEqualTo((mask & 2) != 0); assertThat(resolved.rescuePosition() >= 0).isEqualTo((mask & 4) != 0); @@ -100,12 +132,13 @@ void testFixedValueColumnWithOptionalSystemFields(String format) { } @Test - void testJsonFlatPreservesNestedTypesAndExcludesSystemColumns() { + void testDefaultJsonMappingPreservesNestedTypesAndExcludesSystemColumns() { RowType nested = DataTypes.ROW(DataTypes.FIELD("_kafka_key", DataTypes.STRING())); TableDescriptor table = - enabled( + TableDescriptor.builder() + .schema( Schema.newBuilder() - .column("id", DataTypes.BIGINT().copy(false)) + .column("id", DataTypes.BIGINT()) .column("_kafka_rescue", DataTypes.STRING()) .column("_kafka_metadata", metadataType()) .column("payload", nested) @@ -115,17 +148,17 @@ void testJsonFlatPreservesNestedTypesAndExcludesSystemColumns() { DataTypes.MAP( DataTypes.STRING(), DataTypes.BYTES())) .column("custom_rescue", DataTypes.STRING()) - .build(), - "parse_json") + .build()) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) .build(); KafkaTableSchema resolved = KafkaTableSchema.resolve(table); - assertThat(resolved.valueType()).isEqualTo(ValueType.PARSE_JSON); - assertThat(resolved.valueLayout()).isEqualTo(ValueLayout.FLAT); + assertThat(resolved.valueConversionType()).isEqualTo(ValueConversionType.PARSE_JSON); + assertThat(resolved.valueConversionLayout()).isEqualTo(ValueConversionLayout.FLAT); assertThat(resolved.valueProjection().positions()).containsExactly(0, 3, 6); assertThat(resolved.valueProjection().dataTypeAt(1)).isEqualTo(nested); assertThat(resolved.rescuePosition()).isEqualTo(1); assertThat(resolved.metadataPosition()).isEqualTo(2); - assertThat(table.getProperties()).doesNotContainKey("table.kafka.convert.value.layout"); + assertThat(table.getProperties()).containsOnlyKeys("table.kafka.enabled"); } @Test @@ -145,10 +178,10 @@ void testNestedJsonTargetsOneRow() { .column("_kafka_value", row) .build(), "parse_json") - .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT, "nested") + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT.key(), "nested") .build(); KafkaTableSchema resolved = KafkaTableSchema.resolve(table); - assertThat(resolved.valueLayout()).isEqualTo(ValueLayout.NESTED); + assertThat(resolved.valueConversionLayout()).isEqualTo(ValueConversionLayout.NESTED); assertThat(resolved.valueProjection().positions()).containsExactly(2); assertThat(resolved.valueProjection().dataTypeAt(0)).isEqualTo(row); assertThat(resolved.rescuePosition()).isZero(); @@ -197,8 +230,7 @@ void testJsonAcceptsAllScalarTypesInNestedContainers(String layout) { DataTypes.ARRAY( DataTypes.MAP( DataTypes.STRING().copy(false), - DataTypes.ROW( - DataTypes.FIELD("_kafka_key", scalar.copy(false))))); + DataTypes.ROW(DataTypes.FIELD("_kafka_key", scalar)))); TableDescriptor table = jsonTable(layout, type); assertThatCode(() -> KafkaTableSchema.validate(table)).doesNotThrowAnyException(); } @@ -234,7 +266,7 @@ private static TableDescriptor jsonTable(String layout, DataType businessType) { DataTypes.ROW(DataTypes.FIELD("payload", businessType))) .build(); return enabled(schema, "parse_json") - .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT, layout) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT.key(), layout) .build(); } @@ -247,36 +279,40 @@ private static TableDescriptor jsonTable(String layout, DataType businessType) { "_kafka_metadata", "_kafka_rescue" }) - void testReservedNamesAtValueMappingLevel(String name) { + void testSystemNamesAreBusinessFieldsInsideNestedValue(String name) { assertThat(KafkaTableSchema.isReservedName(name)).isTrue(); + RowType valueType = DataTypes.ROW(DataTypes.FIELD(name, DataTypes.STRING())); TableDescriptor nested = enabled( Schema.newBuilder() - .column( - "_kafka_value", - DataTypes.ROW( - DataTypes.FIELD(name, DataTypes.STRING()))) + .column("_kafka_key", DataTypes.BYTES()) + .column("_kafka_value", valueType) .build(), "parse_json") - .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT, "nested") + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT.key(), "nested") .build(); - assertInvalid(nested, "Reserved name"); + KafkaTableSchema resolved = KafkaTableSchema.resolve(nested); + assertThat(resolved.keyPosition()).isZero(); + assertThat(resolved.valueProjection().positions()).containsExactly(1); + assertThat(resolved.valueProjection().dataTypeAt(0)).isEqualTo(valueType); } @ParameterizedTest @ValueSource(strings = {"raw", "avro", ""}) void testRejectsUnsupportedValueTypes(String type) { - assertInvalid(enabled(valueSchema(), type).build(), "table.kafka.convert.value.type"); + assertThatThrownBy(() -> KafkaTableSchema.resolve(enabled(valueSchema(), type).build())) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("table.kafka.convert.value.type"); } @Test - void testRejectsMissingValueTypeAndTargetConflicts() { + void testRejectsValueTargetConflicts() { assertInvalid( TableDescriptor.builder() .schema(valueSchema()) .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) .build(), - "Missing required"); + "flat JSON"); assertInvalid( enabled(Schema.newBuilder().column("body", DataTypes.BYTES()).build(), "binary") .build(), @@ -284,7 +320,7 @@ void testRejectsMissingValueTypeAndTargetConflicts() { assertInvalid(enabled(valueSchema(), "string").build(), "must be STRING"); assertInvalid( enabled(valueSchema(), "parse_json") - .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT, "nested") + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT.key(), "nested") .build(), "must be ROW"); assertInvalid(enabled(valueSchema(), "parse_json").build(), "flat JSON"); @@ -307,52 +343,46 @@ void testRejectsMissingValueTypeAndTargetConflicts() { .column("unused", DataTypes.STRING()) .build(), "parse_json") - .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT, "nested") + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT.key(), "nested") .build(), "no value mapping"); } @ParameterizedTest @ValueSource(strings = {"binary", "string"}) - void testNonJsonRejectsExplicitLayout(String type) { - assertInvalid( - enabled(valueSchema(), type) - .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT, "flat") - .build(), - "only supported for parse_json"); + void testNonJsonIgnoresDefaultAndExplicitLayouts(String type) { + Schema schema = + Schema.newBuilder() + .column( + "_kafka_value", + "binary".equals(type) ? DataTypes.BYTES() : DataTypes.STRING()) + .build(); + TableDescriptor table = enabled(schema, type).build(); + KafkaTableSchema expected = KafkaTableSchema.resolve(table); + assertThat(expected.valueConversionLayout()).isNull(); + assertThat(expected.valueProjection().positions()).containsExactly(0); + assertThat(table.getProperties()).doesNotContainKey("table.kafka.convert.value.layout"); + for (ValueConversionLayout layout : ValueConversionLayout.values()) { + assertThat( + KafkaTableSchema.resolve( + TableDescriptor.builder(table) + .property( + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT, + layout) + .build())) + .isEqualTo(expected); + } } @Test - void testInvalidLayoutAndKeyConversion() { - assertInvalid( + void testRejectsInvalidJsonLayout() { + TableDescriptor table = enabled(valueSchema(), "parse_json") - .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT, "unknown") - .build(), - "value.layout"); - assertInvalid( - enabled(valueSchema(), "binary") - .property(ConfigOptions.TABLE_KAFKA_CONVERT_KEY_TYPE, "binary") - .build(), - "requires _kafka_key"); - Schema schema = - Schema.newBuilder() - .column("_kafka_value", DataTypes.BYTES()) - .column("_kafka_key", DataTypes.BYTES()) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT.key(), "unknown") .build(); - assertInvalid( - enabled(schema, "binary") - .property(ConfigOptions.TABLE_KAFKA_CONVERT_KEY_TYPE, "string") - .build(), - "only supports binary"); - assertThat( - KafkaTableSchema.resolve( - enabled(schema, "binary") - .property( - ConfigOptions.TABLE_KAFKA_CONVERT_KEY_TYPE, - "binary") - .build()) - .keyPosition()) - .isEqualTo(1); + assertThatThrownBy(() -> KafkaTableSchema.resolve(table)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("table.kafka.convert.value.layout"); } @Test @@ -374,20 +404,95 @@ void testSystemColumnTypes() { } @Test - void testMetadataPreservesDeclaredNullability() { - RowType metadata = - DataTypes.ROW( - DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP_LTZ(3)), - DataTypes.FIELD("timestamp_type", DataTypes.INT().copy(false)), - DataTypes.FIELD("partition", DataTypes.INT().copy(false))); - for (boolean nullable : new boolean[] {true, false}) { - DataType type = metadata.copy(nullable); - TableDescriptor table = withSystem("_kafka_metadata", type); - KafkaTableSchema resolved = KafkaTableSchema.resolve(table); - assertThat(resolved.rowType().getTypeAt(resolved.metadataPosition())).isEqualTo(type); + void testSystemFieldsRequireNullableTypes() { + assertInvalid(withSystem("_kafka_key", DataTypes.BYTES().copy(false)), "_kafka_key"); + assertInvalid( + withSystem( + "_kafka_headers", + DataTypes.MAP(DataTypes.STRING(), DataTypes.BYTES()).copy(false)), + "_kafka_headers"); + assertInvalid( + withSystem( + "_kafka_headers", + DataTypes.MAP(DataTypes.STRING(), DataTypes.BYTES().copy(false))), + "_kafka_headers"); + assertInvalid(withSystem("_kafka_metadata", metadataType().copy(false)), "_kafka_metadata"); + for (int i = 0; i < metadataType().getFieldCount(); i++) { + RowType.Builder metadata = RowType.builder(); + for (int j = 0; j < metadataType().getFieldCount(); j++) { + metadata.field( + metadataType().getFields().get(j).getName(), + metadataType().getTypeAt(j).copy(i != j)); + } + assertInvalid( + withSystem("_kafka_metadata", metadata.build()), + "_kafka_metadata." + metadataType().getFields().get(i).getName()); + } + } + + @ParameterizedTest + @ValueSource(strings = {"flat", "nested"}) + void testJsonFieldsRequireNullableTypesRecursively(String layout) { + DataType notNull = DataTypes.INT().copy(false); + for (DataType type : + Arrays.asList( + notNull, + DataTypes.ROW(DataTypes.FIELD("child", notNull)), + DataTypes.ROW(DataTypes.FIELD("child", DataTypes.INT())).copy(false), + DataTypes.ARRAY(notNull), + DataTypes.ARRAY(DataTypes.INT()).copy(false), + DataTypes.MAP(DataTypes.STRING(), notNull), + DataTypes.MAP(DataTypes.STRING(), DataTypes.INT()).copy(false))) { + assertInvalid(jsonTable(layout, type), "nullable fields"); + } + if ("nested".equals(layout)) { + assertInvalid( + enabled( + Schema.newBuilder() + .column( + "_kafka_value", + DataTypes.ROW( + DataTypes.FIELD( + "id", DataTypes.INT())) + .copy(false)) + .build(), + "parse_json") + .property( + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT, + ValueConversionLayout.NESTED) + .build(), + "_kafka_value"); } } + @Test + void testOptionalBinaryKeyConversion() { + TableDescriptor table = withSystem("_kafka_key", DataTypes.BYTES()); + TableDescriptor explicit = + TableDescriptor.builder(table) + .property( + ConfigOptions.TABLE_KAFKA_CONVERT_KEY_TYPE, + KafkaKeyConversionType.BINARY) + .build(); + assertThat(KafkaTableSchema.resolve(explicit)).isEqualTo(KafkaTableSchema.resolve(table)); + assertThat(table.getProperties()).doesNotContainKey("table.kafka.convert.key.type"); + assertInvalid( + enabled(valueSchema(), "binary") + .property( + ConfigOptions.TABLE_KAFKA_CONVERT_KEY_TYPE, + KafkaKeyConversionType.BINARY) + .build(), + "requires _kafka_key BYTES"); + assertThatThrownBy( + () -> + KafkaTableSchema.resolve( + TableDescriptor.builder(table) + .property("table.kafka.convert.key.type", "string") + .build())) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("table.kafka.convert.key.type"); + } + @Test void testMetadataRejectsInvalidStructureAndOffsetField() { for (DataType invalid : @@ -432,30 +537,29 @@ void testMetadataRejectsInvalidStructureAndOffsetField() { } @Test - void testUnknownAndPrototypeOptions() { - for (String key : - Arrays.asList("table.kafka.unknown", "table.kafka.convert.rescue-column")) { - assertInvalid( - enabled(valueSchema(), "binary").property(key, "anything").build(), - "Unsupported Kafka table property"); - } + void testUnrelatedPropertiesDoNotChangeMapping() { + TableDescriptor table = enabled(valueSchema(), "binary").build(); + KafkaTableSchema expected = KafkaTableSchema.resolve(table); + // Storage option registration is checked by the server, not by this shared resolver. + assertThat( + KafkaTableSchema.resolve( + TableDescriptor.builder(table) + .property("table.kafka.future-option", "anything") + .build())) + .isEqualTo(expected); for (String key : Arrays.asList( "kafka.value.format", - "kafka.key.fields", - "kafka.metadata.headers.column", "fluss.value.format", - "table.kafka.convert.value.type")) { - assertInvalid( - enabled(valueSchema(), "binary").customProperty(key, "raw").build(), - "Unsupported Kafka custom property"); + "table.kafka.convert.value.type", + "owner")) { + assertThat( + KafkaTableSchema.resolve( + TableDescriptor.builder(table) + .customProperty(key, "anything") + .build())) + .isEqualTo(expected); } - assertThat( - KafkaTableSchema.resolve( - enabled(valueSchema(), "binary") - .customProperty("owner", "team") - .build())) - .isNotNull(); } @Test @@ -499,7 +603,7 @@ private static TableDescriptor.Builder enabled(Schema schema, String type) { .schema(schema) .logFormat(LogFormat.ARROW) .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) - .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, type); + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE.key(), type); } private static void assertInvalid(TableDescriptor table, String message) { diff --git a/fluss-kafka/src/main/java/org/apache/fluss/kafka/mapping/KafkaTopicMapper.java b/fluss-kafka/src/main/java/org/apache/fluss/kafka/mapping/KafkaTopicMapper.java index 704d322f2e0..aa31d618986 100644 --- a/fluss-kafka/src/main/java/org/apache/fluss/kafka/mapping/KafkaTopicMapper.java +++ b/fluss-kafka/src/main/java/org/apache/fluss/kafka/mapping/KafkaTopicMapper.java @@ -39,7 +39,7 @@ public final class KafkaTopicMapper { private final String defaultDatabase; /** Creates a mapper using the default database configuration. */ - public KafkaTopicMapper() { + KafkaTopicMapper() { this(new Configuration()); } @@ -62,7 +62,7 @@ public TablePath toTablePath(String topicName) { : TablePath.of( topicName.substring(0, separator), topicName.substring(separator + 1)); - if (!isMappedTable(path)) { + if (!isMappedTableValid(path)) { throw new InvalidTopicException("Canonical Kafka topic name is invalid: " + path); } return path; @@ -70,7 +70,7 @@ public TablePath toTablePath(String topicName) { /** Returns the fully qualified Kafka name of a representable Fluss table. */ public String toTopicName(TablePath tablePath) { - if (!isMappedTable(tablePath)) { + if (!isMappedTableValid(tablePath)) { throw new InvalidTopicException( "Fluss table cannot be represented as a Kafka database.table name: " + tablePath); @@ -78,8 +78,11 @@ public String toTopicName(TablePath tablePath) { return tablePath.toString(); } - /** Returns whether a Fluss user table has a valid, fully qualified Kafka topic name. */ - public boolean isMappedTable(TablePath tablePath) { + /** + * Returns whether a table path maps to a valid, fully qualified Kafka topic name. This does not + * check whether the table exists or enables Kafka compatibility. + */ + public boolean isMappedTableValid(TablePath tablePath) { return KafkaTableSchema.isValidTablePath(tablePath); } diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/mapping/KafkaTopicMapperTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/mapping/KafkaTopicMapperTest.java index f70c7ffca22..c2aa5206640 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/mapping/KafkaTopicMapperTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/mapping/KafkaTopicMapperTest.java @@ -46,7 +46,7 @@ public void testQualifiedNamesAcrossDatabases() { String topicName = database + ".orders_1-2026"; assertThat(mapper.toTablePath(topicName)).isEqualTo(tablePath); assertThat(mapper.toTopicName(tablePath)).isEqualTo(topicName); - assertThat(mapper.isMappedTable(tablePath)).isTrue(); + assertThat(mapper.isMappedTableValid(tablePath)).isTrue(); assertThat(KafkaTopicMapper.isValidTopic(topicName)).isTrue(); } } @@ -121,7 +121,7 @@ public void testNameLengthLimits() { TablePath.of(repeat('a', 201), "orders"), TablePath.of("sales", repeat('a', 201)), TablePath.of(repeat('a', 124), repeat('b', 125)))) { - assertThat(mapper.isMappedTable(invalid)).isFalse(); + assertThat(mapper.isMappedTableValid(invalid)).isFalse(); assertThatThrownBy(() -> mapper.toTablePath(invalid.toString())) .isInstanceOf(InvalidTopicException.class); assertThatThrownBy(() -> mapper.toTopicName(invalid)) @@ -143,7 +143,7 @@ public void testRejectsUnrepresentableTablePaths() { TablePath.of("__system", "orders"), TablePath.of("sales", "__internal"), TablePath.of("sales", "bad name"))) { - assertThat(mapper.isMappedTable(invalid)).isFalse(); + assertThat(mapper.isMappedTableValid(invalid)).isFalse(); assertThatThrownBy(() -> mapper.toTopicName(invalid)) .isInstanceOf(InvalidTopicException.class); } diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java index 0c7ff17eda3..822bf9f60d4 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java @@ -19,6 +19,9 @@ import org.apache.fluss.config.ConfigOptions; import org.apache.fluss.config.Configuration; +import org.apache.fluss.config.ValueConversionLayout; +import org.apache.fluss.config.ValueConversionType; +import org.apache.fluss.exception.InvalidConfigException; import org.apache.fluss.exception.InvalidTableException; import org.apache.fluss.kafka.mapping.KafkaTopicMapper; import org.apache.fluss.metadata.KafkaTableSchema; @@ -223,6 +226,67 @@ private void assertUnchanged(TablePath path, GetTableInfoResponse before) throws assertThat(after.getSchemaId()).isEqualTo(before.getSchemaId()); } + @ParameterizedTest + @ValueSource(strings = {"value", "system", "nested"}) + void testNotNullFieldsAreRejectedBeforePersistence(String field) throws Exception { + boolean system = "system".equals(field); + String column = system ? "_kafka_metadata" : "payload"; + DataType type = + system + ? DataTypes.ROW( + DataTypes.FIELD("timestamp", DataTypes.TIMESTAMP_LTZ(3)), + DataTypes.FIELD("timestamp_type", DataTypes.INT()), + DataTypes.FIELD("partition", DataTypes.INT().copy(false))) + : "nested".equals(field) + ? DataTypes.ARRAY( + DataTypes.MAP( + DataTypes.STRING(), + DataTypes.ROW( + DataTypes.FIELD( + "id", + DataTypes.INT().copy(false))))) + : DataTypes.INT().copy(false); + TableDescriptor descriptor = + TableDescriptor.builder() + .schema(Schema.newBuilder().column(column, type).build()) + .distributedBy(1) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) + .build(); + TablePath path = TablePath.of("ddl", "not_null_" + field); + gateway.createDatabase(newCreateDatabaseRequest("ddl", true)).get(); + assertThatThrownBy( + () -> + gateway.createTable(newCreateTableRequest(path, descriptor, false)) + .get()) + .hasCauseInstanceOf(InvalidTableException.class) + .hasStackTraceContaining("nullable fields"); + assertThat(gateway.tableExists(newTableExistsRequest(path)).get().isExists()).isFalse(); + assertThat(CLUSTER.getZooKeeperClient().getCurrentSchemaId(path)).isZero(); + createTable( + CLUSTER, + path, + TableDescriptor.builder(descriptor) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, false) + .build()); + GetTableInfoResponse before = info(path); + assertThatThrownBy(() -> set(path, Collections.singletonMap("table.kafka.enabled", "true"))) + .hasCauseInstanceOf(InvalidTableException.class) + .hasStackTraceContaining("nullable fields"); + assertUnchanged(path, before); + + // Native schema evolution already rejects new NOT NULL columns. Kafka additionally + // rejects NOT NULL children of otherwise nullable columns. + if (type.isNullable()) { + TablePath alter = TablePath.of("ddl", "alter_not_null_" + field); + createTable(CLUSTER, alter, jsonTable("flat", DataTypes.STRING()).build()); + GetTableInfoResponse beforeAlter = info(alter); + assertThatThrownBy(() -> addColumn(alter, system ? column : "invalid", type)) + .hasCauseInstanceOf(InvalidTableException.class) + .hasStackTraceContaining("nullable fields"); + assertUnchanged(alter, beforeAlter); + } + } + private static TableDescriptor.Builder jsonTable(String layout, DataType businessType) { Schema schema = "flat".equals(layout) @@ -230,15 +294,16 @@ private static TableDescriptor.Builder jsonTable(String layout, DataType busines : Schema.newBuilder() .column( "_kafka_value", - DataTypes.ROW(DataTypes.FIELD("payload", businessType))) + DataTypes.ROW(DataTypes.FIELD("_kafka_key", businessType))) .build(); TableDescriptor.Builder builder = - enabled() + TableDescriptor.builder() .schema(schema) - .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "parse_json"); - // Omitted flat layout must remain omitted through CREATE and ALTER. + .distributedBy(1) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, true); + // Read-time JSON defaults must remain omitted through CREATE and ALTER. if ("nested".equals(layout)) { - builder.property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT, layout); + builder.property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT.key(), layout); } return builder; } @@ -258,16 +323,41 @@ void testOnlyRequestedKafkaPropertiesArePersisted() throws Exception { .column("_kafka_rescue", DataTypes.STRING()) .build(); TableDescriptor descriptor = - enabled() + TableDescriptor.builder() .schema(schema) - .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "parse_json") + .distributedBy(1) + .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) + .customProperty("table.kafka.convert.value.type", "custom-value") + .customProperty("kafka.value.format", "custom-format") + .customProperty("fluss.owner", "team") .build(); createTable(CLUSTER, path, descriptor); TableDescriptor stored = stored(path); assertThat(stored.getSchema()).isEqualTo(schema); - assertThat(kafkaProperties(stored)).isEqualTo(kafkaProperties(descriptor)); + assertThat(stored.getCustomProperties()).isEqualTo(descriptor.getCustomProperties()); + assertThat(kafkaProperties(stored)) + .isEqualTo(Collections.singletonMap("table.kafka.enabled", "true")); assertThat(new KafkaTopicSchemaResolver().resolve(stored).valueProjection().positions()) .containsExactly(1); + set(path, Collections.singletonMap("table.kafka.convert.key.type", "binary")); + assertThat(KafkaTableSchema.resolve(stored(path))) + .isEqualTo(KafkaTableSchema.resolve(stored)); + reset(path, "table.kafka.convert.key.type"); + assertThat(stored(path)).isEqualTo(stored); + TableDescriptor unknownOption = + TableDescriptor.builder(descriptor) + .property("table.kafka.unknown", "anything") + .build(); + assertThatThrownBy( + () -> + gateway.createTable( + newCreateTableRequest( + TablePath.of("ddl", "unknown_option"), + unknownOption, + false)) + .get()) + .hasCauseInstanceOf(InvalidConfigException.class) + .hasStackTraceContaining("table.kafka.unknown"); } @Test @@ -285,10 +375,21 @@ void testEnableDisableAndRejectInvalidProperties() throws Exception { expected.put("table.kafka.enabled", "true"); assertThat(enabled.getProperties()).isEqualTo(expected); assertThat(enabled.getSchema()).isEqualTo(before.getSchema()); - Map invalid = new HashMap<>(); - invalid.put("table.kafka.convert.value.type", "string"); - invalid.put("table.kafka.convert.key.type", "binary"); - assertThatThrownBy(() -> set(path, invalid)) + assertThatThrownBy( + () -> + set( + path, + Collections.singletonMap( + "table.kafka.convert.key.type", "binary"))) + .hasCauseInstanceOf(InvalidTableException.class) + .hasStackTraceContaining("requires _kafka_key BYTES"); + assertThat(stored(path)).isEqualTo(enabled); + assertThatThrownBy( + () -> + set( + path, + Collections.singletonMap( + "table.kafka.convert.value.type", "string"))) .hasCauseInstanceOf(InvalidTableException.class); assertThat(stored(path)).isEqualTo(enabled); assertThatThrownBy(() -> reset(path, "table.kafka.convert.value.type")) @@ -306,6 +407,38 @@ void testEnableDisableAndRejectInvalidProperties() throws Exception { assertThat(stored(path)).isEqualTo(staged); } + @ParameterizedTest + @ValueSource(strings = {"binary", "string"}) + void testNonJsonLayoutDoesNotChangeMapping(String type) throws Exception { + TablePath path = TablePath.of("ddl", type + "_layout"); + TableDescriptor descriptor = + enabled() + .schema( + Schema.newBuilder() + .column( + "_kafka_value", + "binary".equals(type) + ? DataTypes.BYTES() + : DataTypes.STRING()) + .build()) + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE.key(), type) + .build(); + createTable(CLUSTER, path, descriptor); + TableDescriptor before = stored(path); + KafkaTableSchema expected = KafkaTableSchema.resolve(before); + assertThat(expected.valueConversionLayout()).isNull(); + assertThat(stored(path).getProperties()) + .doesNotContainKey("table.kafka.convert.value.layout"); + for (ValueConversionLayout layout : ValueConversionLayout.values()) { + set(path, Collections.singletonMap("table.kafka.convert.value.layout", layout.name())); + assertThat(KafkaTableSchema.resolve(stored(path))).isEqualTo(expected); + assertThat(stored(path).getProperties()) + .containsEntry("table.kafka.convert.value.layout", layout.name()); + } + reset(path, "table.kafka.convert.value.layout"); + assertThat(stored(path)).isEqualTo(before); + } + @Test void testSchemaAlterValidatesBeforePersistenceAndCanBeDisabled() throws Exception { TablePath path = TablePath.of("ddl", "schema_change"); @@ -496,6 +629,6 @@ private static TableDescriptor.Builder enabled() { .distributedBy(1) .logFormat(LogFormat.ARROW) .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) - .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "binary"); + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, ValueConversionType.BINARY); } } diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolverTest.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolverTest.java index 119e8aff38f..7b0d18edbb3 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolverTest.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTopicSchemaResolverTest.java @@ -43,7 +43,8 @@ void testCompatibleSchemaDoesNotEnableTable() { void testInvalidEnabledContractIsNotTreatedAsDisabled() { TableDescriptor table = table().property(ConfigOptions.TABLE_KAFKA_ENABLED, true) - .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "schema_latest") + .property( + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE.key(), "schema_latest") .build(); assertThat(resolver.isEnabled(table)).isTrue(); assertThatThrownBy(() -> resolver.resolve(table)) @@ -62,6 +63,6 @@ void testMalformedPersistedEnableFlagIsAnExplicitContractError() { private static TableDescriptor.Builder table() { return TableDescriptor.builder() .schema(Schema.newBuilder().column("_kafka_value", DataTypes.BYTES()).build()) - .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, "binary"); + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE.key(), "binary"); } } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorServer.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorServer.java index 3b93e2c8c50..faa1367a9f9 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorServer.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorServer.java @@ -64,7 +64,9 @@ import java.util.ArrayList; import java.util.Collection; +import java.util.HashSet; import java.util.List; +import java.util.Set; import java.util.UUID; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutorService; @@ -553,15 +555,15 @@ private void createDefaultDatabase() { @VisibleForTesting static void createDefaultDatabases(MetadataManager metadataManager, Configuration conf) { - List databases = metadataManager.listDatabases(); + Set databases = new HashSet<>(metadataManager.listDatabases()); if (databases.isEmpty()) { metadataManager.createDatabase(DEFAULT_DATABASE, DatabaseDescriptor.EMPTY, true); + databases.add(DEFAULT_DATABASE); LOG.info("Created default database '{}' because no database exists.", DEFAULT_DATABASE); } if (conf.get(ConfigOptions.KAFKA_ENABLED)) { String kafkaDatabase = conf.get(ConfigOptions.KAFKA_DEFAULT_DATABASE); - // The native default database follows the cluster initialization logic above. - if (!DEFAULT_DATABASE.equals(kafkaDatabase) && !databases.contains(kafkaDatabase)) { + if (!databases.contains(kafkaDatabase)) { metadataManager.createDatabase(kafkaDatabase, DatabaseDescriptor.EMPTY, true); LOG.info("Created default database '{}' for Kafka protocol.", kafkaDatabase); } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorService.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorService.java index df8713338a8..545480a9a00 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorService.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorService.java @@ -507,7 +507,7 @@ public CompletableFuture createTable(CreateTableRequest req // validate table descriptor before creating table in lake or fluss metadata, // to avoid orphaned lake tables when validation fails - metadataManager.validateTableDescriptor(tablePath, tableDescriptor); + metadataManager.validateTableDefinition(tablePath, tableDescriptor); // the distribution and bucket count must be set now //noinspection OptionalGetWithoutIsPresent diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/MetadataManager.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/MetadataManager.java index 41259eed034..8588373809e 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/MetadataManager.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/MetadataManager.java @@ -124,17 +124,12 @@ public MetadataManager( this.lakeCatalogDynamicLoader = lakeCatalogDynamicLoader; } - /** Validates the table descriptor. */ - public void validateTableDescriptor(TableDescriptor tableDescriptor) { + /** Validates a descriptor and its Kafka topic identity before persistence. */ + public void validateTableDefinition(TablePath tablePath, TableDescriptor tableDescriptor) { TableDescriptorValidation.validateTableDescriptor( tableDescriptor, maxBucketNum, lakeCatalogDynamicLoader.getLakeCatalogContainer().getDataLakeFormat()); - } - - /** Validates a descriptor and its Kafka topic identity before persistence. */ - public void validateTableDescriptor(TablePath tablePath, TableDescriptor tableDescriptor) { - validateTableDescriptor(tableDescriptor); KafkaTableSchema.validateTablePath(tablePath, tableDescriptor); } @@ -844,7 +839,7 @@ private void doAlterTablePropertiesOnce( // historical partition would be created with the new table-level count while retired // lake data still uses the pre-rescale layout. bucketCountEpoch never decreases, so // this also rejects a table that was rescaled while the feature was temporarily - // disabled. Checked before validateTableDescriptor so the rescale rejection is not + // disabled. Checked before validateTableDefinition so the rescale rejection is not // masked by unrelated option-dependency errors. if (!tableInfo.getTableConfig().isHistoricalPartitionEnabled() && Configuration.fromMap(newDescriptor.getProperties()) @@ -859,7 +854,7 @@ private void doAlterTablePropertiesOnce( } // reuse the same validate logic with the createTable() method - validateTableDescriptor(tablePath, newDescriptor); + validateTableDefinition(tablePath, newDescriptor); beforeUpdate.accept(tableInfo, newDescriptor); diff --git a/fluss-server/src/main/java/org/apache/fluss/server/utils/TableDescriptorValidation.java b/fluss-server/src/main/java/org/apache/fluss/server/utils/TableDescriptorValidation.java index fafbbf9a6c1..2b5d7d82c83 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/utils/TableDescriptorValidation.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/utils/TableDescriptorValidation.java @@ -149,10 +149,6 @@ public static void validateTableDescriptor( /** Validates the schema after altering table columns. */ @Internal public static void validateAlterTableSchema(TableInfo table, Schema newSchema) { - TableDescriptor updated = - TableDescriptor.builder(table.toTableDescriptor()).schema(newSchema).build(); - KafkaTableSchema.validate(updated); - KafkaTableSchema.validateTablePath(table.getTablePath(), updated); checkSystemColumns(newSchema.getRowType()); if (table.getTableConfig() .getMergeEngineType() @@ -162,6 +158,10 @@ public static void validateAlterTableSchema(TableInfo table, Schema newSchema) { } else { validateNoAggregationFunctions(newSchema); } + TableDescriptor updated = + TableDescriptor.builder(table.toTableDescriptor()).schema(newSchema).build(); + KafkaTableSchema.validate(updated); + KafkaTableSchema.validateTablePath(table.getTablePath(), updated); } private static void checkTableLakeFormatMatchesCluster( diff --git a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/KafkaDefaultDatabaseTest.java b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/KafkaDefaultDatabaseTest.java index 5e4cc8bc3a5..ed97d0b5251 100644 --- a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/KafkaDefaultDatabaseTest.java +++ b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/KafkaDefaultDatabaseTest.java @@ -30,10 +30,14 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; import org.junit.jupiter.params.provider.ValueSource; import static org.apache.fluss.server.coordinator.CoordinatorServer.DEFAULT_DATABASE; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; /** Tests startup database creation for native and Kafka default database configurations. */ class KafkaDefaultDatabaseTest { @@ -61,16 +65,26 @@ void setup() throws Exception { } @ParameterizedTest - @ValueSource(booleans = {false, true}) - void testNativeDefaultFollowsClusterInitialization(boolean hasDatabase) { + @CsvSource({"false, false", "false, true", "true, false", "true, true"}) + void testNativeAndKafkaDefaultInitialization(boolean hasDatabase, boolean kafkaEnabled) { if (hasDatabase) { manager.createDatabase("existing", DatabaseDescriptor.EMPTY, false); } Configuration config = kafkaConfiguration(DEFAULT_DATABASE); - CoordinatorServer.createDefaultDatabases(manager, config); - CoordinatorServer.createDefaultDatabases(manager, config); + config.set(ConfigOptions.KAFKA_ENABLED, kafkaEnabled); + MetadataManager observed = spy(manager); + CoordinatorServer.createDefaultDatabases(observed, config); + CoordinatorServer.createDefaultDatabases(observed, config); + boolean createsDefault = !hasDatabase || kafkaEnabled; assertThat(manager.listDatabases()) - .containsExactly(hasDatabase ? "existing" : DEFAULT_DATABASE); + .containsExactlyInAnyOrder( + !hasDatabase + ? new String[] {DEFAULT_DATABASE} + : kafkaEnabled + ? new String[] {"existing", DEFAULT_DATABASE} + : new String[] {"existing"}); + verify(observed, times(createsDefault ? 1 : 0)) + .createDatabase(DEFAULT_DATABASE, DatabaseDescriptor.EMPTY, true); } @ParameterizedTest diff --git a/website/docs/kafka-compatibility.md b/website/docs/kafka-compatibility.md new file mode 100644 index 00000000000..4d1c5d05ac4 --- /dev/null +++ b/website/docs/kafka-compatibility.md @@ -0,0 +1,244 @@ +--- +sidebar_label: Kafka Compatibility +title: Kafka Compatibility +sidebar_position: 5.5 +--- + +# Kafka Compatibility + +Fluss Kafka compatibility maps Kafka topics and records to native Fluss tables. Tables are +created through Fluss DDL or the Flink Fluss catalog, and explicitly enabled for Kafka access. + +:::warning +Kafka protocol compatibility is still in development. The current implementation provides +ApiVersions and the DDL mapping contract described here. Kafka Metadata, Produce, and consumer +APIs are not yet available; enabling a table does not enable these APIs. +::: + +## Supported APIs + +| API or capability | Current status | +|---|---| +| ApiVersions | Implemented. | +| Native CREATE/ALTER and Kafka table mapping validation | Implemented. | +| Kafka Metadata and Produce | Planned for Basic Produce; not yet available. | +| Fetch, consumer groups, and offset commits | Not available. | +| Idempotent and transactional producers | Not available in Basic Produce. | +| CreateTopics and DeleteTopics | Not supported; manage tables through Fluss. | + +The mapping and conversion rules below define the table contract for the Kafka write path. +They do not make an unsupported Kafka API available. + +## Supported tables + +Kafka mapping currently accepts non-partitioned [Log Tables](table-design/table-types/log-table.md) using +the Arrow log format, without a primary key. The initial Kafka write path is designed to +encode Arrow batches and append them to log tables. Supporting other log formats requires +matching encoders; primary key and partitioned tables also require additional write semantics. + +Each table must have a compatible schema and `table.kafka.enabled=true`. System columns are +optional: declare only the Kafka information that the table needs to store. Enabling Kafka +does not add columns or persist default conversion properties. CREATE and ALTER validate the +resulting mapping before persisting it, including schema changes on enabled tables. All declared +columns and nested value fields must be nullable. This includes ROW fields, ARRAY elements, +MAP values, and optional system columns. MAP keys retain Fluss's mandatory non-null constraint; +they are the only exception. Validation rejects NOT NULL fields rather than changing their types. + +## Cluster configuration + +Set `kafka.enabled: true` in the cluster's `server.yaml`. On each TabletServer, add a Kafka +listener alongside its native Fluss listener. For a local TabletServer: + +```yaml title="server.yaml" +kafka.enabled: true +kafka.listener.names: KAFKA +kafka.default.database: fluss +bind.listeners: FLUSS://localhost:9123,KAFKA://localhost:9092 +``` + +Kafka listeners belong on TabletServers. For a distributed deployment, configure reachable +addresses with `bind.listeners` and `advertised.listeners`; see the +[server configuration reference](maintenance/configuration.md#common). All TabletServers +and the coordinator must use the same `kafka.default.database`. + +## Topic-to-table mapping + +Kafka topics map to Fluss tables using `.
`. Topic names without a database prefix +map to the same-named table in the database configured by `kafka.default.database` (default: `fluss`). +Kafka clients can access a topic only after its corresponding table has been created in Fluss. + +For example, `sales.orders` maps to the `orders` table in `sales`, while `orders` maps to +`fluss.orders` with the default configuration. Reverse mapping always uses the fully qualified +name. Changing the default database redirects unqualified names; use fully qualified topic +names when clients need a stable mapping across such changes. + +The native `fluss` database is created when the cluster has no databases. When Kafka is enabled, +the coordinator also ensures that the configured Kafka default database exists, including when +it is `fluss` and other databases already exist. Existing databases are preserved. A database +deleted at runtime is not recreated by a Kafka request; initialization runs when the coordinator +becomes active. +Other databases and all tables must be created explicitly; Kafka does not automatically create +tables, and Kafka CreateTopics/DeleteTopics are not supported. + +### Partition-to-bucket mapping + +A Kafka partition maps to the same-numbered Fluss bucket. The topic's partition count equals +the table's bucket count. Qualified and unqualified names for the same table share its buckets +and log offsets; same-named tables in different databases remain independent. The write path +uses the bucket's leader and replicas rather than a separate Kafka storage layer. + +## Create or enable a table + +The examples below run in an existing [Flink Fluss catalog](engine-flink/ddl.md#create-catalog). +Create a table for JSON values, optionally storing the Kafka key and headers: + +```sql title="Flink SQL" +CREATE DATABASE IF NOT EXISTS sales; + +CREATE TABLE sales.orders ( + order_id BIGINT, + amount DECIMAL(18, 2), + _kafka_key BYTES, + _kafka_headers MAP +) WITH ( + 'table.kafka.enabled' = 'true' +); +``` + +This defines the mapping for `sales.orders`. JSON fields such as `order_id` and `amount` map to +the business columns, while the Kafka record's key and headers map to their system columns. +The omitted conversion type and layout are read as `parse_json` and `flat` without adding +properties to table metadata. + +For an existing table with a compatible JSON schema, enable Kafka: + +```sql title="Flink SQL" +ALTER TABLE sales.orders SET ( + 'table.kafka.enabled' = 'true' +); +``` + +Set `table.kafka.enabled` to `false` to disable Kafka access. Disabled tables are not subject +to Kafka-specific schema validation. + +## Producer configuration and writes + +:::warning +The following example describes the intended Basic Produce usage once Metadata and Produce +are implemented. It cannot write through the current ApiVersions-only endpoint. +::: + +Use the Kafka listener as the bootstrap address and disable idempotence. For the `sales.orders` +table above, a Java producer can send UTF-8 JSON values and string keys as follows: + +```java +import org.apache.kafka.clients.producer.KafkaProducer; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.apache.kafka.common.serialization.StringSerializer; + +import java.nio.charset.StandardCharsets; +import java.util.Properties; + +Properties properties = new Properties(); +properties.put("bootstrap.servers", "localhost:9092"); +properties.put("enable.idempotence", "false"); +properties.put("acks", "all"); + +try (KafkaProducer producer = + new KafkaProducer<>(properties, new StringSerializer(), new StringSerializer())) { + ProducerRecord record = new ProducerRecord<>( + "sales.orders", "order-1", "{\"order_id\":1,\"amount\":12.50}"); + record.headers().add("source", "app".getBytes(StandardCharsets.UTF_8)); + producer.send(record).get(); +} +``` + +The value populates `order_id` and `amount`; the UTF-8 key bytes populate `_kafka_key`, and the +header populates `_kafka_headers`. Binary values should use `ByteArraySerializer`; string values +can use `StringSerializer`. Do not configure `transactional.id`. Basic Produce does not provide +Kafka idempotence or transactional guarantees, so retries can produce duplicate rows. + +## Conversion properties + +These are table properties set through CREATE or ALTER TABLE, rather than `server.yaml` settings. + +| Option | Type | Default | Description | +|---|---|---|---| +| table.kafka.enabled | Boolean | false | Explicitly enable the table's Kafka mapping. | +| table.kafka.convert.value.type | Enum | `parse_json` | `binary`, `string`, or `parse_json`. | +| table.kafka.convert.value.layout | Enum | `flat` | JSON layout: `flat` maps top-level business columns; `nested` targets `_kafka_value ROW<...>`. Ignored for binary/string, whether defaulted or explicitly configured. | +| table.kafka.convert.key.type | Enum | (None) | Optional `binary` conversion; explicitly configuring it requires `_kafka_key BYTES`. | + +Conversion defaults apply only at read time. CREATE and ALTER preserve omitted properties. +Declaring `_kafka_key BYTES` selects binary key mapping even when the key conversion option is +omitted. No key column or conversion property is required when the key is not needed. + +Custom properties are preserved but do not configure the Kafka mapping. Storage properties and +their values are checked by the server against the options supported by that cluster version. + +### Value conversion + +| Conversion | Required value schema | Mapping | +|---|---|---| +| `binary` | `_kafka_value BYTES` | Store the record value as bytes. | +| `string` | `_kafka_value STRING` | Interpret the record value as a UTF-8 string. | +| `parse_json`, `flat` | Business columns at the table's top level | Map JSON object members to business columns by name. | +| `parse_json`, `nested` | `_kafka_value ROW<...>` | Map JSON object members to fields inside the value ROW. | + +Binary, string, and nested JSON map their value to `_kafka_value`; other top-level columns must +be optional system columns. Flat JSON uses top-level business columns and does not allow a +`_kafka_value` column. Both JSON layouts require an object and preserve declared nested ROW, +ARRAY, and MAP structures. + +System names are reserved at the physical table's top level. Inside a nested `_kafka_value` ROW, +they are ordinary business field names: `_kafka_value ROW<_kafka_key STRING>` can coexist with +a top-level `_kafka_key BYTES` column. The inner field comes from the JSON value, and the outer +column comes from the Kafka key. The same rule applies to deeper business structures. + +JSON schemas accept all currently defined Fluss types, subject to the nullable-field rule above +and two additional restrictions: MAP keys +must be STRING, including nested maps, and nesting depth must not exceed 64. Each top-level +business field starts at depth zero; ROW fields, ARRAY elements, and MAP values each add one. +The JSON root and the physical `_kafka_value` ROW container add no level. Scalar value constraints +apply when decoding individual records. + +## System columns + +The four optional columns below may be omitted independently. Unmapped key, headers, or metadata +are not stored in row fields. `_kafka_value` is the value target required by the conversions above. + +| Column | Type | Mapping | +|---|---|---| +| `_kafka_key` | `BYTES` | Kafka record key as bytes. | +| `_kafka_headers` | `MAP` | Kafka headers; the last value wins for duplicate header names. | +| `_kafka_metadata` | `ROW` | Input record timestamp, batch timestamp type, and requested Kafka partition. | +| `_kafka_rescue` | Nullable `STRING` | Compact JSON containing unknown fields, values that cannot be converted, and flat-layout system-name conflicts. | + +The metadata ROW must use the field names, order, and types shown above. Timestamp type is `-1` +when absent, `0` for CreateTime, or `1` for LogAppendTime. An absent timestamp maps to null. +The metadata ROW and its three fields must be nullable. Log offsets remain native log metadata and are not +fields of this ROW. + +A declared rescue column needs no additional property. The conversion contract sends an +unconvertible field's original JSON value to rescue and writes null to that field. For example, +`{"order_id":"invalid","amount":12.50}` writes null to `order_id`, keeps `amount`, and stores +`{"order_id":"invalid"}` in `_kafka_rescue` when that column is declared. Nested errors preserve +their path and structure; valid sibling fields remain usable. If an entire field has the wrong +structure, rescue contains that field's original value and the field is null. + +Without rescue, unknown fields, conversion failures, and flat-layout system-name conflicts +fail conversion. Invalid JSON and a non-object JSON root always fail; rescue requires a parsed +object and does not hide protocol or storage failures. These write-time behaviors require the +planned JSON decoder; this implementation enforces their DDL prerequisites. +Binary/string conversion and JSON records with no fields to rescue write null to this column. + +## Troubleshooting + +| Symptom | What to check | +|---|---| +| Producer cannot discover a topic or send records | Check the API status above first. Metadata and Produce are not yet implemented in this version. | +| Connection fails | Use a TabletServer Kafka listener, enable `kafka.enabled`, and ensure advertised addresses are reachable from the client. | +| A topic does not map to the intended table | Create and enable the Fluss table first. Check the `.
` spelling or `kafka.default.database` for an unqualified name. | +| CREATE or ALTER rejects the Kafka mapping | Use a non-partitioned Arrow log table without a primary key, nullable fields, and the required value column for the chosen conversion. | +| JSON schema validation fails | Check nested MAP key types and the nesting limit. MAP values and ARRAY elements must be nullable. | +| A future Basic Produce client requests unsupported producer APIs | Disable idempotence and transactions. Consumer APIs require separate support. | diff --git a/website/docs/maintenance/configuration.md b/website/docs/maintenance/configuration.md index e451be59f4a..624e670938a 100644 --- a/website/docs/maintenance/configuration.md +++ b/website/docs/maintenance/configuration.md @@ -259,50 +259,10 @@ Kafka protocol compatibility is still in development. |--------------------------------|----------|---------|--------------------------------------------------------------------------------------------------------------------| | kafka.enabled | Boolean | false | Whether enable Fluss Kafka. Disabled by default. When this option is set to true, the Fluss Kafka will be enabled. | | kafka.listener.names | String | KAFKA | The listener names for Kafka wire protocol communication. Support multiple listener names, separated by comma. | -| kafka.default.database | String | fluss | Database used for unqualified Kafka topic names. Fully qualified names can access other databases. When Kafka is enabled, a configured database other than `fluss` is created if missing. The native `fluss` database is created only when the cluster has no databases. | +| kafka.default.database | String | fluss | Database used for unqualified Kafka topic names. When Kafka is enabled, the coordinator creates this database if missing, including `fluss`. | Kafka listeners share `netty.connection.max-idle-time` with native Fluss RPC listeners (see [Netty](#netty)). The default is `10min`; connections close when neither reads nor writes occur for the configured duration. -Kafka topics map to Fluss tables using `.
`. Topic names without a database prefix -map to the same-named table in the database configured by `kafka.default.database` (default: `fluss`). -Kafka clients can access a topic only after its corresponding table has been created in Fluss. - -### Kafka table properties - -| Option | Type | Default | Description | -|---|---|---|---| -| table.kafka.enabled | Boolean | false | Explicitly expose a non-partitioned Arrow log table as a Kafka topic. CREATE and ALTER validate the contract before persisting it. | -| table.kafka.convert.value.type | String | (None) | Required when enabled: `binary`, `string`, or `parse_json`. | -| table.kafka.convert.value.layout | String | (None) | JSON uses `flat` when omitted; `nested` targets `_kafka_value ROW<...>`. Setting this property for binary/string is invalid. | -| table.kafka.convert.key.type | String | (None) | Only `binary` is supported, and requires `_kafka_key BYTES`. Declaring that column implies binary without adding a property. | - -Binary and string values require a user-declared `_kafka_value BYTES` or `_kafka_value STRING` -column respectively. Flat JSON targets business columns at the table level; nested JSON targets -the fields of `_kafka_value ROW<...>`. Both exclude reserved system names from business matching. -JSON business schemas support nested ROW, ARRAY and MAP values. Every MAP key must be STRING, -including maps inside other containers. CREATE and ALTER reject unsupported types or a nesting -depth above 64, counted from each top-level JSON business field at depth zero; ROW fields, -ARRAY elements and MAP values each add one level. The JSON root and the physical nested -`_kafka_value ROW` container add no level, so equivalent flat/nested schemas have the same limit. -Scalar value and nullability checks apply when decoding records. -Optional `_kafka_key BYTES` and `_kafka_headers MAP` columns retain keys and headers. -Duplicate header names use the last value. Optional `_kafka_rescue STRING` must be nullable and -collects unknown or reserved JSON members; it requires no mapping property. Binary/string -conversion writes null to a declared rescue column. - -Enabling Kafka compatibility does not add columns or persist default Kafka properties. When -disabled, ordinary Fluss tables are not subject to Kafka-specific schema validation. Prototype -`kafka.key.*`, `kafka.value.*`, `kafka.metadata.*`, and `fluss.*` conversion properties are not -accepted on enabled tables. - -The `_kafka_metadata` contract is -`ROW`. -This optional physical column stores the input Kafka record timestamp, the batch timestamp type -(`-1` for no timestamp, `0` for CreateTime, `1` for LogAppendTime), and the requested partition. -Without a timestamp, the timestamp field is null; declared NOT NULL constraints still apply -when converting records. These values are available before Arrow encoding. Log offsets remain -native log metadata and are not part of this ROW, so enabling the column requires no append-time -offset backfill. CREATE and ALTER reject extra fields, including the former offset field, and -incorrect field names, ordering or types. -These DDL contracts do not by themselves advertise Kafka Metadata or Produce API support. +See [Kafka Compatibility](../kafka-compatibility.md) for cluster setup, table properties, +topic-to-table mapping, value conversion, and system columns. From f610278073ec2e9775cb200e1b522d00edfa949e Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Sun, 18 Oct 2026 20:00:28 +0800 Subject: [PATCH 13/15] [kafka] Refine mapping docs and conversion names Use Kafka-prefixed value conversion enums and keep path-aware table descriptor validation under its established method name. Simplify the user guide and show topic-to-table examples in a compact mapping table. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 315/315 AI-Contributed/UT: 22/22 --- .../apache/fluss/config/ConfigOptions.java | 12 +- ...t.java => KafkaValueConversionLayout.java} | 2 +- ...ype.java => KafkaValueConversionType.java} | 2 +- .../fluss/metadata/KafkaTableSchema.java | 36 +-- .../fluss/metadata/KafkaTableSchemaTest.java | 14 +- .../kafka/schema/KafkaTableMappingITCase.java | 8 +- .../coordinator/CoordinatorService.java | 2 +- .../server/coordinator/MetadataManager.java | 6 +- website/docs/kafka-compatibility.md | 255 ++++++------------ 9 files changed, 121 insertions(+), 216 deletions(-) rename fluss-common/src/main/java/org/apache/fluss/config/{ValueConversionLayout.java => KafkaValueConversionLayout.java} (96%) rename fluss-common/src/main/java/org/apache/fluss/config/{ValueConversionType.java => KafkaValueConversionType.java} (96%) diff --git a/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java b/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java index 8d1cbe0319e..80dcad59d72 100644 --- a/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java +++ b/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java @@ -2277,19 +2277,19 @@ public class ConfigOptions { "Whether the table is exposed as a Kafka topic. Enabling validates the " + "existing schema and conversion options without adding columns or defaults."); - public static final ConfigOption TABLE_KAFKA_CONVERT_VALUE_TYPE = + public static final ConfigOption TABLE_KAFKA_CONVERT_VALUE_TYPE = key("table.kafka.convert.value.type") - .enumType(ValueConversionType.class) - .defaultValue(ValueConversionType.PARSE_JSON) + .enumType(KafkaValueConversionType.class) + .defaultValue(KafkaValueConversionType.PARSE_JSON) .withDescription( "Kafka value conversion: binary, string, or parse_json. " + "Binary and string target the declared _kafka_value column. " + "The default is used at read time without adding a table property."); - public static final ConfigOption TABLE_KAFKA_CONVERT_VALUE_LAYOUT = + public static final ConfigOption TABLE_KAFKA_CONVERT_VALUE_LAYOUT = key("table.kafka.convert.value.layout") - .enumType(ValueConversionLayout.class) - .defaultValue(ValueConversionLayout.FLAT) + .enumType(KafkaValueConversionLayout.class) + .defaultValue(KafkaValueConversionLayout.FLAT) .withDescription( "JSON layout for parse_json: flat maps top-level business columns; " + "nested targets _kafka_value ROW. Ignored for binary and string, " diff --git a/fluss-common/src/main/java/org/apache/fluss/config/ValueConversionLayout.java b/fluss-common/src/main/java/org/apache/fluss/config/KafkaValueConversionLayout.java similarity index 96% rename from fluss-common/src/main/java/org/apache/fluss/config/ValueConversionLayout.java rename to fluss-common/src/main/java/org/apache/fluss/config/KafkaValueConversionLayout.java index 58e12e85f2f..e1cb4f3aadb 100644 --- a/fluss-common/src/main/java/org/apache/fluss/config/ValueConversionLayout.java +++ b/fluss-common/src/main/java/org/apache/fluss/config/KafkaValueConversionLayout.java @@ -21,7 +21,7 @@ /** Mapping of JSON object members to Fluss table columns. */ @PublicEvolving -public enum ValueConversionLayout { +public enum KafkaValueConversionLayout { /** Map members to top-level business columns. */ FLAT, diff --git a/fluss-common/src/main/java/org/apache/fluss/config/ValueConversionType.java b/fluss-common/src/main/java/org/apache/fluss/config/KafkaValueConversionType.java similarity index 96% rename from fluss-common/src/main/java/org/apache/fluss/config/ValueConversionType.java rename to fluss-common/src/main/java/org/apache/fluss/config/KafkaValueConversionType.java index 9d3683d5372..efa57982b7c 100644 --- a/fluss-common/src/main/java/org/apache/fluss/config/ValueConversionType.java +++ b/fluss-common/src/main/java/org/apache/fluss/config/KafkaValueConversionType.java @@ -21,7 +21,7 @@ /** Conversion applied to Kafka record values. */ @PublicEvolving -public enum ValueConversionType { +public enum KafkaValueConversionType { /** Store the value as bytes in the fixed value column. */ BINARY, diff --git a/fluss-common/src/main/java/org/apache/fluss/metadata/KafkaTableSchema.java b/fluss-common/src/main/java/org/apache/fluss/metadata/KafkaTableSchema.java index e02e5daa2d1..ec3af5a649e 100644 --- a/fluss-common/src/main/java/org/apache/fluss/metadata/KafkaTableSchema.java +++ b/fluss-common/src/main/java/org/apache/fluss/metadata/KafkaTableSchema.java @@ -21,8 +21,8 @@ import org.apache.fluss.config.ConfigOptions; import org.apache.fluss.config.Configuration; import org.apache.fluss.config.TableConfig; -import org.apache.fluss.config.ValueConversionLayout; -import org.apache.fluss.config.ValueConversionType; +import org.apache.fluss.config.KafkaValueConversionLayout; +import org.apache.fluss.config.KafkaValueConversionType; import org.apache.fluss.exception.InvalidTableException; import org.apache.fluss.types.ArrayType; import org.apache.fluss.types.BytesType; @@ -70,8 +70,8 @@ public final class KafkaTableSchema { RESCUE_COLUMN))); private final RowType rowType; - private final ValueConversionType valueConversionType; - private final @Nullable ValueConversionLayout valueConversionLayout; + private final KafkaValueConversionType valueConversionType; + private final @Nullable KafkaValueConversionLayout valueConversionLayout; private final KafkaFieldProjection valueProjection; private final int keyPosition; private final int headersPosition; @@ -80,8 +80,8 @@ public final class KafkaTableSchema { private KafkaTableSchema( RowType rowType, - ValueConversionType valueConversionType, - @Nullable ValueConversionLayout valueConversionLayout, + KafkaValueConversionType valueConversionType, + @Nullable KafkaValueConversionLayout valueConversionLayout, List valuePositions, int keyPosition, int headersPosition, @@ -159,10 +159,10 @@ public static KafkaTableSchema resolve(TableDescriptor table) { "Kafka requires a non-partitioned Arrow log table without a primary key."); } Configuration options = Configuration.fromMap(table.getProperties()); - ValueConversionType valueConversionType = + KafkaValueConversionType valueConversionType = options.get(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE); - ValueConversionLayout valueConversionLayout = - valueConversionType == ValueConversionType.PARSE_JSON + KafkaValueConversionLayout valueConversionLayout = + valueConversionType == KafkaValueConversionType.PARSE_JSON ? options.get(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT) : null; @@ -196,7 +196,7 @@ public static KafkaTableSchema resolve(TableDescriptor table) { } List valuePositions = new ArrayList<>(); - if (valueConversionLayout == ValueConversionLayout.FLAT) { + if (valueConversionLayout == KafkaValueConversionLayout.FLAT) { if (rowType.getFieldIndex(VALUE_COLUMN) >= 0) { throw invalid( "_kafka_value is reserved for binary, string or nested JSON; " @@ -214,17 +214,17 @@ public static KafkaTableSchema resolve(TableDescriptor table) { } DataType type = rowType.getTypeAt(valuePosition); boolean valid = - valueConversionType == ValueConversionType.BINARY + valueConversionType == KafkaValueConversionType.BINARY ? type instanceof BytesType - : valueConversionType == ValueConversionType.STRING + : valueConversionType == KafkaValueConversionType.STRING ? type instanceof StringType : type instanceof RowType; if (!valid) { throw invalid( "_kafka_value must be " - + (valueConversionType == ValueConversionType.BINARY + + (valueConversionType == KafkaValueConversionType.BINARY ? "BYTES" - : valueConversionType == ValueConversionType.STRING + : valueConversionType == KafkaValueConversionType.STRING ? "STRING" : "ROW") + " for " @@ -241,10 +241,10 @@ public static KafkaTableSchema resolve(TableDescriptor table) { } valuePositions.add(valuePosition); } - if (valueConversionType == ValueConversionType.PARSE_JSON) { + if (valueConversionType == KafkaValueConversionType.PARSE_JSON) { for (int position : valuePositions) { DataType type = rowType.getTypeAt(position); - if (valueConversionLayout == ValueConversionLayout.NESTED) { + if (valueConversionLayout == KafkaValueConversionLayout.NESTED) { // The physical value container represents the JSON object root, just like // the table row in flat layout; it must not add a business nesting level. for (DataType field : ((RowType) type).getChildren()) { @@ -371,12 +371,12 @@ public RowType rowType() { } /** Returns the value conversion type. */ - public ValueConversionType valueConversionType() { + public KafkaValueConversionType valueConversionType() { return valueConversionType; } /** Returns the JSON layout, or null for binary/string. */ - public @Nullable ValueConversionLayout valueConversionLayout() { + public @Nullable KafkaValueConversionLayout valueConversionLayout() { return valueConversionLayout; } diff --git a/fluss-common/src/test/java/org/apache/fluss/metadata/KafkaTableSchemaTest.java b/fluss-common/src/test/java/org/apache/fluss/metadata/KafkaTableSchemaTest.java index 4224845b1f4..a8287eda6ee 100644 --- a/fluss-common/src/test/java/org/apache/fluss/metadata/KafkaTableSchemaTest.java +++ b/fluss-common/src/test/java/org/apache/fluss/metadata/KafkaTableSchemaTest.java @@ -19,8 +19,8 @@ import org.apache.fluss.config.ConfigOptions; import org.apache.fluss.config.KafkaKeyConversionType; -import org.apache.fluss.config.ValueConversionLayout; -import org.apache.fluss.config.ValueConversionType; +import org.apache.fluss.config.KafkaValueConversionLayout; +import org.apache.fluss.config.KafkaValueConversionType; import org.apache.fluss.exception.InvalidTableException; import org.apache.fluss.types.DataType; import org.apache.fluss.types.DataTypes; @@ -152,8 +152,8 @@ void testDefaultJsonMappingPreservesNestedTypesAndExcludesSystemColumns() { .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) .build(); KafkaTableSchema resolved = KafkaTableSchema.resolve(table); - assertThat(resolved.valueConversionType()).isEqualTo(ValueConversionType.PARSE_JSON); - assertThat(resolved.valueConversionLayout()).isEqualTo(ValueConversionLayout.FLAT); + assertThat(resolved.valueConversionType()).isEqualTo(KafkaValueConversionType.PARSE_JSON); + assertThat(resolved.valueConversionLayout()).isEqualTo(KafkaValueConversionLayout.FLAT); assertThat(resolved.valueProjection().positions()).containsExactly(0, 3, 6); assertThat(resolved.valueProjection().dataTypeAt(1)).isEqualTo(nested); assertThat(resolved.rescuePosition()).isEqualTo(1); @@ -181,7 +181,7 @@ void testNestedJsonTargetsOneRow() { .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT.key(), "nested") .build(); KafkaTableSchema resolved = KafkaTableSchema.resolve(table); - assertThat(resolved.valueConversionLayout()).isEqualTo(ValueConversionLayout.NESTED); + assertThat(resolved.valueConversionLayout()).isEqualTo(KafkaValueConversionLayout.NESTED); assertThat(resolved.valueProjection().positions()).containsExactly(2); assertThat(resolved.valueProjection().dataTypeAt(0)).isEqualTo(row); assertThat(resolved.rescuePosition()).isZero(); @@ -362,7 +362,7 @@ void testNonJsonIgnoresDefaultAndExplicitLayouts(String type) { assertThat(expected.valueConversionLayout()).isNull(); assertThat(expected.valueProjection().positions()).containsExactly(0); assertThat(table.getProperties()).doesNotContainKey("table.kafka.convert.value.layout"); - for (ValueConversionLayout layout : ValueConversionLayout.values()) { + for (KafkaValueConversionLayout layout : KafkaValueConversionLayout.values()) { assertThat( KafkaTableSchema.resolve( TableDescriptor.builder(table) @@ -459,7 +459,7 @@ void testJsonFieldsRequireNullableTypesRecursively(String layout) { "parse_json") .property( ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_LAYOUT, - ValueConversionLayout.NESTED) + KafkaValueConversionLayout.NESTED) .build(), "_kafka_value"); } diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java index 822bf9f60d4..ba678229be7 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java @@ -19,8 +19,8 @@ import org.apache.fluss.config.ConfigOptions; import org.apache.fluss.config.Configuration; -import org.apache.fluss.config.ValueConversionLayout; -import org.apache.fluss.config.ValueConversionType; +import org.apache.fluss.config.KafkaValueConversionLayout; +import org.apache.fluss.config.KafkaValueConversionType; import org.apache.fluss.exception.InvalidConfigException; import org.apache.fluss.exception.InvalidTableException; import org.apache.fluss.kafka.mapping.KafkaTopicMapper; @@ -429,7 +429,7 @@ void testNonJsonLayoutDoesNotChangeMapping(String type) throws Exception { assertThat(expected.valueConversionLayout()).isNull(); assertThat(stored(path).getProperties()) .doesNotContainKey("table.kafka.convert.value.layout"); - for (ValueConversionLayout layout : ValueConversionLayout.values()) { + for (KafkaValueConversionLayout layout : KafkaValueConversionLayout.values()) { set(path, Collections.singletonMap("table.kafka.convert.value.layout", layout.name())); assertThat(KafkaTableSchema.resolve(stored(path))).isEqualTo(expected); assertThat(stored(path).getProperties()) @@ -629,6 +629,6 @@ private static TableDescriptor.Builder enabled() { .distributedBy(1) .logFormat(LogFormat.ARROW) .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) - .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, ValueConversionType.BINARY); + .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, KafkaValueConversionType.BINARY); } } diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorService.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorService.java index 545480a9a00..df8713338a8 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorService.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorService.java @@ -507,7 +507,7 @@ public CompletableFuture createTable(CreateTableRequest req // validate table descriptor before creating table in lake or fluss metadata, // to avoid orphaned lake tables when validation fails - metadataManager.validateTableDefinition(tablePath, tableDescriptor); + metadataManager.validateTableDescriptor(tablePath, tableDescriptor); // the distribution and bucket count must be set now //noinspection OptionalGetWithoutIsPresent diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/MetadataManager.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/MetadataManager.java index 8588373809e..52a89625dac 100644 --- a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/MetadataManager.java +++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/MetadataManager.java @@ -125,7 +125,7 @@ public MetadataManager( } /** Validates a descriptor and its Kafka topic identity before persistence. */ - public void validateTableDefinition(TablePath tablePath, TableDescriptor tableDescriptor) { + public void validateTableDescriptor(TablePath tablePath, TableDescriptor tableDescriptor) { TableDescriptorValidation.validateTableDescriptor( tableDescriptor, maxBucketNum, @@ -839,7 +839,7 @@ private void doAlterTablePropertiesOnce( // historical partition would be created with the new table-level count while retired // lake data still uses the pre-rescale layout. bucketCountEpoch never decreases, so // this also rejects a table that was rescaled while the feature was temporarily - // disabled. Checked before validateTableDefinition so the rescale rejection is not + // disabled. Checked before validateTableDescriptor so the rescale rejection is not // masked by unrelated option-dependency errors. if (!tableInfo.getTableConfig().isHistoricalPartitionEnabled() && Configuration.fromMap(newDescriptor.getProperties()) @@ -854,7 +854,7 @@ private void doAlterTablePropertiesOnce( } // reuse the same validate logic with the createTable() method - validateTableDefinition(tablePath, newDescriptor); + validateTableDescriptor(tablePath, newDescriptor); beforeUpdate.accept(tableInfo, newDescriptor); diff --git a/website/docs/kafka-compatibility.md b/website/docs/kafka-compatibility.md index 4d1c5d05ac4..6bb898a2f17 100644 --- a/website/docs/kafka-compatibility.md +++ b/website/docs/kafka-compatibility.md @@ -6,48 +6,21 @@ sidebar_position: 5.5 # Kafka Compatibility -Fluss Kafka compatibility maps Kafka topics and records to native Fluss tables. Tables are -created through Fluss DDL or the Flink Fluss catalog, and explicitly enabled for Kafka access. - -:::warning -Kafka protocol compatibility is still in development. The current implementation provides -ApiVersions and the DDL mapping contract described here. Kafka Metadata, Produce, and consumer -APIs are not yet available; enabling a table does not enable these APIs. -::: - -## Supported APIs - -| API or capability | Current status | -|---|---| -| ApiVersions | Implemented. | -| Native CREATE/ALTER and Kafka table mapping validation | Implemented. | -| Kafka Metadata and Produce | Planned for Basic Produce; not yet available. | -| Fetch, consumer groups, and offset commits | Not available. | -| Idempotent and transactional producers | Not available in Basic Produce. | -| CreateTopics and DeleteTopics | Not supported; manage tables through Fluss. | - -The mapping and conversion rules below define the table contract for the Kafka write path. -They do not make an unsupported Kafka API available. +Fluss can expose eligible Fluss tables as Kafka topics. Create the table in Fluss and enable its +Kafka mapping with `table.kafka.enabled=true`. ## Supported tables -Kafka mapping currently accepts non-partitioned [Log Tables](table-design/table-types/log-table.md) using -the Arrow log format, without a primary key. The initial Kafka write path is designed to -encode Arrow batches and append them to log tables. Supporting other log formats requires -matching encoders; primary key and partitioned tables also require additional write semantics. - -Each table must have a compatible schema and `table.kafka.enabled=true`. System columns are -optional: declare only the Kafka information that the table needs to store. Enabling Kafka -does not add columns or persist default conversion properties. CREATE and ALTER validate the -resulting mapping before persisting it, including schema changes on enabled tables. All declared -columns and nested value fields must be nullable. This includes ROW fields, ARRAY elements, -MAP values, and optional system columns. MAP keys retain Fluss's mandatory non-null constraint; -they are the only exception. Validation rejects NOT NULL fields rather than changing their types. +Kafka mapping supports non-partitioned [Log Tables](table-design/table-types/log-table.md) with +the Arrow log format and no primary key. CREATE and ALTER validate the mapping before saving it. +All declared columns and nested value fields must be nullable, including ROW fields, ARRAY +elements, MAP values, and system columns. MAP keys retain Fluss's non-null constraint. The four +system columns are optional and can be declared independently. ## Cluster configuration -Set `kafka.enabled: true` in the cluster's `server.yaml`. On each TabletServer, add a Kafka -listener alongside its native Fluss listener. For a local TabletServer: +Enable Kafka on the cluster and configure a Kafka listener on each TabletServer. For a local +TabletServer: ```yaml title="server.yaml" kafka.enabled: true @@ -56,41 +29,34 @@ kafka.default.database: fluss bind.listeners: FLUSS://localhost:9123,KAFKA://localhost:9092 ``` -Kafka listeners belong on TabletServers. For a distributed deployment, configure reachable -addresses with `bind.listeners` and `advertised.listeners`; see the -[server configuration reference](maintenance/configuration.md#common). All TabletServers -and the coordinator must use the same `kafka.default.database`. +For distributed deployments, configure reachable `bind.listeners` and `advertised.listeners`. +The coordinator and all TabletServers must use the same `kafka.default.database`; see the +[server configuration reference](maintenance/configuration.md#common). ## Topic-to-table mapping -Kafka topics map to Fluss tables using `.
`. Topic names without a database prefix -map to the same-named table in the database configured by `kafka.default.database` (default: `fluss`). -Kafka clients can access a topic only after its corresponding table has been created in Fluss. +Topic names use `.
`. Names without a database prefix use +`kafka.default.database` (default: `fluss`). With the default setting, mappings are: -For example, `sales.orders` maps to the `orders` table in `sales`, while `orders` maps to -`fluss.orders` with the default configuration. Reverse mapping always uses the fully qualified -name. Changing the default database redirects unqualified names; use fully qualified topic -names when clients need a stable mapping across such changes. - -The native `fluss` database is created when the cluster has no databases. When Kafka is enabled, -the coordinator also ensures that the configured Kafka default database exists, including when -it is `fluss` and other databases already exist. Existing databases are preserved. A database -deleted at runtime is not recreated by a Kafka request; initialization runs when the coordinator -becomes active. -Other databases and all tables must be created explicitly; Kafka does not automatically create -tables, and Kafka CreateTopics/DeleteTopics are not supported. +| Kafka topic name | Fluss table path | +|---|---| +| `orders` | `fluss.orders` | +| `fluss.orders` | `fluss.orders` | +| `sales.orders` | `sales.orders` | +| `warehouse.events` | `warehouse.events` | -### Partition-to-bucket mapping +The native `fluss` database is created when the cluster has no databases. With Kafka enabled, the +coordinator also creates the configured default database if it is missing, even in a non-empty +cluster. Create other databases and all tables through Fluss; Kafka CreateTopics and DeleteTopics +are not supported. -A Kafka partition maps to the same-numbered Fluss bucket. The topic's partition count equals -the table's bucket count. Qualified and unqualified names for the same table share its buckets -and log offsets; same-named tables in different databases remain independent. The write path -uses the bucket's leader and replicas rather than a separate Kafka storage layer. +A Kafka partition maps to the Fluss bucket with the same number, so the topic partition count must +match the table bucket count. ## Create or enable a table -The examples below run in an existing [Flink Fluss catalog](engine-flink/ddl.md#create-catalog). -Create a table for JSON values, optionally storing the Kafka key and headers: +The examples use an existing [Flink Fluss catalog](engine-flink/ddl.md#create-catalog). Create a +table for JSON values, optionally storing Kafka keys and headers: ```sql title="Flink SQL" CREATE DATABASE IF NOT EXISTS sales; @@ -105,12 +71,12 @@ CREATE TABLE sales.orders ( ); ``` -This defines the mapping for `sales.orders`. JSON fields such as `order_id` and `amount` map to -the business columns, while the Kafka record's key and headers map to their system columns. -The omitted conversion type and layout are read as `parse_json` and `flat` without adding -properties to table metadata. +This maps the `sales.orders` topic to the table. JSON members map to `order_id` and `amount`; the +Kafka key and headers map to their system columns. With no conversion properties, values use +`parse_json` with the `flat` layout. These defaults are applied when resolving the mapping and are +not added to table properties. -For an existing table with a compatible JSON schema, enable Kafka: +Enable Kafka mapping for an existing compatible table with: ```sql title="Flink SQL" ALTER TABLE sales.orders SET ( @@ -118,127 +84,66 @@ ALTER TABLE sales.orders SET ( ); ``` -Set `table.kafka.enabled` to `false` to disable Kafka access. Disabled tables are not subject -to Kafka-specific schema validation. - -## Producer configuration and writes - -:::warning -The following example describes the intended Basic Produce usage once Metadata and Produce -are implemented. It cannot write through the current ApiVersions-only endpoint. -::: - -Use the Kafka listener as the bootstrap address and disable idempotence. For the `sales.orders` -table above, a Java producer can send UTF-8 JSON values and string keys as follows: - -```java -import org.apache.kafka.clients.producer.KafkaProducer; -import org.apache.kafka.clients.producer.ProducerRecord; -import org.apache.kafka.common.serialization.StringSerializer; - -import java.nio.charset.StandardCharsets; -import java.util.Properties; - -Properties properties = new Properties(); -properties.put("bootstrap.servers", "localhost:9092"); -properties.put("enable.idempotence", "false"); -properties.put("acks", "all"); - -try (KafkaProducer producer = - new KafkaProducer<>(properties, new StringSerializer(), new StringSerializer())) { - ProducerRecord record = new ProducerRecord<>( - "sales.orders", "order-1", "{\"order_id\":1,\"amount\":12.50}"); - record.headers().add("source", "app".getBytes(StandardCharsets.UTF_8)); - producer.send(record).get(); -} -``` - -The value populates `order_id` and `amount`; the UTF-8 key bytes populate `_kafka_key`, and the -header populates `_kafka_headers`. Binary values should use `ByteArraySerializer`; string values -can use `StringSerializer`. Do not configure `transactional.id`. Basic Produce does not provide -Kafka idempotence or transactional guarantees, so retries can produce duplicate rows. +Set the property to `false` to disable Kafka mapping. Disabled tables are not subject to Kafka +schema validation. ## Conversion properties -These are table properties set through CREATE or ALTER TABLE, rather than `server.yaml` settings. +Set these table properties with CREATE or ALTER TABLE. | Option | Type | Default | Description | |---|---|---|---| -| table.kafka.enabled | Boolean | false | Explicitly enable the table's Kafka mapping. | -| table.kafka.convert.value.type | Enum | `parse_json` | `binary`, `string`, or `parse_json`. | -| table.kafka.convert.value.layout | Enum | `flat` | JSON layout: `flat` maps top-level business columns; `nested` targets `_kafka_value ROW<...>`. Ignored for binary/string, whether defaulted or explicitly configured. | -| table.kafka.convert.key.type | Enum | (None) | Optional `binary` conversion; explicitly configuring it requires `_kafka_key BYTES`. | +| `table.kafka.enabled` | Boolean | `false` | Enable Kafka mapping for the table. | +| `table.kafka.convert.value.type` | Enum | `parse_json` | Convert values as `binary`, `string`, or `parse_json`. | +| `table.kafka.convert.value.layout` | Enum | `flat` | JSON layout: `flat` maps top-level columns; `nested` maps fields in `_kafka_value ROW<...>`. Ignored for binary and string. | +| `table.kafka.convert.key.type` | Enum | None | Optional `binary` key conversion; requires `_kafka_key BYTES`. | -Conversion defaults apply only at read time. CREATE and ALTER preserve omitted properties. -Declaring `_kafka_key BYTES` selects binary key mapping even when the key conversion option is -omitted. No key column or conversion property is required when the key is not needed. +Defaults apply when resolving the mapping; omitted properties stay omitted. Declaring +`_kafka_key BYTES` selects binary key mapping without a key conversion property. No key column or +property is required when keys are not stored. Custom properties do not configure Kafka mapping; +the server validates registered table properties. -Custom properties are preserved but do not configure the Kafka mapping. Storage properties and -their values are checked by the server against the options supported by that cluster version. +| Value conversion | Required schema | Mapping | +|---|---|---| +| `binary` | `_kafka_value BYTES` | Store the value as bytes. | +| `string` | `_kafka_value STRING` | Decode the value as UTF-8. | +| `parse_json` with `flat` | Top-level business columns | Map JSON object members to columns by name. | +| `parse_json` with `nested` | `_kafka_value ROW<...>` | Map JSON members to fields in the value ROW. | -### Value conversion +Binary, string, and nested JSON use `_kafka_value`; other top-level columns must be optional +system columns. Flat JSON uses top-level business columns and cannot declare `_kafka_value`. +JSON values must be objects. Nested ROW, ARRAY, and MAP schemas are supported. System names are +reserved at the table's top level, but are ordinary names inside business ROWs. -| Conversion | Required value schema | Mapping | -|---|---|---| -| `binary` | `_kafka_value BYTES` | Store the record value as bytes. | -| `string` | `_kafka_value STRING` | Interpret the record value as a UTF-8 string. | -| `parse_json`, `flat` | Business columns at the table's top level | Map JSON object members to business columns by name. | -| `parse_json`, `nested` | `_kafka_value ROW<...>` | Map JSON object members to fields inside the value ROW. | - -Binary, string, and nested JSON map their value to `_kafka_value`; other top-level columns must -be optional system columns. Flat JSON uses top-level business columns and does not allow a -`_kafka_value` column. Both JSON layouts require an object and preserve declared nested ROW, -ARRAY, and MAP structures. - -System names are reserved at the physical table's top level. Inside a nested `_kafka_value` ROW, -they are ordinary business field names: `_kafka_value ROW<_kafka_key STRING>` can coexist with -a top-level `_kafka_key BYTES` column. The inner field comes from the JSON value, and the outer -column comes from the Kafka key. The same rule applies to deeper business structures. - -JSON schemas accept all currently defined Fluss types, subject to the nullable-field rule above -and two additional restrictions: MAP keys -must be STRING, including nested maps, and nesting depth must not exceed 64. Each top-level -business field starts at depth zero; ROW fields, ARRAY elements, and MAP values each add one. -The JSON root and the physical `_kafka_value` ROW container add no level. Scalar value constraints -apply when decoding individual records. +JSON schema limits: + +- MAP keys must be STRING, including nested MAPs. +- Nesting depth is at most 64. Each top-level business field starts at depth zero; ROW fields, + ARRAY elements, and MAP values each add one. The JSON root and `_kafka_value` container do not + count. +- Fields must be nullable as described above. Value constraints are checked when records are + decoded. ## System columns -The four optional columns below may be omitted independently. Unmapped key, headers, or metadata -are not stored in row fields. `_kafka_value` is the value target required by the conversions above. +Declare only the Kafka information the table needs to store. Unmapped keys, headers, and metadata +are not stored in row fields. `_kafka_value` is the conversion target described above. | Column | Type | Mapping | |---|---|---| -| `_kafka_key` | `BYTES` | Kafka record key as bytes. | -| `_kafka_headers` | `MAP` | Kafka headers; the last value wins for duplicate header names. | -| `_kafka_metadata` | `ROW` | Input record timestamp, batch timestamp type, and requested Kafka partition. | -| `_kafka_rescue` | Nullable `STRING` | Compact JSON containing unknown fields, values that cannot be converted, and flat-layout system-name conflicts. | - -The metadata ROW must use the field names, order, and types shown above. Timestamp type is `-1` -when absent, `0` for CreateTime, or `1` for LogAppendTime. An absent timestamp maps to null. -The metadata ROW and its three fields must be nullable. Log offsets remain native log metadata and are not -fields of this ROW. - -A declared rescue column needs no additional property. The conversion contract sends an -unconvertible field's original JSON value to rescue and writes null to that field. For example, -`{"order_id":"invalid","amount":12.50}` writes null to `order_id`, keeps `amount`, and stores -`{"order_id":"invalid"}` in `_kafka_rescue` when that column is declared. Nested errors preserve -their path and structure; valid sibling fields remain usable. If an entire field has the wrong -structure, rescue contains that field's original value and the field is null. - -Without rescue, unknown fields, conversion failures, and flat-layout system-name conflicts -fail conversion. Invalid JSON and a non-object JSON root always fail; rescue requires a parsed -object and does not hide protocol or storage failures. These write-time behaviors require the -planned JSON decoder; this implementation enforces their DDL prerequisites. -Binary/string conversion and JSON records with no fields to rescue write null to this column. - -## Troubleshooting - -| Symptom | What to check | -|---|---| -| Producer cannot discover a topic or send records | Check the API status above first. Metadata and Produce are not yet implemented in this version. | -| Connection fails | Use a TabletServer Kafka listener, enable `kafka.enabled`, and ensure advertised addresses are reachable from the client. | -| A topic does not map to the intended table | Create and enable the Fluss table first. Check the `.
` spelling or `kafka.default.database` for an unqualified name. | -| CREATE or ALTER rejects the Kafka mapping | Use a non-partitioned Arrow log table without a primary key, nullable fields, and the required value column for the chosen conversion. | -| JSON schema validation fails | Check nested MAP key types and the nesting limit. MAP values and ARRAY elements must be nullable. | -| A future Basic Produce client requests unsupported producer APIs | Disable idempotence and transactions. Consumer APIs require separate support. | +| `_kafka_key` | `BYTES` | Kafka record key. | +| `_kafka_headers` | `MAP` | Kafka headers; the last value wins for duplicate names. | +| `_kafka_metadata` | `ROW` | Input timestamp, timestamp type, and requested partition. | +| `_kafka_rescue` | Nullable `STRING` | Compact JSON for unknown or unconvertible fields and flat-layout system-name conflicts. | + +The metadata ROW must use the field names, order, and types shown above; the ROW and its fields +must be nullable. Timestamp type is `-1` when absent, `0` for CreateTime, or `1` for LogAppendTime. +An absent timestamp maps to null. Log offsets remain native log metadata and are not included in +this ROW. + +`_kafka_rescue` needs no additional property. The planned JSON decoder will store unknown fields, +flat-layout name conflicts, and conversion failures there as compact JSON. It will set only an +unconvertible target field to null and preserve valid siblings. Without a rescue column, these +errors fail conversion; invalid JSON and non-object roots fail even when rescue is declared. +PR03 validates the DDL prerequisites for this behavior; it does not implement the decoder. For +binary/string conversion or records with nothing to rescue, the column is null. From b98ca1c29cc4678bb85b0594434854f6f982bdcc Mon Sep 17 00:00:00 2001 From: Yang Guo Date: Sun, 18 Oct 2026 20:06:07 +0800 Subject: [PATCH 14/15] [kafka] Apply Spotless formatting fixes Reorder the conversion enum imports and wrap the long test builder call to match the repository's Spotless format. Co-Authored-By: Codex AI-Model: gpt-6 AI-Contributed/Feature: 2/2 AI-Contributed/UT: 4/4 --- .../main/java/org/apache/fluss/metadata/KafkaTableSchema.java | 2 +- .../apache/fluss/kafka/schema/KafkaTableMappingITCase.java | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/fluss-common/src/main/java/org/apache/fluss/metadata/KafkaTableSchema.java b/fluss-common/src/main/java/org/apache/fluss/metadata/KafkaTableSchema.java index ec3af5a649e..0005111a4d1 100644 --- a/fluss-common/src/main/java/org/apache/fluss/metadata/KafkaTableSchema.java +++ b/fluss-common/src/main/java/org/apache/fluss/metadata/KafkaTableSchema.java @@ -20,9 +20,9 @@ import org.apache.fluss.annotation.Internal; import org.apache.fluss.config.ConfigOptions; import org.apache.fluss.config.Configuration; -import org.apache.fluss.config.TableConfig; import org.apache.fluss.config.KafkaValueConversionLayout; import org.apache.fluss.config.KafkaValueConversionType; +import org.apache.fluss.config.TableConfig; import org.apache.fluss.exception.InvalidTableException; import org.apache.fluss.types.ArrayType; import org.apache.fluss.types.BytesType; diff --git a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java index ba678229be7..43eb7c9a40e 100644 --- a/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java +++ b/fluss-kafka/src/test/java/org/apache/fluss/kafka/schema/KafkaTableMappingITCase.java @@ -629,6 +629,8 @@ private static TableDescriptor.Builder enabled() { .distributedBy(1) .logFormat(LogFormat.ARROW) .property(ConfigOptions.TABLE_KAFKA_ENABLED, true) - .property(ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, KafkaValueConversionType.BINARY); + .property( + ConfigOptions.TABLE_KAFKA_CONVERT_VALUE_TYPE, + KafkaValueConversionType.BINARY); } } From 3af2b1ea294035c44b9dc8a2744931e211505fcf Mon Sep 17 00:00:00 2001 From: Jark Wu Date: Mon, 19 Oct 2026 00:17:47 +0800 Subject: [PATCH 15/15] remove AI slop Co-Authored-By: Codex AI-Contributed/Feature: 48/48 AI-Contributed/UT: 0/0 --- .../apache/fluss/config/ConfigOptions.java | 26 ++++++++++--------- website/docs/kafka-compatibility.md | 22 +++++++--------- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java b/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java index 80dcad59d72..8f4dad6d777 100644 --- a/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java +++ b/fluss-common/src/main/java/org/apache/fluss/config/ConfigOptions.java @@ -2274,36 +2274,38 @@ public class ConfigOptions { .booleanType() .defaultValue(false) .withDescription( - "Whether the table is exposed as a Kafka topic. Enabling validates the " - + "existing schema and conversion options without adding columns or defaults."); + "Whether to expose this table as a Kafka topic. Requires a non-partitioned " + + "log table using the Arrow log format."); public static final ConfigOption TABLE_KAFKA_CONVERT_VALUE_TYPE = key("table.kafka.convert.value.type") .enumType(KafkaValueConversionType.class) .defaultValue(KafkaValueConversionType.PARSE_JSON) .withDescription( - "Kafka value conversion: binary, string, or parse_json. " - + "Binary and string target the declared _kafka_value column. " - + "The default is used at read time without adding a table property."); + "How to convert Kafka record values. 'binary' stores the raw bytes in " + + "_kafka_value (BYTES); 'string' decodes UTF-8 text into _kafka_value " + + "(STRING); 'parse_json' parses a JSON object and maps its fields " + + "according to table.kafka.convert.value.layout. Defaults to 'parse_json'."); public static final ConfigOption TABLE_KAFKA_CONVERT_VALUE_LAYOUT = key("table.kafka.convert.value.layout") .enumType(KafkaValueConversionLayout.class) .defaultValue(KafkaValueConversionLayout.FLAT) .withDescription( - "JSON layout for parse_json: flat maps top-level business columns; " - + "nested targets _kafka_value ROW. Ignored for binary and string, " - + "whether defaulted or explicitly configured. The default is used " - + "at read time without adding a table property."); + "How to map JSON object fields when table.kafka.convert.value.type is " + + "'parse_json'. 'flat' maps fields by name to top-level table columns; " + + "'nested' maps them to fields in the _kafka_value column, which must " + + "have ROW type. Defaults to 'flat'. Ignored for 'binary' and 'string' conversion."); public static final ConfigOption TABLE_KAFKA_CONVERT_KEY_TYPE = key("table.kafka.convert.key.type") .enumType(KafkaKeyConversionType.class) .noDefaultValue() .withDescription( - "Optional Kafka key conversion. Only binary is supported and requires " - + "_kafka_key BYTES. Declaring that column implies binary without " - + "adding a table property."); + "How to convert Kafka record keys. Only 'binary' is supported, which stores " + + "raw key bytes in the _kafka_key column of type BYTES. If this column " + + "is declared, keys are stored even when this option is omitted. " + + "Without this column, keys are not stored and this option must be left unset."); // ------------------------------------------------------------------------ // ConfigOptions for Consumer Offsets diff --git a/website/docs/kafka-compatibility.md b/website/docs/kafka-compatibility.md index 6bb898a2f17..bfb7aa47fde 100644 --- a/website/docs/kafka-compatibility.md +++ b/website/docs/kafka-compatibility.md @@ -73,8 +73,7 @@ CREATE TABLE sales.orders ( This maps the `sales.orders` topic to the table. JSON members map to `order_id` and `amount`; the Kafka key and headers map to their system columns. With no conversion properties, values use -`parse_json` with the `flat` layout. These defaults are applied when resolving the mapping and are -not added to table properties. +`parse_json` with the `flat` layout. Enable Kafka mapping for an existing compatible table with: @@ -98,10 +97,8 @@ Set these table properties with CREATE or ALTER TABLE. | `table.kafka.convert.value.layout` | Enum | `flat` | JSON layout: `flat` maps top-level columns; `nested` maps fields in `_kafka_value ROW<...>`. Ignored for binary and string. | | `table.kafka.convert.key.type` | Enum | None | Optional `binary` key conversion; requires `_kafka_key BYTES`. | -Defaults apply when resolving the mapping; omitted properties stay omitted. Declaring -`_kafka_key BYTES` selects binary key mapping without a key conversion property. No key column or -property is required when keys are not stored. Custom properties do not configure Kafka mapping; -the server validates registered table properties. +Declare `_kafka_key BYTES` to store Kafka keys as bytes; `table.kafka.convert.key.type` can be +omitted. To discard keys, omit both the column and the property. | Value conversion | Required schema | Mapping | |---|---|---| @@ -141,9 +138,10 @@ must be nullable. Timestamp type is `-1` when absent, `0` for CreateTime, or `1` An absent timestamp maps to null. Log offsets remain native log metadata and are not included in this ROW. -`_kafka_rescue` needs no additional property. The planned JSON decoder will store unknown fields, -flat-layout name conflicts, and conversion failures there as compact JSON. It will set only an -unconvertible target field to null and preserve valid siblings. Without a rescue column, these -errors fail conversion; invalid JSON and non-object roots fail even when rescue is declared. -PR03 validates the DDL prerequisites for this behavior; it does not implement the decoder. For -binary/string conversion or records with nothing to rescue, the column is null. +Use the optional `_kafka_rescue` column to retain JSON fields that cannot be mapped to the table +schema for later inspection. It stores unknown fields, values that cannot be converted to the +declared type, and fields whose names conflict with system columns in the flat layout as compact +JSON. When a value cannot be converted, the corresponding table field is set to null while valid +fields are preserved. Without `_kafka_rescue`, these errors cause conversion to fail. Invalid JSON +and non-object roots always fail conversion. The column is null for binary/string conversion or +when there is nothing to rescue.