From 4992c92b4ee5e1d3449dc82117a9ba093c4f02f4 Mon Sep 17 00:00:00 2001 From: Zhigao Tong Date: Mon, 21 Sep 2026 12:49:05 +0800 Subject: [PATCH 01/32] fts: support MATCH AGAINST boolean pushdown on release-8.5 --- contrib/tipb | 2 +- .../Coprocessor/DAGStorageInterpreter.cpp | 5 +- dbms/src/Flash/Coprocessor/DAGUtils.cpp | 2 + dbms/src/Flash/Coprocessor/RemoteRequest.cpp | 2 +- dbms/src/Flash/Coprocessor/TiDBTableScan.cpp | 28 + dbms/src/Flash/Coprocessor/TiDBTableScan.h | 12 + .../Flash/Planner/Plans/PhysicalTableScan.cpp | 147 +++- .../Flash/Planner/Plans/PhysicalTableScan.h | 7 +- dbms/src/Functions/FunctionsFullText.cpp | 747 ++++++++++++++++++ dbms/src/Functions/FunctionsFullText.h | 22 + dbms/src/Functions/registerFunctions.cpp | 2 + dbms/src/Functions/tests/gtest_fulltext.cpp | 179 +++++ dbms/src/TiDB/Schema/TiDB.cpp | 19 +- .../2026-09-18-match-against-tiflash.md | 115 +++ libs/libcommon/include/common/demangle.h | 1 + 15 files changed, 1267 insertions(+), 23 deletions(-) create mode 100644 dbms/src/Functions/FunctionsFullText.cpp create mode 100644 dbms/src/Functions/FunctionsFullText.h create mode 100644 dbms/src/Functions/tests/gtest_fulltext.cpp create mode 100644 docs/design/2026-09-18-match-against-tiflash.md diff --git a/contrib/tipb b/contrib/tipb index 0607513e7fa..0a9c803d4e7 160000 --- a/contrib/tipb +++ b/contrib/tipb @@ -1 +1 @@ -Subproject commit 0607513e7fa40f3b564f0b269da76c1c8dc90032 +Subproject commit 0a9c803d4e7bf37a7d67a3e74609b8dd56399d09 diff --git a/dbms/src/Flash/Coprocessor/DAGStorageInterpreter.cpp b/dbms/src/Flash/Coprocessor/DAGStorageInterpreter.cpp index df67e11030f..1d849dd0b1c 100644 --- a/dbms/src/Flash/Coprocessor/DAGStorageInterpreter.cpp +++ b/dbms/src/Flash/Coprocessor/DAGStorageInterpreter.cpp @@ -1584,7 +1584,7 @@ std::pair> DAGStorageInterpreter::getColumnsForTableSc auto const & ci = table_scan.getColumns()[i]; const ColumnID cid = ci.id; - if (ci.hasGeneratedColumnFlag()) + if (ci.hasGeneratedColumnFlag() || isTiDBFTSScoreColumn(cid)) { LOG_DEBUG(log, "got column({}) with generated column flag", i); const auto & data_type = getDataTypeByColumnInfoForComputingLayer(ci); @@ -1617,7 +1617,8 @@ std::pair> DAGStorageInterpreter::getColumnsForTableSc // If the column is not generated column, not in the filter columns and column id is not -1, then it may need cast. for (const auto & col : table_scan.getColumns()) may_need_add_cast_column_tmp.push_back( - !col.hasGeneratedColumnFlag() && !filter_col_id_set.contains(col.id) && col.id != -1); + !col.hasGeneratedColumnFlag() && !isTiDBFTSScoreColumn(col.id) && !filter_col_id_set.contains(col.id) + && col.id != -1); return {required_columns_tmp, may_need_add_cast_column_tmp}; } diff --git a/dbms/src/Flash/Coprocessor/DAGUtils.cpp b/dbms/src/Flash/Coprocessor/DAGUtils.cpp index efacecc9b93..487ed9a8eed 100644 --- a/dbms/src/Flash/Coprocessor/DAGUtils.cpp +++ b/dbms/src/Flash/Coprocessor/DAGUtils.cpp @@ -455,6 +455,8 @@ const std::unordered_map scalar_func_map({ {tipb::ScalarFuncSig::RegexpInStrSig, "regexp_instr"}, {tipb::ScalarFuncSig::RegexpReplaceSig, "regexp_replace"}, {tipb::ScalarFuncSig::RegexpSubstrSig, "regexp_substr"}, + {tipb::ScalarFuncSig::FTSMatchWord, "fts_match_word"}, + {tipb::ScalarFuncSig::FTSMatchExpression, "fts_match_expression"}, {tipb::ScalarFuncSig::JsonExtractSig, "json_extract"}, {tipb::ScalarFuncSig::JsonUnquoteSig, "json_unquote"}, diff --git a/dbms/src/Flash/Coprocessor/RemoteRequest.cpp b/dbms/src/Flash/Coprocessor/RemoteRequest.cpp index 637203a728c..129fb5695a5 100644 --- a/dbms/src/Flash/Coprocessor/RemoteRequest.cpp +++ b/dbms/src/Flash/Coprocessor/RemoteRequest.cpp @@ -54,7 +54,7 @@ RemoteRequest RemoteRequest::build( const auto & col = table_scan.getColumns()[i]; auto col_id = col.id; - if (col.hasGeneratedColumnFlag()) + if (col.hasGeneratedColumnFlag() || isTiDBFTSScoreColumn(col_id)) { const auto & col_name = GeneratedColumnPlaceholderBlockInputStream::getColumnName(i); schema.emplace_back(std::make_pair(col_name, std::move(col))); diff --git a/dbms/src/Flash/Coprocessor/TiDBTableScan.cpp b/dbms/src/Flash/Coprocessor/TiDBTableScan.cpp index d1545dc24e6..de6d6ee9855 100644 --- a/dbms/src/Flash/Coprocessor/TiDBTableScan.cpp +++ b/dbms/src/Flash/Coprocessor/TiDBTableScan.cpp @@ -17,6 +17,31 @@ namespace DB { +namespace +{ +tipb::FTSQueryInfo extractFtsQuery(const tipb::Executor * table_scan) +{ + const auto & indexes = table_scan->tp() == tipb::TypePartitionTableScan + ? table_scan->partition_table_scan().used_columnar_indexes() + : table_scan->tbl_scan().used_columnar_indexes(); + const tipb::ColumnarIndexInfo * fts_index = nullptr; + for (const auto & index : indexes) + { + if (index.index_type() != tipb::ColumnarIndexType::TypeFulltext) + continue; + if (fts_index != nullptr) + throw TiFlashException( + "A table scan cannot contain multiple full-text indexes", + Errors::Coprocessor::BadRequest); + fts_index = &index; + } + if (fts_index == nullptr) + return {}; + RUNTIME_CHECK(fts_index->has_fts_query_info()); + return fts_index->fts_query_info(); +} +} + TiDBTableScan::TiDBTableScan( const tipb::Executor * table_scan_, const String & executor_id_, @@ -32,6 +57,7 @@ TiDBTableScan::TiDBTableScan( : table_scan->tbl_scan().pushed_down_filter_conditions()) , ann_query_info( is_partition_table_scan ? table_scan->partition_table_scan().ann_query() : table_scan->tbl_scan().ann_query()) + , fts_query_info(extractFtsQuery(table_scan)) // Only No-partition table need keep order when tablescan executor required keep order. // If keep_order is not set, keep order for safety. , keep_order( @@ -99,6 +125,8 @@ void TiDBTableScan::constructTableScanForRemoteRead(tipb::TableScan * tipb_table *tipb_table_scan->add_columns() = column; for (const auto & filter : partition_table_scan.pushed_down_filter_conditions()) *tipb_table_scan->add_pushed_down_filter_conditions() = filter; + for (const auto & index : partition_table_scan.used_columnar_indexes()) + *tipb_table_scan->add_used_columnar_indexes() = index; tipb_table_scan->set_desc(partition_table_scan.desc()); for (auto id : partition_table_scan.primary_column_ids()) tipb_table_scan->add_primary_column_ids(id); diff --git a/dbms/src/Flash/Coprocessor/TiDBTableScan.h b/dbms/src/Flash/Coprocessor/TiDBTableScan.h index 9d9164c5f29..b17b336e1c5 100644 --- a/dbms/src/Flash/Coprocessor/TiDBTableScan.h +++ b/dbms/src/Flash/Coprocessor/TiDBTableScan.h @@ -23,6 +23,16 @@ namespace DB { class DAGContext; +// TiDB allocates this negative column ID for the synthetic MATCH_AGAINST score +// column. It is not part of the physical table schema and must be represented +// by a generated-column placeholder in the TiFlash scan pipeline. +inline constexpr ColumnID TiDBVirtualColFTSScoreID = -2050; + +inline bool isTiDBFTSScoreColumn(ColumnID column_id) +{ + return column_id == TiDBVirtualColFTSScoreID; +} + /// TiDBTableScan is a wrap to hide the difference of `TableScan` and `PartitionTableScan` class TiDBTableScan { @@ -46,6 +56,7 @@ class TiDBTableScan const google::protobuf::RepeatedPtrField & getPushedDownFilters() const { return pushed_down_filters; } const tipb::ANNQueryInfo & getANNQueryInfo() const { return ann_query_info; } + const tipb::FTSQueryInfo & getFTSQueryInfo() const { return fts_query_info; } private: const tipb::Executor * table_scan; @@ -68,6 +79,7 @@ class TiDBTableScan const google::protobuf::RepeatedPtrField pushed_down_filters; const tipb::ANNQueryInfo ann_query_info; + const tipb::FTSQueryInfo fts_query_info; bool keep_order; bool is_fast_scan; diff --git a/dbms/src/Flash/Planner/Plans/PhysicalTableScan.cpp b/dbms/src/Flash/Planner/Plans/PhysicalTableScan.cpp index 364cd2e94a9..8da0af6a565 100644 --- a/dbms/src/Flash/Planner/Plans/PhysicalTableScan.cpp +++ b/dbms/src/Flash/Planner/Plans/PhysicalTableScan.cpp @@ -14,8 +14,11 @@ #include #include +#include +#include #include #include +#include #include #include #include @@ -23,14 +26,112 @@ #include #include #include +#include #include #include #include +#include + namespace DB { namespace { +constexpr std::string_view fts_boolean_query_marker = "__tiflash_fts_boolean_query__:"; + +tipb::Expr buildFTSExpression(const TiDBTableScan & table_scan, Int32 result_type) +{ + const auto & query_info = table_scan.getFTSQueryInfo(); + const auto query_func = query_info.has_query_func() ? query_info.query_func() : tipb::ScalarFuncSig::FTSMatchWord; + if (query_func == tipb::ScalarFuncSig::FTSMatchWord) + RUNTIME_CHECK_MSG(query_info.columns_size() == 1, "FTS_MATCH_WORD currently supports exactly one column"); + else + RUNTIME_CHECK_MSG( + query_func == tipb::ScalarFuncSig::FTSMatchExpression && query_info.columns_size() > 0, + "Unsupported full-text query function or empty MATCH column list"); + + tipb::Expr expression; + expression.set_tp(tipb::ExprType::ScalarFunc); + expression.set_sig(query_func); + expression.mutable_field_type()->set_tp(result_type); + expression.mutable_field_type()->set_flag(TiDB::ColumnFlagNotNull); + *expression.add_children() = constructStringLiteralTiExpr(query_info.query_text()); + + for (const auto & query_column : query_info.columns()) + { + const auto column_id = query_column.column_id(); + const TiDB::ColumnInfo * column_info = nullptr; + Int64 column_index = -1; + for (size_t i = 0; i < table_scan.getColumns().size(); ++i) + { + const auto & column = table_scan.getColumns()[i]; + if (column.id == column_id) + { + column_info = &column; + column_index = i; + break; + } + } + RUNTIME_CHECK_MSG(column_info != nullptr, "Full-text column is not present in table scan columns"); + + tipb::Expr column_ref; + column_ref.set_tp(tipb::ExprType::ColumnRef); + WriteBufferFromOwnString ss; + encodeDAGInt64(column_index, ss); + column_ref.set_val(ss.releaseStr()); + *column_ref.mutable_field_type() = TiDB::columnInfoToFieldType(*column_info); + *expression.add_children() = std::move(column_ref); + } + + if (query_func == tipb::ScalarFuncSig::FTSMatchExpression && query_info.has_boolean_query()) + { + *expression.add_children() + = constructStringLiteralTiExpr(String(fts_boolean_query_marker) + query_info.boolean_query().SerializeAsString()); + } + return expression; +} + +tipb::Expr buildFTSFilter(const TiDBTableScan & table_scan) +{ + return buildFTSExpression(table_scan, TiDB::TypeDouble); +} + +String getFTSScorePlaceholderName(const TiDBTableScan & table_scan) +{ + for (size_t i = 0; i < table_scan.getColumns().size(); ++i) + { + if (isTiDBFTSScoreColumn(table_scan.getColumns()[i].id)) + return GeneratedColumnPlaceholderBlockInputStream::getColumnName(i); + } + throw TiFlashException("FTS score column is missing from table scan", Errors::Coprocessor::BadRequest); +} + +ExpressionActionsPtr buildFTSScoreActions( + const Block & input_header, + const TiDBTableScan & table_scan, + const Context & context) +{ + auto actions = std::make_shared(input_header.getColumnsWithTypeAndName()); + DAGExpressionAnalyzer analyzer(input_header, context); + const auto score_expr = buildFTSExpression(table_scan, TiDB::TypeFloat); + const auto score_expr_name = analyzer.getActions(score_expr, actions); + const auto score_column_name = getFTSScorePlaceholderName(table_scan); + + RUNTIME_CHECK_MSG( + actions->getSampleBlock().has(score_column_name), + "FTS score placeholder is missing from table scan input"); + actions->add(ExpressionAction::removeColumn(score_column_name)); + actions->add(ExpressionAction::copyColumn(score_expr_name, score_column_name)); + + NamesWithAliases project_columns; + project_columns.reserve(input_header.columns()); + for (const auto & column : input_header) + project_columns.emplace_back(column.name, column.name); + actions->add(ExpressionAction::project(project_columns)); + actions->finalize(input_header.getNames()); + return actions; +} + NamesWithAliases buildTableScanProjectionCols( Int64 logical_table_id, const NamesAndTypes & schema, @@ -82,7 +183,14 @@ PhysicalTableScan::PhysicalTableScan( : PhysicalLeaf(executor_id_, PlanType::TableScan, schema_, FineGrainedShuffle{}, req_id) , tidb_table_scan(tidb_table_scan_) , sample_block(sample_block_) -{} +{ + if (tidb_table_scan.getFTSQueryInfo().columns_size() > 0) + { + google::protobuf::RepeatedPtrField conditions; + *conditions.Add() = buildFTSFilter(tidb_table_scan); + filter_conditions = FilterConditions(executor_id_, conditions); + } +} PhysicalPlanNodePtr PhysicalTableScan::build( const String & executor_id, @@ -113,7 +221,7 @@ void PhysicalTableScan::buildBlockInputStreamImpl(DAGPipeline & pipeline, Contex DAGStorageInterpreter storage_interpreter(context, tidb_table_scan, filter_conditions, max_streams); storage_interpreter.execute(pipeline); } - buildProjection(pipeline); + buildProjection(pipeline, context); } void PhysicalTableScan::buildPipeline( @@ -136,7 +244,7 @@ void PhysicalTableScan::buildPipeline( DAGStorageInterpreter storage_interpreter(context, tidb_table_scan, filter_conditions, context.getMaxStreams()); storage_interpreter.execute(exec_context, pipeline_exec_builder); } - buildProjection(exec_context, pipeline_exec_builder); + buildProjection(exec_context, pipeline_exec_builder, context); PhysicalPlanNode::buildPipeline(builder, context, exec_context); } @@ -151,8 +259,17 @@ void PhysicalTableScan::buildPipelineExecGroupImpl( group_builder = std::move(pipeline_exec_builder); } -void PhysicalTableScan::buildProjection(DAGPipeline & pipeline) +void PhysicalTableScan::buildProjection(DAGPipeline & pipeline, Context & context) { + if (tidb_table_scan.getFTSQueryInfo().query_type() == tipb::FTSQueryType::FTSQueryTypeWithScore) + { + RUNTIME_CHECK_MSG( + !pipeline.streams.empty(), + "FTS score cannot be materialized without a table scan stream"); + auto score_actions = buildFTSScoreActions(pipeline.firstStream()->getHeader(), tidb_table_scan, context); + executeExpression(pipeline, score_actions, log, "full-text score"); + } + const auto & schema_project_cols = buildTableScanProjectionCols( tidb_table_scan.getLogicalTableID(), schema, @@ -165,8 +282,15 @@ void PhysicalTableScan::buildProjection(DAGPipeline & pipeline) void PhysicalTableScan::buildProjection( PipelineExecutorContext & exec_context, - PipelineExecGroupBuilder & group_builder) + PipelineExecGroupBuilder & group_builder, + Context & context) { + if (tidb_table_scan.getFTSQueryInfo().query_type() == tipb::FTSQueryType::FTSQueryTypeWithScore) + { + auto score_actions = buildFTSScoreActions(group_builder.getCurrentHeader(), tidb_table_scan, context); + executeExpression(exec_context, group_builder, score_actions, log); + } + auto header = group_builder.getCurrentHeader(); const auto & schema_project_cols = buildTableScanProjectionCols(tidb_table_scan.getLogicalTableID(), schema, header); @@ -190,14 +314,11 @@ const Block & PhysicalTableScan::getSampleBlock() const bool PhysicalTableScan::setFilterConditions(const String & filter_executor_id, const tipb::Selection & selection) { - /// Since there is at most one selection on the table scan, setFilterConditions() will only be called at most once. - /// So in this case hasFilterConditions() is always false. - if (unlikely(hasFilterConditions())) - { - return false; - } - - filter_conditions = FilterConditions::filterConditionsFrom(filter_executor_id, selection); + if (!hasFilterConditions()) + filter_conditions = FilterConditions::filterConditionsFrom(filter_executor_id, selection); + else + for (const auto & condition : selection.conditions()) + *filter_conditions.conditions.Add() = condition; return true; } diff --git a/dbms/src/Flash/Planner/Plans/PhysicalTableScan.h b/dbms/src/Flash/Planner/Plans/PhysicalTableScan.h index 284b75d0c73..eff25aa37bd 100644 --- a/dbms/src/Flash/Planner/Plans/PhysicalTableScan.h +++ b/dbms/src/Flash/Planner/Plans/PhysicalTableScan.h @@ -59,8 +59,11 @@ class PhysicalTableScan : public PhysicalLeaf Context & /*context*/, size_t /*concurrency*/) override; - void buildProjection(DAGPipeline & pipeline); - void buildProjection(PipelineExecutorContext & exec_context, PipelineExecGroupBuilder & group_builder); + void buildProjection(DAGPipeline & pipeline, Context & context); + void buildProjection( + PipelineExecutorContext & exec_context, + PipelineExecGroupBuilder & group_builder, + Context & context); private: FilterConditions filter_conditions; diff --git a/dbms/src/Functions/FunctionsFullText.cpp b/dbms/src/Functions/FunctionsFullText.cpp new file mode 100644 index 00000000000..65adbd34991 --- /dev/null +++ b/dbms/src/Functions/FunctionsFullText.cpp @@ -0,0 +1,747 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ +namespace ErrorCodes +{ +extern const int ILLEGAL_COLUMN; +} + +namespace +{ +constexpr size_t default_min_token_size = 3; +constexpr size_t default_max_token_size = 84; +constexpr std::string_view fts_boolean_query_marker = "__tiflash_fts_boolean_query__:"; + +struct FullTextToken +{ + String text; + size_t position = 0; +}; + +struct BooleanClause +{ + enum class Modifier + { + Should, + Must, + MustNot, + }; + + Modifier modifier = Modifier::Should; + bool phrase = false; + bool prefix = false; + std::vector terms; + std::vector offsets; +}; + +using FullTextColumn = std::vector; +using FullTextDocument = std::vector; + +bool isFullTextToken(UInt32 code_point) +{ + return isAlphaNumericASCII(static_cast(code_point)) || code_point == '_' || Poco::Unicode::isAlpha(code_point) + || Poco::Unicode::isDigit(code_point); +} + +std::pair decodeCodePoint(std::string_view text, size_t offset) +{ + const auto decoded = UTF8::utf8Decode(text.data() + offset, text.size() - offset); + if (decoded.second == 0 || decoded.first == UTF8::UTF8_Error || decoded.second > text.size() - offset) + return {static_cast(text[offset]), 1}; + return {decoded.first, decoded.second}; +} + +std::vector tokenizeText(std::string_view text) +{ + std::vector result; + size_t position = 0; + for (size_t i = 0; i < text.size();) + { + const auto [code_point, length] = decodeCodePoint(text, i); + if (!isFullTextToken(code_point)) + { + i += length; + continue; + } + + String token; + while (i < text.size()) + { + const auto [code_point, length] = decodeCodePoint(text, i); + if (!isFullTextToken(code_point)) + break; + token.append(text.data() + i, length); + i += length; + } + + // The TiDB analyzer lower-cases after tokenization. Poco uses the + // same Unicode tables for the C++ implementation, while also + // preserving the ASCII fast path used by the old evaluator. + token = Poco::UTF8::toLower(token); + result.push_back({std::move(token), position++}); + } + return result; +} + +bool isDefaultStopword(const String & token) +{ + static const std::unordered_set stopwords{ + "a", "about", "an", "are", "as", "at", "be", "by", "com", "de", "en", "for", + "from", "how", "i", "in", "is", "it", "la", "of", "on", "or", "that", "the", + "this", "to", "was", "what", "when", "where", "who", "will", "with", "und", "www"}; + return stopwords.contains(token); +} + +std::vector analyzeText(std::string_view text) +{ + std::vector result; + for (auto & token : tokenizeText(text)) + { + const auto code_points = UTF8::countCodePoints( + reinterpret_cast(token.text.data()), + token.text.size()); + if (code_points >= default_min_token_size && code_points <= default_max_token_size + && !isDefaultStopword(token.text)) + result.push_back(std::move(token)); + } + return result; +} + +bool isBooleanWhitespace(char c) +{ + return c == ' ' || c == '\t' || c == '\n' || c == '\r'; +} + +bool parseBooleanQuery(std::string_view query, std::vector & clauses) +{ + for (size_t i = 0; i < query.size();) + { + while (i < query.size() && isBooleanWhitespace(query[i])) + ++i; + if (i == query.size()) + break; + + BooleanClause clause; + // The STANDARD parser used by #70485 accepts only the operators that + // affect filtering. Other InnoDB operators affect scoring or phrase + // proximity and must not be silently treated as ordinary text. + if (query[i] == '%' || query[i] == '(' || query[i] == ')' || query[i] == '<' || query[i] == '>' + || query[i] == '~' || query[i] == '@') + return false; + + if (query[i] == '+' || query[i] == '-') + { + clause.modifier = query[i] == '+' ? BooleanClause::Modifier::Must : BooleanClause::Modifier::MustNot; + ++i; + } + + if (i == query.size()) + return false; + + // InnoDB accepts a leading wildcard as a no-op. A trailing wildcard + // is handled below and turns a term into a prefix query. + if (query[i] == '*') + { + ++i; + if (i == query.size()) + continue; + } + + String raw; + if (query[i] == '"') + { + clause.phrase = true; + ++i; + const size_t start = i; + while (i < query.size() && query[i] != '"' && query[i] != '\n') + ++i; + if (i == query.size() || query[i] != '"') + return false; + raw.assign(query.substr(start, i - start)); + ++i; + } + else + { + const size_t start = i; + while (i < query.size() && !isBooleanWhitespace(query[i]) && query[i] != '+' && query[i] != '-' + && query[i] != '*' && query[i] != '%' && query[i] != '(' && query[i] != ')' && query[i] != '<' + && query[i] != '>' && query[i] != '~' && query[i] != '@') + ++i; + if (start == i) + return false; + raw.assign(query.substr(start, i - start)); + if (!raw.empty() && raw.back() == '*') + { + clause.prefix = true; + raw.pop_back(); + } + } + + const auto terms = clause.prefix ? tokenizeText(raw) : analyzeText(raw); + if (clause.prefix && terms.size() != 1) + continue; + + const size_t first_position = terms.empty() ? 0 : terms.front().position; + for (const auto & term : terms) + { + const auto code_points = UTF8::countCodePoints( + reinterpret_cast(term.text.data()), + term.text.size()); + if (code_points <= default_max_token_size && (clause.prefix || code_points >= default_min_token_size) + && (clause.prefix || !isDefaultStopword(term.text))) + { + clause.terms.push_back(term.text); + if (clause.phrase) + clause.offsets.push_back(term.position - first_position); + } + } + if (!clause.terms.empty() || clause.modifier == BooleanClause::Modifier::Must) + clauses.push_back(std::move(clause)); + } + return true; +} + +bool matchesPhraseInColumn(const BooleanClause & clause, const FullTextColumn & document) +{ + if (clause.terms.empty() || clause.terms.size() != clause.offsets.size()) + return false; + + for (const auto & start : document) + { + bool matched = true; + for (size_t i = 0; i < clause.terms.size(); ++i) + { + const auto expected_position = start.position + clause.offsets[i]; + const auto it = std::find_if(document.begin(), document.end(), [&](const FullTextToken & token) { + return token.position == expected_position; + }); + if (it == document.end() || it->text != clause.terms[i]) + { + matched = false; + break; + } + } + if (matched) + return true; + } + return false; +} + +bool matchesClause(const BooleanClause & clause, const FullTextDocument & document) +{ + if (clause.terms.empty()) + return false; + + if (clause.phrase) + return std::any_of(document.begin(), document.end(), [&](const FullTextColumn & column) { + return matchesPhraseInColumn(clause, column); + }); + + return std::all_of(clause.terms.begin(), clause.terms.end(), [&](const String & term) { + return std::any_of(document.begin(), document.end(), [&](const FullTextColumn & column) { + return std::any_of(column.begin(), column.end(), [&](const FullTextToken & token) { + return clause.prefix ? token.text.starts_with(term) : token.text == term; + }); + }); + }); +} + +size_t countClauseMatches(const BooleanClause & clause, const FullTextDocument & document) +{ + if (!matchesClause(clause, document)) + return 0; + + if (!clause.phrase) + { + size_t count = 0; + for (const auto & term : clause.terms) + { + for (const auto & column : document) + { + count += std::count_if(column.begin(), column.end(), [&](const FullTextToken & token) { + return clause.prefix ? token.text.starts_with(term) : token.text == term; + }); + } + } + return count; + } + + size_t count = 0; + for (const auto & column : document) + { + for (const auto & start : column) + { + bool matched = true; + for (size_t i = 0; i < clause.terms.size(); ++i) + { + const auto expected_position = start.position + clause.offsets[i]; + const auto it = std::find_if(column.begin(), column.end(), [&](const FullTextToken & token) { + return token.position == expected_position; + }); + if (it == column.end() || it->text != clause.terms[i]) + { + matched = false; + break; + } + } + if (matched) + ++count; + } + } + return count; +} + +FullTextColumn analyzeColumn(std::string_view document) +{ + return analyzeText(document); +} + +Float64 matchBooleanScore(const std::vector & clauses, const FullTextDocument & document) +{ + if (clauses.empty()) + return 0; + + bool has_positive = false; + bool has_must = false; + bool positive_match = false; + Float64 score = 0; + for (const auto & clause : clauses) + { + const bool matched = matchesClause(clause, document); + const auto clause_score = static_cast(countClauseMatches(clause, document)); + switch (clause.modifier) + { + case BooleanClause::Modifier::Must: + has_must = true; + if (!matched) + return 0; + score += clause_score; + break; + case BooleanClause::Modifier::MustNot: + if (matched) + return 0; + break; + case BooleanClause::Modifier::Should: + has_positive = true; + positive_match = positive_match || matched; + score += clause_score; + break; + } + } + + // With a required term, unprefixed terms are optional. Without one, at + // least one unprefixed term must match, matching BOOLEAN MODE semantics. + if (!(has_must || !has_positive || positive_match)) + return 0; + // A query containing only prohibited terms has no positive term to score, + // but an accepted row still has to pass the boolean filter. + return score > 0 ? score : 1; +} + +Float64 matchBooleanScore(std::string_view query, const FullTextDocument & document) +{ + std::vector clauses; + if (!parseBooleanQuery(query, clauses)) + return 0; + return matchBooleanScore(clauses, document); +} + +Float64 matchBooleanScore(const tipb::FTSBooleanQuery & query, const FullTextDocument & document) +{ + std::vector clauses; + for (const auto & node : query.nodes()) + { + if (!node.has_term()) + return 0; + + const auto & term = node.term(); + BooleanClause clause; + switch (node.occur()) + { + case tipb::FTSBooleanOccur::FTSBooleanOccurMust: + clause.modifier = BooleanClause::Modifier::Must; + break; + case tipb::FTSBooleanOccur::FTSBooleanOccurMustNot: + clause.modifier = BooleanClause::Modifier::MustNot; + break; + case tipb::FTSBooleanOccur::FTSBooleanOccurShould: + clause.modifier = BooleanClause::Modifier::Should; + break; + default: + return 0; + } + + switch (term.term_type()) + { + case tipb::FTSBooleanTermType::FTSBooleanTermWord: + for (const auto & token : analyzeText(term.text())) + clause.terms.push_back(token.text); + break; + case tipb::FTSBooleanTermType::FTSBooleanTermPrefix: + { + clause.prefix = true; + const auto terms = tokenizeText(term.text()); + if (terms.size() != 1) + { + if (clause.modifier == BooleanClause::Modifier::Must) + clauses.push_back(std::move(clause)); + continue; + } + const auto code_points = UTF8::countCodePoints( + reinterpret_cast(terms.front().text.data()), + terms.front().text.size()); + if (code_points > default_max_token_size) + { + if (clause.modifier == BooleanClause::Modifier::Must) + clauses.push_back(std::move(clause)); + continue; + } + clause.terms.push_back(terms.front().text); + break; + } + case tipb::FTSBooleanTermType::FTSBooleanTermPhrase: + { + clause.phrase = true; + const auto terms = analyzeText(term.text()); + const size_t first_position = terms.empty() ? 0 : terms.front().position; + for (const auto & token : terms) + { + clause.terms.push_back(token.text); + clause.offsets.push_back(token.position - first_position); + } + break; + } + default: + return 0; + } + + if (!clause.terms.empty() || clause.modifier == BooleanClause::Modifier::Must) + clauses.push_back(std::move(clause)); + } + + // A BOOLEAN MODE query containing only prohibited terms has no positive + // branch. TiDB's local evaluator treats it as matching no rows. + if (std::none_of(clauses.begin(), clauses.end(), [](const BooleanClause & clause) { + return clause.modifier != BooleanClause::Modifier::MustNot; + })) + return 0; + + // The protocol path is the no-score MATCH ... AGAINST BOOLEAN MODE + // predicate introduced by #70484/#70485. TiDB's local evaluator returns + // a boolean 0/1 result, so do not expose term-frequency counts here. + return matchBooleanScore(clauses, document) > 0 ? 1 : 0; +} + +Float64 matchBooleanScore(std::string_view query, std::string_view document) +{ + return matchBooleanScore(query, FullTextDocument{analyzeColumn(document)}); +} + +size_t countTermMatches(const String & term, const FullTextColumn & document) +{ + return std::count_if(document.begin(), document.end(), [&](const FullTextToken & token) { + return token.text == term; + }); +} + +Float64 matchNaturalLanguageScore(std::string_view query, const FullTextDocument & document) +{ + const auto query_tokens = analyzeText(query); + if (query_tokens.empty()) + return 0; + + std::unordered_set unique_query_tokens; + Float64 score = 0; + for (const auto & query_token : query_tokens) + { + if (!unique_query_tokens.insert(query_token.text).second) + continue; + for (const auto & column : document) + score += static_cast(countTermMatches(query_token.text, column)); + } + return score; +} + +bool queryUsesBooleanSyntax(std::string_view query) +{ + return query.find_first_of("+-\"*%()<>~@") != std::string_view::npos; +} + +String getStringAt(const IColumn & column, size_t row) +{ + if (const auto * column_nullable = typeid_cast(&column)) + { + if (column_nullable->getNullMapData()[row]) + return {}; + return getStringAt(column_nullable->getNestedColumn(), row); + } + + if (const auto * column_const = typeid_cast(&column)) + return column_const->getValue(); + + const auto * column_string = checkAndGetColumn(&column); + if (column_string == nullptr) + throw Exception("Full-text arguments must be string columns", ErrorCodes::ILLEGAL_COLUMN); + + const auto & chars = column_string->getChars(); + const auto & offsets = column_string->getOffsets(); + const size_t begin = row == 0 ? 0 : offsets[row - 1]; + const size_t end = offsets[row]; + return String(reinterpret_cast(&chars[begin]), end - begin - 1); +} + +bool isNullAt(const IColumn & column, size_t row) +{ + if (const auto * column_nullable = typeid_cast(&column)) + return column_nullable->getNullMapData()[row]; + if (const auto * column_const = typeid_cast(&column)) + { + if (const auto * nested_nullable = typeid_cast(&column_const->getDataColumn())) + return nested_nullable->getNullMapData()[0]; + } + return false; +} + +bool decodeFTSBooleanQuery(const IColumn & column, tipb::FTSBooleanQuery & query) +{ + const auto * constant = typeid_cast(&column); + if (constant == nullptr) + return false; + const auto encoded = constant->getValue(); + const std::string_view value(encoded); + if (!value.starts_with(fts_boolean_query_marker)) + return false; + return query.ParseFromString(std::string(value.substr(fts_boolean_query_marker.size()))); +} + +class FunctionFTSMatchWord final : public IFunction +{ +public: + static constexpr auto name = "fts_match_word"; + static FunctionPtr create(const Context &) { return std::make_shared(); } + + String getName() const override { return name; } + size_t getNumberOfArguments() const override { return 2; } + bool useDefaultImplementationForConstants() const override { return false; } + ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {0}; } + + DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override + { + bool nullable = false; + for (const auto & argument : arguments) + { + if (!removeNullable(argument)->isString()) + throw Exception( + "Illegal type " + argument->getName() + " of argument of function " + getName(), + ErrorCodes::ILLEGAL_COLUMN); + nullable = nullable || argument->isNullable(); + } + DataTypePtr result_type = std::make_shared(); + return nullable ? std::make_shared(result_type) : result_type; + } + + void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result) const override + { + const auto * query_column = typeid_cast(&*block.getByPosition(arguments[0]).column); + if (query_column == nullptr) + throw Exception( + "The query argument of fts_match_word must be constant", + ErrorCodes::ILLEGAL_COLUMN); + + const auto & document_column = block.getByPosition(arguments[1]).column; + auto output = ColumnFloat64::create(document_column->size()); + auto & output_data = output->getData(); + + if (isNullAt(*query_column, 0)) + { + auto null_map = ColumnUInt8::create(document_column->size(), 1); + block.getByPosition(result).column = ColumnNullable::create(std::move(output), std::move(null_map)); + return; + } + const auto query = query_column->getValue(); + + if (block.getByPosition(arguments[1]).type->isNullable() || document_column->isColumnNullable()) + { + auto null_map = ColumnUInt8::create(document_column->size(), 0); + auto & null_map_data = null_map->getData(); + for (size_t row = 0; row < document_column->size(); ++row) + { + if (isNullAt(*document_column, row)) + null_map_data[row] = 1; + else + output_data[row] = matchBooleanScore(query, getStringAt(*document_column, row)); + } + block.getByPosition(result).column = ColumnNullable::create(std::move(output), std::move(null_map)); + return; + } + + if (const auto * document = checkAndGetColumn(&*document_column)) + { + const auto & chars = document->getChars(); + const auto & offsets = document->getOffsets(); + for (size_t row = 0, begin = 0; row < offsets.size(); ++row) + { + const size_t end = offsets[row]; + const size_t length = end - begin - 1; + output_data[row] + = matchBooleanScore(query, std::string_view(reinterpret_cast(&chars[begin]), length)); + begin = end; + } + } + else if (const auto * document = typeid_cast(&*document_column)) + { + const auto value = document->getValue(); + const Float64 matched = matchBooleanScore(query, value); + std::fill(output_data.begin(), output_data.end(), matched); + } + else + { + throw Exception( + "Illegal column " + document_column->getName() + " of argument of function " + getName(), + ErrorCodes::ILLEGAL_COLUMN); + } + block.getByPosition(result).column = std::move(output); + } +}; + +class FunctionFTSMatchExpression final : public IFunction +{ +public: + static constexpr auto name = "fts_match_expression"; + static FunctionPtr create(const Context &) { return std::make_shared(); } + + String getName() const override { return name; } + size_t getNumberOfArguments() const override { return 0; } + bool isVariadic() const override { return true; } + bool useDefaultImplementationForConstants() const override { return false; } + ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {0}; } + + DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override + { + if (arguments.size() < 2) + throw Exception("fts_match_expression requires a query and at least one column"); + bool nullable = false; + for (const auto & argument : arguments) + { + if (!removeNullable(argument)->isString()) + throw Exception( + "Illegal type " + argument->getName() + " of argument of function " + getName(), + ErrorCodes::ILLEGAL_COLUMN); + nullable = nullable || argument->isNullable(); + } + DataTypePtr result_type = std::make_shared(); + return nullable ? std::make_shared(result_type) : result_type; + } + + void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result) const override + { + const auto * query_column = typeid_cast(&*block.getByPosition(arguments[0]).column); + if (query_column == nullptr) + throw Exception( + "The query argument of fts_match_expression must be constant", + ErrorCodes::ILLEGAL_COLUMN); + + const size_t rows = block.getByPosition(arguments[1]).column->size(); + auto output = ColumnFloat64::create(rows, 0); + auto & output_data = output->getData(); + const bool nullable = block.getByPosition(result).type->isNullable(); + auto null_map = nullable ? ColumnUInt8::create(rows, 0) : nullptr; + auto * null_map_data = null_map ? &null_map->getData() : nullptr; + + if (isNullAt(*query_column, 0)) + { + if (null_map_data != nullptr) + std::fill(null_map_data->begin(), null_map_data->end(), 1); + ColumnPtr result_column; + if (null_map) + result_column = ColumnNullable::create(std::move(output), std::move(null_map)); + else + result_column = std::move(output); + block.getByPosition(result).column = std::move(result_column); + return; + } + const auto query = query_column->getValue(); + size_t document_argument_end = arguments.size(); + tipb::FTSBooleanQuery protocol_boolean_query; + const bool has_protocol_boolean_query = arguments.size() > 2 + && decodeFTSBooleanQuery(*block.getByPosition(arguments.back()).column, protocol_boolean_query); + if (has_protocol_boolean_query) + --document_argument_end; + + for (size_t row = 0; row < rows; ++row) + { + FullTextDocument document; + document.reserve(document_argument_end - 1); + for (size_t arg = 1; arg < document_argument_end; ++arg) + { + // A NULL MATCH column contributes no tokens. This is + // intentional: #70485 relies on a row with a NULL body + // still matching a required term from another MATCH column. + if (isNullAt(*block.getByPosition(arguments[arg]).column, row)) + { + document.emplace_back(); + continue; + } + document.push_back(analyzeColumn(getStringAt(*block.getByPosition(arguments[arg]).column, row))); + } + if (has_protocol_boolean_query) + output_data[row] = matchBooleanScore(protocol_boolean_query, document); + else if (queryUsesBooleanSyntax(query)) + output_data[row] = matchBooleanScore(query, document); + else + output_data[row] = matchNaturalLanguageScore(query, document); + } + ColumnPtr result_column; + if (null_map) + result_column = ColumnNullable::create(std::move(output), std::move(null_map)); + else + result_column = std::move(output); + block.getByPosition(result).column = std::move(result_column); + } +}; +} + +void registerFunctionsFullText(FunctionFactory & factory) +{ + factory.registerFunction(FunctionFactory::CaseInsensitive); + factory.registerFunction(FunctionFactory::CaseInsensitive); +} +} // namespace DB diff --git a/dbms/src/Functions/FunctionsFullText.h b/dbms/src/Functions/FunctionsFullText.h new file mode 100644 index 00000000000..797560b0e9e --- /dev/null +++ b/dbms/src/Functions/FunctionsFullText.h @@ -0,0 +1,22 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include + +namespace DB +{ +void registerFunctionsFullText(FunctionFactory & factory); +} diff --git a/dbms/src/Functions/registerFunctions.cpp b/dbms/src/Functions/registerFunctions.cpp index cb43ae7760d..e3f833811d5 100644 --- a/dbms/src/Functions/registerFunctions.cpp +++ b/dbms/src/Functions/registerFunctions.cpp @@ -51,6 +51,7 @@ void registerFunctionsRegexpSubstr(FunctionFactory &); void registerFunctionsRegexpReplace(FunctionFactory &); void registerFunctionsGrouping(FunctionFactory &); void registerFunctionsVector(FunctionFactory &); +void registerFunctionsFullText(FunctionFactory &); void registerFunctions() { @@ -85,6 +86,7 @@ void registerFunctions() registerFunctionsIsIPAddr(factory); registerFunctionsGrouping(factory); registerFunctionsVector(factory); + registerFunctionsFullText(factory); } } // namespace DB diff --git a/dbms/src/Functions/tests/gtest_fulltext.cpp b/dbms/src/Functions/tests/gtest_fulltext.cpp new file mode 100644 index 00000000000..89eedffeda5 --- /dev/null +++ b/dbms/src/Functions/tests/gtest_fulltext.cpp @@ -0,0 +1,179 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include + +namespace DB::tests +{ +class TestFullText : public DB::tests::FunctionTest +{ +}; + +TEST_F(TestFullText, MatchWordBoolean) +try +{ + const auto query = createConstColumn(4, "+quick -slow"); + const auto documents = createColumn({ + "A quick brown fox", + "A slow brown fox", + "A brown fox", + "A QUICK runner", + }); + ASSERT_COLUMN_EQ(createColumn({1, 0, 0, 1}), executeFunction("fts_match_word", {query, documents})); +} +CATCH + +TEST_F(TestFullText, MatchWordPhraseAndPrefix) +try +{ + ASSERT_COLUMN_EQ( + createColumn({1, 0}), + executeFunction( + "fts_match_word", + {createConstColumn(2, "\"quick brown\""), createColumn({"quick brown fox", "quick fox brown"})})); + ASSERT_COLUMN_EQ( + createColumn({1, 0}), + executeFunction( + "fts_match_word", + {createConstColumn(2, "run*"), createColumn({"runner", "walk"})})); +} +CATCH + +TEST_F(TestFullText, MatchWordTokenBoundaryAndPhrasePositions) +try +{ + ASSERT_COLUMN_EQ( + createColumn({0, 1}), + executeFunction( + "fts_match_word", + {createConstColumn(2, "cat"), createColumn({"concatenate", "a cat"})})); + ASSERT_COLUMN_EQ( + createColumn({0, 1}), + executeFunction( + "fts_match_word", + {createConstColumn(2, "\"quick the brown\""), + createColumn({"quick brown", "quick the brown"})})); +} +CATCH + +TEST_F(TestFullText, MatchWordNull) +try +{ + ASSERT_COLUMN_EQ( + createColumn>({1, {}}), + executeFunction( + "fts_match_word", + {createConstColumn(2, "hello"), createColumn>({"hello world", {}})})); +} +CATCH + +TEST_F(TestFullText, MatchWordScore) +try +{ + ASSERT_COLUMN_EQ( + createColumn({1, 2, 0}), + executeFunction( + "fts_match_word", + {createConstColumn(3, "quick"), createColumn({"quick", "quick quick", "slow"})})); +} +CATCH + +TEST_F(TestFullText, MatchExpression) +try +{ + ASSERT_COLUMN_EQ( + createColumn({1, 0}), + executeFunction( + "fts_match_expression", + {createConstColumn(2, "quick fox"), createColumn({"quick brown", "slow turtle"})})); +} +CATCH + +TEST_F(TestFullText, MatchExpressionProtocolBooleanQuery) +try +{ + tipb::FTSBooleanQuery boolean_query; + auto * required = boolean_query.add_nodes(); + required->set_occur(tipb::FTSBooleanOccurMust); + required->mutable_term()->set_term_type(tipb::FTSBooleanTermWord); + required->mutable_term()->set_text("quick"); + auto * prohibited = boolean_query.add_nodes(); + prohibited->set_occur(tipb::FTSBooleanOccurMustNot); + prohibited->mutable_term()->set_term_type(tipb::FTSBooleanTermWord); + prohibited->mutable_term()->set_text("slow"); + + const String metadata = "__tiflash_fts_boolean_query__:" + boolean_query.SerializeAsString(); + ASSERT_COLUMN_EQ( + createColumn({1, 0, 0}), + executeFunction( + "fts_match_expression", + {createConstColumn(3, "+quick -slow"), + createColumn({"quick brown", "slow fox", "brown fox"}), + createConstColumn(3, metadata)})); + + tipb::FTSBooleanQuery prohibited_query; + auto * prohibited_only = prohibited_query.add_nodes(); + prohibited_only->set_occur(tipb::FTSBooleanOccurMustNot); + prohibited_only->mutable_term()->set_term_type(tipb::FTSBooleanTermWord); + prohibited_only->mutable_term()->set_text("slow"); + const String prohibited_metadata = "__tiflash_fts_boolean_query__:" + prohibited_query.SerializeAsString(); + ASSERT_COLUMN_EQ( + createColumn({0, 0, 0}), + executeFunction( + "fts_match_expression", + {createConstColumn(3, "-slow"), + createColumn({"quick brown", "slow fox", "brown fox"}), + createConstColumn(3, prohibited_metadata)})); +} +CATCH + +TEST_F(TestFullText, MatchExpressionScore) +try +{ + ASSERT_COLUMN_EQ( + createColumn({1, 3, 0}), + executeFunction( + "fts_match_expression", + {createConstColumn(3, "quick fox"), + createColumn({"quick brown", "quick fox fox", "slow turtle"})})); +} +CATCH + +TEST_F(TestFullText, MatchExpressionNullColumn) +try +{ + ASSERT_COLUMN_EQ( + createColumn>({0, 1}), + executeFunction( + "fts_match_expression", + {createConstColumn(2, "+indexing -postgresql"), + createColumn({"Indexing", "Indexing"}), + createColumn>({"PostgreSQL", {}})})); +} +CATCH + +TEST_F(TestFullText, MatchBooleanUnsupportedScoreOperator) +try +{ + ASSERT_COLUMN_EQ( + createColumn({0}), + executeFunction( + "fts_match_word", + {createConstColumn(1, ">quick"), createColumn({"quick"})})); +} +CATCH +} // namespace DB::tests diff --git a/dbms/src/TiDB/Schema/TiDB.cpp b/dbms/src/TiDB/Schema/TiDB.cpp index 4402215c6dc..f55b2b7f29a 100644 --- a/dbms/src/TiDB/Schema/TiDB.cpp +++ b/dbms/src/TiDB/Schema/TiDB.cpp @@ -1363,10 +1363,21 @@ tipb::FieldType columnInfoToFieldType(const ColumnInfo & ci) ret.set_decimal(ci.decimal); if (!ci.collate.isEmpty()) { - auto collator_name = ci.collate.convert(); - TiDBCollatorPtr collator = ITiDBCollator::getCollator(collator_name); - RUNTIME_CHECK_MSG(collator, "cannot find collator: {}", collator_name); - ret.set_collate(collator->getCollatorId()); + if (ci.collate.isInteger()) + { + // Table-scan ColumnInfo comes directly from tipb and carries the + // signed collation ID used by TiDB's new-collation protocol. + // Preserve it for expression field types instead of treating it + // as a collation name. + ret.set_collate(ci.collate.convert()); + } + else + { + auto collator_name = ci.collate.convert(); + TiDBCollatorPtr collator = ITiDBCollator::getCollator(collator_name); + RUNTIME_CHECK_MSG(collator, "cannot find collator: {}", collator_name); + ret.set_collate(collator->getCollatorId()); + } } for (const auto & elem : ci.elems) { diff --git a/docs/design/2026-09-18-match-against-tiflash.md b/docs/design/2026-09-18-match-against-tiflash.md new file mode 100644 index 00000000000..93c0fd9aafb --- /dev/null +++ b/docs/design/2026-09-18-match-against-tiflash.md @@ -0,0 +1,115 @@ +# TiFlash MATCH AGAINST pushdown + +## Introduction + +This document describes the TiFlash-side implementation for evaluating TiDB +`MATCH ... AGAINST` and `FTS_MATCH_WORD` expressions on the read snapshot. +The implementation is intentionally limited to the existing release-8.5 +TiDB-to-TiFlash protocol. TiKV, TICI, and TiDB-side changes are out of scope. + +## Motivation or Background + +The TiDB planner can serialize full-text scalar signatures and the +`FTSQueryTypeWithScore` table-scan metadata into tipb, but the release-8.5 +TiFlash branch did not recognize those signatures. A snapshot-local evaluator +allows TiFlash to execute the predicate and materialize `_FTS_SCORE` after +reading the row, without depending on an asynchronously maintained external +index. + +## Detailed Design + +### Protocol and planner path + +The `contrib/tipb` submodule is advanced to the upstream protocol revision that +contains `ScalarFuncSig_FTSMatchWord`, `ScalarFuncSig_FTSMatchExpression`, +`FTSQueryInfo`, and `used_columnar_indexes`. + +TiFlash maps the two scalar signatures as follows: + +| tipb signature | TiFlash function | semantics | +| --- | --- | --- | +| `FTSMatchWord` | `fts_match_word` | Boolean-mode filter evaluator | +| `FTSMatchExpression` | `fts_match_expression` | Boolean-aware evaluator over MATCH columns | + +For an index-style `FTSQueryInfo`, `PhysicalTableScan` reconstructs a +`fts_match_word(query, column)` filter and merges it with ordinary Selection +conditions. For `FTSQueryTypeWithScore`, it evaluates the same expression after +the scan and replaces the `_FTS_SCORE` generated-column placeholder with the +resulting `FLOAT` column. This keeps the existing table-scan schema and lets +Projection, Filter, and TopN consume the same score column. The expression +pipeline runs on the same read snapshot as the row data. + +### Analyzer and matcher + +The evaluator implements the STANDARD_V1-compatible Boolean matching subset +used by #70484/#70485: + +- Unicode letter/number token runs with `_` preserved; +- Unicode lower-casing; +- default token length range 3..84; +- the default InnoDB stopword set; +- Boolean `+` required terms, `-` prohibited terms, quoted phrases, and a + trailing `*` prefix; +- phrase positions are retained across analyzer filtering, so removed + stopwords do not close phrase gaps; +- NULL MATCH columns contribute no tokens and do not nullify the whole row. + +The predicate path consumes the numeric result as a Boolean value (`0` means +no match, a positive value means match). Queries containing Boolean syntax use +the Boolean matcher; plain queries retain the existing token-overlap behavior +for compatibility with the current scalar signature. The implementation +still returns a deterministic positive term-frequency value for the existing +`_FTS_SCORE` placeholder path, but that score is not part of the +#70484/#70485 acceptance scope and is not claimed to be MySQL/InnoDB +relevance. Nullable query arguments preserve NULL; nullable MATCH columns +contribute no tokens. + +### Consistency boundary + +This is a scan evaluator, not a DeltaMerge native full-text index. It therefore +does not introduce a second mutable index and cannot return rows that are +invisible at the read timestamp. The index id in `FTSQueryInfo` is used only to +identify the query contract in this phase; it is not read by a separate index +reader. The score is calculated from the query and row text, so it does not +provide corpus-wide IDF normalization. + +## Test Design + +### Functional Tests + +`dbms/src/Functions/tests/gtest_fulltext.cpp` covers required/prohibited terms, +phrases with stopword gaps, prefixes, word boundaries, nullable input, NULL +MATCH columns, and unsupported score modifiers. + +### Compatibility Tests + +The normal local TiFlash read path and region-retry remote-read path use the +same expression evaluator. TICI/Disaggregated execution is intentionally not +part of this phase. + +### Benchmark Tests + +No benchmark is included in this phase. The scan evaluator is expected to be +slower than an inverted index and should be treated as compute pushdown, not +index-accelerated search. + +## Impacts & Risks + +The implementation adds per-row tokenization and matching CPU cost. Its score +is deterministic but is not a native MySQL relevance score. Custom analyzer +sysvars, custom stopword lists, NGRAM_V1, and non-default `AGAINST` modifiers +are not represented in the current wire contract and must not be treated as +supported by this phase. + +## Investigation & Alternatives + +An asynchronous external index was rejected because it cannot provide the +required snapshot consistency. A native DeltaMerge full-text index remains a +long-term performance design, but it requires a larger write-path, snapshot, +and index-reader change set. + +## Unresolved Questions + +If exact corpus-wide ranking is required, a future phase must define a +TiDB-compatible relevance-score contract and add a native DeltaMerge full-text +index. diff --git a/libs/libcommon/include/common/demangle.h b/libs/libcommon/include/common/demangle.h index d47fa290179..e3772b4ce01 100644 --- a/libs/libcommon/include/common/demangle.h +++ b/libs/libcommon/include/common/demangle.h @@ -14,6 +14,7 @@ #pragma once +#include #include #include From 55c2b6810c174be0ecf29621248f5847c720da2f Mon Sep 17 00:00:00 2001 From: Zhigao Tong Date: Tue, 22 Sep 2026 13:38:02 +0800 Subject: [PATCH 02/32] fts: support nullable MATCH columns in TiFlash --- .../Flash/Planner/Plans/PhysicalTableScan.cpp | 19 ++ dbms/src/Functions/FunctionsFullText.cpp | 168 +++++++++++++----- dbms/src/Functions/tests/gtest_fulltext.cpp | 29 +++ 3 files changed, 170 insertions(+), 46 deletions(-) diff --git a/dbms/src/Flash/Planner/Plans/PhysicalTableScan.cpp b/dbms/src/Flash/Planner/Plans/PhysicalTableScan.cpp index 8da0af6a565..b9aa22f1ac8 100644 --- a/dbms/src/Flash/Planner/Plans/PhysicalTableScan.cpp +++ b/dbms/src/Flash/Planner/Plans/PhysicalTableScan.cpp @@ -57,6 +57,8 @@ tipb::Expr buildFTSExpression(const TiDBTableScan & table_scan, Int32 result_typ expression.mutable_field_type()->set_flag(TiDB::ColumnFlagNotNull); *expression.add_children() = constructStringLiteralTiExpr(query_info.query_text()); + bool fts_collation_initialized = false; + bool fts_result_nullable = false; for (const auto & query_column : query_info.columns()) { const auto column_id = query_column.column_id(); @@ -80,9 +82,26 @@ tipb::Expr buildFTSExpression(const TiDBTableScan & table_scan, Int32 result_typ encodeDAGInt64(column_index, ss); column_ref.set_val(ss.releaseStr()); *column_ref.mutable_field_type() = TiDB::columnInfoToFieldType(*column_info); + fts_result_nullable = fts_result_nullable || !column_info->hasNotNullFlag(); + // FTS returns a numeric score, but its string matching semantics are + // determined by the MATCH column collation. Carry the first column's + // protocol collation on the scalar expression so the DAG analyzer + // passes the same collator to fts_match_expression as it does for + // LIKE and comparison functions. + if (!fts_collation_initialized && column_ref.field_type().collate() != 0) + { + expression.mutable_field_type()->set_collate(column_ref.field_type().collate()); + fts_collation_initialized = true; + } *expression.add_children() = std::move(column_ref); } + // MATCH returns NULL when any input column is NULL. Keep the result type + // nullable so the analyzer does not insert a cast from Nullable(Float64) + // to Float64, which would fail at runtime for a NULL row. + if (fts_result_nullable) + expression.mutable_field_type()->set_flag(0); + if (query_func == tipb::ScalarFuncSig::FTSMatchExpression && query_info.has_boolean_query()) { *expression.add_children() diff --git a/dbms/src/Functions/FunctionsFullText.cpp b/dbms/src/Functions/FunctionsFullText.cpp index 65adbd34991..407d6ef2c2e 100644 --- a/dbms/src/Functions/FunctionsFullText.cpp +++ b/dbms/src/Functions/FunctionsFullText.cpp @@ -27,6 +27,7 @@ #include #include #include +#include #include #include @@ -89,7 +90,7 @@ std::pair decodeCodePoint(std::string_view text, size_t offset) return {decoded.first, decoded.second}; } -std::vector tokenizeText(std::string_view text) +std::vector tokenizeText(std::string_view text, const TiDB::TiDBCollatorPtr & collator = nullptr) { std::vector result; size_t position = 0; @@ -112,10 +113,13 @@ std::vector tokenizeText(std::string_view text) i += length; } - // The TiDB analyzer lower-cases after tokenization. Poco uses the - // same Unicode tables for the C++ implementation, while also - // preserving the ASCII fast path used by the old evaluator. - token = Poco::UTF8::toLower(token); + // If a collation is present, retain the source spelling and let the + // collator decide case and accent equivalence during matching. This + // is important for binary collations, where lower-casing would make + // a case-sensitive MATCH unexpectedly case-insensitive. Keep the + // legacy lower-case behavior for callers without collation metadata. + if (!collator) + token = Poco::UTF8::toLower(token); result.push_back({std::move(token), position++}); } return result; @@ -127,13 +131,15 @@ bool isDefaultStopword(const String & token) "a", "about", "an", "are", "as", "at", "be", "by", "com", "de", "en", "for", "from", "how", "i", "in", "is", "it", "la", "of", "on", "or", "that", "the", "this", "to", "was", "what", "when", "where", "who", "will", "with", "und", "www"}; - return stopwords.contains(token); + // Stopwords are stored in canonical lower-case form. Keep their existing + // case-insensitive behavior independently from document matching. + return stopwords.contains(Poco::UTF8::toLower(token)); } -std::vector analyzeText(std::string_view text) +std::vector analyzeText(std::string_view text, const TiDB::TiDBCollatorPtr & collator = nullptr) { std::vector result; - for (auto & token : tokenizeText(text)) + for (auto & token : tokenizeText(text, collator)) { const auto code_points = UTF8::countCodePoints( reinterpret_cast(token.text.data()), @@ -145,12 +151,43 @@ std::vector analyzeText(std::string_view text) return result; } +bool textEquals(std::string_view lhs, std::string_view rhs, const TiDB::TiDBCollatorPtr & collator) +{ + if (!collator) + return lhs == rhs; + return collator->compare(lhs.data(), lhs.size(), rhs.data(), rhs.size()) == 0; +} + +bool textStartsWith(std::string_view value, std::string_view prefix, const TiDB::TiDBCollatorPtr & collator) +{ + if (!collator) + return value.starts_with(prefix); + + // Reuse the same collation-aware pattern implementation as LIKE. Escape + // token characters that have LIKE meaning before appending the wildcard. + String pattern; + pattern.reserve(prefix.size() + 1); + for (const char c : prefix) + { + if (c == '\\' || c == '%' || c == '_') + pattern.push_back('\\'); + pattern.push_back(c); + } + pattern.push_back('%'); + auto matcher = collator->pattern(); + matcher->compile(pattern, '\\'); + return matcher->match(value.data(), value.size()); +} + bool isBooleanWhitespace(char c) { return c == ' ' || c == '\t' || c == '\n' || c == '\r'; } -bool parseBooleanQuery(std::string_view query, std::vector & clauses) +bool parseBooleanQuery( + std::string_view query, + std::vector & clauses, + const TiDB::TiDBCollatorPtr & collator = nullptr) { for (size_t i = 0; i < query.size();) { @@ -215,7 +252,7 @@ bool parseBooleanQuery(std::string_view query, std::vector & clau } } - const auto terms = clause.prefix ? tokenizeText(raw) : analyzeText(raw); + const auto terms = clause.prefix ? tokenizeText(raw, collator) : analyzeText(raw, collator); if (clause.prefix && terms.size() != 1) continue; @@ -239,7 +276,10 @@ bool parseBooleanQuery(std::string_view query, std::vector & clau return true; } -bool matchesPhraseInColumn(const BooleanClause & clause, const FullTextColumn & document) +bool matchesPhraseInColumn( + const BooleanClause & clause, + const FullTextColumn & document, + const TiDB::TiDBCollatorPtr & collator) { if (clause.terms.empty() || clause.terms.size() != clause.offsets.size()) return false; @@ -253,7 +293,7 @@ bool matchesPhraseInColumn(const BooleanClause & clause, const FullTextColumn & const auto it = std::find_if(document.begin(), document.end(), [&](const FullTextToken & token) { return token.position == expected_position; }); - if (it == document.end() || it->text != clause.terms[i]) + if (it == document.end() || !textEquals(it->text, clause.terms[i], collator)) { matched = false; break; @@ -265,28 +305,34 @@ bool matchesPhraseInColumn(const BooleanClause & clause, const FullTextColumn & return false; } -bool matchesClause(const BooleanClause & clause, const FullTextDocument & document) +bool matchesClause( + const BooleanClause & clause, + const FullTextDocument & document, + const TiDB::TiDBCollatorPtr & collator) { if (clause.terms.empty()) return false; if (clause.phrase) return std::any_of(document.begin(), document.end(), [&](const FullTextColumn & column) { - return matchesPhraseInColumn(clause, column); + return matchesPhraseInColumn(clause, column, collator); }); return std::all_of(clause.terms.begin(), clause.terms.end(), [&](const String & term) { return std::any_of(document.begin(), document.end(), [&](const FullTextColumn & column) { return std::any_of(column.begin(), column.end(), [&](const FullTextToken & token) { - return clause.prefix ? token.text.starts_with(term) : token.text == term; + return clause.prefix ? textStartsWith(token.text, term, collator) : textEquals(token.text, term, collator); }); }); }); } -size_t countClauseMatches(const BooleanClause & clause, const FullTextDocument & document) +size_t countClauseMatches( + const BooleanClause & clause, + const FullTextDocument & document, + const TiDB::TiDBCollatorPtr & collator) { - if (!matchesClause(clause, document)) + if (!matchesClause(clause, document, collator)) return 0; if (!clause.phrase) @@ -297,7 +343,7 @@ size_t countClauseMatches(const BooleanClause & clause, const FullTextDocument & for (const auto & column : document) { count += std::count_if(column.begin(), column.end(), [&](const FullTextToken & token) { - return clause.prefix ? token.text.starts_with(term) : token.text == term; + return clause.prefix ? textStartsWith(token.text, term, collator) : textEquals(token.text, term, collator); }); } } @@ -316,7 +362,7 @@ size_t countClauseMatches(const BooleanClause & clause, const FullTextDocument & const auto it = std::find_if(column.begin(), column.end(), [&](const FullTextToken & token) { return token.position == expected_position; }); - if (it == column.end() || it->text != clause.terms[i]) + if (it == column.end() || !textEquals(it->text, clause.terms[i], collator)) { matched = false; break; @@ -329,12 +375,15 @@ size_t countClauseMatches(const BooleanClause & clause, const FullTextDocument & return count; } -FullTextColumn analyzeColumn(std::string_view document) +FullTextColumn analyzeColumn(std::string_view document, const TiDB::TiDBCollatorPtr & collator = nullptr) { - return analyzeText(document); + return analyzeText(document, collator); } -Float64 matchBooleanScore(const std::vector & clauses, const FullTextDocument & document) +Float64 matchBooleanScore( + const std::vector & clauses, + const FullTextDocument & document, + const TiDB::TiDBCollatorPtr & collator = nullptr) { if (clauses.empty()) return 0; @@ -345,8 +394,8 @@ Float64 matchBooleanScore(const std::vector & clauses, const Full Float64 score = 0; for (const auto & clause : clauses) { - const bool matched = matchesClause(clause, document); - const auto clause_score = static_cast(countClauseMatches(clause, document)); + const bool matched = matchesClause(clause, document, collator); + const auto clause_score = static_cast(countClauseMatches(clause, document, collator)); switch (clause.modifier) { case BooleanClause::Modifier::Must: @@ -376,15 +425,21 @@ Float64 matchBooleanScore(const std::vector & clauses, const Full return score > 0 ? score : 1; } -Float64 matchBooleanScore(std::string_view query, const FullTextDocument & document) +Float64 matchBooleanScore( + std::string_view query, + const FullTextDocument & document, + const TiDB::TiDBCollatorPtr & collator = nullptr) { std::vector clauses; - if (!parseBooleanQuery(query, clauses)) + if (!parseBooleanQuery(query, clauses, collator)) return 0; - return matchBooleanScore(clauses, document); + return matchBooleanScore(clauses, document, collator); } -Float64 matchBooleanScore(const tipb::FTSBooleanQuery & query, const FullTextDocument & document) +Float64 matchBooleanScore( + const tipb::FTSBooleanQuery & query, + const FullTextDocument & document, + const TiDB::TiDBCollatorPtr & collator = nullptr) { std::vector clauses; for (const auto & node : query.nodes()) @@ -412,13 +467,13 @@ Float64 matchBooleanScore(const tipb::FTSBooleanQuery & query, const FullTextDoc switch (term.term_type()) { case tipb::FTSBooleanTermType::FTSBooleanTermWord: - for (const auto & token : analyzeText(term.text())) + for (const auto & token : analyzeText(term.text(), collator)) clause.terms.push_back(token.text); break; case tipb::FTSBooleanTermType::FTSBooleanTermPrefix: { clause.prefix = true; - const auto terms = tokenizeText(term.text()); + const auto terms = tokenizeText(term.text(), collator); if (terms.size() != 1) { if (clause.modifier == BooleanClause::Modifier::Must) @@ -440,7 +495,7 @@ Float64 matchBooleanScore(const tipb::FTSBooleanQuery & query, const FullTextDoc case tipb::FTSBooleanTermType::FTSBooleanTermPhrase: { clause.phrase = true; - const auto terms = analyzeText(term.text()); + const auto terms = analyzeText(term.text(), collator); const size_t first_position = terms.empty() ? 0 : terms.front().position; for (const auto & token : terms) { @@ -467,24 +522,33 @@ Float64 matchBooleanScore(const tipb::FTSBooleanQuery & query, const FullTextDoc // The protocol path is the no-score MATCH ... AGAINST BOOLEAN MODE // predicate introduced by #70484/#70485. TiDB's local evaluator returns // a boolean 0/1 result, so do not expose term-frequency counts here. - return matchBooleanScore(clauses, document) > 0 ? 1 : 0; + return matchBooleanScore(clauses, document, collator) > 0 ? 1 : 0; } -Float64 matchBooleanScore(std::string_view query, std::string_view document) +Float64 matchBooleanScore( + std::string_view query, + std::string_view document, + const TiDB::TiDBCollatorPtr & collator = nullptr) { - return matchBooleanScore(query, FullTextDocument{analyzeColumn(document)}); + return matchBooleanScore(query, FullTextDocument{analyzeColumn(document, collator)}, collator); } -size_t countTermMatches(const String & term, const FullTextColumn & document) +size_t countTermMatches( + const String & term, + const FullTextColumn & document, + const TiDB::TiDBCollatorPtr & collator) { return std::count_if(document.begin(), document.end(), [&](const FullTextToken & token) { - return token.text == term; + return textEquals(token.text, term, collator); }); } -Float64 matchNaturalLanguageScore(std::string_view query, const FullTextDocument & document) +Float64 matchNaturalLanguageScore( + std::string_view query, + const FullTextDocument & document, + const TiDB::TiDBCollatorPtr & collator = nullptr) { - const auto query_tokens = analyzeText(query); + const auto query_tokens = analyzeText(query, collator); if (query_tokens.empty()) return 0; @@ -495,7 +559,7 @@ Float64 matchNaturalLanguageScore(std::string_view query, const FullTextDocument if (!unique_query_tokens.insert(query_token.text).second) continue; for (const auto & column : document) - score += static_cast(countTermMatches(query_token.text, column)); + score += static_cast(countTermMatches(query_token.text, column, collator)); } return score; } @@ -562,6 +626,7 @@ class FunctionFTSMatchWord final : public IFunction size_t getNumberOfArguments() const override { return 2; } bool useDefaultImplementationForConstants() const override { return false; } ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {0}; } + void setCollator(const TiDB::TiDBCollatorPtr & collator_) override { collator = collator_; } DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override { @@ -607,7 +672,7 @@ class FunctionFTSMatchWord final : public IFunction if (isNullAt(*document_column, row)) null_map_data[row] = 1; else - output_data[row] = matchBooleanScore(query, getStringAt(*document_column, row)); + output_data[row] = matchBooleanScore(query, getStringAt(*document_column, row), collator); } block.getByPosition(result).column = ColumnNullable::create(std::move(output), std::move(null_map)); return; @@ -622,14 +687,17 @@ class FunctionFTSMatchWord final : public IFunction const size_t end = offsets[row]; const size_t length = end - begin - 1; output_data[row] - = matchBooleanScore(query, std::string_view(reinterpret_cast(&chars[begin]), length)); + = matchBooleanScore( + query, + std::string_view(reinterpret_cast(&chars[begin]), length), + collator); begin = end; } } else if (const auto * document = typeid_cast(&*document_column)) { const auto value = document->getValue(); - const Float64 matched = matchBooleanScore(query, value); + const Float64 matched = matchBooleanScore(query, value, collator); std::fill(output_data.begin(), output_data.end(), matched); } else @@ -640,6 +708,9 @@ class FunctionFTSMatchWord final : public IFunction } block.getByPosition(result).column = std::move(output); } + +private: + TiDB::TiDBCollatorPtr collator; }; class FunctionFTSMatchExpression final : public IFunction @@ -653,6 +724,7 @@ class FunctionFTSMatchExpression final : public IFunction bool isVariadic() const override { return true; } bool useDefaultImplementationForConstants() const override { return false; } ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {0}; } + void setCollator(const TiDB::TiDBCollatorPtr & collator_) override { collator = collator_; } DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override { @@ -720,14 +792,15 @@ class FunctionFTSMatchExpression final : public IFunction document.emplace_back(); continue; } - document.push_back(analyzeColumn(getStringAt(*block.getByPosition(arguments[arg]).column, row))); + document.push_back( + analyzeColumn(getStringAt(*block.getByPosition(arguments[arg]).column, row), collator)); } if (has_protocol_boolean_query) - output_data[row] = matchBooleanScore(protocol_boolean_query, document); + output_data[row] = matchBooleanScore(protocol_boolean_query, document, collator); else if (queryUsesBooleanSyntax(query)) - output_data[row] = matchBooleanScore(query, document); + output_data[row] = matchBooleanScore(query, document, collator); else - output_data[row] = matchNaturalLanguageScore(query, document); + output_data[row] = matchNaturalLanguageScore(query, document, collator); } ColumnPtr result_column; if (null_map) @@ -736,6 +809,9 @@ class FunctionFTSMatchExpression final : public IFunction result_column = std::move(output); block.getByPosition(result).column = std::move(result_column); } + +private: + TiDB::TiDBCollatorPtr collator; }; } diff --git a/dbms/src/Functions/tests/gtest_fulltext.cpp b/dbms/src/Functions/tests/gtest_fulltext.cpp index 89eedffeda5..533c29ecf27 100644 --- a/dbms/src/Functions/tests/gtest_fulltext.cpp +++ b/dbms/src/Functions/tests/gtest_fulltext.cpp @@ -14,6 +14,7 @@ #include #include +#include #include #include @@ -153,6 +154,34 @@ try } CATCH +TEST_F(TestFullText, MatchExpressionCollation) +try +{ + const auto ci_collator = TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_GENERAL_CI); + ASSERT_COLUMN_EQ( + createColumn({1, 1}), + executeFunction( + "fts_match_expression", + {createConstColumn(2, "+quick"), createColumn({"QUICK runner", "quick runner"})}, + ci_collator)); + + const auto binary_collator = TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_BIN); + ASSERT_COLUMN_EQ( + createColumn({0, 1}), + executeFunction( + "fts_match_expression", + {createConstColumn(2, "+quick"), createColumn({"QUICK runner", "quick runner"})}, + binary_collator)); + + ASSERT_COLUMN_EQ( + createColumn({1, 0}), + executeFunction( + "fts_match_expression", + {createConstColumn(2, "+run*"), createColumn({"RUNNER", "walk"})}, + ci_collator)); +} +CATCH + TEST_F(TestFullText, MatchExpressionNullColumn) try { From b6f1477fa184f51533af6ac3adf55743c15374fb Mon Sep 17 00:00:00 2001 From: Zhigao Tong Date: Tue, 22 Sep 2026 13:44:23 +0800 Subject: [PATCH 03/32] docs: update TiFlash MATCH AGAINST design --- .../2026-09-18-match-against-tiflash.md | 173 +++++++++++++++--- 1 file changed, 148 insertions(+), 25 deletions(-) diff --git a/docs/design/2026-09-18-match-against-tiflash.md b/docs/design/2026-09-18-match-against-tiflash.md index 93c0fd9aafb..efbe46971d5 100644 --- a/docs/design/2026-09-18-match-against-tiflash.md +++ b/docs/design/2026-09-18-match-against-tiflash.md @@ -1,11 +1,30 @@ # TiFlash MATCH AGAINST pushdown +## Implementation status + +This document describes the implementation currently on the `match_against` +branch. The verified TiFlash commit is `55c2b6810c`: +`fts: support nullable MATCH columns in TiFlash`. + +The delivery target is the release-8.5 family and the supported product +surface is: + +```sql +MATCH(col) AGAINST('+tidb -mysql' IN BOOLEAN MODE) +``` + +The implementation is a snapshot-local scan evaluator. It does not require a +FULLTEXT index and it is not an inverted-index implementation. TiFlash is +used when a table has an available replica; without a TiFlash replica, TiDB +uses its local evaluator through the normal TiKV read path. + ## Introduction This document describes the TiFlash-side implementation for evaluating TiDB `MATCH ... AGAINST` and `FTS_MATCH_WORD` expressions on the read snapshot. The implementation is intentionally limited to the existing release-8.5 -TiDB-to-TiFlash protocol. TiKV, TICI, and TiDB-side changes are out of scope. +TiDB-to-TiFlash protocol. TiKV, TICI, and TiDB-side changes are out of scope +for this TiFlash change. ## Motivation or Background @@ -20,24 +39,37 @@ index. ### Protocol and planner path -The `contrib/tipb` submodule is advanced to the upstream protocol revision that -contains `ScalarFuncSig_FTSMatchWord`, `ScalarFuncSig_FTSMatchExpression`, -`FTSQueryInfo`, and `used_columnar_indexes`. +The `contrib/tipb` submodule is pinned to the FTS protocol commit +`0a9c803d4e` on the `release-8.5-match-against` line. The protocol carries +`ScalarFuncSig_FTSMatchWord`, `ScalarFuncSig_FTSMatchExpression`, +`FTSQueryInfo`, `FTSBooleanQuery`, and `used_columnar_indexes`. TiFlash maps the two scalar signatures as follows: | tipb signature | TiFlash function | semantics | | --- | --- | --- | -| `FTSMatchWord` | `fts_match_word` | Boolean-mode filter evaluator | +| `FTSMatchWord` | `fts_match_word` | Legacy one-column full-text evaluator | | `FTSMatchExpression` | `fts_match_expression` | Boolean-aware evaluator over MATCH columns | -For an index-style `FTSQueryInfo`, `PhysicalTableScan` reconstructs a -`fts_match_word(query, column)` filter and merges it with ordinary Selection -conditions. For `FTSQueryTypeWithScore`, it evaluates the same expression after -the scan and replaces the `_FTS_SCORE` generated-column placeholder with the -resulting `FLOAT` column. This keeps the existing table-scan schema and lets -Projection, Filter, and TopN consume the same score column. The expression -pipeline runs on the same read snapshot as the row data. +For the #70484/#70485 boolean predicate, TiDB serializes a `FTSQueryInfo` +into the table scan. `TiDBTableScan` extracts it and `PhysicalTableScan` +builds an `fts_match_expression` filter containing: + +1. the original query text; +2. MATCH column references and their TiDB field types; +3. the encoded `FTSBooleanQuery` containing required and prohibited terms. + +The filter is attached to the TiFlash table-scan pipeline rather than added +as a root TiDB Selection. The expression carries the first MATCH column's +protocol collation to the TiFlash function. The score path still supports the +existing `_FTS_SCORE` placeholder, but score semantics are outside the +boolean-mode delivery target. + +The expression pipeline runs on the same read snapshot as the row data. The +FTS query is evaluated by the scan executor after rows are read; it is not +used by the DeltaMerge rough-set index. Consequently, current logs may report +`FTSMatchExpression is not supported` for rough-set pruning while the MATCH +predicate itself is still evaluated successfully by the TiFlash scan. ### Analyzer and matcher @@ -45,7 +77,7 @@ The evaluator implements the STANDARD_V1-compatible Boolean matching subset used by #70484/#70485: - Unicode letter/number token runs with `_` preserved; -- Unicode lower-casing; +- Unicode lower-casing when no protocol collator is available; - default token length range 3..84; - the default InnoDB stopword set; - Boolean `+` required terms, `-` prohibited terms, quoted phrases, and a @@ -55,14 +87,38 @@ used by #70484/#70485: - NULL MATCH columns contribute no tokens and do not nullify the whole row. The predicate path consumes the numeric result as a Boolean value (`0` means -no match, a positive value means match). Queries containing Boolean syntax use -the Boolean matcher; plain queries retain the existing token-overlap behavior -for compatibility with the current scalar signature. The implementation -still returns a deterministic positive term-frequency value for the existing -`_FTS_SCORE` placeholder path, but that score is not part of the -#70484/#70485 acceptance scope and is not claimed to be MySQL/InnoDB -relevance. Nullable query arguments preserve NULL; nullable MATCH columns -contribute no tokens. +no match, a positive value means match). Queries containing Boolean syntax +use the Boolean matcher; plain queries retain the existing token-overlap +behavior for compatibility with the current scalar signature. The +implementation still returns a deterministic positive term-frequency value +for the existing `_FTS_SCORE` placeholder path, but that score is not part of +the #70484/#70485 acceptance scope and is not claimed to be MySQL/InnoDB +relevance. Nullable query arguments preserve NULL. + +### Collation handling + +When a protocol collation is available, TiFlash preserves the source spelling +of tokens and compares terms through the TiDB collator. This makes matching +case and accent behavior follow the column collation. Prefix matching reuses +the same collator-aware comparison path. + +The local TiDB fallback is a separate implementation. The current fallback +still lower-cases tokens and therefore does not fully preserve +`utf8mb4_bin` case sensitivity. This is a TiDB-side consistency limitation; +it is not changed by the TiFlash implementation. + +### Nullable columns + +`FunctionsFullText.cpp` handles nullable string columns by treating a NULL +MATCH column as an empty document. The table-scan expression must nevertheless +declare a nullable result type: otherwise the DAG analyzer inserts a cast from +`Nullable(Float64)` to `Float64` and a NULL row fails with +`Cannot convert NULL value to non-Nullable type`. + +`PhysicalTableScan` now clears `ColumnFlagNotNull` when any MATCH column is +nullable. This keeps the planner metadata aligned with the function return +type and lets a nullable column be evaluated as a normal no-match row in a +WHERE predicate. ### Consistency boundary @@ -79,7 +135,66 @@ provide corpus-wide IDF normalization. `dbms/src/Functions/tests/gtest_fulltext.cpp` covers required/prohibited terms, phrases with stopword gaps, prefixes, word boundaries, nullable input, NULL -MATCH columns, and unsupported score modifiers. +MATCH columns, collation behavior, and unsupported score modifiers. + +### Local E2E validation + +The local E2E test used TiUP playground v8.5.8 with locally built TiDB and +TiFlash binaries. The TiFlash replica was waited to `AVAILABLE=1` and +`PROGRESS=1` before querying. + +The main query: + +```sql +SELECT id, body +FROM articles +WHERE MATCH(body) AGAINST('+tidb -mysql' IN BOOLEAN MODE) +ORDER BY id; +``` + +returned rows `2`, `4`, and `6`. Its plan was: + +```text +TableReader +└─ExchangeSender mpp[tiflash] + └─TableFullScan mpp[tiflash] +``` + +There was no root `Selection(match_against(...))`. `EXPLAIN ANALYZE` showed +three actual output rows from the TiFlash task. + +The currently supported TiDB collations were tested on TiFlash: + +- `ascii_bin` +- `latin1_bin` +- `utf8_bin` +- `utf8_general_ci` +- `utf8_unicode_ci` +- `utf8mb4_bin` +- `utf8mb4_general_ci` +- `utf8mb4_unicode_ci` +- `utf8mb4_0900_bin` +- `utf8mb4_0900_ai_ci` + +The matrix covered exact terms, `*` prefixes, case behavior, and +`cafe`/`café` accent behavior. Binary collations matched only the exact case +and spelling; CI/AI collations matched the expected case or accent variants. + +The nullable regression test used a `utf8mb4_bin NULL` column containing +`quick runner`, `NULL`, and `QUICK runner`. The query returned only +`quick runner`, used `mpp[tiflash]`, and no longer raised the NULL conversion +error. + +The no-replica test returned successfully through: + +```text +Selection +└─TableReader + └─TableFullScan cop[tikv] +``` + +This confirms the TiDB fallback path remains available when TiFlash cannot be +used. ### Compatibility Tests @@ -101,6 +216,12 @@ sysvars, custom stopword lists, NGRAM_V1, and non-default `AGAINST` modifiers are not represented in the current wire contract and must not be treated as supported by this phase. +The scan evaluator currently does not perform inverted-index lookup or +rough-set pruning for FTS. Large tables may therefore scan many rows even +when the final MATCH result is small. Chinese/CJK tokenization is also not +claimed by this phase; the tested STANDARD_V1 sample produced no match for a +Chinese term on both TiFlash and the TiDB fallback. + ## Investigation & Alternatives An asynchronous external index was rejected because it cannot provide the @@ -108,8 +229,10 @@ required snapshot consistency. A native DeltaMerge full-text index remains a long-term performance design, but it requires a larger write-path, snapshot, and index-reader change set. -## Unresolved Questions +## Remaining work -If exact corpus-wide ranking is required, a future phase must define a -TiDB-compatible relevance-score contract and add a native DeltaMerge full-text +Before product release, the paired TiDB branch and TiFlash branch still need +CI coverage, release-8.5 build verification, and code review. If exact +corpus-wide ranking or index-level performance is required, a future phase +must define a relevance-score contract and add a native DeltaMerge full-text index. From 6405b452c14d09619cc6fc53933e25046ba222a8 Mon Sep 17 00:00:00 2001 From: Zhigao Tong Date: Tue, 22 Sep 2026 14:51:10 +0800 Subject: [PATCH 04/32] docs: refresh MATCH AGAINST E2E status --- .../2026-09-18-match-against-tiflash.md | 41 +++++++++++-------- 1 file changed, 25 insertions(+), 16 deletions(-) diff --git a/docs/design/2026-09-18-match-against-tiflash.md b/docs/design/2026-09-18-match-against-tiflash.md index efbe46971d5..be8e4cc8944 100644 --- a/docs/design/2026-09-18-match-against-tiflash.md +++ b/docs/design/2026-09-18-match-against-tiflash.md @@ -3,8 +3,9 @@ ## Implementation status This document describes the implementation currently on the `match_against` -branch. The verified TiFlash commit is `55c2b6810c`: -`fts: support nullable MATCH columns in TiFlash`. +branch. The verified TiFlash commit is `b6f1477fa1` on `origin/match_against`. +It includes the Boolean pushdown implementation, nullable MATCH-column +handling, and the current design-document update. The delivery target is the release-8.5 family and the supported product surface is: @@ -13,10 +14,12 @@ surface is: MATCH(col) AGAINST('+tidb -mysql' IN BOOLEAN MODE) ``` -The implementation is a snapshot-local scan evaluator. It does not require a -FULLTEXT index and it is not an inverted-index implementation. TiFlash is -used when a table has an available replica; without a TiFlash replica, TiDB -uses its local evaluator through the normal TiKV read path. +The implementation is a snapshot-local scan evaluator. TiFlash does not read +or maintain a physical FULLTEXT inverted index in this phase. TiDB still +requires a public FULLTEXT index as planner metadata and as the source of the +parser configuration. TiFlash is used when a table has an available replica; +without a TiFlash replica, TiDB uses its local evaluator through the normal +TiKV read path. ## Introduction @@ -102,10 +105,11 @@ of tokens and compares terms through the TiDB collator. This makes matching case and accent behavior follow the column collation. Prefix matching reuses the same collator-aware comparison path. -The local TiDB fallback is a separate implementation. The current fallback -still lower-cases tokens and therefore does not fully preserve -`utf8mb4_bin` case sensitivity. This is a TiDB-side consistency limitation; -it is not changed by the TiFlash implementation. +The local TiDB fallback is a separate implementation, but it now receives the +MATCH column collation in its analyzer configuration and uses the same +collator-aware token, prefix, and phrase comparisons. The fallback therefore +preserves the tested `utf8mb4_bin`, `utf8mb4_general_ci`, and +`utf8mb4_0900_ai_ci` behavior when TiFlash is unavailable. ### Nullable columns @@ -152,7 +156,8 @@ WHERE MATCH(body) AGAINST('+tidb -mysql' IN BOOLEAN MODE) ORDER BY id; ``` -returned rows `2`, `4`, and `6`. Its plan was: +returned row `2` with the data set used for the release-8.5 verification. Its +plan was: ```text TableReader @@ -161,7 +166,7 @@ TableReader ``` There was no root `Selection(match_against(...))`. `EXPLAIN ANALYZE` showed -three actual output rows from the TiFlash task. +five rows scanned and one row output by the TiFlash task. The currently supported TiDB collations were tested on TiFlash: @@ -194,7 +199,9 @@ Selection ``` This confirms the TiDB fallback path remains available when TiFlash cannot be -used. +used and returns the same result as the native path. The fallback E2E also +verified that `utf8mb4_bin` distinguishes case while CI/AI collations match +the expected case and accent variants. ### Compatibility Tests @@ -218,9 +225,11 @@ supported by this phase. The scan evaluator currently does not perform inverted-index lookup or rough-set pruning for FTS. Large tables may therefore scan many rows even -when the final MATCH result is small. Chinese/CJK tokenization is also not -claimed by this phase; the tested STANDARD_V1 sample produced no match for a -Chinese term on both TiFlash and the TiDB fallback. +when the final MATCH result is small. Chinese/CJK behavior follows the +STANDARD_V1 tokenization and default minimum token length: the tested +`+数据库` query matched standalone `数据库` tokens, while the two-character +`+中文` query was filtered by the default minimum token length. This phase +does not claim general Chinese linguistic segmentation. ## Investigation & Alternatives From 0ca2f3f33630c6fe948b1dd214e8136d8fb5367b Mon Sep 17 00:00:00 2001 From: Zhigao Tong Date: Wed, 23 Sep 2026 13:28:02 +0800 Subject: [PATCH 05/32] fts: support NGRAM boolean pushdown --- contrib/tipb | 2 +- .../Flash/Planner/Plans/PhysicalTableScan.cpp | 2 +- dbms/src/Functions/FunctionsFullText.cpp | 134 ++++++++++++++++-- dbms/src/Functions/tests/gtest_fulltext.cpp | 46 +++++- .../2026-09-18-match-against-tiflash.md | 54 ++++--- 5 files changed, 204 insertions(+), 34 deletions(-) diff --git a/contrib/tipb b/contrib/tipb index 0a9c803d4e7..26695da26ed 160000 --- a/contrib/tipb +++ b/contrib/tipb @@ -1 +1 @@ -Subproject commit 0a9c803d4e7bf37a7d67a3e74609b8dd56399d09 +Subproject commit 26695da26ed8f8d6347f74a4f1f2731ed4d22cfe diff --git a/dbms/src/Flash/Planner/Plans/PhysicalTableScan.cpp b/dbms/src/Flash/Planner/Plans/PhysicalTableScan.cpp index b9aa22f1ac8..d64b8b4d606 100644 --- a/dbms/src/Flash/Planner/Plans/PhysicalTableScan.cpp +++ b/dbms/src/Flash/Planner/Plans/PhysicalTableScan.cpp @@ -37,7 +37,7 @@ namespace DB { namespace { -constexpr std::string_view fts_boolean_query_marker = "__tiflash_fts_boolean_query__:"; +constexpr std::string_view fts_boolean_query_marker = "__tiflash_fts_bool_query__:"; tipb::Expr buildFTSExpression(const TiDBTableScan & table_scan, Int32 result_type) { diff --git a/dbms/src/Functions/FunctionsFullText.cpp b/dbms/src/Functions/FunctionsFullText.cpp index 407d6ef2c2e..3df1ae5ca41 100644 --- a/dbms/src/Functions/FunctionsFullText.cpp +++ b/dbms/src/Functions/FunctionsFullText.cpp @@ -49,7 +49,9 @@ namespace { constexpr size_t default_min_token_size = 3; constexpr size_t default_max_token_size = 84; -constexpr std::string_view fts_boolean_query_marker = "__tiflash_fts_boolean_query__:"; +constexpr size_t default_ngram_token_size = 2; +constexpr std::string_view ngram_parser = "NGRAM_V1"; +constexpr std::string_view fts_boolean_query_marker = "__tiflash_fts_bool_query__:"; struct FullTextToken { @@ -151,6 +153,48 @@ std::vector analyzeText(std::string_view text, const TiDB::TiDBCo return result; } +std::vector analyzeNgramText( + std::string_view text, + size_t ngram_token_size, + const TiDB::TiDBCollatorPtr & collator = nullptr) +{ + if (ngram_token_size == 0) + return {}; + + std::vector result; + size_t next_position_base = 0; + for (auto & token : tokenizeText(text, collator)) + { + std::vector char_boundaries; + char_boundaries.reserve(token.text.size() + 1); + char_boundaries.push_back(0); + for (size_t offset = 0; offset < token.text.size();) + { + const auto [code_point, length] = decodeCodePoint(token.text, offset); + (void)code_point; + offset += length; + char_boundaries.push_back(offset); + } + + const size_t char_count = char_boundaries.size() - 1; + const size_t base_position = std::max(token.position, next_position_base); + if (char_count < ngram_token_size) + { + next_position_base = std::max(next_position_base, token.position + 1); + continue; + } + + for (size_t start = 0; start + ngram_token_size <= char_count; ++start) + { + const size_t begin = char_boundaries[start]; + const size_t end = char_boundaries[start + ngram_token_size]; + result.push_back({token.text.substr(begin, end - begin), base_position + start}); + } + next_position_base = base_position + char_count - ngram_token_size + 1; + } + return result; +} + bool textEquals(std::string_view lhs, std::string_view rhs, const TiDB::TiDBCollatorPtr & collator) { if (!collator) @@ -380,6 +424,15 @@ FullTextColumn analyzeColumn(std::string_view document, const TiDB::TiDBCollator return analyzeText(document, collator); } +FullTextColumn analyzeColumn( + std::string_view document, + bool use_ngram, + size_t ngram_token_size, + const TiDB::TiDBCollatorPtr & collator = nullptr) +{ + return use_ngram ? analyzeNgramText(document, ngram_token_size, collator) : analyzeText(document, collator); +} + Float64 matchBooleanScore( const std::vector & clauses, const FullTextDocument & document, @@ -422,7 +475,42 @@ Float64 matchBooleanScore( return 0; // A query containing only prohibited terms has no positive term to score, // but an accepted row still has to pass the boolean filter. - return score > 0 ? score : 1; + return score > 0 ? score : 1; +} + +bool matchBooleanPredicate( + const std::vector & clauses, + const FullTextDocument & document, + const TiDB::TiDBCollatorPtr & collator = nullptr) +{ + if (clauses.empty()) + return false; + + bool has_positive = false; + bool has_must = false; + bool positive_match = false; + for (const auto & clause : clauses) + { + const bool matched = matchesClause(clause, document, collator); + switch (clause.modifier) + { + case BooleanClause::Modifier::Must: + has_must = true; + if (!matched) + return false; + break; + case BooleanClause::Modifier::MustNot: + if (matched) + return false; + break; + case BooleanClause::Modifier::Should: + has_positive = true; + positive_match = positive_match || matched; + break; + } + } + + return has_must || !has_positive || positive_match; } Float64 matchBooleanScore( @@ -442,6 +530,11 @@ Float64 matchBooleanScore( const TiDB::TiDBCollatorPtr & collator = nullptr) { std::vector clauses; + const bool use_ngram = query.query_tokenizer() == ngram_parser; + const size_t ngram_token_size = query.ngram_token_size() == 0 ? default_ngram_token_size : query.ngram_token_size(); + auto analyze_query = [&](std::string_view text) { + return use_ngram ? analyzeNgramText(text, ngram_token_size, collator) : analyzeText(text, collator); + }; for (const auto & node : query.nodes()) { if (!node.has_term()) @@ -467,13 +560,30 @@ Float64 matchBooleanScore( switch (term.term_type()) { case tipb::FTSBooleanTermType::FTSBooleanTermWord: - for (const auto & token : analyzeText(term.text(), collator)) - clause.terms.push_back(token.text); + { + const auto terms = analyze_query(term.text()); + if (use_ngram) + { + clause.phrase = true; + const size_t first_position = terms.empty() ? 0 : terms.front().position; + for (const auto & token : terms) + { + clause.terms.push_back(token.text); + clause.offsets.push_back(token.position - first_position); + } + } + else + { + for (const auto & token : terms) + clause.terms.push_back(token.text); + } break; + } case tipb::FTSBooleanTermType::FTSBooleanTermPrefix: { clause.prefix = true; - const auto terms = tokenizeText(term.text(), collator); + const auto terms = use_ngram ? analyzeNgramText(term.text(), ngram_token_size, collator) + : tokenizeText(term.text(), collator); if (terms.size() != 1) { if (clause.modifier == BooleanClause::Modifier::Must) @@ -495,7 +605,7 @@ Float64 matchBooleanScore( case tipb::FTSBooleanTermType::FTSBooleanTermPhrase: { clause.phrase = true; - const auto terms = analyzeText(term.text(), collator); + const auto terms = analyze_query(term.text()); const size_t first_position = terms.empty() ? 0 : terms.front().position; for (const auto & token : terms) { @@ -522,7 +632,7 @@ Float64 matchBooleanScore( // The protocol path is the no-score MATCH ... AGAINST BOOLEAN MODE // predicate introduced by #70484/#70485. TiDB's local evaluator returns // a boolean 0/1 result, so do not expose term-frequency counts here. - return matchBooleanScore(clauses, document, collator) > 0 ? 1 : 0; + return matchBooleanPredicate(clauses, document, collator) ? 1 : 0; } Float64 matchBooleanScore( @@ -777,6 +887,10 @@ class FunctionFTSMatchExpression final : public IFunction && decodeFTSBooleanQuery(*block.getByPosition(arguments.back()).column, protocol_boolean_query); if (has_protocol_boolean_query) --document_argument_end; + const bool use_ngram = has_protocol_boolean_query && protocol_boolean_query.query_tokenizer() == ngram_parser; + const size_t ngram_token_size = !use_ngram || protocol_boolean_query.ngram_token_size() == 0 + ? default_ngram_token_size + : protocol_boolean_query.ngram_token_size(); for (size_t row = 0; row < rows; ++row) { @@ -793,7 +907,11 @@ class FunctionFTSMatchExpression final : public IFunction continue; } document.push_back( - analyzeColumn(getStringAt(*block.getByPosition(arguments[arg]).column, row), collator)); + analyzeColumn( + getStringAt(*block.getByPosition(arguments[arg]).column, row), + use_ngram, + ngram_token_size, + collator)); } if (has_protocol_boolean_query) output_data[row] = matchBooleanScore(protocol_boolean_query, document, collator); diff --git a/dbms/src/Functions/tests/gtest_fulltext.cpp b/dbms/src/Functions/tests/gtest_fulltext.cpp index 533c29ecf27..f19a4cd5f5b 100644 --- a/dbms/src/Functions/tests/gtest_fulltext.cpp +++ b/dbms/src/Functions/tests/gtest_fulltext.cpp @@ -117,7 +117,7 @@ try prohibited->mutable_term()->set_term_type(tipb::FTSBooleanTermWord); prohibited->mutable_term()->set_text("slow"); - const String metadata = "__tiflash_fts_boolean_query__:" + boolean_query.SerializeAsString(); + const String metadata = "__tiflash_fts_bool_query__:" + boolean_query.SerializeAsString(); ASSERT_COLUMN_EQ( createColumn({1, 0, 0}), executeFunction( @@ -131,7 +131,7 @@ try prohibited_only->set_occur(tipb::FTSBooleanOccurMustNot); prohibited_only->mutable_term()->set_term_type(tipb::FTSBooleanTermWord); prohibited_only->mutable_term()->set_text("slow"); - const String prohibited_metadata = "__tiflash_fts_boolean_query__:" + prohibited_query.SerializeAsString(); + const String prohibited_metadata = "__tiflash_fts_bool_query__:" + prohibited_query.SerializeAsString(); ASSERT_COLUMN_EQ( createColumn({0, 0, 0}), executeFunction( @@ -142,6 +142,48 @@ try } CATCH +TEST_F(TestFullText, MatchExpressionNgramProtocolBooleanQuery) +try +{ + tipb::FTSBooleanQuery boolean_query; + boolean_query.set_query_tokenizer("NGRAM_V1"); + boolean_query.set_ngram_token_size(2); + auto * required = boolean_query.add_nodes(); + required->set_occur(tipb::FTSBooleanOccurMust); + required->mutable_term()->set_term_type(tipb::FTSBooleanTermWord); + required->mutable_term()->set_text("数据库"); + auto * prohibited = boolean_query.add_nodes(); + prohibited->set_occur(tipb::FTSBooleanOccurMustNot); + prohibited->mutable_term()->set_term_type(tipb::FTSBooleanTermWord); + prohibited->mutable_term()->set_text("mysql"); + + const String metadata = "__tiflash_fts_bool_query__:" + boolean_query.SerializeAsString(); + ASSERT_COLUMN_EQ( + createColumn({1, 0, 1, 0}), + executeFunction( + "fts_match_expression", + {createConstColumn(4, "+数据库 -mysql"), + createColumn({"数据库系统", "MySQL 数据库", "数据库", "数据科学"}), + createConstColumn(4, metadata)})); + + const auto ci_collator = TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_GENERAL_CI); + tipb::FTSBooleanQuery case_query; + case_query.set_query_tokenizer("NGRAM_V1"); + case_query.set_ngram_token_size(2); + auto * case_term = case_query.add_nodes(); + case_term->set_occur(tipb::FTSBooleanOccurMust); + case_term->mutable_term()->set_term_type(tipb::FTSBooleanTermWord); + case_term->mutable_term()->set_text("mysql"); + const String case_metadata = "__tiflash_fts_bool_query__:" + case_query.SerializeAsString(); + ASSERT_COLUMN_EQ( + createColumn({1, 0}), + executeFunction( + "fts_match_expression", + {createConstColumn(2, "+mysql"), createColumn({"MySQL", "PostgreSQL"}), createConstColumn(2, case_metadata)}, + ci_collator)); +} +CATCH + TEST_F(TestFullText, MatchExpressionScore) try { diff --git a/docs/design/2026-09-18-match-against-tiflash.md b/docs/design/2026-09-18-match-against-tiflash.md index be8e4cc8944..dcb60bc4bc9 100644 --- a/docs/design/2026-09-18-match-against-tiflash.md +++ b/docs/design/2026-09-18-match-against-tiflash.md @@ -3,9 +3,10 @@ ## Implementation status This document describes the implementation currently on the `match_against` -branch. The verified TiFlash commit is `b6f1477fa1` on `origin/match_against`. -It includes the Boolean pushdown implementation, nullable MATCH-column -handling, and the current design-document update. +branch. Validation was performed with the release-8.5 TiDB/TiFlash branches +and TiUP playground v8.5.8. It includes Boolean pushdown, STANDARD_V1 and +NGRAM_V1 analysis, nullable MATCH-column handling, and collation-aware +fallback behavior. The delivery target is the release-8.5 family and the supported product surface is: @@ -43,9 +44,11 @@ index. ### Protocol and planner path The `contrib/tipb` submodule is pinned to the FTS protocol commit -`0a9c803d4e` on the `release-8.5-match-against` line. The protocol carries +`26695da26e` on the `release-8.5-match-against` line. The protocol carries `ScalarFuncSig_FTSMatchWord`, `ScalarFuncSig_FTSMatchExpression`, -`FTSQueryInfo`, `FTSBooleanQuery`, and `used_columnar_indexes`. +`FTSQueryInfo`, `FTSBooleanQuery`, and `used_columnar_indexes`. `FTSBooleanQuery` +also carries `query_tokenizer` and `ngram_token_size`, so TiFlash uses the +same analyzer selected by the FULLTEXT index and TiDB session configuration. TiFlash maps the two scalar signatures as follows: @@ -76,13 +79,14 @@ predicate itself is still evaluated successfully by the TiFlash scan. ### Analyzer and matcher -The evaluator implements the STANDARD_V1-compatible Boolean matching subset +The evaluator implements the STANDARD_V1 and NGRAM_V1 Boolean matching subset used by #70484/#70485: -- Unicode letter/number token runs with `_` preserved; -- Unicode lower-casing when no protocol collator is available; -- default token length range 3..84; -- the default InnoDB stopword set; +- Both analyzers recognize Unicode letter/number token runs with `_` preserved; +- STANDARD_V1 uses Unicode lower-casing when no protocol collator is available, + a default token length range of 3..84, and the default InnoDB stopword set; +- NGRAM_V1 emits fixed-size Unicode code-point ngrams. The token size is read + from `ngram_token_size`, whose release-8.5 default is 2; - Boolean `+` required terms, `-` prohibited terms, quoted phrases, and a trailing `*` prefix; - phrase positions are retained across analyzer filtering, so removed @@ -199,9 +203,16 @@ Selection ``` This confirms the TiDB fallback path remains available when TiFlash cannot be -used and returns the same result as the native path. The fallback E2E also -verified that `utf8mb4_bin` distinguishes case while CI/AI collations match -the expected case and accent variants. +used and returns the same result as the native path after +`tidb_enable_local_match_against` is enabled. With the session variable left +`OFF`, TiDB intentionally returns the existing error requiring either a +TiFlash FULLTEXT replica or local MATCH AGAINST evaluation. + +The NGRAM E2E used a public FULLTEXT index with `WITH PARSER NGRAM`, +`ngram_token_size = 2`, and the query `+数据库 -mysql`. TiFlash pushdown and +TiDB fallback returned identical rows under both `utf8mb4_bin` and +`utf8mb4_general_ci`; the binary collation remained case-sensitive while the +CI collation matched `mysql` and `MySQL` equivalently. ### Compatibility Tests @@ -218,18 +229,17 @@ index-accelerated search. ## Impacts & Risks The implementation adds per-row tokenization and matching CPU cost. Its score -is deterministic but is not a native MySQL relevance score. Custom analyzer -sysvars, custom stopword lists, NGRAM_V1, and non-default `AGAINST` modifiers -are not represented in the current wire contract and must not be treated as -supported by this phase. +is deterministic but is not a native MySQL relevance score. Custom stopword +lists and non-default `AGAINST` modifiers are not represented in the current +wire contract and must not be treated as supported by this phase. NGRAM_V1 is +supported for the configured `ngram_token_size`; it is still a scan evaluator, +not an inverted-index implementation. The scan evaluator currently does not perform inverted-index lookup or rough-set pruning for FTS. Large tables may therefore scan many rows even -when the final MATCH result is small. Chinese/CJK behavior follows the -STANDARD_V1 tokenization and default minimum token length: the tested -`+数据库` query matched standalone `数据库` tokens, while the two-character -`+中文` query was filtered by the default minimum token length. This phase -does not claim general Chinese linguistic segmentation. +when the final MATCH result is small. STANDARD_V1 Chinese/CJK behavior follows +its default minimum token length, while NGRAM_V1 handles CJK through fixed-size +code-point windows. This phase does not claim general linguistic segmentation. ## Investigation & Alternatives From 2d7d7c8685992567ea1022cc508fdedd05a0877a Mon Sep 17 00:00:00 2001 From: Zhigao Tong Date: Wed, 23 Sep 2026 13:55:42 +0800 Subject: [PATCH 06/32] test: cover FTS collation matrix on TiFlash --- dbms/src/Functions/tests/gtest_fulltext.cpp | 62 +++++++++++++++++++ .../2026-09-18-match-against-tiflash.md | 5 +- 2 files changed, 66 insertions(+), 1 deletion(-) diff --git a/dbms/src/Functions/tests/gtest_fulltext.cpp b/dbms/src/Functions/tests/gtest_fulltext.cpp index f19a4cd5f5b..1b708a66104 100644 --- a/dbms/src/Functions/tests/gtest_fulltext.cpp +++ b/dbms/src/Functions/tests/gtest_fulltext.cpp @@ -224,6 +224,68 @@ try } CATCH +TEST_F(TestFullText, MatchExpressionCollationMatrix) +try +{ + const auto query = createConstColumn(3, "+cafe"); + const auto documents = createColumn({"CAFE", "café", "cafe"}); + const auto prefix_query = createConstColumn(3, "+caf*"); + const auto make_metadata = [](tipb::FTSBooleanTermType term_type, const String & term) { + tipb::FTSBooleanQuery boolean_query; + boolean_query.set_query_tokenizer("STANDARD_V1"); + auto * node = boolean_query.add_nodes(); + node->set_occur(tipb::FTSBooleanOccurMust); + node->mutable_term()->set_term_type(term_type); + node->mutable_term()->set_text(term); + return String("__tiflash_fts_bool_query__:") + boolean_query.SerializeAsString(); + }; + const auto word_metadata = createConstColumn( + 3, make_metadata(tipb::FTSBooleanTermWord, "cafe")); + const auto prefix_metadata = createConstColumn( + 3, make_metadata(tipb::FTSBooleanTermPrefix, "caf")); + + const auto utf8mb4_bin = TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_BIN); + ASSERT_COLUMN_EQ( + createColumn({0, 0, 1}), + executeFunction("fts_match_expression", {query, documents, word_metadata}, utf8mb4_bin)); + ASSERT_COLUMN_EQ( + createColumn({0, 1, 1}), + executeFunction("fts_match_expression", {prefix_query, documents, prefix_metadata}, utf8mb4_bin)); + + const auto utf8mb4_0900_bin = TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_0900_BIN); + ASSERT_COLUMN_EQ( + createColumn({0, 0, 1}), + executeFunction("fts_match_expression", {query, documents, word_metadata}, utf8mb4_0900_bin)); + ASSERT_COLUMN_EQ( + createColumn({0, 1, 1}), + executeFunction("fts_match_expression", {prefix_query, documents, prefix_metadata}, utf8mb4_0900_bin)); + + const auto utf8mb4_general_ci = TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_GENERAL_CI); + ASSERT_COLUMN_EQ( + createColumn({1, 1, 1}), + executeFunction("fts_match_expression", {query, documents, word_metadata}, utf8mb4_general_ci)); + ASSERT_COLUMN_EQ( + createColumn({1, 1, 1}), + executeFunction("fts_match_expression", {prefix_query, documents, prefix_metadata}, utf8mb4_general_ci)); + + const auto utf8mb4_unicode_ci = TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_UNICODE_CI); + ASSERT_COLUMN_EQ( + createColumn({1, 1, 1}), + executeFunction("fts_match_expression", {query, documents, word_metadata}, utf8mb4_unicode_ci)); + ASSERT_COLUMN_EQ( + createColumn({1, 1, 1}), + executeFunction("fts_match_expression", {prefix_query, documents, prefix_metadata}, utf8mb4_unicode_ci)); + + const auto utf8mb4_0900_ai_ci = TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_0900_AI_CI); + ASSERT_COLUMN_EQ( + createColumn({1, 1, 1}), + executeFunction("fts_match_expression", {query, documents, word_metadata}, utf8mb4_0900_ai_ci)); + ASSERT_COLUMN_EQ( + createColumn({1, 1, 1}), + executeFunction("fts_match_expression", {prefix_query, documents, prefix_metadata}, utf8mb4_0900_ai_ci)); +} +CATCH + TEST_F(TestFullText, MatchExpressionNullColumn) try { diff --git a/docs/design/2026-09-18-match-against-tiflash.md b/docs/design/2026-09-18-match-against-tiflash.md index dcb60bc4bc9..c2e22b6fcd7 100644 --- a/docs/design/2026-09-18-match-against-tiflash.md +++ b/docs/design/2026-09-18-match-against-tiflash.md @@ -143,7 +143,10 @@ provide corpus-wide IDF normalization. `dbms/src/Functions/tests/gtest_fulltext.cpp` covers required/prohibited terms, phrases with stopword gaps, prefixes, word boundaries, nullable input, NULL -MATCH columns, collation behavior, and unsupported score modifiers. +MATCH columns, collation behavior, and unsupported score modifiers. The +`MatchExpressionCollationMatrix` case covers `utf8mb4_bin`, +`utf8mb4_0900_bin`, `utf8mb4_general_ci`, `utf8mb4_unicode_ci`, and +`utf8mb4_0900_ai_ci` for both ordinary terms and prefix terms. ### Local E2E validation From a3fca18c1e0055265c17627df6027e3aabbe9fad Mon Sep 17 00:00:00 2001 From: Zhigao Tong Date: Wed, 23 Sep 2026 19:47:25 +0800 Subject: [PATCH 07/32] build: include direct RaftLogManager dependencies --- dbms/src/Storages/KVStore/MultiRaft/Disagg/RaftLogManager.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dbms/src/Storages/KVStore/MultiRaft/Disagg/RaftLogManager.h b/dbms/src/Storages/KVStore/MultiRaft/Disagg/RaftLogManager.h index 09b6a60e5d7..3b151d8062e 100644 --- a/dbms/src/Storages/KVStore/MultiRaft/Disagg/RaftLogManager.h +++ b/dbms/src/Storages/KVStore/MultiRaft/Disagg/RaftLogManager.h @@ -18,6 +18,8 @@ #include #include +#include +#include #include namespace DB From 2145aac2e53a5bf78b0e75c119f74748d8041828 Mon Sep 17 00:00:00 2001 From: Zhigao Tong Date: Wed, 23 Sep 2026 19:47:44 +0800 Subject: [PATCH 08/32] fts: fix TiFlash MATCH evaluation edge cases --- dbms/src/Functions/FunctionsFullText.cpp | 12 +++++-- dbms/src/Functions/tests/gtest_fulltext.cpp | 21 +++++++++++++ .../2026-09-18-match-against-tiflash.md | 31 +++++++++++++++---- 3 files changed, 56 insertions(+), 8 deletions(-) diff --git a/dbms/src/Functions/FunctionsFullText.cpp b/dbms/src/Functions/FunctionsFullText.cpp index 3df1ae5ca41..e87c25925a6 100644 --- a/dbms/src/Functions/FunctionsFullText.cpp +++ b/dbms/src/Functions/FunctionsFullText.cpp @@ -289,10 +289,14 @@ bool parseBooleanQuery( if (start == i) return false; raw.assign(query.substr(start, i - start)); - if (!raw.empty() && raw.back() == '*') + // The scan above stops before `*`, so consume the wildcard here + // and attach prefix semantics to this term. Leaving it for the + // next iteration incorrectly turns `run*` into an exact `run` + // match (and makes it fail against `runner`). + if (i < query.size() && query[i] == '*') { clause.prefix = true; - raw.pop_back(); + ++i; } } @@ -833,6 +837,10 @@ class FunctionFTSMatchExpression final : public IFunction size_t getNumberOfArguments() const override { return 0; } bool isVariadic() const override { return true; } bool useDefaultImplementationForConstants() const override { return false; } + // NULL MATCH columns are empty documents, not NULL MATCH results. The + // implementation below handles nullable columns per row, so the generic + // IFunction NULL wrapper must not short-circuit the whole expression. + bool useDefaultImplementationForNulls() const override { return false; } ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {0}; } void setCollator(const TiDB::TiDBCollatorPtr & collator_) override { collator = collator_; } diff --git a/dbms/src/Functions/tests/gtest_fulltext.cpp b/dbms/src/Functions/tests/gtest_fulltext.cpp index 1b708a66104..4512ea62344 100644 --- a/dbms/src/Functions/tests/gtest_fulltext.cpp +++ b/dbms/src/Functions/tests/gtest_fulltext.cpp @@ -104,6 +104,27 @@ try } CATCH +TEST_F(TestFullText, MatchExpressionMultiColumnNullable) +try +{ + tipb::FTSBooleanQuery boolean_query; + auto * required = boolean_query.add_nodes(); + required->set_occur(tipb::FTSBooleanOccurMust); + required->mutable_term()->set_term_type(tipb::FTSBooleanTermWord); + required->mutable_term()->set_text("quick"); + + const String metadata = "__tiflash_fts_bool_query__:" + boolean_query.SerializeAsString(); + ASSERT_COLUMN_EQ( + createColumn>({1, 1, 0, 0}), + executeFunction( + "fts_match_expression", + {createConstColumn(4, "+quick"), + createColumn>({{}, "quick fox", {}, "slow fox"}), + createColumn>({"quick fox", {}, "slow fox", {}}), + createConstColumn(4, metadata)})); +} +CATCH + TEST_F(TestFullText, MatchExpressionProtocolBooleanQuery) try { diff --git a/docs/design/2026-09-18-match-against-tiflash.md b/docs/design/2026-09-18-match-against-tiflash.md index c2e22b6fcd7..fae8220581c 100644 --- a/docs/design/2026-09-18-match-against-tiflash.md +++ b/docs/design/2026-09-18-match-against-tiflash.md @@ -8,13 +8,19 @@ and TiUP playground v8.5.8. It includes Boolean pushdown, STANDARD_V1 and NGRAM_V1 analysis, nullable MATCH-column handling, and collation-aware fallback behavior. -The delivery target is the release-8.5 family and the supported product -surface is: +The delivery target is the release-8.5 family. The supported product surface +is single- or multi-column MATCH in BOOLEAN MODE, provided the TiDB planner can +resolve one matching public FULLTEXT index: ```sql MATCH(col) AGAINST('+tidb -mysql' IN BOOLEAN MODE) +MATCH(title, body) AGAINST('+tidb -mysql' IN BOOLEAN MODE) ``` +For multi-column queries, the ordered MATCH column list must exactly match the +columns of one public composite FULLTEXT index on the same table. Independent +single-column indexes do not qualify for native TiFlash pushdown. + The implementation is a snapshot-local scan evaluator. TiFlash does not read or maintain a physical FULLTEXT inverted index in this phase. TiDB still requires a public FULLTEXT index as planner metadata and as the source of the @@ -58,8 +64,11 @@ TiFlash maps the two scalar signatures as follows: | `FTSMatchExpression` | `fts_match_expression` | Boolean-aware evaluator over MATCH columns | For the #70484/#70485 boolean predicate, TiDB serializes a `FTSQueryInfo` -into the table scan. `TiDBTableScan` extracts it and `PhysicalTableScan` -builds an `fts_match_expression` filter containing: +into the table scan. For native pushdown, the MATCH columns must resolve in +order to one public composite FULLTEXT index on the same table; this prevents +the planner from substituting unrelated per-column indexes. `TiDBTableScan` +extracts the query and `PhysicalTableScan` builds an `fts_match_expression` +filter containing: 1. the original query text; 2. MATCH column references and their TiDB field types; @@ -148,6 +157,15 @@ MATCH columns, collation behavior, and unsupported score modifiers. The `utf8mb4_0900_bin`, `utf8mb4_general_ci`, `utf8mb4_unicode_ci`, and `utf8mb4_0900_ai_ci` for both ordinary terms and prefix terms. +On 2026-09-23, the `gtests_dbms --gtest_filter='TestFullText.*'` suite passed +all 14 tests. This includes regressions for parsing `run*` as a prefix query +and matching `+run*` against `RUNNER` under a case-insensitive collation. The +fix was in Boolean query parsing: the scanner stopped before `*` but did not +attach it to the preceding term. TiUP SQL E2E was not rerun after this parser +change; the E2E results below are from the earlier validation run. The +multi-column planner resolver and TiFlash evaluator have targeted unit +coverage, but multi-column MATCH has not yet been verified through TiUP SQL. + ### Local E2E validation The local E2E test used TiUP playground v8.5.8 with locally built TiDB and @@ -253,8 +271,9 @@ and index-reader change set. ## Remaining work -Before product release, the paired TiDB branch and TiFlash branch still need -CI coverage, release-8.5 build verification, and code review. If exact +Before product release, multi-column MATCH still needs TiUP SQL E2E coverage; +the paired TiDB branch and TiFlash branch also need CI coverage, release-8.5 +build verification, and code review. If exact corpus-wide ranking or index-level performance is required, a future phase must define a relevance-score contract and add a native DeltaMerge full-text index. From 2913916f1b48f5866ae752a2e59eb27b6dabfab1 Mon Sep 17 00:00:00 2001 From: Zhigao Tong Date: Mon, 28 Sep 2026 16:57:52 +0800 Subject: [PATCH 09/32] fts: align NGRAM prefix evaluation --- dbms/src/Functions/FunctionsFullText.cpp | 44 ++++++++++++++++++++- dbms/src/Functions/tests/gtest_fulltext.cpp | 35 ++++++++++++++++ 2 files changed, 77 insertions(+), 2 deletions(-) diff --git a/dbms/src/Functions/FunctionsFullText.cpp b/dbms/src/Functions/FunctionsFullText.cpp index e87c25925a6..41306a8fc95 100644 --- a/dbms/src/Functions/FunctionsFullText.cpp +++ b/dbms/src/Functions/FunctionsFullText.cpp @@ -585,9 +585,49 @@ Float64 matchBooleanScore( } case tipb::FTSBooleanTermType::FTSBooleanTermPrefix: { - clause.prefix = true; const auto terms = use_ngram ? analyzeNgramText(term.text(), ngram_token_size, collator) - : tokenizeText(term.text(), collator); + : tokenizeText(term.text(), collator); + if (use_ngram) + { + if (!terms.empty()) + { + // TiDB expands an NGRAM prefix at least as long as one + // gram into a positional phrase of query ngrams. Matching + // only one analyzed term here makes prefixes such as + // "caf*" ("ca", "af" at token size 2) never match. + clause.phrase = true; + const size_t first_position = terms.front().position; + for (const auto & token : terms) + { + clause.terms.push_back(token.text); + clause.offsets.push_back(token.position - first_position); + } + break; + } + + // A query shorter than the configured gram size is retained + // by TiDB as a prefix over document ngrams instead of being + // discarded by the analyzer. + const auto source_terms = tokenizeText(term.text(), collator); + if (source_terms.size() == 1) + { + const auto code_points = UTF8::countCodePoints( + reinterpret_cast(source_terms.front().text.data()), + source_terms.front().text.size()); + if (code_points < ngram_token_size) + { + clause.prefix = true; + clause.terms.push_back(source_terms.front().text); + break; + } + } + + if (clause.modifier == BooleanClause::Modifier::Must) + clauses.push_back(std::move(clause)); + continue; + } + + clause.prefix = true; if (terms.size() != 1) { if (clause.modifier == BooleanClause::Modifier::Must) diff --git a/dbms/src/Functions/tests/gtest_fulltext.cpp b/dbms/src/Functions/tests/gtest_fulltext.cpp index 4512ea62344..01415216b43 100644 --- a/dbms/src/Functions/tests/gtest_fulltext.cpp +++ b/dbms/src/Functions/tests/gtest_fulltext.cpp @@ -205,6 +205,41 @@ try } CATCH +TEST_F(TestFullText, MatchExpressionNgramPrefix) +try +{ + const auto make_metadata = [](UInt32 token_size, const String & prefix) { + tipb::FTSBooleanQuery boolean_query; + boolean_query.set_query_tokenizer("NGRAM_V1"); + boolean_query.set_ngram_token_size(token_size); + auto * node = boolean_query.add_nodes(); + node->set_occur(tipb::FTSBooleanOccurMust); + node->mutable_term()->set_term_type(tipb::FTSBooleanTermPrefix); + node->mutable_term()->set_text(prefix); + return String("__tiflash_fts_bool_query__:") + boolean_query.SerializeAsString(); + }; + + const auto query = createConstColumn(4, "+caf*"); + const auto documents = createColumn({"CAFE", "café", "cafe", "decaf"}); + const auto metadata = createConstColumn(4, make_metadata(2, "caf")); + const auto utf8mb4_bin = TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_BIN); + ASSERT_COLUMN_EQ( + createColumn({0, 1, 1, 1}), + executeFunction("fts_match_expression", {query, documents, metadata}, utf8mb4_bin)); + + const auto utf8mb4_general_ci = TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_GENERAL_CI); + ASSERT_COLUMN_EQ( + createColumn({1, 1, 1, 1}), + executeFunction("fts_match_expression", {query, documents, metadata}, utf8mb4_general_ci)); + + const auto short_query = createConstColumn(4, "+c*"); + const auto short_metadata = createConstColumn(4, make_metadata(2, "c")); + ASSERT_COLUMN_EQ( + createColumn({1, 1, 1, 1}), + executeFunction("fts_match_expression", {short_query, documents, short_metadata}, utf8mb4_general_ci)); +} +CATCH + TEST_F(TestFullText, MatchExpressionScore) try { From 50aea27110a8a5c557f4b3104684a61bd4021dcb Mon Sep 17 00:00:00 2001 From: Zhigao Tong Date: Mon, 28 Sep 2026 19:20:02 +0800 Subject: [PATCH 10/32] fts: forward STANDARD_V1 analyzer settings --- contrib/tipb | 2 +- dbms/src/Functions/FunctionsFullText.cpp | 57 +++++++++++++++++---- dbms/src/Functions/tests/gtest_fulltext.cpp | 55 ++++++++++++++++++++ 3 files changed, 103 insertions(+), 11 deletions(-) diff --git a/contrib/tipb b/contrib/tipb index 26695da26ed..1b94a80baeb 160000 --- a/contrib/tipb +++ b/contrib/tipb @@ -1 +1 @@ -Subproject commit 26695da26ed8f8d6347f74a4f1f2731ed4d22cfe +Subproject commit 1b94a80baeb1d47e8a28f5eb287fae1801c3f5d1 diff --git a/dbms/src/Functions/FunctionsFullText.cpp b/dbms/src/Functions/FunctionsFullText.cpp index 41306a8fc95..e792596468d 100644 --- a/dbms/src/Functions/FunctionsFullText.cpp +++ b/dbms/src/Functions/FunctionsFullText.cpp @@ -127,7 +127,7 @@ std::vector tokenizeText(std::string_view text, const TiDB::TiDBC return result; } -bool isDefaultStopword(const String & token) +bool isDefaultStopword(const String & token, const TiDB::TiDBCollatorPtr & collator = nullptr) { static const std::unordered_set stopwords{ "a", "about", "an", "are", "as", "at", "be", "by", "com", "de", "en", "for", @@ -135,10 +135,23 @@ bool isDefaultStopword(const String & token) "this", "to", "was", "what", "when", "where", "who", "will", "with", "und", "www"}; // Stopwords are stored in canonical lower-case form. Keep their existing // case-insensitive behavior independently from document matching. - return stopwords.contains(Poco::UTF8::toLower(token)); + if (stopwords.contains(Poco::UTF8::toLower(token))) + return true; + if (collator) + { + return std::any_of(stopwords.begin(), stopwords.end(), [&](const String & stopword) { + return collator->compare(token.data(), token.size(), stopword.data(), stopword.size()) == 0; + }); + } + return false; } -std::vector analyzeText(std::string_view text, const TiDB::TiDBCollatorPtr & collator = nullptr) +std::vector analyzeText( + std::string_view text, + const TiDB::TiDBCollatorPtr & collator = nullptr, + size_t min_token_size = default_min_token_size, + size_t max_token_size = default_max_token_size, + bool enable_stopword = true) { std::vector result; for (auto & token : tokenizeText(text, collator)) @@ -146,8 +159,8 @@ std::vector analyzeText(std::string_view text, const TiDB::TiDBCo const auto code_points = UTF8::countCodePoints( reinterpret_cast(token.text.data()), token.text.size()); - if (code_points >= default_min_token_size && code_points <= default_max_token_size - && !isDefaultStopword(token.text)) + if (code_points >= min_token_size && code_points <= max_token_size + && (!enable_stopword || !isDefaultStopword(token.text, collator))) result.push_back(std::move(token)); } return result; @@ -432,9 +445,13 @@ FullTextColumn analyzeColumn( std::string_view document, bool use_ngram, size_t ngram_token_size, - const TiDB::TiDBCollatorPtr & collator = nullptr) + const TiDB::TiDBCollatorPtr & collator = nullptr, + size_t min_token_size = default_min_token_size, + size_t max_token_size = default_max_token_size, + bool enable_stopword = true) { - return use_ngram ? analyzeNgramText(document, ngram_token_size, collator) : analyzeText(document, collator); + return use_ngram ? analyzeNgramText(document, ngram_token_size, collator) + : analyzeText(document, collator, min_token_size, max_token_size, enable_stopword); } Float64 matchBooleanScore( @@ -536,8 +553,15 @@ Float64 matchBooleanScore( std::vector clauses; const bool use_ngram = query.query_tokenizer() == ngram_parser; const size_t ngram_token_size = query.ngram_token_size() == 0 ? default_ngram_token_size : query.ngram_token_size(); + const bool has_standard_config = query.innodb_ft_min_token_size() != 0 || query.innodb_ft_max_token_size() != 0; + const size_t min_token_size = has_standard_config ? query.innodb_ft_min_token_size() : default_min_token_size; + const size_t max_token_size = has_standard_config ? query.innodb_ft_max_token_size() : default_max_token_size; + const bool enable_stopword = has_standard_config ? query.innodb_ft_enable_stopword() : true; + if (!use_ngram && (max_token_size == 0 || min_token_size > max_token_size)) + return 0; auto analyze_query = [&](std::string_view text) { - return use_ngram ? analyzeNgramText(text, ngram_token_size, collator) : analyzeText(text, collator); + return use_ngram ? analyzeNgramText(text, ngram_token_size, collator) + : analyzeText(text, collator, min_token_size, max_token_size, enable_stopword); }; for (const auto & node : query.nodes()) { @@ -637,7 +661,7 @@ Float64 matchBooleanScore( const auto code_points = UTF8::countCodePoints( reinterpret_cast(terms.front().text.data()), terms.front().text.size()); - if (code_points > default_max_token_size) + if (code_points > max_token_size) { if (clause.modifier == BooleanClause::Modifier::Must) clauses.push_back(std::move(clause)); @@ -939,6 +963,16 @@ class FunctionFTSMatchExpression final : public IFunction const size_t ngram_token_size = !use_ngram || protocol_boolean_query.ngram_token_size() == 0 ? default_ngram_token_size : protocol_boolean_query.ngram_token_size(); + const bool has_standard_config = has_protocol_boolean_query + && (protocol_boolean_query.innodb_ft_min_token_size() != 0 + || protocol_boolean_query.innodb_ft_max_token_size() != 0); + const size_t min_token_size = has_standard_config + ? protocol_boolean_query.innodb_ft_min_token_size() + : default_min_token_size; + const size_t max_token_size = has_standard_config + ? protocol_boolean_query.innodb_ft_max_token_size() + : default_max_token_size; + const bool enable_stopword = !has_standard_config || protocol_boolean_query.innodb_ft_enable_stopword(); for (size_t row = 0; row < rows; ++row) { @@ -959,7 +993,10 @@ class FunctionFTSMatchExpression final : public IFunction getStringAt(*block.getByPosition(arguments[arg]).column, row), use_ngram, ngram_token_size, - collator)); + collator, + min_token_size, + max_token_size, + enable_stopword)); } if (has_protocol_boolean_query) output_data[row] = matchBooleanScore(protocol_boolean_query, document, collator); diff --git a/dbms/src/Functions/tests/gtest_fulltext.cpp b/dbms/src/Functions/tests/gtest_fulltext.cpp index 01415216b43..e897ec2f72f 100644 --- a/dbms/src/Functions/tests/gtest_fulltext.cpp +++ b/dbms/src/Functions/tests/gtest_fulltext.cpp @@ -163,6 +163,61 @@ try } CATCH +TEST_F(TestFullText, MatchExpressionStandardAnalyzerProtocolSettings) +try +{ + const auto make_metadata = [](const String & term, UInt32 min_token_size, UInt32 max_token_size, bool enable_stopword) { + tipb::FTSBooleanQuery boolean_query; + boolean_query.set_query_tokenizer("STANDARD_V1"); + boolean_query.set_innodb_ft_min_token_size(min_token_size); + boolean_query.set_innodb_ft_max_token_size(max_token_size); + boolean_query.set_innodb_ft_enable_stopword(enable_stopword); + auto * required = boolean_query.add_nodes(); + required->set_occur(tipb::FTSBooleanOccurMust); + required->mutable_term()->set_term_type(tipb::FTSBooleanTermWord); + required->mutable_term()->set_text(term); + return String("__tiflash_fts_bool_query__:") + boolean_query.SerializeAsString(); + }; + + const auto short_term_metadata = make_metadata("go", 1, 84, true); + ASSERT_COLUMN_EQ( + createColumn({1, 0}), + executeFunction( + "fts_match_expression", + {createConstColumn(2, "+go"), + createColumn({"go", "good"}), + createConstColumn(2, short_term_metadata)})); + + const auto stopword_metadata = make_metadata("the", 1, 84, false); + ASSERT_COLUMN_EQ( + createColumn({1, 0}), + executeFunction( + "fts_match_expression", + {createConstColumn(2, "+the"), + createColumn({"the", "there"}), + createConstColumn(2, stopword_metadata)})); + + const auto collated_stopword_metadata = make_metadata("thé", 3, 84, true); + ASSERT_COLUMN_EQ( + createColumn({0}), + executeFunction( + "fts_match_expression", + {createConstColumn(1, "+thé"), + createColumn({"thé"}), + createConstColumn(1, collated_stopword_metadata)}, + TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_GENERAL_CI))); + + const auto max_size_metadata = make_metadata("extraordinary", 1, 10, false); + ASSERT_COLUMN_EQ( + createColumn({0}), + executeFunction( + "fts_match_expression", + {createConstColumn(1, "+extraordinary"), + createColumn({"extraordinary"}), + createConstColumn(1, max_size_metadata)})); +} +CATCH + TEST_F(TestFullText, MatchExpressionNgramProtocolBooleanQuery) try { From ea7f96cb01987635b6130e97168007ea38d09c3d Mon Sep 17 00:00:00 2001 From: Zhigao Tong Date: Mon, 28 Sep 2026 19:39:48 +0800 Subject: [PATCH 11/32] build: sync tipb default-value comments --- contrib/tipb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/tipb b/contrib/tipb index 1b94a80baeb..6364e49b4c7 160000 --- a/contrib/tipb +++ b/contrib/tipb @@ -1 +1 @@ -Subproject commit 1b94a80baeb1d47e8a28f5eb287fae1801c3f5d1 +Subproject commit 6364e49b4c7049cfed6a3328daf3525b737802c3 From f8a84f4f2c5f514975db56819fccd60ebcb57930 Mon Sep 17 00:00:00 2001 From: Zhigao Tong Date: Wed, 30 Sep 2026 14:01:25 +0800 Subject: [PATCH 12/32] fts: read Boolean query from structured Tipb expression --- contrib/tipb | 2 +- .../DAGExpressionAnalyzerHelper.cpp | 12 ++- .../Flash/Planner/Plans/PhysicalTableScan.cpp | 7 +- dbms/src/Functions/FunctionsFullText.cpp | 36 +++++-- dbms/src/Functions/tests/gtest_fulltext.cpp | 94 +++++++++++-------- 5 files changed, 96 insertions(+), 55 deletions(-) diff --git a/contrib/tipb b/contrib/tipb index 6364e49b4c7..5e00c3bc4a3 160000 --- a/contrib/tipb +++ b/contrib/tipb @@ -1 +1 @@ -Subproject commit 6364e49b4c7049cfed6a3328daf3525b737802c3 +Subproject commit 5e00c3bc4a3f6ad75833dcc10b2378ab670334ab diff --git a/dbms/src/Flash/Coprocessor/DAGExpressionAnalyzerHelper.cpp b/dbms/src/Flash/Coprocessor/DAGExpressionAnalyzerHelper.cpp index 14f5e55c4bb..b570eabc5ee 100644 --- a/dbms/src/Flash/Coprocessor/DAGExpressionAnalyzerHelper.cpp +++ b/dbms/src/Flash/Coprocessor/DAGExpressionAnalyzerHelper.cpp @@ -513,13 +513,23 @@ String DAGExpressionAnalyzerHelper::buildDefaultFunction( const tipb::Expr & expr, const ExpressionActionsPtr & actions) { - const String & func_name = getFunctionName(expr); + String func_name = getFunctionName(expr); Names argument_names; for (const auto & child : expr.children()) { String name = analyzer->getActions(child, actions); argument_names.push_back(name); } + if (expr.sig() == tipb::ScalarFuncSig::FTSMatchExpression && expr.has_fts_boolean_query()) + { + // Keep the wire representation typed on Expr. The scalar function's + // runtime interface uses a private function variant with a required + // final constant argument, so the query cannot be confused with a + // constant MATCH document column. + func_name = "fts_match_expression_with_boolean_query"; + const auto metadata_expr = constructStringLiteralTiExpr(expr.fts_boolean_query().SerializeAsString()); + argument_names.push_back(analyzer->getActions(metadata_expr, actions)); + } return analyzer->applyFunction(func_name, argument_names, actions, getCollatorFromExpr(expr)); } diff --git a/dbms/src/Flash/Planner/Plans/PhysicalTableScan.cpp b/dbms/src/Flash/Planner/Plans/PhysicalTableScan.cpp index d64b8b4d606..673260e6bda 100644 --- a/dbms/src/Flash/Planner/Plans/PhysicalTableScan.cpp +++ b/dbms/src/Flash/Planner/Plans/PhysicalTableScan.cpp @@ -37,8 +37,6 @@ namespace DB { namespace { -constexpr std::string_view fts_boolean_query_marker = "__tiflash_fts_bool_query__:"; - tipb::Expr buildFTSExpression(const TiDBTableScan & table_scan, Int32 result_type) { const auto & query_info = table_scan.getFTSQueryInfo(); @@ -103,10 +101,7 @@ tipb::Expr buildFTSExpression(const TiDBTableScan & table_scan, Int32 result_typ expression.mutable_field_type()->set_flag(0); if (query_func == tipb::ScalarFuncSig::FTSMatchExpression && query_info.has_boolean_query()) - { - *expression.add_children() - = constructStringLiteralTiExpr(String(fts_boolean_query_marker) + query_info.boolean_query().SerializeAsString()); - } + *expression.mutable_fts_boolean_query() = query_info.boolean_query(); return expression; } diff --git a/dbms/src/Functions/FunctionsFullText.cpp b/dbms/src/Functions/FunctionsFullText.cpp index e792596468d..115b4870e51 100644 --- a/dbms/src/Functions/FunctionsFullText.cpp +++ b/dbms/src/Functions/FunctionsFullText.cpp @@ -51,7 +51,6 @@ constexpr size_t default_min_token_size = 3; constexpr size_t default_max_token_size = 84; constexpr size_t default_ngram_token_size = 2; constexpr std::string_view ngram_parser = "NGRAM_V1"; -constexpr std::string_view fts_boolean_query_marker = "__tiflash_fts_bool_query__:"; struct FullTextToken { @@ -788,10 +787,7 @@ bool decodeFTSBooleanQuery(const IColumn & column, tipb::FTSBooleanQuery & query if (constant == nullptr) return false; const auto encoded = constant->getValue(); - const std::string_view value(encoded); - if (!value.starts_with(fts_boolean_query_marker)) - return false; - return query.ParseFromString(std::string(value.substr(fts_boolean_query_marker.size()))); + return query.ParseFromString(encoded); } class FunctionFTSMatchWord final : public IFunction @@ -891,12 +887,16 @@ class FunctionFTSMatchWord final : public IFunction TiDB::TiDBCollatorPtr collator; }; -class FunctionFTSMatchExpression final : public IFunction +class FunctionFTSMatchExpression : public IFunction { public: static constexpr auto name = "fts_match_expression"; static FunctionPtr create(const Context &) { return std::make_shared(); } + explicit FunctionFTSMatchExpression(bool require_boolean_query = false) + : require_boolean_query(require_boolean_query) + {} + String getName() const override { return name; } size_t getNumberOfArguments() const override { return 0; } bool isVariadic() const override { return true; } @@ -955,8 +955,13 @@ class FunctionFTSMatchExpression final : public IFunction const auto query = query_column->getValue(); size_t document_argument_end = arguments.size(); tipb::FTSBooleanQuery protocol_boolean_query; - const bool has_protocol_boolean_query = arguments.size() > 2 - && decodeFTSBooleanQuery(*block.getByPosition(arguments.back()).column, protocol_boolean_query); + const bool has_protocol_boolean_query = require_boolean_query; + if (require_boolean_query + && (arguments.size() <= 2 + || !decodeFTSBooleanQuery(*block.getByPosition(arguments.back()).column, protocol_boolean_query))) + throw Exception( + "fts_match_expression_with_boolean_query requires valid Boolean query metadata", + ErrorCodes::ILLEGAL_COLUMN); if (has_protocol_boolean_query) --document_argument_end; const bool use_ngram = has_protocol_boolean_query && protocol_boolean_query.query_tokenizer() == ngram_parser; @@ -1015,6 +1020,20 @@ class FunctionFTSMatchExpression final : public IFunction private: TiDB::TiDBCollatorPtr collator; + bool require_boolean_query; +}; + +class FunctionFTSMatchExpressionWithBooleanQuery final : public FunctionFTSMatchExpression +{ +public: + static constexpr auto name = "fts_match_expression_with_boolean_query"; + static FunctionPtr create(const Context &) { return std::make_shared(); } + + FunctionFTSMatchExpressionWithBooleanQuery() + : FunctionFTSMatchExpression(true) + {} + + String getName() const override { return name; } }; } @@ -1022,5 +1041,6 @@ void registerFunctionsFullText(FunctionFactory & factory) { factory.registerFunction(FunctionFactory::CaseInsensitive); factory.registerFunction(FunctionFactory::CaseInsensitive); + factory.registerFunction(FunctionFactory::CaseInsensitive); } } // namespace DB diff --git a/dbms/src/Functions/tests/gtest_fulltext.cpp b/dbms/src/Functions/tests/gtest_fulltext.cpp index e897ec2f72f..2c3812c5a98 100644 --- a/dbms/src/Functions/tests/gtest_fulltext.cpp +++ b/dbms/src/Functions/tests/gtest_fulltext.cpp @@ -113,15 +113,17 @@ try required->mutable_term()->set_term_type(tipb::FTSBooleanTermWord); required->mutable_term()->set_text("quick"); - const String metadata = "__tiflash_fts_bool_query__:" + boolean_query.SerializeAsString(); + const String metadata = boolean_query.SerializeAsString(); ASSERT_COLUMN_EQ( createColumn>({1, 1, 0, 0}), executeFunction( - "fts_match_expression", + "fts_match_expression_with_boolean_query", {createConstColumn(4, "+quick"), createColumn>({{}, "quick fox", {}, "slow fox"}), createColumn>({"quick fox", {}, "slow fox", {}}), - createConstColumn(4, metadata)})); + createConstColumn(4, metadata)}, + nullptr, + true)); } CATCH @@ -138,28 +140,32 @@ try prohibited->mutable_term()->set_term_type(tipb::FTSBooleanTermWord); prohibited->mutable_term()->set_text("slow"); - const String metadata = "__tiflash_fts_bool_query__:" + boolean_query.SerializeAsString(); + const String metadata = boolean_query.SerializeAsString(); ASSERT_COLUMN_EQ( createColumn({1, 0, 0}), executeFunction( - "fts_match_expression", + "fts_match_expression_with_boolean_query", {createConstColumn(3, "+quick -slow"), createColumn({"quick brown", "slow fox", "brown fox"}), - createConstColumn(3, metadata)})); + createConstColumn(3, metadata)}, + nullptr, + true)); tipb::FTSBooleanQuery prohibited_query; auto * prohibited_only = prohibited_query.add_nodes(); prohibited_only->set_occur(tipb::FTSBooleanOccurMustNot); prohibited_only->mutable_term()->set_term_type(tipb::FTSBooleanTermWord); prohibited_only->mutable_term()->set_text("slow"); - const String prohibited_metadata = "__tiflash_fts_bool_query__:" + prohibited_query.SerializeAsString(); + const String prohibited_metadata = prohibited_query.SerializeAsString(); ASSERT_COLUMN_EQ( createColumn({0, 0, 0}), executeFunction( - "fts_match_expression", + "fts_match_expression_with_boolean_query", {createConstColumn(3, "-slow"), createColumn({"quick brown", "slow fox", "brown fox"}), - createConstColumn(3, prohibited_metadata)})); + createConstColumn(3, prohibited_metadata)}, + nullptr, + true)); } CATCH @@ -176,45 +182,52 @@ try required->set_occur(tipb::FTSBooleanOccurMust); required->mutable_term()->set_term_type(tipb::FTSBooleanTermWord); required->mutable_term()->set_text(term); - return String("__tiflash_fts_bool_query__:") + boolean_query.SerializeAsString(); + return boolean_query.SerializeAsString(); }; const auto short_term_metadata = make_metadata("go", 1, 84, true); ASSERT_COLUMN_EQ( createColumn({1, 0}), executeFunction( - "fts_match_expression", + "fts_match_expression_with_boolean_query", {createConstColumn(2, "+go"), createColumn({"go", "good"}), - createConstColumn(2, short_term_metadata)})); + createConstColumn(2, short_term_metadata)}, + nullptr, + true)); const auto stopword_metadata = make_metadata("the", 1, 84, false); ASSERT_COLUMN_EQ( createColumn({1, 0}), executeFunction( - "fts_match_expression", + "fts_match_expression_with_boolean_query", {createConstColumn(2, "+the"), createColumn({"the", "there"}), - createConstColumn(2, stopword_metadata)})); + createConstColumn(2, stopword_metadata)}, + nullptr, + true)); const auto collated_stopword_metadata = make_metadata("thé", 3, 84, true); ASSERT_COLUMN_EQ( createColumn({0}), executeFunction( - "fts_match_expression", + "fts_match_expression_with_boolean_query", {createConstColumn(1, "+thé"), createColumn({"thé"}), createConstColumn(1, collated_stopword_metadata)}, - TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_GENERAL_CI))); + TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_GENERAL_CI), + true)); const auto max_size_metadata = make_metadata("extraordinary", 1, 10, false); ASSERT_COLUMN_EQ( createColumn({0}), executeFunction( - "fts_match_expression", + "fts_match_expression_with_boolean_query", {createConstColumn(1, "+extraordinary"), createColumn({"extraordinary"}), - createConstColumn(1, max_size_metadata)})); + createConstColumn(1, max_size_metadata)}, + nullptr, + true)); } CATCH @@ -233,14 +246,16 @@ try prohibited->mutable_term()->set_term_type(tipb::FTSBooleanTermWord); prohibited->mutable_term()->set_text("mysql"); - const String metadata = "__tiflash_fts_bool_query__:" + boolean_query.SerializeAsString(); + const String metadata = boolean_query.SerializeAsString(); ASSERT_COLUMN_EQ( createColumn({1, 0, 1, 0}), executeFunction( - "fts_match_expression", + "fts_match_expression_with_boolean_query", {createConstColumn(4, "+数据库 -mysql"), createColumn({"数据库系统", "MySQL 数据库", "数据库", "数据科学"}), - createConstColumn(4, metadata)})); + createConstColumn(4, metadata)}, + nullptr, + true)); const auto ci_collator = TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_GENERAL_CI); tipb::FTSBooleanQuery case_query; @@ -250,13 +265,14 @@ try case_term->set_occur(tipb::FTSBooleanOccurMust); case_term->mutable_term()->set_term_type(tipb::FTSBooleanTermWord); case_term->mutable_term()->set_text("mysql"); - const String case_metadata = "__tiflash_fts_bool_query__:" + case_query.SerializeAsString(); + const String case_metadata = case_query.SerializeAsString(); ASSERT_COLUMN_EQ( createColumn({1, 0}), executeFunction( - "fts_match_expression", + "fts_match_expression_with_boolean_query", {createConstColumn(2, "+mysql"), createColumn({"MySQL", "PostgreSQL"}), createConstColumn(2, case_metadata)}, - ci_collator)); + ci_collator, + true)); } CATCH @@ -271,7 +287,7 @@ try node->set_occur(tipb::FTSBooleanOccurMust); node->mutable_term()->set_term_type(tipb::FTSBooleanTermPrefix); node->mutable_term()->set_text(prefix); - return String("__tiflash_fts_bool_query__:") + boolean_query.SerializeAsString(); + return boolean_query.SerializeAsString(); }; const auto query = createConstColumn(4, "+caf*"); @@ -280,18 +296,18 @@ try const auto utf8mb4_bin = TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_BIN); ASSERT_COLUMN_EQ( createColumn({0, 1, 1, 1}), - executeFunction("fts_match_expression", {query, documents, metadata}, utf8mb4_bin)); + executeFunction("fts_match_expression_with_boolean_query", {query, documents, metadata}, utf8mb4_bin, true)); const auto utf8mb4_general_ci = TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_GENERAL_CI); ASSERT_COLUMN_EQ( createColumn({1, 1, 1, 1}), - executeFunction("fts_match_expression", {query, documents, metadata}, utf8mb4_general_ci)); + executeFunction("fts_match_expression_with_boolean_query", {query, documents, metadata}, utf8mb4_general_ci, true)); const auto short_query = createConstColumn(4, "+c*"); const auto short_metadata = createConstColumn(4, make_metadata(2, "c")); ASSERT_COLUMN_EQ( createColumn({1, 1, 1, 1}), - executeFunction("fts_match_expression", {short_query, documents, short_metadata}, utf8mb4_general_ci)); + executeFunction("fts_match_expression_with_boolean_query", {short_query, documents, short_metadata}, utf8mb4_general_ci, true)); } CATCH @@ -348,7 +364,7 @@ try node->set_occur(tipb::FTSBooleanOccurMust); node->mutable_term()->set_term_type(term_type); node->mutable_term()->set_text(term); - return String("__tiflash_fts_bool_query__:") + boolean_query.SerializeAsString(); + return boolean_query.SerializeAsString(); }; const auto word_metadata = createConstColumn( 3, make_metadata(tipb::FTSBooleanTermWord, "cafe")); @@ -358,42 +374,42 @@ try const auto utf8mb4_bin = TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_BIN); ASSERT_COLUMN_EQ( createColumn({0, 0, 1}), - executeFunction("fts_match_expression", {query, documents, word_metadata}, utf8mb4_bin)); + executeFunction("fts_match_expression_with_boolean_query", {query, documents, word_metadata}, utf8mb4_bin, true)); ASSERT_COLUMN_EQ( createColumn({0, 1, 1}), - executeFunction("fts_match_expression", {prefix_query, documents, prefix_metadata}, utf8mb4_bin)); + executeFunction("fts_match_expression_with_boolean_query", {prefix_query, documents, prefix_metadata}, utf8mb4_bin, true)); const auto utf8mb4_0900_bin = TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_0900_BIN); ASSERT_COLUMN_EQ( createColumn({0, 0, 1}), - executeFunction("fts_match_expression", {query, documents, word_metadata}, utf8mb4_0900_bin)); + executeFunction("fts_match_expression_with_boolean_query", {query, documents, word_metadata}, utf8mb4_0900_bin, true)); ASSERT_COLUMN_EQ( createColumn({0, 1, 1}), - executeFunction("fts_match_expression", {prefix_query, documents, prefix_metadata}, utf8mb4_0900_bin)); + executeFunction("fts_match_expression_with_boolean_query", {prefix_query, documents, prefix_metadata}, utf8mb4_0900_bin, true)); const auto utf8mb4_general_ci = TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_GENERAL_CI); ASSERT_COLUMN_EQ( createColumn({1, 1, 1}), - executeFunction("fts_match_expression", {query, documents, word_metadata}, utf8mb4_general_ci)); + executeFunction("fts_match_expression_with_boolean_query", {query, documents, word_metadata}, utf8mb4_general_ci, true)); ASSERT_COLUMN_EQ( createColumn({1, 1, 1}), - executeFunction("fts_match_expression", {prefix_query, documents, prefix_metadata}, utf8mb4_general_ci)); + executeFunction("fts_match_expression_with_boolean_query", {prefix_query, documents, prefix_metadata}, utf8mb4_general_ci, true)); const auto utf8mb4_unicode_ci = TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_UNICODE_CI); ASSERT_COLUMN_EQ( createColumn({1, 1, 1}), - executeFunction("fts_match_expression", {query, documents, word_metadata}, utf8mb4_unicode_ci)); + executeFunction("fts_match_expression_with_boolean_query", {query, documents, word_metadata}, utf8mb4_unicode_ci, true)); ASSERT_COLUMN_EQ( createColumn({1, 1, 1}), - executeFunction("fts_match_expression", {prefix_query, documents, prefix_metadata}, utf8mb4_unicode_ci)); + executeFunction("fts_match_expression_with_boolean_query", {prefix_query, documents, prefix_metadata}, utf8mb4_unicode_ci, true)); const auto utf8mb4_0900_ai_ci = TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_0900_AI_CI); ASSERT_COLUMN_EQ( createColumn({1, 1, 1}), - executeFunction("fts_match_expression", {query, documents, word_metadata}, utf8mb4_0900_ai_ci)); + executeFunction("fts_match_expression_with_boolean_query", {query, documents, word_metadata}, utf8mb4_0900_ai_ci, true)); ASSERT_COLUMN_EQ( createColumn({1, 1, 1}), - executeFunction("fts_match_expression", {prefix_query, documents, prefix_metadata}, utf8mb4_0900_ai_ci)); + executeFunction("fts_match_expression_with_boolean_query", {prefix_query, documents, prefix_metadata}, utf8mb4_0900_ai_ci, true)); } CATCH From 3a504cbb0d1d1b209b287d99ef3bdd158c33481c Mon Sep 17 00:00:00 2001 From: Zhigao Tong Date: Wed, 30 Sep 2026 16:06:33 +0800 Subject: [PATCH 13/32] fts: decode versioned metadata from Expr.val --- contrib/tipb | 2 +- .../DAGExpressionAnalyzerHelper.cpp | 11 ++-- .../Flash/Planner/Plans/PhysicalTableScan.cpp | 7 ++- dbms/src/Functions/FunctionsFullText.cpp | 8 ++- dbms/src/Functions/tests/gtest_fulltext.cpp | 63 ++++++++++++++++--- 5 files changed, 74 insertions(+), 17 deletions(-) diff --git a/contrib/tipb b/contrib/tipb index 5e00c3bc4a3..2c0cddbdaca 160000 --- a/contrib/tipb +++ b/contrib/tipb @@ -1 +1 @@ -Subproject commit 5e00c3bc4a3f6ad75833dcc10b2378ab670334ab +Subproject commit 2c0cddbdaca8dbcd45bf48d49750a90fa68d44cc diff --git a/dbms/src/Flash/Coprocessor/DAGExpressionAnalyzerHelper.cpp b/dbms/src/Flash/Coprocessor/DAGExpressionAnalyzerHelper.cpp index b570eabc5ee..3f97344b941 100644 --- a/dbms/src/Flash/Coprocessor/DAGExpressionAnalyzerHelper.cpp +++ b/dbms/src/Flash/Coprocessor/DAGExpressionAnalyzerHelper.cpp @@ -520,14 +520,13 @@ String DAGExpressionAnalyzerHelper::buildDefaultFunction( String name = analyzer->getActions(child, actions); argument_names.push_back(name); } - if (expr.sig() == tipb::ScalarFuncSig::FTSMatchExpression && expr.has_fts_boolean_query()) + if (expr.sig() == tipb::ScalarFuncSig::FTSMatchExpression && !expr.val().empty()) { - // Keep the wire representation typed on Expr. The scalar function's - // runtime interface uses a private function variant with a required - // final constant argument, so the query cannot be confused with a - // constant MATCH document column. + // FTS metadata is carried in the generic scalar-function metadata slot + // and must not be confused with SQL arguments. The runtime function + // interface uses a private variant with a required final constant. func_name = "fts_match_expression_with_boolean_query"; - const auto metadata_expr = constructStringLiteralTiExpr(expr.fts_boolean_query().SerializeAsString()); + const auto metadata_expr = constructStringLiteralTiExpr(expr.val()); argument_names.push_back(analyzer->getActions(metadata_expr, actions)); } return analyzer->applyFunction(func_name, argument_names, actions, getCollatorFromExpr(expr)); diff --git a/dbms/src/Flash/Planner/Plans/PhysicalTableScan.cpp b/dbms/src/Flash/Planner/Plans/PhysicalTableScan.cpp index 673260e6bda..928ee14afb9 100644 --- a/dbms/src/Flash/Planner/Plans/PhysicalTableScan.cpp +++ b/dbms/src/Flash/Planner/Plans/PhysicalTableScan.cpp @@ -101,7 +101,12 @@ tipb::Expr buildFTSExpression(const TiDBTableScan & table_scan, Int32 result_typ expression.mutable_field_type()->set_flag(0); if (query_func == tipb::ScalarFuncSig::FTSMatchExpression && query_info.has_boolean_query()) - *expression.mutable_fts_boolean_query() = query_info.boolean_query(); + { + tipb::FTSMatchExpressionMetadata metadata; + metadata.set_version(1); + *metadata.mutable_boolean_query() = query_info.boolean_query(); + expression.set_val(metadata.SerializeAsString()); + } return expression; } diff --git a/dbms/src/Functions/FunctionsFullText.cpp b/dbms/src/Functions/FunctionsFullText.cpp index 115b4870e51..57468963409 100644 --- a/dbms/src/Functions/FunctionsFullText.cpp +++ b/dbms/src/Functions/FunctionsFullText.cpp @@ -51,6 +51,7 @@ constexpr size_t default_min_token_size = 3; constexpr size_t default_max_token_size = 84; constexpr size_t default_ngram_token_size = 2; constexpr std::string_view ngram_parser = "NGRAM_V1"; +constexpr UInt32 fts_match_expression_metadata_version = 1; struct FullTextToken { @@ -787,7 +788,12 @@ bool decodeFTSBooleanQuery(const IColumn & column, tipb::FTSBooleanQuery & query if (constant == nullptr) return false; const auto encoded = constant->getValue(); - return query.ParseFromString(encoded); + tipb::FTSMatchExpressionMetadata metadata; + if (!metadata.ParseFromString(encoded) || metadata.version() != fts_match_expression_metadata_version + || !metadata.has_boolean_query()) + return false; + query = metadata.boolean_query(); + return true; } class FunctionFTSMatchWord final : public IFunction diff --git a/dbms/src/Functions/tests/gtest_fulltext.cpp b/dbms/src/Functions/tests/gtest_fulltext.cpp index 2c3812c5a98..cb53c9546c9 100644 --- a/dbms/src/Functions/tests/gtest_fulltext.cpp +++ b/dbms/src/Functions/tests/gtest_fulltext.cpp @@ -24,6 +24,17 @@ class TestFullText : public DB::tests::FunctionTest { }; +namespace +{ +String serializeFTSBooleanQuery(const tipb::FTSBooleanQuery & query) +{ + tipb::FTSMatchExpressionMetadata metadata; + metadata.set_version(1); + *metadata.mutable_boolean_query() = query; + return metadata.SerializeAsString(); +} +} // namespace + TEST_F(TestFullText, MatchWordBoolean) try { @@ -113,7 +124,7 @@ try required->mutable_term()->set_term_type(tipb::FTSBooleanTermWord); required->mutable_term()->set_text("quick"); - const String metadata = boolean_query.SerializeAsString(); + const String metadata = serializeFTSBooleanQuery(boolean_query); ASSERT_COLUMN_EQ( createColumn>({1, 1, 0, 0}), executeFunction( @@ -140,7 +151,7 @@ try prohibited->mutable_term()->set_term_type(tipb::FTSBooleanTermWord); prohibited->mutable_term()->set_text("slow"); - const String metadata = boolean_query.SerializeAsString(); + const String metadata = serializeFTSBooleanQuery(boolean_query); ASSERT_COLUMN_EQ( createColumn({1, 0, 0}), executeFunction( @@ -156,7 +167,7 @@ try prohibited_only->set_occur(tipb::FTSBooleanOccurMustNot); prohibited_only->mutable_term()->set_term_type(tipb::FTSBooleanTermWord); prohibited_only->mutable_term()->set_text("slow"); - const String prohibited_metadata = prohibited_query.SerializeAsString(); + const String prohibited_metadata = serializeFTSBooleanQuery(prohibited_query); ASSERT_COLUMN_EQ( createColumn({0, 0, 0}), executeFunction( @@ -169,6 +180,42 @@ try } CATCH +TEST_F(TestFullText, MatchExpressionRejectsInvalidMetadata) +try +{ + tipb::FTSBooleanQuery boolean_query; + auto * required = boolean_query.add_nodes(); + required->set_occur(tipb::FTSBooleanOccurMust); + required->mutable_term()->set_term_type(tipb::FTSBooleanTermWord); + required->mutable_term()->set_text("quick"); + + tipb::FTSMatchExpressionMetadata metadata; + metadata.set_version(2); + *metadata.mutable_boolean_query() = boolean_query; + ASSERT_THROW( + executeFunction( + "fts_match_expression_with_boolean_query", + {createConstColumn(1, "+quick"), + createColumn({"quick brown"}), + createConstColumn(1, metadata.SerializeAsString())}, + nullptr, + true), + Exception); + + metadata.set_version(1); + metadata.clear_boolean_query(); + ASSERT_THROW( + executeFunction( + "fts_match_expression_with_boolean_query", + {createConstColumn(1, "+quick"), + createColumn({"quick brown"}), + createConstColumn(1, metadata.SerializeAsString())}, + nullptr, + true), + Exception); +} +CATCH + TEST_F(TestFullText, MatchExpressionStandardAnalyzerProtocolSettings) try { @@ -182,7 +229,7 @@ try required->set_occur(tipb::FTSBooleanOccurMust); required->mutable_term()->set_term_type(tipb::FTSBooleanTermWord); required->mutable_term()->set_text(term); - return boolean_query.SerializeAsString(); + return serializeFTSBooleanQuery(boolean_query); }; const auto short_term_metadata = make_metadata("go", 1, 84, true); @@ -246,7 +293,7 @@ try prohibited->mutable_term()->set_term_type(tipb::FTSBooleanTermWord); prohibited->mutable_term()->set_text("mysql"); - const String metadata = boolean_query.SerializeAsString(); + const String metadata = serializeFTSBooleanQuery(boolean_query); ASSERT_COLUMN_EQ( createColumn({1, 0, 1, 0}), executeFunction( @@ -265,7 +312,7 @@ try case_term->set_occur(tipb::FTSBooleanOccurMust); case_term->mutable_term()->set_term_type(tipb::FTSBooleanTermWord); case_term->mutable_term()->set_text("mysql"); - const String case_metadata = case_query.SerializeAsString(); + const String case_metadata = serializeFTSBooleanQuery(case_query); ASSERT_COLUMN_EQ( createColumn({1, 0}), executeFunction( @@ -287,7 +334,7 @@ try node->set_occur(tipb::FTSBooleanOccurMust); node->mutable_term()->set_term_type(tipb::FTSBooleanTermPrefix); node->mutable_term()->set_text(prefix); - return boolean_query.SerializeAsString(); + return serializeFTSBooleanQuery(boolean_query); }; const auto query = createConstColumn(4, "+caf*"); @@ -364,7 +411,7 @@ try node->set_occur(tipb::FTSBooleanOccurMust); node->mutable_term()->set_term_type(term_type); node->mutable_term()->set_text(term); - return boolean_query.SerializeAsString(); + return serializeFTSBooleanQuery(boolean_query); }; const auto word_metadata = createConstColumn( 3, make_metadata(tipb::FTSBooleanTermWord, "cafe")); From 22bdbe48c6edcf2d3221e8aaa0d02392c4c34d00 Mon Sep 17 00:00:00 2001 From: Zhigao Tong Date: Thu, 8 Oct 2026 14:47:43 +0800 Subject: [PATCH 14/32] fts: remove TiCI scan-level MATCH path --- contrib/tipb | 2 +- .../DAGExpressionAnalyzerHelper.cpp | 4 +- .../Coprocessor/DAGStorageInterpreter.cpp | 5 +- dbms/src/Flash/Coprocessor/DAGUtils.cpp | 3 +- dbms/src/Flash/Coprocessor/RemoteRequest.cpp | 2 +- dbms/src/Flash/Coprocessor/TiDBTableScan.cpp | 28 -- dbms/src/Flash/Coprocessor/TiDBTableScan.h | 12 - .../Flash/Planner/Plans/PhysicalTableScan.cpp | 145 +----- dbms/src/Functions/FunctionsFullText.cpp | 421 +----------------- dbms/src/Functions/tests/gtest_fulltext.cpp | 211 ++------- 10 files changed, 65 insertions(+), 768 deletions(-) diff --git a/contrib/tipb b/contrib/tipb index 2c0cddbdaca..e2791d0a2b5 160000 --- a/contrib/tipb +++ b/contrib/tipb @@ -1 +1 @@ -Subproject commit 2c0cddbdaca8dbcd45bf48d49750a90fa68d44cc +Subproject commit e2791d0a2b58299ee333e63709eefa1700f166d2 diff --git a/dbms/src/Flash/Coprocessor/DAGExpressionAnalyzerHelper.cpp b/dbms/src/Flash/Coprocessor/DAGExpressionAnalyzerHelper.cpp index 3f97344b941..1029a0e80e9 100644 --- a/dbms/src/Flash/Coprocessor/DAGExpressionAnalyzerHelper.cpp +++ b/dbms/src/Flash/Coprocessor/DAGExpressionAnalyzerHelper.cpp @@ -520,12 +520,12 @@ String DAGExpressionAnalyzerHelper::buildDefaultFunction( String name = analyzer->getActions(child, actions); argument_names.push_back(name); } - if (expr.sig() == tipb::ScalarFuncSig::FTSMatchExpression && !expr.val().empty()) + if (expr.sig() == tipb::ScalarFuncSig::FTSMatchBooleanExpression) { // FTS metadata is carried in the generic scalar-function metadata slot // and must not be confused with SQL arguments. The runtime function // interface uses a private variant with a required final constant. - func_name = "fts_match_expression_with_boolean_query"; + func_name = "fts_match_boolean_expression"; const auto metadata_expr = constructStringLiteralTiExpr(expr.val()); argument_names.push_back(analyzer->getActions(metadata_expr, actions)); } diff --git a/dbms/src/Flash/Coprocessor/DAGStorageInterpreter.cpp b/dbms/src/Flash/Coprocessor/DAGStorageInterpreter.cpp index 1d849dd0b1c..df67e11030f 100644 --- a/dbms/src/Flash/Coprocessor/DAGStorageInterpreter.cpp +++ b/dbms/src/Flash/Coprocessor/DAGStorageInterpreter.cpp @@ -1584,7 +1584,7 @@ std::pair> DAGStorageInterpreter::getColumnsForTableSc auto const & ci = table_scan.getColumns()[i]; const ColumnID cid = ci.id; - if (ci.hasGeneratedColumnFlag() || isTiDBFTSScoreColumn(cid)) + if (ci.hasGeneratedColumnFlag()) { LOG_DEBUG(log, "got column({}) with generated column flag", i); const auto & data_type = getDataTypeByColumnInfoForComputingLayer(ci); @@ -1617,8 +1617,7 @@ std::pair> DAGStorageInterpreter::getColumnsForTableSc // If the column is not generated column, not in the filter columns and column id is not -1, then it may need cast. for (const auto & col : table_scan.getColumns()) may_need_add_cast_column_tmp.push_back( - !col.hasGeneratedColumnFlag() && !isTiDBFTSScoreColumn(col.id) && !filter_col_id_set.contains(col.id) - && col.id != -1); + !col.hasGeneratedColumnFlag() && !filter_col_id_set.contains(col.id) && col.id != -1); return {required_columns_tmp, may_need_add_cast_column_tmp}; } diff --git a/dbms/src/Flash/Coprocessor/DAGUtils.cpp b/dbms/src/Flash/Coprocessor/DAGUtils.cpp index 487ed9a8eed..febe7644289 100644 --- a/dbms/src/Flash/Coprocessor/DAGUtils.cpp +++ b/dbms/src/Flash/Coprocessor/DAGUtils.cpp @@ -455,8 +455,7 @@ const std::unordered_map scalar_func_map({ {tipb::ScalarFuncSig::RegexpInStrSig, "regexp_instr"}, {tipb::ScalarFuncSig::RegexpReplaceSig, "regexp_replace"}, {tipb::ScalarFuncSig::RegexpSubstrSig, "regexp_substr"}, - {tipb::ScalarFuncSig::FTSMatchWord, "fts_match_word"}, - {tipb::ScalarFuncSig::FTSMatchExpression, "fts_match_expression"}, + {tipb::ScalarFuncSig::FTSMatchBooleanExpression, "fts_match_boolean_expression"}, {tipb::ScalarFuncSig::JsonExtractSig, "json_extract"}, {tipb::ScalarFuncSig::JsonUnquoteSig, "json_unquote"}, diff --git a/dbms/src/Flash/Coprocessor/RemoteRequest.cpp b/dbms/src/Flash/Coprocessor/RemoteRequest.cpp index 129fb5695a5..637203a728c 100644 --- a/dbms/src/Flash/Coprocessor/RemoteRequest.cpp +++ b/dbms/src/Flash/Coprocessor/RemoteRequest.cpp @@ -54,7 +54,7 @@ RemoteRequest RemoteRequest::build( const auto & col = table_scan.getColumns()[i]; auto col_id = col.id; - if (col.hasGeneratedColumnFlag() || isTiDBFTSScoreColumn(col_id)) + if (col.hasGeneratedColumnFlag()) { const auto & col_name = GeneratedColumnPlaceholderBlockInputStream::getColumnName(i); schema.emplace_back(std::make_pair(col_name, std::move(col))); diff --git a/dbms/src/Flash/Coprocessor/TiDBTableScan.cpp b/dbms/src/Flash/Coprocessor/TiDBTableScan.cpp index de6d6ee9855..d1545dc24e6 100644 --- a/dbms/src/Flash/Coprocessor/TiDBTableScan.cpp +++ b/dbms/src/Flash/Coprocessor/TiDBTableScan.cpp @@ -17,31 +17,6 @@ namespace DB { -namespace -{ -tipb::FTSQueryInfo extractFtsQuery(const tipb::Executor * table_scan) -{ - const auto & indexes = table_scan->tp() == tipb::TypePartitionTableScan - ? table_scan->partition_table_scan().used_columnar_indexes() - : table_scan->tbl_scan().used_columnar_indexes(); - const tipb::ColumnarIndexInfo * fts_index = nullptr; - for (const auto & index : indexes) - { - if (index.index_type() != tipb::ColumnarIndexType::TypeFulltext) - continue; - if (fts_index != nullptr) - throw TiFlashException( - "A table scan cannot contain multiple full-text indexes", - Errors::Coprocessor::BadRequest); - fts_index = &index; - } - if (fts_index == nullptr) - return {}; - RUNTIME_CHECK(fts_index->has_fts_query_info()); - return fts_index->fts_query_info(); -} -} - TiDBTableScan::TiDBTableScan( const tipb::Executor * table_scan_, const String & executor_id_, @@ -57,7 +32,6 @@ TiDBTableScan::TiDBTableScan( : table_scan->tbl_scan().pushed_down_filter_conditions()) , ann_query_info( is_partition_table_scan ? table_scan->partition_table_scan().ann_query() : table_scan->tbl_scan().ann_query()) - , fts_query_info(extractFtsQuery(table_scan)) // Only No-partition table need keep order when tablescan executor required keep order. // If keep_order is not set, keep order for safety. , keep_order( @@ -125,8 +99,6 @@ void TiDBTableScan::constructTableScanForRemoteRead(tipb::TableScan * tipb_table *tipb_table_scan->add_columns() = column; for (const auto & filter : partition_table_scan.pushed_down_filter_conditions()) *tipb_table_scan->add_pushed_down_filter_conditions() = filter; - for (const auto & index : partition_table_scan.used_columnar_indexes()) - *tipb_table_scan->add_used_columnar_indexes() = index; tipb_table_scan->set_desc(partition_table_scan.desc()); for (auto id : partition_table_scan.primary_column_ids()) tipb_table_scan->add_primary_column_ids(id); diff --git a/dbms/src/Flash/Coprocessor/TiDBTableScan.h b/dbms/src/Flash/Coprocessor/TiDBTableScan.h index b17b336e1c5..9d9164c5f29 100644 --- a/dbms/src/Flash/Coprocessor/TiDBTableScan.h +++ b/dbms/src/Flash/Coprocessor/TiDBTableScan.h @@ -23,16 +23,6 @@ namespace DB { class DAGContext; -// TiDB allocates this negative column ID for the synthetic MATCH_AGAINST score -// column. It is not part of the physical table schema and must be represented -// by a generated-column placeholder in the TiFlash scan pipeline. -inline constexpr ColumnID TiDBVirtualColFTSScoreID = -2050; - -inline bool isTiDBFTSScoreColumn(ColumnID column_id) -{ - return column_id == TiDBVirtualColFTSScoreID; -} - /// TiDBTableScan is a wrap to hide the difference of `TableScan` and `PartitionTableScan` class TiDBTableScan { @@ -56,7 +46,6 @@ class TiDBTableScan const google::protobuf::RepeatedPtrField & getPushedDownFilters() const { return pushed_down_filters; } const tipb::ANNQueryInfo & getANNQueryInfo() const { return ann_query_info; } - const tipb::FTSQueryInfo & getFTSQueryInfo() const { return fts_query_info; } private: const tipb::Executor * table_scan; @@ -79,7 +68,6 @@ class TiDBTableScan const google::protobuf::RepeatedPtrField pushed_down_filters; const tipb::ANNQueryInfo ann_query_info; - const tipb::FTSQueryInfo fts_query_info; bool keep_order; bool is_fast_scan; diff --git a/dbms/src/Flash/Planner/Plans/PhysicalTableScan.cpp b/dbms/src/Flash/Planner/Plans/PhysicalTableScan.cpp index 928ee14afb9..39b4c6432da 100644 --- a/dbms/src/Flash/Planner/Plans/PhysicalTableScan.cpp +++ b/dbms/src/Flash/Planner/Plans/PhysicalTableScan.cpp @@ -26,131 +26,14 @@ #include #include #include -#include #include #include #include -#include - namespace DB { namespace { -tipb::Expr buildFTSExpression(const TiDBTableScan & table_scan, Int32 result_type) -{ - const auto & query_info = table_scan.getFTSQueryInfo(); - const auto query_func = query_info.has_query_func() ? query_info.query_func() : tipb::ScalarFuncSig::FTSMatchWord; - if (query_func == tipb::ScalarFuncSig::FTSMatchWord) - RUNTIME_CHECK_MSG(query_info.columns_size() == 1, "FTS_MATCH_WORD currently supports exactly one column"); - else - RUNTIME_CHECK_MSG( - query_func == tipb::ScalarFuncSig::FTSMatchExpression && query_info.columns_size() > 0, - "Unsupported full-text query function or empty MATCH column list"); - - tipb::Expr expression; - expression.set_tp(tipb::ExprType::ScalarFunc); - expression.set_sig(query_func); - expression.mutable_field_type()->set_tp(result_type); - expression.mutable_field_type()->set_flag(TiDB::ColumnFlagNotNull); - *expression.add_children() = constructStringLiteralTiExpr(query_info.query_text()); - - bool fts_collation_initialized = false; - bool fts_result_nullable = false; - for (const auto & query_column : query_info.columns()) - { - const auto column_id = query_column.column_id(); - const TiDB::ColumnInfo * column_info = nullptr; - Int64 column_index = -1; - for (size_t i = 0; i < table_scan.getColumns().size(); ++i) - { - const auto & column = table_scan.getColumns()[i]; - if (column.id == column_id) - { - column_info = &column; - column_index = i; - break; - } - } - RUNTIME_CHECK_MSG(column_info != nullptr, "Full-text column is not present in table scan columns"); - - tipb::Expr column_ref; - column_ref.set_tp(tipb::ExprType::ColumnRef); - WriteBufferFromOwnString ss; - encodeDAGInt64(column_index, ss); - column_ref.set_val(ss.releaseStr()); - *column_ref.mutable_field_type() = TiDB::columnInfoToFieldType(*column_info); - fts_result_nullable = fts_result_nullable || !column_info->hasNotNullFlag(); - // FTS returns a numeric score, but its string matching semantics are - // determined by the MATCH column collation. Carry the first column's - // protocol collation on the scalar expression so the DAG analyzer - // passes the same collator to fts_match_expression as it does for - // LIKE and comparison functions. - if (!fts_collation_initialized && column_ref.field_type().collate() != 0) - { - expression.mutable_field_type()->set_collate(column_ref.field_type().collate()); - fts_collation_initialized = true; - } - *expression.add_children() = std::move(column_ref); - } - - // MATCH returns NULL when any input column is NULL. Keep the result type - // nullable so the analyzer does not insert a cast from Nullable(Float64) - // to Float64, which would fail at runtime for a NULL row. - if (fts_result_nullable) - expression.mutable_field_type()->set_flag(0); - - if (query_func == tipb::ScalarFuncSig::FTSMatchExpression && query_info.has_boolean_query()) - { - tipb::FTSMatchExpressionMetadata metadata; - metadata.set_version(1); - *metadata.mutable_boolean_query() = query_info.boolean_query(); - expression.set_val(metadata.SerializeAsString()); - } - return expression; -} - -tipb::Expr buildFTSFilter(const TiDBTableScan & table_scan) -{ - return buildFTSExpression(table_scan, TiDB::TypeDouble); -} - -String getFTSScorePlaceholderName(const TiDBTableScan & table_scan) -{ - for (size_t i = 0; i < table_scan.getColumns().size(); ++i) - { - if (isTiDBFTSScoreColumn(table_scan.getColumns()[i].id)) - return GeneratedColumnPlaceholderBlockInputStream::getColumnName(i); - } - throw TiFlashException("FTS score column is missing from table scan", Errors::Coprocessor::BadRequest); -} - -ExpressionActionsPtr buildFTSScoreActions( - const Block & input_header, - const TiDBTableScan & table_scan, - const Context & context) -{ - auto actions = std::make_shared(input_header.getColumnsWithTypeAndName()); - DAGExpressionAnalyzer analyzer(input_header, context); - const auto score_expr = buildFTSExpression(table_scan, TiDB::TypeFloat); - const auto score_expr_name = analyzer.getActions(score_expr, actions); - const auto score_column_name = getFTSScorePlaceholderName(table_scan); - - RUNTIME_CHECK_MSG( - actions->getSampleBlock().has(score_column_name), - "FTS score placeholder is missing from table scan input"); - actions->add(ExpressionAction::removeColumn(score_column_name)); - actions->add(ExpressionAction::copyColumn(score_expr_name, score_column_name)); - - NamesWithAliases project_columns; - project_columns.reserve(input_header.columns()); - for (const auto & column : input_header) - project_columns.emplace_back(column.name, column.name); - actions->add(ExpressionAction::project(project_columns)); - actions->finalize(input_header.getNames()); - return actions; -} - NamesWithAliases buildTableScanProjectionCols( Int64 logical_table_id, const NamesAndTypes & schema, @@ -202,14 +85,7 @@ PhysicalTableScan::PhysicalTableScan( : PhysicalLeaf(executor_id_, PlanType::TableScan, schema_, FineGrainedShuffle{}, req_id) , tidb_table_scan(tidb_table_scan_) , sample_block(sample_block_) -{ - if (tidb_table_scan.getFTSQueryInfo().columns_size() > 0) - { - google::protobuf::RepeatedPtrField conditions; - *conditions.Add() = buildFTSFilter(tidb_table_scan); - filter_conditions = FilterConditions(executor_id_, conditions); - } -} +{} PhysicalPlanNodePtr PhysicalTableScan::build( const String & executor_id, @@ -278,17 +154,8 @@ void PhysicalTableScan::buildPipelineExecGroupImpl( group_builder = std::move(pipeline_exec_builder); } -void PhysicalTableScan::buildProjection(DAGPipeline & pipeline, Context & context) +void PhysicalTableScan::buildProjection(DAGPipeline & pipeline, Context & /*context*/) { - if (tidb_table_scan.getFTSQueryInfo().query_type() == tipb::FTSQueryType::FTSQueryTypeWithScore) - { - RUNTIME_CHECK_MSG( - !pipeline.streams.empty(), - "FTS score cannot be materialized without a table scan stream"); - auto score_actions = buildFTSScoreActions(pipeline.firstStream()->getHeader(), tidb_table_scan, context); - executeExpression(pipeline, score_actions, log, "full-text score"); - } - const auto & schema_project_cols = buildTableScanProjectionCols( tidb_table_scan.getLogicalTableID(), schema, @@ -302,14 +169,8 @@ void PhysicalTableScan::buildProjection(DAGPipeline & pipeline, Context & contex void PhysicalTableScan::buildProjection( PipelineExecutorContext & exec_context, PipelineExecGroupBuilder & group_builder, - Context & context) + Context & /*context*/) { - if (tidb_table_scan.getFTSQueryInfo().query_type() == tipb::FTSQueryType::FTSQueryTypeWithScore) - { - auto score_actions = buildFTSScoreActions(group_builder.getCurrentHeader(), tidb_table_scan, context); - executeExpression(exec_context, group_builder, score_actions, log); - } - auto header = group_builder.getCurrentHeader(); const auto & schema_project_cols = buildTableScanProjectionCols(tidb_table_scan.getLogicalTableID(), schema, header); diff --git a/dbms/src/Functions/FunctionsFullText.cpp b/dbms/src/Functions/FunctionsFullText.cpp index 57468963409..56b1687f8c6 100644 --- a/dbms/src/Functions/FunctionsFullText.cpp +++ b/dbms/src/Functions/FunctionsFullText.cpp @@ -34,7 +34,6 @@ #include #include #include -#include #include #include @@ -51,7 +50,7 @@ constexpr size_t default_min_token_size = 3; constexpr size_t default_max_token_size = 84; constexpr size_t default_ngram_token_size = 2; constexpr std::string_view ngram_parser = "NGRAM_V1"; -constexpr UInt32 fts_match_expression_metadata_version = 1; +constexpr UInt32 fts_match_boolean_metadata_version = 1; struct FullTextToken { @@ -236,107 +235,6 @@ bool textStartsWith(std::string_view value, std::string_view prefix, const TiDB: return matcher->match(value.data(), value.size()); } -bool isBooleanWhitespace(char c) -{ - return c == ' ' || c == '\t' || c == '\n' || c == '\r'; -} - -bool parseBooleanQuery( - std::string_view query, - std::vector & clauses, - const TiDB::TiDBCollatorPtr & collator = nullptr) -{ - for (size_t i = 0; i < query.size();) - { - while (i < query.size() && isBooleanWhitespace(query[i])) - ++i; - if (i == query.size()) - break; - - BooleanClause clause; - // The STANDARD parser used by #70485 accepts only the operators that - // affect filtering. Other InnoDB operators affect scoring or phrase - // proximity and must not be silently treated as ordinary text. - if (query[i] == '%' || query[i] == '(' || query[i] == ')' || query[i] == '<' || query[i] == '>' - || query[i] == '~' || query[i] == '@') - return false; - - if (query[i] == '+' || query[i] == '-') - { - clause.modifier = query[i] == '+' ? BooleanClause::Modifier::Must : BooleanClause::Modifier::MustNot; - ++i; - } - - if (i == query.size()) - return false; - - // InnoDB accepts a leading wildcard as a no-op. A trailing wildcard - // is handled below and turns a term into a prefix query. - if (query[i] == '*') - { - ++i; - if (i == query.size()) - continue; - } - - String raw; - if (query[i] == '"') - { - clause.phrase = true; - ++i; - const size_t start = i; - while (i < query.size() && query[i] != '"' && query[i] != '\n') - ++i; - if (i == query.size() || query[i] != '"') - return false; - raw.assign(query.substr(start, i - start)); - ++i; - } - else - { - const size_t start = i; - while (i < query.size() && !isBooleanWhitespace(query[i]) && query[i] != '+' && query[i] != '-' - && query[i] != '*' && query[i] != '%' && query[i] != '(' && query[i] != ')' && query[i] != '<' - && query[i] != '>' && query[i] != '~' && query[i] != '@') - ++i; - if (start == i) - return false; - raw.assign(query.substr(start, i - start)); - // The scan above stops before `*`, so consume the wildcard here - // and attach prefix semantics to this term. Leaving it for the - // next iteration incorrectly turns `run*` into an exact `run` - // match (and makes it fail against `runner`). - if (i < query.size() && query[i] == '*') - { - clause.prefix = true; - ++i; - } - } - - const auto terms = clause.prefix ? tokenizeText(raw, collator) : analyzeText(raw, collator); - if (clause.prefix && terms.size() != 1) - continue; - - const size_t first_position = terms.empty() ? 0 : terms.front().position; - for (const auto & term : terms) - { - const auto code_points = UTF8::countCodePoints( - reinterpret_cast(term.text.data()), - term.text.size()); - if (code_points <= default_max_token_size && (clause.prefix || code_points >= default_min_token_size) - && (clause.prefix || !isDefaultStopword(term.text))) - { - clause.terms.push_back(term.text); - if (clause.phrase) - clause.offsets.push_back(term.position - first_position); - } - } - if (!clause.terms.empty() || clause.modifier == BooleanClause::Modifier::Must) - clauses.push_back(std::move(clause)); - } - return true; -} - bool matchesPhraseInColumn( const BooleanClause & clause, const FullTextColumn & document, @@ -388,59 +286,6 @@ bool matchesClause( }); } -size_t countClauseMatches( - const BooleanClause & clause, - const FullTextDocument & document, - const TiDB::TiDBCollatorPtr & collator) -{ - if (!matchesClause(clause, document, collator)) - return 0; - - if (!clause.phrase) - { - size_t count = 0; - for (const auto & term : clause.terms) - { - for (const auto & column : document) - { - count += std::count_if(column.begin(), column.end(), [&](const FullTextToken & token) { - return clause.prefix ? textStartsWith(token.text, term, collator) : textEquals(token.text, term, collator); - }); - } - } - return count; - } - - size_t count = 0; - for (const auto & column : document) - { - for (const auto & start : column) - { - bool matched = true; - for (size_t i = 0; i < clause.terms.size(); ++i) - { - const auto expected_position = start.position + clause.offsets[i]; - const auto it = std::find_if(column.begin(), column.end(), [&](const FullTextToken & token) { - return token.position == expected_position; - }); - if (it == column.end() || !textEquals(it->text, clause.terms[i], collator)) - { - matched = false; - break; - } - } - if (matched) - ++count; - } - } - return count; -} - -FullTextColumn analyzeColumn(std::string_view document, const TiDB::TiDBCollatorPtr & collator = nullptr) -{ - return analyzeText(document, collator); -} - FullTextColumn analyzeColumn( std::string_view document, bool use_ngram, @@ -454,51 +299,6 @@ FullTextColumn analyzeColumn( : analyzeText(document, collator, min_token_size, max_token_size, enable_stopword); } -Float64 matchBooleanScore( - const std::vector & clauses, - const FullTextDocument & document, - const TiDB::TiDBCollatorPtr & collator = nullptr) -{ - if (clauses.empty()) - return 0; - - bool has_positive = false; - bool has_must = false; - bool positive_match = false; - Float64 score = 0; - for (const auto & clause : clauses) - { - const bool matched = matchesClause(clause, document, collator); - const auto clause_score = static_cast(countClauseMatches(clause, document, collator)); - switch (clause.modifier) - { - case BooleanClause::Modifier::Must: - has_must = true; - if (!matched) - return 0; - score += clause_score; - break; - case BooleanClause::Modifier::MustNot: - if (matched) - return 0; - break; - case BooleanClause::Modifier::Should: - has_positive = true; - positive_match = positive_match || matched; - score += clause_score; - break; - } - } - - // With a required term, unprefixed terms are optional. Without one, at - // least one unprefixed term must match, matching BOOLEAN MODE semantics. - if (!(has_must || !has_positive || positive_match)) - return 0; - // A query containing only prohibited terms has no positive term to score, - // but an accepted row still has to pass the boolean filter. - return score > 0 ? score : 1; -} - bool matchBooleanPredicate( const std::vector & clauses, const FullTextDocument & document, @@ -534,17 +334,6 @@ bool matchBooleanPredicate( return has_must || !has_positive || positive_match; } -Float64 matchBooleanScore( - std::string_view query, - const FullTextDocument & document, - const TiDB::TiDBCollatorPtr & collator = nullptr) -{ - std::vector clauses; - if (!parseBooleanQuery(query, clauses, collator)) - return 0; - return matchBooleanScore(clauses, document, collator); -} - Float64 matchBooleanScore( const tipb::FTSBooleanQuery & query, const FullTextDocument & document, @@ -703,50 +492,6 @@ Float64 matchBooleanScore( return matchBooleanPredicate(clauses, document, collator) ? 1 : 0; } -Float64 matchBooleanScore( - std::string_view query, - std::string_view document, - const TiDB::TiDBCollatorPtr & collator = nullptr) -{ - return matchBooleanScore(query, FullTextDocument{analyzeColumn(document, collator)}, collator); -} - -size_t countTermMatches( - const String & term, - const FullTextColumn & document, - const TiDB::TiDBCollatorPtr & collator) -{ - return std::count_if(document.begin(), document.end(), [&](const FullTextToken & token) { - return textEquals(token.text, term, collator); - }); -} - -Float64 matchNaturalLanguageScore( - std::string_view query, - const FullTextDocument & document, - const TiDB::TiDBCollatorPtr & collator = nullptr) -{ - const auto query_tokens = analyzeText(query, collator); - if (query_tokens.empty()) - return 0; - - std::unordered_set unique_query_tokens; - Float64 score = 0; - for (const auto & query_token : query_tokens) - { - if (!unique_query_tokens.insert(query_token.text).second) - continue; - for (const auto & column : document) - score += static_cast(countTermMatches(query_token.text, column, collator)); - } - return score; -} - -bool queryUsesBooleanSyntax(std::string_view query) -{ - return query.find_first_of("+-\"*%()<>~@") != std::string_view::npos; -} - String getStringAt(const IColumn & column, size_t row) { if (const auto * column_nullable = typeid_cast(&column)) @@ -788,120 +533,21 @@ bool decodeFTSBooleanQuery(const IColumn & column, tipb::FTSBooleanQuery & query if (constant == nullptr) return false; const auto encoded = constant->getValue(); - tipb::FTSMatchExpressionMetadata metadata; - if (!metadata.ParseFromString(encoded) || metadata.version() != fts_match_expression_metadata_version + tipb::FTSMatchBooleanMetadata metadata; + if (!metadata.ParseFromString(encoded) || metadata.version() != fts_match_boolean_metadata_version || !metadata.has_boolean_query()) return false; query = metadata.boolean_query(); + if (query.query_tokenizer() != "STANDARD_V1" && query.query_tokenizer() != ngram_parser) + return false; return true; } -class FunctionFTSMatchWord final : public IFunction -{ -public: - static constexpr auto name = "fts_match_word"; - static FunctionPtr create(const Context &) { return std::make_shared(); } - - String getName() const override { return name; } - size_t getNumberOfArguments() const override { return 2; } - bool useDefaultImplementationForConstants() const override { return false; } - ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {0}; } - void setCollator(const TiDB::TiDBCollatorPtr & collator_) override { collator = collator_; } - - DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override - { - bool nullable = false; - for (const auto & argument : arguments) - { - if (!removeNullable(argument)->isString()) - throw Exception( - "Illegal type " + argument->getName() + " of argument of function " + getName(), - ErrorCodes::ILLEGAL_COLUMN); - nullable = nullable || argument->isNullable(); - } - DataTypePtr result_type = std::make_shared(); - return nullable ? std::make_shared(result_type) : result_type; - } - - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result) const override - { - const auto * query_column = typeid_cast(&*block.getByPosition(arguments[0]).column); - if (query_column == nullptr) - throw Exception( - "The query argument of fts_match_word must be constant", - ErrorCodes::ILLEGAL_COLUMN); - - const auto & document_column = block.getByPosition(arguments[1]).column; - auto output = ColumnFloat64::create(document_column->size()); - auto & output_data = output->getData(); - - if (isNullAt(*query_column, 0)) - { - auto null_map = ColumnUInt8::create(document_column->size(), 1); - block.getByPosition(result).column = ColumnNullable::create(std::move(output), std::move(null_map)); - return; - } - const auto query = query_column->getValue(); - - if (block.getByPosition(arguments[1]).type->isNullable() || document_column->isColumnNullable()) - { - auto null_map = ColumnUInt8::create(document_column->size(), 0); - auto & null_map_data = null_map->getData(); - for (size_t row = 0; row < document_column->size(); ++row) - { - if (isNullAt(*document_column, row)) - null_map_data[row] = 1; - else - output_data[row] = matchBooleanScore(query, getStringAt(*document_column, row), collator); - } - block.getByPosition(result).column = ColumnNullable::create(std::move(output), std::move(null_map)); - return; - } - - if (const auto * document = checkAndGetColumn(&*document_column)) - { - const auto & chars = document->getChars(); - const auto & offsets = document->getOffsets(); - for (size_t row = 0, begin = 0; row < offsets.size(); ++row) - { - const size_t end = offsets[row]; - const size_t length = end - begin - 1; - output_data[row] - = matchBooleanScore( - query, - std::string_view(reinterpret_cast(&chars[begin]), length), - collator); - begin = end; - } - } - else if (const auto * document = typeid_cast(&*document_column)) - { - const auto value = document->getValue(); - const Float64 matched = matchBooleanScore(query, value, collator); - std::fill(output_data.begin(), output_data.end(), matched); - } - else - { - throw Exception( - "Illegal column " + document_column->getName() + " of argument of function " + getName(), - ErrorCodes::ILLEGAL_COLUMN); - } - block.getByPosition(result).column = std::move(output); - } - -private: - TiDB::TiDBCollatorPtr collator; -}; - -class FunctionFTSMatchExpression : public IFunction +class FunctionFTSMatchBooleanExpression final : public IFunction { public: - static constexpr auto name = "fts_match_expression"; - static FunctionPtr create(const Context &) { return std::make_shared(); } - - explicit FunctionFTSMatchExpression(bool require_boolean_query = false) - : require_boolean_query(require_boolean_query) - {} + static constexpr auto name = "fts_match_boolean_expression"; + static FunctionPtr create(const Context &) { return std::make_shared(); } String getName() const override { return name; } size_t getNumberOfArguments() const override { return 0; } @@ -916,8 +562,8 @@ class FunctionFTSMatchExpression : public IFunction DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override { - if (arguments.size() < 2) - throw Exception("fts_match_expression requires a query and at least one column"); + if (arguments.size() < 3) + throw Exception("fts_match_boolean_expression requires a query, at least one column, and metadata"); bool nullable = false; for (const auto & argument : arguments) { @@ -936,7 +582,7 @@ class FunctionFTSMatchExpression : public IFunction const auto * query_column = typeid_cast(&*block.getByPosition(arguments[0]).column); if (query_column == nullptr) throw Exception( - "The query argument of fts_match_expression must be constant", + "The query argument of fts_match_boolean_expression must be constant", ErrorCodes::ILLEGAL_COLUMN); const size_t rows = block.getByPosition(arguments[1]).column->size(); @@ -958,25 +604,19 @@ class FunctionFTSMatchExpression : public IFunction block.getByPosition(result).column = std::move(result_column); return; } - const auto query = query_column->getValue(); - size_t document_argument_end = arguments.size(); + size_t document_argument_end = arguments.size() - 1; tipb::FTSBooleanQuery protocol_boolean_query; - const bool has_protocol_boolean_query = require_boolean_query; - if (require_boolean_query - && (arguments.size() <= 2 - || !decodeFTSBooleanQuery(*block.getByPosition(arguments.back()).column, protocol_boolean_query))) + if (arguments.size() <= 2 + || !decodeFTSBooleanQuery(*block.getByPosition(arguments.back()).column, protocol_boolean_query)) throw Exception( - "fts_match_expression_with_boolean_query requires valid Boolean query metadata", + "fts_match_boolean_expression requires valid Boolean query metadata", ErrorCodes::ILLEGAL_COLUMN); - if (has_protocol_boolean_query) - --document_argument_end; - const bool use_ngram = has_protocol_boolean_query && protocol_boolean_query.query_tokenizer() == ngram_parser; + const bool use_ngram = protocol_boolean_query.query_tokenizer() == ngram_parser; const size_t ngram_token_size = !use_ngram || protocol_boolean_query.ngram_token_size() == 0 ? default_ngram_token_size : protocol_boolean_query.ngram_token_size(); - const bool has_standard_config = has_protocol_boolean_query - && (protocol_boolean_query.innodb_ft_min_token_size() != 0 - || protocol_boolean_query.innodb_ft_max_token_size() != 0); + const bool has_standard_config = protocol_boolean_query.innodb_ft_min_token_size() != 0 + || protocol_boolean_query.innodb_ft_max_token_size() != 0; const size_t min_token_size = has_standard_config ? protocol_boolean_query.innodb_ft_min_token_size() : default_min_token_size; @@ -1009,12 +649,7 @@ class FunctionFTSMatchExpression : public IFunction max_token_size, enable_stopword)); } - if (has_protocol_boolean_query) - output_data[row] = matchBooleanScore(protocol_boolean_query, document, collator); - else if (queryUsesBooleanSyntax(query)) - output_data[row] = matchBooleanScore(query, document, collator); - else - output_data[row] = matchNaturalLanguageScore(query, document, collator); + output_data[row] = matchBooleanScore(protocol_boolean_query, document, collator); } ColumnPtr result_column; if (null_map) @@ -1026,27 +661,11 @@ class FunctionFTSMatchExpression : public IFunction private: TiDB::TiDBCollatorPtr collator; - bool require_boolean_query; -}; - -class FunctionFTSMatchExpressionWithBooleanQuery final : public FunctionFTSMatchExpression -{ -public: - static constexpr auto name = "fts_match_expression_with_boolean_query"; - static FunctionPtr create(const Context &) { return std::make_shared(); } - - FunctionFTSMatchExpressionWithBooleanQuery() - : FunctionFTSMatchExpression(true) - {} - - String getName() const override { return name; } }; } void registerFunctionsFullText(FunctionFactory & factory) { - factory.registerFunction(FunctionFactory::CaseInsensitive); - factory.registerFunction(FunctionFactory::CaseInsensitive); - factory.registerFunction(FunctionFactory::CaseInsensitive); + factory.registerFunction(FunctionFactory::CaseInsensitive); } } // namespace DB diff --git a/dbms/src/Functions/tests/gtest_fulltext.cpp b/dbms/src/Functions/tests/gtest_fulltext.cpp index cb53c9546c9..8640ef10344 100644 --- a/dbms/src/Functions/tests/gtest_fulltext.cpp +++ b/dbms/src/Functions/tests/gtest_fulltext.cpp @@ -28,94 +28,16 @@ namespace { String serializeFTSBooleanQuery(const tipb::FTSBooleanQuery & query) { - tipb::FTSMatchExpressionMetadata metadata; + tipb::FTSMatchBooleanMetadata metadata; metadata.set_version(1); *metadata.mutable_boolean_query() = query; + if (metadata.boolean_query().query_tokenizer().empty()) + metadata.mutable_boolean_query()->set_query_tokenizer("STANDARD_V1"); return metadata.SerializeAsString(); } } // namespace -TEST_F(TestFullText, MatchWordBoolean) -try -{ - const auto query = createConstColumn(4, "+quick -slow"); - const auto documents = createColumn({ - "A quick brown fox", - "A slow brown fox", - "A brown fox", - "A QUICK runner", - }); - ASSERT_COLUMN_EQ(createColumn({1, 0, 0, 1}), executeFunction("fts_match_word", {query, documents})); -} -CATCH - -TEST_F(TestFullText, MatchWordPhraseAndPrefix) -try -{ - ASSERT_COLUMN_EQ( - createColumn({1, 0}), - executeFunction( - "fts_match_word", - {createConstColumn(2, "\"quick brown\""), createColumn({"quick brown fox", "quick fox brown"})})); - ASSERT_COLUMN_EQ( - createColumn({1, 0}), - executeFunction( - "fts_match_word", - {createConstColumn(2, "run*"), createColumn({"runner", "walk"})})); -} -CATCH - -TEST_F(TestFullText, MatchWordTokenBoundaryAndPhrasePositions) -try -{ - ASSERT_COLUMN_EQ( - createColumn({0, 1}), - executeFunction( - "fts_match_word", - {createConstColumn(2, "cat"), createColumn({"concatenate", "a cat"})})); - ASSERT_COLUMN_EQ( - createColumn({0, 1}), - executeFunction( - "fts_match_word", - {createConstColumn(2, "\"quick the brown\""), - createColumn({"quick brown", "quick the brown"})})); -} -CATCH - -TEST_F(TestFullText, MatchWordNull) -try -{ - ASSERT_COLUMN_EQ( - createColumn>({1, {}}), - executeFunction( - "fts_match_word", - {createConstColumn(2, "hello"), createColumn>({"hello world", {}})})); -} -CATCH - -TEST_F(TestFullText, MatchWordScore) -try -{ - ASSERT_COLUMN_EQ( - createColumn({1, 2, 0}), - executeFunction( - "fts_match_word", - {createConstColumn(3, "quick"), createColumn({"quick", "quick quick", "slow"})})); -} -CATCH - -TEST_F(TestFullText, MatchExpression) -try -{ - ASSERT_COLUMN_EQ( - createColumn({1, 0}), - executeFunction( - "fts_match_expression", - {createConstColumn(2, "quick fox"), createColumn({"quick brown", "slow turtle"})})); -} -CATCH - -TEST_F(TestFullText, MatchExpressionMultiColumnNullable) +TEST_F(TestFullText, MatchBooleanMultiColumnNullable) try { tipb::FTSBooleanQuery boolean_query; @@ -128,7 +50,7 @@ try ASSERT_COLUMN_EQ( createColumn>({1, 1, 0, 0}), executeFunction( - "fts_match_expression_with_boolean_query", + "fts_match_boolean_expression", {createConstColumn(4, "+quick"), createColumn>({{}, "quick fox", {}, "slow fox"}), createColumn>({"quick fox", {}, "slow fox", {}}), @@ -138,7 +60,7 @@ try } CATCH -TEST_F(TestFullText, MatchExpressionProtocolBooleanQuery) +TEST_F(TestFullText, MatchBooleanProtocolQuery) try { tipb::FTSBooleanQuery boolean_query; @@ -155,7 +77,7 @@ try ASSERT_COLUMN_EQ( createColumn({1, 0, 0}), executeFunction( - "fts_match_expression_with_boolean_query", + "fts_match_boolean_expression", {createConstColumn(3, "+quick -slow"), createColumn({"quick brown", "slow fox", "brown fox"}), createConstColumn(3, metadata)}, @@ -171,7 +93,7 @@ try ASSERT_COLUMN_EQ( createColumn({0, 0, 0}), executeFunction( - "fts_match_expression_with_boolean_query", + "fts_match_boolean_expression", {createConstColumn(3, "-slow"), createColumn({"quick brown", "slow fox", "brown fox"}), createConstColumn(3, prohibited_metadata)}, @@ -180,7 +102,7 @@ try } CATCH -TEST_F(TestFullText, MatchExpressionRejectsInvalidMetadata) +TEST_F(TestFullText, MatchBooleanRejectsInvalidMetadata) try { tipb::FTSBooleanQuery boolean_query; @@ -189,12 +111,12 @@ try required->mutable_term()->set_term_type(tipb::FTSBooleanTermWord); required->mutable_term()->set_text("quick"); - tipb::FTSMatchExpressionMetadata metadata; + tipb::FTSMatchBooleanMetadata metadata; metadata.set_version(2); *metadata.mutable_boolean_query() = boolean_query; ASSERT_THROW( executeFunction( - "fts_match_expression_with_boolean_query", + "fts_match_boolean_expression", {createConstColumn(1, "+quick"), createColumn({"quick brown"}), createConstColumn(1, metadata.SerializeAsString())}, @@ -206,7 +128,7 @@ try metadata.clear_boolean_query(); ASSERT_THROW( executeFunction( - "fts_match_expression_with_boolean_query", + "fts_match_boolean_expression", {createConstColumn(1, "+quick"), createColumn({"quick brown"}), createConstColumn(1, metadata.SerializeAsString())}, @@ -216,7 +138,7 @@ try } CATCH -TEST_F(TestFullText, MatchExpressionStandardAnalyzerProtocolSettings) +TEST_F(TestFullText, MatchBooleanStandardAnalyzerProtocolSettings) try { const auto make_metadata = [](const String & term, UInt32 min_token_size, UInt32 max_token_size, bool enable_stopword) { @@ -236,7 +158,7 @@ try ASSERT_COLUMN_EQ( createColumn({1, 0}), executeFunction( - "fts_match_expression_with_boolean_query", + "fts_match_boolean_expression", {createConstColumn(2, "+go"), createColumn({"go", "good"}), createConstColumn(2, short_term_metadata)}, @@ -247,7 +169,7 @@ try ASSERT_COLUMN_EQ( createColumn({1, 0}), executeFunction( - "fts_match_expression_with_boolean_query", + "fts_match_boolean_expression", {createConstColumn(2, "+the"), createColumn({"the", "there"}), createConstColumn(2, stopword_metadata)}, @@ -258,7 +180,7 @@ try ASSERT_COLUMN_EQ( createColumn({0}), executeFunction( - "fts_match_expression_with_boolean_query", + "fts_match_boolean_expression", {createConstColumn(1, "+thé"), createColumn({"thé"}), createConstColumn(1, collated_stopword_metadata)}, @@ -269,7 +191,7 @@ try ASSERT_COLUMN_EQ( createColumn({0}), executeFunction( - "fts_match_expression_with_boolean_query", + "fts_match_boolean_expression", {createConstColumn(1, "+extraordinary"), createColumn({"extraordinary"}), createConstColumn(1, max_size_metadata)}, @@ -278,7 +200,7 @@ try } CATCH -TEST_F(TestFullText, MatchExpressionNgramProtocolBooleanQuery) +TEST_F(TestFullText, MatchBooleanNgramProtocolQuery) try { tipb::FTSBooleanQuery boolean_query; @@ -297,7 +219,7 @@ try ASSERT_COLUMN_EQ( createColumn({1, 0, 1, 0}), executeFunction( - "fts_match_expression_with_boolean_query", + "fts_match_boolean_expression", {createConstColumn(4, "+数据库 -mysql"), createColumn({"数据库系统", "MySQL 数据库", "数据库", "数据科学"}), createConstColumn(4, metadata)}, @@ -316,14 +238,14 @@ try ASSERT_COLUMN_EQ( createColumn({1, 0}), executeFunction( - "fts_match_expression_with_boolean_query", + "fts_match_boolean_expression", {createConstColumn(2, "+mysql"), createColumn({"MySQL", "PostgreSQL"}), createConstColumn(2, case_metadata)}, ci_collator, true)); } CATCH -TEST_F(TestFullText, MatchExpressionNgramPrefix) +TEST_F(TestFullText, MatchBooleanNgramPrefix) try { const auto make_metadata = [](UInt32 token_size, const String & prefix) { @@ -343,62 +265,22 @@ try const auto utf8mb4_bin = TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_BIN); ASSERT_COLUMN_EQ( createColumn({0, 1, 1, 1}), - executeFunction("fts_match_expression_with_boolean_query", {query, documents, metadata}, utf8mb4_bin, true)); + executeFunction("fts_match_boolean_expression", {query, documents, metadata}, utf8mb4_bin, true)); const auto utf8mb4_general_ci = TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_GENERAL_CI); ASSERT_COLUMN_EQ( createColumn({1, 1, 1, 1}), - executeFunction("fts_match_expression_with_boolean_query", {query, documents, metadata}, utf8mb4_general_ci, true)); + executeFunction("fts_match_boolean_expression", {query, documents, metadata}, utf8mb4_general_ci, true)); const auto short_query = createConstColumn(4, "+c*"); const auto short_metadata = createConstColumn(4, make_metadata(2, "c")); ASSERT_COLUMN_EQ( createColumn({1, 1, 1, 1}), - executeFunction("fts_match_expression_with_boolean_query", {short_query, documents, short_metadata}, utf8mb4_general_ci, true)); + executeFunction("fts_match_boolean_expression", {short_query, documents, short_metadata}, utf8mb4_general_ci, true)); } CATCH -TEST_F(TestFullText, MatchExpressionScore) -try -{ - ASSERT_COLUMN_EQ( - createColumn({1, 3, 0}), - executeFunction( - "fts_match_expression", - {createConstColumn(3, "quick fox"), - createColumn({"quick brown", "quick fox fox", "slow turtle"})})); -} -CATCH - -TEST_F(TestFullText, MatchExpressionCollation) -try -{ - const auto ci_collator = TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_GENERAL_CI); - ASSERT_COLUMN_EQ( - createColumn({1, 1}), - executeFunction( - "fts_match_expression", - {createConstColumn(2, "+quick"), createColumn({"QUICK runner", "quick runner"})}, - ci_collator)); - - const auto binary_collator = TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_BIN); - ASSERT_COLUMN_EQ( - createColumn({0, 1}), - executeFunction( - "fts_match_expression", - {createConstColumn(2, "+quick"), createColumn({"QUICK runner", "quick runner"})}, - binary_collator)); - - ASSERT_COLUMN_EQ( - createColumn({1, 0}), - executeFunction( - "fts_match_expression", - {createConstColumn(2, "+run*"), createColumn({"RUNNER", "walk"})}, - ci_collator)); -} -CATCH - -TEST_F(TestFullText, MatchExpressionCollationMatrix) +TEST_F(TestFullText, MatchBooleanCollationMatrix) try { const auto query = createConstColumn(3, "+cafe"); @@ -421,66 +303,43 @@ try const auto utf8mb4_bin = TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_BIN); ASSERT_COLUMN_EQ( createColumn({0, 0, 1}), - executeFunction("fts_match_expression_with_boolean_query", {query, documents, word_metadata}, utf8mb4_bin, true)); + executeFunction("fts_match_boolean_expression", {query, documents, word_metadata}, utf8mb4_bin, true)); ASSERT_COLUMN_EQ( createColumn({0, 1, 1}), - executeFunction("fts_match_expression_with_boolean_query", {prefix_query, documents, prefix_metadata}, utf8mb4_bin, true)); + executeFunction("fts_match_boolean_expression", {prefix_query, documents, prefix_metadata}, utf8mb4_bin, true)); const auto utf8mb4_0900_bin = TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_0900_BIN); ASSERT_COLUMN_EQ( createColumn({0, 0, 1}), - executeFunction("fts_match_expression_with_boolean_query", {query, documents, word_metadata}, utf8mb4_0900_bin, true)); + executeFunction("fts_match_boolean_expression", {query, documents, word_metadata}, utf8mb4_0900_bin, true)); ASSERT_COLUMN_EQ( createColumn({0, 1, 1}), - executeFunction("fts_match_expression_with_boolean_query", {prefix_query, documents, prefix_metadata}, utf8mb4_0900_bin, true)); + executeFunction("fts_match_boolean_expression", {prefix_query, documents, prefix_metadata}, utf8mb4_0900_bin, true)); const auto utf8mb4_general_ci = TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_GENERAL_CI); ASSERT_COLUMN_EQ( createColumn({1, 1, 1}), - executeFunction("fts_match_expression_with_boolean_query", {query, documents, word_metadata}, utf8mb4_general_ci, true)); + executeFunction("fts_match_boolean_expression", {query, documents, word_metadata}, utf8mb4_general_ci, true)); ASSERT_COLUMN_EQ( createColumn({1, 1, 1}), - executeFunction("fts_match_expression_with_boolean_query", {prefix_query, documents, prefix_metadata}, utf8mb4_general_ci, true)); + executeFunction("fts_match_boolean_expression", {prefix_query, documents, prefix_metadata}, utf8mb4_general_ci, true)); const auto utf8mb4_unicode_ci = TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_UNICODE_CI); ASSERT_COLUMN_EQ( createColumn({1, 1, 1}), - executeFunction("fts_match_expression_with_boolean_query", {query, documents, word_metadata}, utf8mb4_unicode_ci, true)); + executeFunction("fts_match_boolean_expression", {query, documents, word_metadata}, utf8mb4_unicode_ci, true)); ASSERT_COLUMN_EQ( createColumn({1, 1, 1}), - executeFunction("fts_match_expression_with_boolean_query", {prefix_query, documents, prefix_metadata}, utf8mb4_unicode_ci, true)); + executeFunction("fts_match_boolean_expression", {prefix_query, documents, prefix_metadata}, utf8mb4_unicode_ci, true)); const auto utf8mb4_0900_ai_ci = TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_0900_AI_CI); ASSERT_COLUMN_EQ( createColumn({1, 1, 1}), - executeFunction("fts_match_expression_with_boolean_query", {query, documents, word_metadata}, utf8mb4_0900_ai_ci, true)); + executeFunction("fts_match_boolean_expression", {query, documents, word_metadata}, utf8mb4_0900_ai_ci, true)); ASSERT_COLUMN_EQ( createColumn({1, 1, 1}), - executeFunction("fts_match_expression_with_boolean_query", {prefix_query, documents, prefix_metadata}, utf8mb4_0900_ai_ci, true)); -} -CATCH - -TEST_F(TestFullText, MatchExpressionNullColumn) -try -{ - ASSERT_COLUMN_EQ( - createColumn>({0, 1}), - executeFunction( - "fts_match_expression", - {createConstColumn(2, "+indexing -postgresql"), - createColumn({"Indexing", "Indexing"}), - createColumn>({"PostgreSQL", {}})})); + executeFunction("fts_match_boolean_expression", {prefix_query, documents, prefix_metadata}, utf8mb4_0900_ai_ci, true)); } CATCH -TEST_F(TestFullText, MatchBooleanUnsupportedScoreOperator) -try -{ - ASSERT_COLUMN_EQ( - createColumn({0}), - executeFunction( - "fts_match_word", - {createConstColumn(1, ">quick"), createColumn({"quick"})})); -} -CATCH } // namespace DB::tests From 7f0cac25d70235a6acfa51e3ee8548587fadf754 Mon Sep 17 00:00:00 2001 From: Zhigao Tong Date: Thu, 8 Oct 2026 22:32:02 +0800 Subject: [PATCH 15/32] tiflash: execute Local MATCH through scalar evaluation --- contrib/tipb | 2 +- .../DAGExpressionAnalyzerHelper.cpp | 6 +- dbms/src/Flash/Coprocessor/DAGUtils.cpp | 2 +- dbms/src/Functions/FunctionsFullText.cpp | 671 ----------- .../Functions/FunctionsLocalMatchAgainst.cpp | 1013 +++++++++++++++++ ...ullText.h => FunctionsLocalMatchAgainst.h} | 2 +- dbms/src/Functions/registerFunctions.cpp | 4 +- dbms/src/Functions/tests/gtest_fulltext.cpp | 345 ------ .../tests/gtest_local_match_against.cpp | 555 +++++++++ 9 files changed, 1576 insertions(+), 1024 deletions(-) delete mode 100644 dbms/src/Functions/FunctionsFullText.cpp create mode 100644 dbms/src/Functions/FunctionsLocalMatchAgainst.cpp rename dbms/src/Functions/{FunctionsFullText.h => FunctionsLocalMatchAgainst.h} (90%) delete mode 100644 dbms/src/Functions/tests/gtest_fulltext.cpp create mode 100644 dbms/src/Functions/tests/gtest_local_match_against.cpp diff --git a/contrib/tipb b/contrib/tipb index e2791d0a2b5..f7dbf3d45cd 160000 --- a/contrib/tipb +++ b/contrib/tipb @@ -1 +1 @@ -Subproject commit e2791d0a2b58299ee333e63709eefa1700f166d2 +Subproject commit f7dbf3d45cd28bd47b41d9b80033f127a3a98de9 diff --git a/dbms/src/Flash/Coprocessor/DAGExpressionAnalyzerHelper.cpp b/dbms/src/Flash/Coprocessor/DAGExpressionAnalyzerHelper.cpp index 1029a0e80e9..a64f3bea5c8 100644 --- a/dbms/src/Flash/Coprocessor/DAGExpressionAnalyzerHelper.cpp +++ b/dbms/src/Flash/Coprocessor/DAGExpressionAnalyzerHelper.cpp @@ -520,12 +520,12 @@ String DAGExpressionAnalyzerHelper::buildDefaultFunction( String name = analyzer->getActions(child, actions); argument_names.push_back(name); } - if (expr.sig() == tipb::ScalarFuncSig::FTSMatchBooleanExpression) + if (expr.sig() == tipb::ScalarFuncSig::LocalMatchAgainstBoolean) { - // FTS metadata is carried in the generic scalar-function metadata slot + // Local MATCH metadata is carried in the generic scalar-function metadata slot // and must not be confused with SQL arguments. The runtime function // interface uses a private variant with a required final constant. - func_name = "fts_match_boolean_expression"; + func_name = "local_match_against_boolean"; const auto metadata_expr = constructStringLiteralTiExpr(expr.val()); argument_names.push_back(analyzer->getActions(metadata_expr, actions)); } diff --git a/dbms/src/Flash/Coprocessor/DAGUtils.cpp b/dbms/src/Flash/Coprocessor/DAGUtils.cpp index febe7644289..79abe601636 100644 --- a/dbms/src/Flash/Coprocessor/DAGUtils.cpp +++ b/dbms/src/Flash/Coprocessor/DAGUtils.cpp @@ -455,7 +455,7 @@ const std::unordered_map scalar_func_map({ {tipb::ScalarFuncSig::RegexpInStrSig, "regexp_instr"}, {tipb::ScalarFuncSig::RegexpReplaceSig, "regexp_replace"}, {tipb::ScalarFuncSig::RegexpSubstrSig, "regexp_substr"}, - {tipb::ScalarFuncSig::FTSMatchBooleanExpression, "fts_match_boolean_expression"}, + {tipb::ScalarFuncSig::LocalMatchAgainstBoolean, "local_match_against_boolean"}, {tipb::ScalarFuncSig::JsonExtractSig, "json_extract"}, {tipb::ScalarFuncSig::JsonUnquoteSig, "json_unquote"}, diff --git a/dbms/src/Functions/FunctionsFullText.cpp b/dbms/src/Functions/FunctionsFullText.cpp deleted file mode 100644 index 56b1687f8c6..00000000000 --- a/dbms/src/Functions/FunctionsFullText.cpp +++ /dev/null @@ -1,671 +0,0 @@ -// Copyright 2026 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include - -namespace DB -{ -namespace ErrorCodes -{ -extern const int ILLEGAL_COLUMN; -} - -namespace -{ -constexpr size_t default_min_token_size = 3; -constexpr size_t default_max_token_size = 84; -constexpr size_t default_ngram_token_size = 2; -constexpr std::string_view ngram_parser = "NGRAM_V1"; -constexpr UInt32 fts_match_boolean_metadata_version = 1; - -struct FullTextToken -{ - String text; - size_t position = 0; -}; - -struct BooleanClause -{ - enum class Modifier - { - Should, - Must, - MustNot, - }; - - Modifier modifier = Modifier::Should; - bool phrase = false; - bool prefix = false; - std::vector terms; - std::vector offsets; -}; - -using FullTextColumn = std::vector; -using FullTextDocument = std::vector; - -bool isFullTextToken(UInt32 code_point) -{ - return isAlphaNumericASCII(static_cast(code_point)) || code_point == '_' || Poco::Unicode::isAlpha(code_point) - || Poco::Unicode::isDigit(code_point); -} - -std::pair decodeCodePoint(std::string_view text, size_t offset) -{ - const auto decoded = UTF8::utf8Decode(text.data() + offset, text.size() - offset); - if (decoded.second == 0 || decoded.first == UTF8::UTF8_Error || decoded.second > text.size() - offset) - return {static_cast(text[offset]), 1}; - return {decoded.first, decoded.second}; -} - -std::vector tokenizeText(std::string_view text, const TiDB::TiDBCollatorPtr & collator = nullptr) -{ - std::vector result; - size_t position = 0; - for (size_t i = 0; i < text.size();) - { - const auto [code_point, length] = decodeCodePoint(text, i); - if (!isFullTextToken(code_point)) - { - i += length; - continue; - } - - String token; - while (i < text.size()) - { - const auto [code_point, length] = decodeCodePoint(text, i); - if (!isFullTextToken(code_point)) - break; - token.append(text.data() + i, length); - i += length; - } - - // If a collation is present, retain the source spelling and let the - // collator decide case and accent equivalence during matching. This - // is important for binary collations, where lower-casing would make - // a case-sensitive MATCH unexpectedly case-insensitive. Keep the - // legacy lower-case behavior for callers without collation metadata. - if (!collator) - token = Poco::UTF8::toLower(token); - result.push_back({std::move(token), position++}); - } - return result; -} - -bool isDefaultStopword(const String & token, const TiDB::TiDBCollatorPtr & collator = nullptr) -{ - static const std::unordered_set stopwords{ - "a", "about", "an", "are", "as", "at", "be", "by", "com", "de", "en", "for", - "from", "how", "i", "in", "is", "it", "la", "of", "on", "or", "that", "the", - "this", "to", "was", "what", "when", "where", "who", "will", "with", "und", "www"}; - // Stopwords are stored in canonical lower-case form. Keep their existing - // case-insensitive behavior independently from document matching. - if (stopwords.contains(Poco::UTF8::toLower(token))) - return true; - if (collator) - { - return std::any_of(stopwords.begin(), stopwords.end(), [&](const String & stopword) { - return collator->compare(token.data(), token.size(), stopword.data(), stopword.size()) == 0; - }); - } - return false; -} - -std::vector analyzeText( - std::string_view text, - const TiDB::TiDBCollatorPtr & collator = nullptr, - size_t min_token_size = default_min_token_size, - size_t max_token_size = default_max_token_size, - bool enable_stopword = true) -{ - std::vector result; - for (auto & token : tokenizeText(text, collator)) - { - const auto code_points = UTF8::countCodePoints( - reinterpret_cast(token.text.data()), - token.text.size()); - if (code_points >= min_token_size && code_points <= max_token_size - && (!enable_stopword || !isDefaultStopword(token.text, collator))) - result.push_back(std::move(token)); - } - return result; -} - -std::vector analyzeNgramText( - std::string_view text, - size_t ngram_token_size, - const TiDB::TiDBCollatorPtr & collator = nullptr) -{ - if (ngram_token_size == 0) - return {}; - - std::vector result; - size_t next_position_base = 0; - for (auto & token : tokenizeText(text, collator)) - { - std::vector char_boundaries; - char_boundaries.reserve(token.text.size() + 1); - char_boundaries.push_back(0); - for (size_t offset = 0; offset < token.text.size();) - { - const auto [code_point, length] = decodeCodePoint(token.text, offset); - (void)code_point; - offset += length; - char_boundaries.push_back(offset); - } - - const size_t char_count = char_boundaries.size() - 1; - const size_t base_position = std::max(token.position, next_position_base); - if (char_count < ngram_token_size) - { - next_position_base = std::max(next_position_base, token.position + 1); - continue; - } - - for (size_t start = 0; start + ngram_token_size <= char_count; ++start) - { - const size_t begin = char_boundaries[start]; - const size_t end = char_boundaries[start + ngram_token_size]; - result.push_back({token.text.substr(begin, end - begin), base_position + start}); - } - next_position_base = base_position + char_count - ngram_token_size + 1; - } - return result; -} - -bool textEquals(std::string_view lhs, std::string_view rhs, const TiDB::TiDBCollatorPtr & collator) -{ - if (!collator) - return lhs == rhs; - return collator->compare(lhs.data(), lhs.size(), rhs.data(), rhs.size()) == 0; -} - -bool textStartsWith(std::string_view value, std::string_view prefix, const TiDB::TiDBCollatorPtr & collator) -{ - if (!collator) - return value.starts_with(prefix); - - // Reuse the same collation-aware pattern implementation as LIKE. Escape - // token characters that have LIKE meaning before appending the wildcard. - String pattern; - pattern.reserve(prefix.size() + 1); - for (const char c : prefix) - { - if (c == '\\' || c == '%' || c == '_') - pattern.push_back('\\'); - pattern.push_back(c); - } - pattern.push_back('%'); - auto matcher = collator->pattern(); - matcher->compile(pattern, '\\'); - return matcher->match(value.data(), value.size()); -} - -bool matchesPhraseInColumn( - const BooleanClause & clause, - const FullTextColumn & document, - const TiDB::TiDBCollatorPtr & collator) -{ - if (clause.terms.empty() || clause.terms.size() != clause.offsets.size()) - return false; - - for (const auto & start : document) - { - bool matched = true; - for (size_t i = 0; i < clause.terms.size(); ++i) - { - const auto expected_position = start.position + clause.offsets[i]; - const auto it = std::find_if(document.begin(), document.end(), [&](const FullTextToken & token) { - return token.position == expected_position; - }); - if (it == document.end() || !textEquals(it->text, clause.terms[i], collator)) - { - matched = false; - break; - } - } - if (matched) - return true; - } - return false; -} - -bool matchesClause( - const BooleanClause & clause, - const FullTextDocument & document, - const TiDB::TiDBCollatorPtr & collator) -{ - if (clause.terms.empty()) - return false; - - if (clause.phrase) - return std::any_of(document.begin(), document.end(), [&](const FullTextColumn & column) { - return matchesPhraseInColumn(clause, column, collator); - }); - - return std::all_of(clause.terms.begin(), clause.terms.end(), [&](const String & term) { - return std::any_of(document.begin(), document.end(), [&](const FullTextColumn & column) { - return std::any_of(column.begin(), column.end(), [&](const FullTextToken & token) { - return clause.prefix ? textStartsWith(token.text, term, collator) : textEquals(token.text, term, collator); - }); - }); - }); -} - -FullTextColumn analyzeColumn( - std::string_view document, - bool use_ngram, - size_t ngram_token_size, - const TiDB::TiDBCollatorPtr & collator = nullptr, - size_t min_token_size = default_min_token_size, - size_t max_token_size = default_max_token_size, - bool enable_stopword = true) -{ - return use_ngram ? analyzeNgramText(document, ngram_token_size, collator) - : analyzeText(document, collator, min_token_size, max_token_size, enable_stopword); -} - -bool matchBooleanPredicate( - const std::vector & clauses, - const FullTextDocument & document, - const TiDB::TiDBCollatorPtr & collator = nullptr) -{ - if (clauses.empty()) - return false; - - bool has_positive = false; - bool has_must = false; - bool positive_match = false; - for (const auto & clause : clauses) - { - const bool matched = matchesClause(clause, document, collator); - switch (clause.modifier) - { - case BooleanClause::Modifier::Must: - has_must = true; - if (!matched) - return false; - break; - case BooleanClause::Modifier::MustNot: - if (matched) - return false; - break; - case BooleanClause::Modifier::Should: - has_positive = true; - positive_match = positive_match || matched; - break; - } - } - - return has_must || !has_positive || positive_match; -} - -Float64 matchBooleanScore( - const tipb::FTSBooleanQuery & query, - const FullTextDocument & document, - const TiDB::TiDBCollatorPtr & collator = nullptr) -{ - std::vector clauses; - const bool use_ngram = query.query_tokenizer() == ngram_parser; - const size_t ngram_token_size = query.ngram_token_size() == 0 ? default_ngram_token_size : query.ngram_token_size(); - const bool has_standard_config = query.innodb_ft_min_token_size() != 0 || query.innodb_ft_max_token_size() != 0; - const size_t min_token_size = has_standard_config ? query.innodb_ft_min_token_size() : default_min_token_size; - const size_t max_token_size = has_standard_config ? query.innodb_ft_max_token_size() : default_max_token_size; - const bool enable_stopword = has_standard_config ? query.innodb_ft_enable_stopword() : true; - if (!use_ngram && (max_token_size == 0 || min_token_size > max_token_size)) - return 0; - auto analyze_query = [&](std::string_view text) { - return use_ngram ? analyzeNgramText(text, ngram_token_size, collator) - : analyzeText(text, collator, min_token_size, max_token_size, enable_stopword); - }; - for (const auto & node : query.nodes()) - { - if (!node.has_term()) - return 0; - - const auto & term = node.term(); - BooleanClause clause; - switch (node.occur()) - { - case tipb::FTSBooleanOccur::FTSBooleanOccurMust: - clause.modifier = BooleanClause::Modifier::Must; - break; - case tipb::FTSBooleanOccur::FTSBooleanOccurMustNot: - clause.modifier = BooleanClause::Modifier::MustNot; - break; - case tipb::FTSBooleanOccur::FTSBooleanOccurShould: - clause.modifier = BooleanClause::Modifier::Should; - break; - default: - return 0; - } - - switch (term.term_type()) - { - case tipb::FTSBooleanTermType::FTSBooleanTermWord: - { - const auto terms = analyze_query(term.text()); - if (use_ngram) - { - clause.phrase = true; - const size_t first_position = terms.empty() ? 0 : terms.front().position; - for (const auto & token : terms) - { - clause.terms.push_back(token.text); - clause.offsets.push_back(token.position - first_position); - } - } - else - { - for (const auto & token : terms) - clause.terms.push_back(token.text); - } - break; - } - case tipb::FTSBooleanTermType::FTSBooleanTermPrefix: - { - const auto terms = use_ngram ? analyzeNgramText(term.text(), ngram_token_size, collator) - : tokenizeText(term.text(), collator); - if (use_ngram) - { - if (!terms.empty()) - { - // TiDB expands an NGRAM prefix at least as long as one - // gram into a positional phrase of query ngrams. Matching - // only one analyzed term here makes prefixes such as - // "caf*" ("ca", "af" at token size 2) never match. - clause.phrase = true; - const size_t first_position = terms.front().position; - for (const auto & token : terms) - { - clause.terms.push_back(token.text); - clause.offsets.push_back(token.position - first_position); - } - break; - } - - // A query shorter than the configured gram size is retained - // by TiDB as a prefix over document ngrams instead of being - // discarded by the analyzer. - const auto source_terms = tokenizeText(term.text(), collator); - if (source_terms.size() == 1) - { - const auto code_points = UTF8::countCodePoints( - reinterpret_cast(source_terms.front().text.data()), - source_terms.front().text.size()); - if (code_points < ngram_token_size) - { - clause.prefix = true; - clause.terms.push_back(source_terms.front().text); - break; - } - } - - if (clause.modifier == BooleanClause::Modifier::Must) - clauses.push_back(std::move(clause)); - continue; - } - - clause.prefix = true; - if (terms.size() != 1) - { - if (clause.modifier == BooleanClause::Modifier::Must) - clauses.push_back(std::move(clause)); - continue; - } - const auto code_points = UTF8::countCodePoints( - reinterpret_cast(terms.front().text.data()), - terms.front().text.size()); - if (code_points > max_token_size) - { - if (clause.modifier == BooleanClause::Modifier::Must) - clauses.push_back(std::move(clause)); - continue; - } - clause.terms.push_back(terms.front().text); - break; - } - case tipb::FTSBooleanTermType::FTSBooleanTermPhrase: - { - clause.phrase = true; - const auto terms = analyze_query(term.text()); - const size_t first_position = terms.empty() ? 0 : terms.front().position; - for (const auto & token : terms) - { - clause.terms.push_back(token.text); - clause.offsets.push_back(token.position - first_position); - } - break; - } - default: - return 0; - } - - if (!clause.terms.empty() || clause.modifier == BooleanClause::Modifier::Must) - clauses.push_back(std::move(clause)); - } - - // A BOOLEAN MODE query containing only prohibited terms has no positive - // branch. TiDB's local evaluator treats it as matching no rows. - if (std::none_of(clauses.begin(), clauses.end(), [](const BooleanClause & clause) { - return clause.modifier != BooleanClause::Modifier::MustNot; - })) - return 0; - - // The protocol path is the no-score MATCH ... AGAINST BOOLEAN MODE - // predicate introduced by #70484/#70485. TiDB's local evaluator returns - // a boolean 0/1 result, so do not expose term-frequency counts here. - return matchBooleanPredicate(clauses, document, collator) ? 1 : 0; -} - -String getStringAt(const IColumn & column, size_t row) -{ - if (const auto * column_nullable = typeid_cast(&column)) - { - if (column_nullable->getNullMapData()[row]) - return {}; - return getStringAt(column_nullable->getNestedColumn(), row); - } - - if (const auto * column_const = typeid_cast(&column)) - return column_const->getValue(); - - const auto * column_string = checkAndGetColumn(&column); - if (column_string == nullptr) - throw Exception("Full-text arguments must be string columns", ErrorCodes::ILLEGAL_COLUMN); - - const auto & chars = column_string->getChars(); - const auto & offsets = column_string->getOffsets(); - const size_t begin = row == 0 ? 0 : offsets[row - 1]; - const size_t end = offsets[row]; - return String(reinterpret_cast(&chars[begin]), end - begin - 1); -} - -bool isNullAt(const IColumn & column, size_t row) -{ - if (const auto * column_nullable = typeid_cast(&column)) - return column_nullable->getNullMapData()[row]; - if (const auto * column_const = typeid_cast(&column)) - { - if (const auto * nested_nullable = typeid_cast(&column_const->getDataColumn())) - return nested_nullable->getNullMapData()[0]; - } - return false; -} - -bool decodeFTSBooleanQuery(const IColumn & column, tipb::FTSBooleanQuery & query) -{ - const auto * constant = typeid_cast(&column); - if (constant == nullptr) - return false; - const auto encoded = constant->getValue(); - tipb::FTSMatchBooleanMetadata metadata; - if (!metadata.ParseFromString(encoded) || metadata.version() != fts_match_boolean_metadata_version - || !metadata.has_boolean_query()) - return false; - query = metadata.boolean_query(); - if (query.query_tokenizer() != "STANDARD_V1" && query.query_tokenizer() != ngram_parser) - return false; - return true; -} - -class FunctionFTSMatchBooleanExpression final : public IFunction -{ -public: - static constexpr auto name = "fts_match_boolean_expression"; - static FunctionPtr create(const Context &) { return std::make_shared(); } - - String getName() const override { return name; } - size_t getNumberOfArguments() const override { return 0; } - bool isVariadic() const override { return true; } - bool useDefaultImplementationForConstants() const override { return false; } - // NULL MATCH columns are empty documents, not NULL MATCH results. The - // implementation below handles nullable columns per row, so the generic - // IFunction NULL wrapper must not short-circuit the whole expression. - bool useDefaultImplementationForNulls() const override { return false; } - ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {0}; } - void setCollator(const TiDB::TiDBCollatorPtr & collator_) override { collator = collator_; } - - DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override - { - if (arguments.size() < 3) - throw Exception("fts_match_boolean_expression requires a query, at least one column, and metadata"); - bool nullable = false; - for (const auto & argument : arguments) - { - if (!removeNullable(argument)->isString()) - throw Exception( - "Illegal type " + argument->getName() + " of argument of function " + getName(), - ErrorCodes::ILLEGAL_COLUMN); - nullable = nullable || argument->isNullable(); - } - DataTypePtr result_type = std::make_shared(); - return nullable ? std::make_shared(result_type) : result_type; - } - - void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result) const override - { - const auto * query_column = typeid_cast(&*block.getByPosition(arguments[0]).column); - if (query_column == nullptr) - throw Exception( - "The query argument of fts_match_boolean_expression must be constant", - ErrorCodes::ILLEGAL_COLUMN); - - const size_t rows = block.getByPosition(arguments[1]).column->size(); - auto output = ColumnFloat64::create(rows, 0); - auto & output_data = output->getData(); - const bool nullable = block.getByPosition(result).type->isNullable(); - auto null_map = nullable ? ColumnUInt8::create(rows, 0) : nullptr; - auto * null_map_data = null_map ? &null_map->getData() : nullptr; - - if (isNullAt(*query_column, 0)) - { - if (null_map_data != nullptr) - std::fill(null_map_data->begin(), null_map_data->end(), 1); - ColumnPtr result_column; - if (null_map) - result_column = ColumnNullable::create(std::move(output), std::move(null_map)); - else - result_column = std::move(output); - block.getByPosition(result).column = std::move(result_column); - return; - } - size_t document_argument_end = arguments.size() - 1; - tipb::FTSBooleanQuery protocol_boolean_query; - if (arguments.size() <= 2 - || !decodeFTSBooleanQuery(*block.getByPosition(arguments.back()).column, protocol_boolean_query)) - throw Exception( - "fts_match_boolean_expression requires valid Boolean query metadata", - ErrorCodes::ILLEGAL_COLUMN); - const bool use_ngram = protocol_boolean_query.query_tokenizer() == ngram_parser; - const size_t ngram_token_size = !use_ngram || protocol_boolean_query.ngram_token_size() == 0 - ? default_ngram_token_size - : protocol_boolean_query.ngram_token_size(); - const bool has_standard_config = protocol_boolean_query.innodb_ft_min_token_size() != 0 - || protocol_boolean_query.innodb_ft_max_token_size() != 0; - const size_t min_token_size = has_standard_config - ? protocol_boolean_query.innodb_ft_min_token_size() - : default_min_token_size; - const size_t max_token_size = has_standard_config - ? protocol_boolean_query.innodb_ft_max_token_size() - : default_max_token_size; - const bool enable_stopword = !has_standard_config || protocol_boolean_query.innodb_ft_enable_stopword(); - - for (size_t row = 0; row < rows; ++row) - { - FullTextDocument document; - document.reserve(document_argument_end - 1); - for (size_t arg = 1; arg < document_argument_end; ++arg) - { - // A NULL MATCH column contributes no tokens. This is - // intentional: #70485 relies on a row with a NULL body - // still matching a required term from another MATCH column. - if (isNullAt(*block.getByPosition(arguments[arg]).column, row)) - { - document.emplace_back(); - continue; - } - document.push_back( - analyzeColumn( - getStringAt(*block.getByPosition(arguments[arg]).column, row), - use_ngram, - ngram_token_size, - collator, - min_token_size, - max_token_size, - enable_stopword)); - } - output_data[row] = matchBooleanScore(protocol_boolean_query, document, collator); - } - ColumnPtr result_column; - if (null_map) - result_column = ColumnNullable::create(std::move(output), std::move(null_map)); - else - result_column = std::move(output); - block.getByPosition(result).column = std::move(result_column); - } - -private: - TiDB::TiDBCollatorPtr collator; -}; -} - -void registerFunctionsFullText(FunctionFactory & factory) -{ - factory.registerFunction(FunctionFactory::CaseInsensitive); -} -} // namespace DB diff --git a/dbms/src/Functions/FunctionsLocalMatchAgainst.cpp b/dbms/src/Functions/FunctionsLocalMatchAgainst.cpp new file mode 100644 index 00000000000..2c6d34daad5 --- /dev/null +++ b/dbms/src/Functions/FunctionsLocalMatchAgainst.cpp @@ -0,0 +1,1013 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ +namespace ErrorCodes +{ +extern const int ILLEGAL_COLUMN; +} + +namespace +{ +constexpr size_t default_min_token_size = 3; +constexpr size_t default_max_token_size = 84; +constexpr size_t default_ngram_token_size = 2; +// This is the Local MATCH semantic protocol version, not the TiDB/TiFlash +// product version. Version 1 fixes the Boolean AST interpretation, analyzer +// behavior, and built-in stopword set. TiFlash must support a version before +// TiDB emits it; unknown versions are rejected, and existing versions must +// never be reinterpreted with changed semantics. +constexpr UInt32 local_match_against_protocol_version = 1; + +struct FullTextToken +{ + String text; + size_t position = 0; + size_t code_points = 0; +}; + +class FullTextColumn +{ +public: + using Container = std::vector; + using iterator = Container::iterator; + using const_iterator = Container::const_iterator; + + void clear() { used = 0; } + + void reserve(size_t size) { tokens.reserve(size); } + + void append(std::string_view text, size_t position, size_t code_points) + { + if (used == tokens.size()) + tokens.emplace_back(); + auto & token = tokens[used++]; + token.text.assign(text.data(), text.size()); + token.position = position; + token.code_points = code_points; + } + + template + void filterInPlace(Predicate predicate) + { + size_t write = 0; + for (size_t read = 0; read < used; ++read) + { + if (!predicate(tokens[read])) + continue; + if (write != read) + std::swap(tokens[write], tokens[read]); + ++write; + } + used = write; + } + + bool empty() const { return used == 0; } + size_t size() const { return used; } + FullTextToken & front() { return tokens.front(); } + const FullTextToken & front() const { return tokens.front(); } + FullTextToken & operator[](size_t index) { return tokens[index]; } + const FullTextToken & operator[](size_t index) const { return tokens[index]; } + iterator begin() { return tokens.begin(); } + iterator end() { return tokens.begin() + used; } + const_iterator begin() const { return tokens.begin(); } + const_iterator end() const { return tokens.begin() + used; } + +private: + Container tokens; + size_t used = 0; +}; + +struct BooleanClause +{ + enum class Modifier + { + Should, + Must, + MustNot, + }; + + Modifier modifier = Modifier::Should; + bool phrase = false; + bool prefix = false; + std::vector terms; + std::vector offsets; + std::vector> prefix_matchers; +}; + +struct AnalyzerScratch +{ + FullTextColumn raw_tokens; + std::vector char_boundaries; +}; + +struct CompiledBooleanQuery +{ + std::vector clauses; + bool matches_nothing = false; +}; + +void lowercaseTokenInPlace(String & token) +{ + const bool is_ascii = std::all_of(token.begin(), token.end(), [](unsigned char c) { return c < 0x80; }); + if (is_ascii) + { + for (char & c : token) + if (c >= 'A' && c <= 'Z') + c = static_cast(c + ('a' - 'A')); + return; + } + Poco::UTF8::toLowerInPlace(token); +} + +bool isFullTextToken(UInt32 code_point) +{ + return isAlphaNumericASCII(static_cast(code_point)) || code_point == '_' || Poco::Unicode::isAlpha(code_point) + || Poco::Unicode::isDigit(code_point); +} + +std::pair decodeCodePoint(std::string_view text, size_t offset) +{ + const auto decoded = UTF8::utf8Decode(text.data() + offset, text.size() - offset); + if (decoded.second == 0 || decoded.first == UTF8::UTF8_Error || decoded.second > text.size() - offset) + return {static_cast(text[offset]), 1}; + return {decoded.first, decoded.second}; +} + +void tokenizeTextInto( + std::string_view text, + FullTextColumn & result, + const TiDB::TiDBCollatorPtr & collator = nullptr, + bool preserve_case = false) +{ + result.clear(); + size_t position = 0; + for (size_t i = 0; i < text.size();) + { + const auto [code_point, length] = decodeCodePoint(text, i); + if (!isFullTextToken(code_point)) + { + i += length; + continue; + } + + const size_t token_start = i; + size_t code_points = 0; + while (i < text.size()) + { + const auto [code_point, length] = decodeCodePoint(text, i); + if (!isFullTextToken(code_point)) + break; + ++code_points; + i += length; + } + + // If a collation is present, retain the source spelling and let the + // collator decide case and accent equivalence during matching. This + // is important for binary collations, where lower-casing would make + // a case-sensitive MATCH unexpectedly case-insensitive. Keep the + // legacy lower-case behavior for callers without collation metadata. + result.append(text.substr(token_start, i - token_start), position++, code_points); + if (!collator && !preserve_case) + lowercaseTokenInPlace(result[result.size() - 1].text); + } +} + +FullTextColumn tokenizeText( + std::string_view text, + const TiDB::TiDBCollatorPtr & collator = nullptr, + bool preserve_case = false) +{ + FullTextColumn result; + tokenizeTextInto(text, result, collator, preserve_case); + return result; +} + +const std::unordered_set & defaultStopwords() +{ + static const std::unordered_set stopwords{ + "a", "about", "an", "are", "as", "at", "be", "by", "com", "de", "en", "for", + "from", "how", "i", "in", "is", "it", "la", "of", "on", "or", "that", "the", + "this", "to", "was", "what", "when", "where", "who", "will", "with", "und", "www"}; + return stopwords; +} + +const std::unordered_map> & defaultStopwordsByLength() +{ + static const auto stopwords_by_length = [] { + std::unordered_map> result; + for (const auto & stopword : defaultStopwords()) + { + const size_t length + = UTF8::countCodePoints(reinterpret_cast(stopword.data()), stopword.size()); + result[length].push_back(stopword); + } + return result; + }(); + return stopwords_by_length; +} + +bool isDefaultStopword(std::string_view token, const TiDB::TiDBCollatorPtr & stopword_collator) +{ + const auto & stopwords = defaultStopwords(); + if (!stopword_collator) + return stopwords.contains(Poco::UTF8::toLower(String(token))); + // MySQL compares stopwords using collation_server. This is independent of + // the MATCH column collation and may be case-sensitive (for example + // utf8mb4_bin). + return std::any_of(stopwords.begin(), stopwords.end(), [&](const String & stopword) { + return stopword_collator->compare(token.data(), token.size(), stopword.data(), stopword.size()) == 0; + }); +} + +bool containsNgramStopword( + std::string_view token, + const std::vector & char_boundaries, + size_t ngram_start, + size_t ngram_token_size, + const TiDB::TiDBCollatorPtr & stopword_collator) +{ + const size_t ngram_end = ngram_start + ngram_token_size; + const auto & stopwords = defaultStopwords(); + const auto & stopwords_by_length = defaultStopwordsByLength(); + for (size_t start = ngram_start; start < ngram_end; ++start) + { + for (size_t end = start + 1; end <= ngram_end; ++end) + { + const auto candidate = token.substr(char_boundaries[start], char_boundaries[end] - char_boundaries[start]); + if (!stopword_collator && stopwords.contains(Poco::UTF8::toLower(String(candidate)))) + return true; + if (stopword_collator) + { + const auto found = stopwords_by_length.find(end - start); + if (found != stopwords_by_length.end() + && std::any_of(found->second.begin(), found->second.end(), [&](const String & stopword) { + return stopword_collator + ->compare(candidate.data(), candidate.size(), stopword.data(), stopword.size()) + == 0; + })) + return true; + } + } + } + return false; +} + +void analyzeTextInto( + std::string_view text, + FullTextColumn & result, + const TiDB::TiDBCollatorPtr & collator = nullptr, + const TiDB::TiDBCollatorPtr & stopword_collator = nullptr, + size_t min_token_size = default_min_token_size, + size_t max_token_size = default_max_token_size, + bool enable_stopword = true) +{ + const bool preserve_case = enable_stopword && static_cast(stopword_collator); + tokenizeTextInto(text, result, collator, preserve_case); + result.filterInPlace([&](const FullTextToken & token) { + return token.code_points >= min_token_size && token.code_points <= max_token_size + && (!enable_stopword || !isDefaultStopword(token.text, stopword_collator)); + }); + // When stopwords are compared with collation_server, tokenization must + // retain source case until filtering is complete. MATCH without an + // explicit column collation still uses the existing case-insensitive + // behavior afterwards. + if (!collator && preserve_case) + for (auto & token : result) + lowercaseTokenInPlace(token.text); +} + +FullTextColumn analyzeText( + std::string_view text, + const TiDB::TiDBCollatorPtr & collator = nullptr, + const TiDB::TiDBCollatorPtr & stopword_collator = nullptr, + size_t min_token_size = default_min_token_size, + size_t max_token_size = default_max_token_size, + bool enable_stopword = true) +{ + FullTextColumn result; + analyzeTextInto(text, result, collator, stopword_collator, min_token_size, max_token_size, enable_stopword); + return result; +} + +void analyzeNgramTextInto( + std::string_view text, + FullTextColumn & result, + AnalyzerScratch & scratch, + size_t ngram_token_size, + const TiDB::TiDBCollatorPtr & collator = nullptr, + const TiDB::TiDBCollatorPtr & stopword_collator = nullptr, + bool enable_stopword = true) +{ + result.clear(); + if (ngram_token_size == 0) + return; + + const bool preserve_case = enable_stopword && static_cast(stopword_collator); + tokenizeTextInto(text, scratch.raw_tokens, collator, preserve_case); + size_t next_position_base = 0; + auto & char_boundaries = scratch.char_boundaries; + for (auto & token : scratch.raw_tokens) + { + char_boundaries.clear(); + char_boundaries.reserve(token.code_points + 1); + char_boundaries.push_back(0); + for (size_t offset = 0; offset < token.text.size();) + { + const auto [code_point, length] = decodeCodePoint(token.text, offset); + (void)code_point; + offset += length; + char_boundaries.push_back(offset); + } + + const size_t char_count = token.code_points; + const size_t base_position = std::max(token.position, next_position_base); + if (char_count < ngram_token_size) + { + next_position_base = std::max(next_position_base, token.position + 1); + continue; + } + + for (size_t start = 0; start + ngram_token_size <= char_count; ++start) + { + const size_t begin = char_boundaries[start]; + const size_t end = char_boundaries[start + ngram_token_size]; + if (!enable_stopword + || !containsNgramStopword(token.text, char_boundaries, start, ngram_token_size, stopword_collator)) + { + result.append( + std::string_view(token.text).substr(begin, end - begin), + base_position + start, + ngram_token_size); + if (!collator && preserve_case) + lowercaseTokenInPlace(result[result.size() - 1].text); + } + } + next_position_base = base_position + char_count - ngram_token_size + 1; + } +} + +FullTextColumn analyzeNgramText( + std::string_view text, + size_t ngram_token_size, + const TiDB::TiDBCollatorPtr & collator = nullptr, + const TiDB::TiDBCollatorPtr & stopword_collator = nullptr, + bool enable_stopword = true) +{ + FullTextColumn result; + AnalyzerScratch scratch; + analyzeNgramTextInto(text, result, scratch, ngram_token_size, collator, stopword_collator, enable_stopword); + return result; +} + +bool textEquals(std::string_view lhs, std::string_view rhs, const TiDB::TiDBCollatorPtr & collator) +{ + if (!collator) + return lhs == rhs; + return collator->compare(lhs.data(), lhs.size(), rhs.data(), rhs.size()) == 0; +} + +String makePrefixPattern(std::string_view prefix) +{ + // Reuse the same collation-aware pattern implementation as LIKE. Escape + // token characters that have LIKE meaning before appending the wildcard. + String pattern; + pattern.reserve(prefix.size() + 1); + for (const char c : prefix) + { + if (c == '\\' || c == '%' || c == '_') + pattern.push_back('\\'); + pattern.push_back(c); + } + pattern.push_back('%'); + return pattern; +} + +std::unique_ptr compilePrefixMatcher( + std::string_view prefix, + const TiDB::TiDBCollatorPtr & collator) +{ + auto matcher = collator->pattern(); + matcher->compile(makePrefixPattern(prefix), '\\'); + return matcher; +} + +bool textStartsWith(std::string_view value, std::string_view prefix, const TiDB::TiDBCollatorPtr & collator) +{ + if (!collator) + return value.starts_with(prefix); + const auto matcher = compilePrefixMatcher(prefix, collator); + return matcher->match(value.data(), value.size()); +} + +bool matchesTermAt( + const BooleanClause & clause, + size_t term_index, + const FullTextToken & token, + const TiDB::TiDBCollatorPtr & collator) +{ + if (!clause.prefix) + return textEquals(token.text, clause.terms[term_index], collator); + if (collator && term_index < clause.prefix_matchers.size()) + return clause.prefix_matchers[term_index]->match(token.text.data(), token.text.size()); + return textStartsWith(token.text, clause.terms[term_index], collator); +} + +struct ClauseMatchState +{ + std::vector matched_terms; + size_t remaining_terms = 0; + bool matched = false; +}; + +void initializeClauseMatchStates( + const std::vector & clauses, + std::vector & states) +{ + states.resize(clauses.size()); + for (size_t i = 0; i < clauses.size(); ++i) + if (!clauses[i].phrase) + { + states[i].matched_terms.resize(clauses[i].terms.size()); + states[i].remaining_terms = clauses[i].terms.size(); + } +} + +bool matchesPhraseEndingAt( + const BooleanClause & clause, + const FullTextColumn & column, + const FullTextToken & end, + const TiDB::TiDBCollatorPtr & collator) +{ + if (clause.terms.empty() || clause.terms.size() != clause.offsets.size()) + return false; + + const size_t last_term = clause.terms.size() - 1; + const size_t last_offset = clause.offsets[last_term]; + if (end.position < last_offset || !matchesTermAt(clause, last_term, end, collator)) + return false; + + const size_t start_position = end.position - last_offset; + for (size_t i = 0; i < last_term; ++i) + { + const size_t expected_position = start_position + clause.offsets[i]; + const auto it = std::lower_bound( + column.begin(), + column.end(), + expected_position, + [](const FullTextToken & token, size_t position) { return token.position < position; }); + if (it == column.end() || it->position != expected_position || !textEquals(it->text, clause.terms[i], collator)) + return false; + } + return true; +} + +void analyzeColumnInto( + std::string_view document, + FullTextColumn & output, + AnalyzerScratch & scratch, + bool use_ngram, + size_t ngram_token_size, + const TiDB::TiDBCollatorPtr & collator = nullptr, + const TiDB::TiDBCollatorPtr & stopword_collator = nullptr, + size_t min_token_size = default_min_token_size, + size_t max_token_size = default_max_token_size, + bool enable_stopword = true) +{ + if (use_ngram) + analyzeNgramTextInto( + document, output, scratch, ngram_token_size, collator, stopword_collator, enable_stopword); + else + analyzeTextInto( + document, output, collator, stopword_collator, min_token_size, max_token_size, enable_stopword); +} + +void resetClauseMatchStates(std::vector & states) +{ + for (auto & state : states) + { + state.matched = false; + state.remaining_terms = state.matched_terms.size(); + std::fill(state.matched_terms.begin(), state.matched_terms.end(), 0); + } +} + +bool updateBooleanMatchStatesForColumn( + const CompiledBooleanQuery & query, + const FullTextColumn & column, + const TiDB::TiDBCollatorPtr & collator, + std::vector & states) +{ + const auto & clauses = query.clauses; + if (query.matches_nothing || clauses.empty()) + return false; + + // Walk this MATCH column's analyzed tokens once and update every clause's + // state. Phrase clauses are checked only when their final term is + // encountered; positional lookups preserve stopword gaps and phrases do + // not cross MATCH columns. + for (const auto & token : column) + { + for (size_t clause_index = 0; clause_index < clauses.size(); ++clause_index) + { + const auto & clause = clauses[clause_index]; + auto & state = states[clause_index]; + if (state.matched || clause.terms.empty()) + continue; + + if (clause.phrase) + { + if (matchesPhraseEndingAt(clause, column, token, collator)) + state.matched = true; + continue; + } + + for (size_t term_index = 0; term_index < clause.terms.size(); ++term_index) + if (!state.matched_terms[term_index] && matchesTermAt(clause, term_index, token, collator)) + { + state.matched_terms[term_index] = 1; + --state.remaining_terms; + } + state.matched = state.remaining_terms == 0; + if (state.matched && clause.modifier == BooleanClause::Modifier::MustNot) + return true; + } + } + return false; +} + +bool evaluateBooleanMatchResult( + const CompiledBooleanQuery & query, + const std::vector & states) +{ + if (query.matches_nothing || query.clauses.empty()) + return false; + bool has_positive = false; + bool has_must = false; + bool positive_match = false; + for (size_t i = 0; i < query.clauses.size(); ++i) + { + const auto & clause = query.clauses[i]; + const bool matched = states[i].matched; + switch (clause.modifier) + { + case BooleanClause::Modifier::Must: + has_must = true; + if (!matched) + return false; + break; + case BooleanClause::Modifier::MustNot: + if (matched) + return false; + break; + case BooleanClause::Modifier::Should: + has_positive = true; + positive_match = positive_match || matched; + break; + } + } + + return has_must || !has_positive || positive_match; +} + +CompiledBooleanQuery compileBooleanQuery( + const tipb::LocalMatchAgainstBooleanQuery & query, + bool use_ngram, + size_t ngram_token_size, + size_t min_token_size, + size_t max_token_size, + bool enable_stopword, + const TiDB::TiDBCollatorPtr & collator = nullptr, + const TiDB::TiDBCollatorPtr & stopword_collator = nullptr) +{ + CompiledBooleanQuery compiled; + if (!use_ngram && (max_token_size == 0 || min_token_size > max_token_size)) + { + compiled.matches_nothing = true; + return compiled; + } + auto analyze_query = [&](std::string_view text) { + return use_ngram + ? analyzeNgramText(text, ngram_token_size, collator, stopword_collator, enable_stopword) + : analyzeText(text, collator, stopword_collator, min_token_size, max_token_size, enable_stopword); + }; + for (const auto & node : query.nodes()) + { + BooleanClause clause; + switch (node.occur()) + { + case tipb::LocalMatchAgainstBooleanOccur::LocalMatchAgainstBooleanOccurMust: + clause.modifier = BooleanClause::Modifier::Must; + break; + case tipb::LocalMatchAgainstBooleanOccur::LocalMatchAgainstBooleanOccurMustNot: + clause.modifier = BooleanClause::Modifier::MustNot; + break; + case tipb::LocalMatchAgainstBooleanOccur::LocalMatchAgainstBooleanOccurShould: + clause.modifier = BooleanClause::Modifier::Should; + break; + default: + compiled.matches_nothing = true; + return compiled; + } + + switch (node.term_type()) + { + case tipb::LocalMatchAgainstBooleanTermType::LocalMatchAgainstBooleanTermWord: + { + const auto terms = analyze_query(node.text()); + if (use_ngram) + { + clause.phrase = true; + const size_t first_position = terms.empty() ? 0 : terms.front().position; + for (const auto & token : terms) + { + clause.terms.push_back(token.text); + clause.offsets.push_back(token.position - first_position); + } + } + else + { + for (const auto & token : terms) + clause.terms.push_back(token.text); + } + break; + } + case tipb::LocalMatchAgainstBooleanTermType::LocalMatchAgainstBooleanTermPrefix: + { + const auto terms = use_ngram + ? analyzeNgramText(node.text(), ngram_token_size, collator, stopword_collator, enable_stopword) + : tokenizeText(node.text(), collator); + if (use_ngram) + { + if (!terms.empty()) + { + // TiDB expands an NGRAM prefix at least as long as one + // gram into a positional phrase of query ngrams. Matching + // only one analyzed term here makes prefixes such as + // "caf*" ("ca", "af" at token size 2) never match. + clause.phrase = true; + const size_t first_position = terms.front().position; + for (const auto & token : terms) + { + clause.terms.push_back(token.text); + clause.offsets.push_back(token.position - first_position); + } + break; + } + + // A query shorter than the configured gram size is retained + // by TiDB as a prefix over document ngrams instead of being + // discarded by the analyzer. + const auto source_terms = tokenizeText(node.text(), collator); + if (source_terms.size() == 1) + { + const auto code_points = UTF8::countCodePoints( + reinterpret_cast(source_terms.front().text.data()), + source_terms.front().text.size()); + if (code_points < ngram_token_size) + { + clause.prefix = true; + clause.terms.push_back(source_terms.front().text); + break; + } + } + + if (clause.modifier == BooleanClause::Modifier::Must) + compiled.clauses.push_back(std::move(clause)); + continue; + } + + clause.prefix = true; + if (terms.size() != 1) + { + if (clause.modifier == BooleanClause::Modifier::Must) + compiled.clauses.push_back(std::move(clause)); + continue; + } + const auto code_points = UTF8::countCodePoints( + reinterpret_cast(terms.front().text.data()), + terms.front().text.size()); + if (code_points > max_token_size) + { + if (clause.modifier == BooleanClause::Modifier::Must) + compiled.clauses.push_back(std::move(clause)); + continue; + } + clause.terms.push_back(terms.front().text); + break; + } + case tipb::LocalMatchAgainstBooleanTermType::LocalMatchAgainstBooleanTermPhrase: + { + clause.phrase = true; + const auto terms = analyze_query(node.text()); + const size_t first_position = terms.empty() ? 0 : terms.front().position; + for (const auto & token : terms) + { + clause.terms.push_back(token.text); + clause.offsets.push_back(token.position - first_position); + } + break; + } + default: + compiled.matches_nothing = true; + return compiled; + } + + if (clause.prefix && collator) + { + clause.prefix_matchers.reserve(clause.terms.size()); + for (const auto & term : clause.terms) + clause.prefix_matchers.push_back(compilePrefixMatcher(term, collator)); + } + + if (!clause.terms.empty() || clause.modifier == BooleanClause::Modifier::Must) + compiled.clauses.push_back(std::move(clause)); + } + + // A BOOLEAN MODE query containing only prohibited terms has no positive + // branch. TiDB's local evaluator treats it as matching no rows. + if (std::none_of(compiled.clauses.begin(), compiled.clauses.end(), [](const BooleanClause & clause) { + return clause.modifier != BooleanClause::Modifier::MustNot; + })) + compiled.matches_nothing = true; + + return compiled; +} + +Float64 matchBooleanScore( + const CompiledBooleanQuery & query, + const std::vector & states) +{ + // The protocol path is the no-score MATCH ... AGAINST BOOLEAN MODE + // predicate introduced by #70484/#70485. TiDB's local evaluator returns + // a boolean 0/1 result, so do not expose term-frequency counts here. + return evaluateBooleanMatchResult(query, states) ? 1 : 0; +} + +class FullTextColumnAccessor +{ +public: + explicit FullTextColumnAccessor(const IColumn & column) { initialize(column, false); } + + bool isNull(size_t row) const + { + return nullable_column != nullptr + && nullable_column->getNullMapData()[nullable_is_constant ? 0 : row] != 0; + } + + std::string_view getString(size_t row) const + { + const auto & chars = string_column->getChars(); + const auto & offsets = string_column->getOffsets(); + const size_t string_row = string_is_constant ? 0 : row; + const size_t begin = string_row == 0 ? 0 : offsets[string_row - 1]; + const size_t end = offsets[string_row]; + return {reinterpret_cast(&chars[begin]), end - begin - 1}; + } + +private: + void initialize(const IColumn & column, bool row_is_constant) + { + if (const auto * column_const = typeid_cast(&column)) + { + string_is_constant = true; + initialize(column_const->getDataColumn(), true); + return; + } + if (const auto * column_nullable = typeid_cast(&column)) + { + nullable_column = column_nullable; + nullable_is_constant = row_is_constant; + initialize(column_nullable->getNestedColumn(), row_is_constant); + return; + } + string_column = checkAndGetColumn(&column); + if (string_column == nullptr) + throw Exception("Full-text arguments must be string columns", ErrorCodes::ILLEGAL_COLUMN); + } + + const ColumnString * string_column = nullptr; + const ColumnNullable * nullable_column = nullptr; + bool string_is_constant = false; + bool nullable_is_constant = false; +}; + +bool decodeLocalMatchAgainstBooleanQuery(const IColumn & column, tipb::LocalMatchAgainstBooleanQuery & query) +{ + const auto * constant = typeid_cast(&column); + if (constant == nullptr) + return false; + const auto encoded = constant->getValue(); + // Expr.val directly contains the versioned Local MATCH query. Validate the + // semantic version before interpreting parser or stopword fields; never + // guess a default for a version this TiFlash binary does not understand. + if (!query.ParseFromString(encoded) || query.version() != local_match_against_protocol_version) + return false; + if (query.parser() != tipb::LocalMatchAgainstParser::LocalMatchAgainstParserStandard + && query.parser() != tipb::LocalMatchAgainstParser::LocalMatchAgainstParserNgram) + return false; + if (query.stopword_mode() != tipb::LocalMatchAgainstStopwordMode::LocalMatchAgainstStopwordModeDisabled + && query.stopword_mode() != tipb::LocalMatchAgainstStopwordMode::LocalMatchAgainstStopwordModeBuiltin) + return false; + return true; +} + +class FunctionLocalMatchAgainstBoolean final : public IFunction +{ +public: + static constexpr auto name = "local_match_against_boolean"; + static FunctionPtr create(const Context &) { return std::make_shared(); } + + String getName() const override { return name; } + size_t getNumberOfArguments() const override { return 0; } + bool isVariadic() const override { return true; } + bool useDefaultImplementationForConstants() const override { return false; } + // NULL MATCH columns are empty documents, not NULL MATCH results. The + // implementation below handles nullable columns per row, so the generic + // IFunction NULL wrapper must not short-circuit the whole expression. + bool useDefaultImplementationForNulls() const override { return false; } + ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {0}; } + void setCollator(const TiDB::TiDBCollatorPtr & collator_) override { collator = collator_; } + + DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override + { + if (arguments.size() < 3) + throw Exception("local_match_against_boolean requires a query, at least one column, and metadata"); + bool nullable = false; + for (const auto & argument : arguments) + { + if (!removeNullable(argument)->isString()) + throw Exception( + "Illegal type " + argument->getName() + " of argument of function " + getName(), + ErrorCodes::ILLEGAL_COLUMN); + nullable = nullable || argument->isNullable(); + } + DataTypePtr result_type = std::make_shared(); + return nullable ? std::make_shared(result_type) : result_type; + } + + void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result) const override + { + const auto * query_column = typeid_cast(&*block.getByPosition(arguments[0]).column); + if (query_column == nullptr) + throw Exception( + "The query argument of local_match_against_boolean must be constant", + ErrorCodes::ILLEGAL_COLUMN); + + const size_t rows = block.getByPosition(arguments[1]).column->size(); + auto output = ColumnFloat64::create(rows, 0); + auto & output_data = output->getData(); + const bool nullable = block.getByPosition(result).type->isNullable(); + auto null_map = nullable ? ColumnUInt8::create(rows, 0) : nullptr; + auto * null_map_data = null_map ? &null_map->getData() : nullptr; + + const FullTextColumnAccessor query_accessor(*query_column); + if (query_accessor.isNull(0)) + { + if (null_map_data != nullptr) + std::fill(null_map_data->begin(), null_map_data->end(), 1); + ColumnPtr result_column; + if (null_map) + result_column = ColumnNullable::create(std::move(output), std::move(null_map)); + else + result_column = std::move(output); + block.getByPosition(result).column = std::move(result_column); + return; + } + size_t document_argument_end = arguments.size() - 1; + tipb::LocalMatchAgainstBooleanQuery protocol_boolean_query; + if (arguments.size() <= 2 + || !decodeLocalMatchAgainstBooleanQuery( + *block.getByPosition(arguments.back()).column, + protocol_boolean_query)) + throw Exception( + "local_match_against_boolean requires valid Boolean query metadata", + ErrorCodes::ILLEGAL_COLUMN); + const bool use_ngram + = protocol_boolean_query.parser() == tipb::LocalMatchAgainstParser::LocalMatchAgainstParserNgram; + const size_t ngram_token_size = !use_ngram || protocol_boolean_query.ngram_token_size() == 0 + ? default_ngram_token_size + : protocol_boolean_query.ngram_token_size(); + const bool has_standard_config = protocol_boolean_query.innodb_ft_min_token_size() != 0 + || protocol_boolean_query.innodb_ft_max_token_size() != 0; + const size_t min_token_size + = has_standard_config ? protocol_boolean_query.innodb_ft_min_token_size() : default_min_token_size; + const size_t max_token_size + = has_standard_config ? protocol_boolean_query.innodb_ft_max_token_size() : default_max_token_size; + const bool enable_stopword + = protocol_boolean_query.stopword_mode() + == tipb::LocalMatchAgainstStopwordMode::LocalMatchAgainstStopwordModeBuiltin; + const auto stopword_collator + = enable_stopword ? TiDB::ITiDBCollator::getCollator(protocol_boolean_query.stopword_collation()) : nullptr; + if (enable_stopword && !stopword_collator) + throw Exception( + "local_match_against_boolean requires a supported stopword collation when stopwords are enabled", + ErrorCodes::ILLEGAL_COLUMN); + const CompiledBooleanQuery compiled_query = compileBooleanQuery( + protocol_boolean_query, + use_ngram, + ngram_token_size, + min_token_size, + max_token_size, + enable_stopword, + collator, + stopword_collator); + std::vector document_columns; + document_columns.reserve(document_argument_end - 1); + for (size_t arg = 1; arg < document_argument_end; ++arg) + document_columns.emplace_back(*block.getByPosition(arguments[arg]).column); + std::vector clause_states; + initializeClauseMatchStates(compiled_query.clauses, clause_states); + FullTextColumn analyzed_column; + AnalyzerScratch analyzer_scratch; + + for (size_t row = 0; row < rows; ++row) + { + resetClauseMatchStates(clause_states); + bool rejected_by_prohibited_clause = false; + for (const auto & document_column : document_columns) + { + // A NULL MATCH column contributes no tokens. This is + // intentional: #70485 relies on a row with a NULL body + // still matching a required term from another MATCH column. + if (document_column.isNull(row)) + { + analyzed_column.clear(); + continue; + } + analyzeColumnInto( + document_column.getString(row), + analyzed_column, + analyzer_scratch, + use_ngram, + ngram_token_size, + collator, + stopword_collator, + min_token_size, + max_token_size, + enable_stopword); + if (updateBooleanMatchStatesForColumn(compiled_query, analyzed_column, collator, clause_states)) + { + rejected_by_prohibited_clause = true; + break; + } + } + output_data[row] = rejected_by_prohibited_clause ? 0 : matchBooleanScore(compiled_query, clause_states); + } + ColumnPtr result_column; + if (null_map) + result_column = ColumnNullable::create(std::move(output), std::move(null_map)); + else + result_column = std::move(output); + block.getByPosition(result).column = std::move(result_column); + } + +private: + TiDB::TiDBCollatorPtr collator; +}; +} // namespace + +void registerFunctionsLocalMatchAgainst(FunctionFactory & factory) +{ + factory.registerFunction(FunctionFactory::CaseInsensitive); +} +} // namespace DB diff --git a/dbms/src/Functions/FunctionsFullText.h b/dbms/src/Functions/FunctionsLocalMatchAgainst.h similarity index 90% rename from dbms/src/Functions/FunctionsFullText.h rename to dbms/src/Functions/FunctionsLocalMatchAgainst.h index 797560b0e9e..064c503718f 100644 --- a/dbms/src/Functions/FunctionsFullText.h +++ b/dbms/src/Functions/FunctionsLocalMatchAgainst.h @@ -18,5 +18,5 @@ namespace DB { -void registerFunctionsFullText(FunctionFactory & factory); +void registerFunctionsLocalMatchAgainst(FunctionFactory & factory); } diff --git a/dbms/src/Functions/registerFunctions.cpp b/dbms/src/Functions/registerFunctions.cpp index e3f833811d5..7a9193c392c 100644 --- a/dbms/src/Functions/registerFunctions.cpp +++ b/dbms/src/Functions/registerFunctions.cpp @@ -51,7 +51,7 @@ void registerFunctionsRegexpSubstr(FunctionFactory &); void registerFunctionsRegexpReplace(FunctionFactory &); void registerFunctionsGrouping(FunctionFactory &); void registerFunctionsVector(FunctionFactory &); -void registerFunctionsFullText(FunctionFactory &); +void registerFunctionsLocalMatchAgainst(FunctionFactory &); void registerFunctions() { @@ -86,7 +86,7 @@ void registerFunctions() registerFunctionsIsIPAddr(factory); registerFunctionsGrouping(factory); registerFunctionsVector(factory); - registerFunctionsFullText(factory); + registerFunctionsLocalMatchAgainst(factory); } } // namespace DB diff --git a/dbms/src/Functions/tests/gtest_fulltext.cpp b/dbms/src/Functions/tests/gtest_fulltext.cpp deleted file mode 100644 index 8640ef10344..00000000000 --- a/dbms/src/Functions/tests/gtest_fulltext.cpp +++ /dev/null @@ -1,345 +0,0 @@ -// Copyright 2026 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include -#include -#include -#include - -namespace DB::tests -{ -class TestFullText : public DB::tests::FunctionTest -{ -}; - -namespace -{ -String serializeFTSBooleanQuery(const tipb::FTSBooleanQuery & query) -{ - tipb::FTSMatchBooleanMetadata metadata; - metadata.set_version(1); - *metadata.mutable_boolean_query() = query; - if (metadata.boolean_query().query_tokenizer().empty()) - metadata.mutable_boolean_query()->set_query_tokenizer("STANDARD_V1"); - return metadata.SerializeAsString(); -} -} // namespace - -TEST_F(TestFullText, MatchBooleanMultiColumnNullable) -try -{ - tipb::FTSBooleanQuery boolean_query; - auto * required = boolean_query.add_nodes(); - required->set_occur(tipb::FTSBooleanOccurMust); - required->mutable_term()->set_term_type(tipb::FTSBooleanTermWord); - required->mutable_term()->set_text("quick"); - - const String metadata = serializeFTSBooleanQuery(boolean_query); - ASSERT_COLUMN_EQ( - createColumn>({1, 1, 0, 0}), - executeFunction( - "fts_match_boolean_expression", - {createConstColumn(4, "+quick"), - createColumn>({{}, "quick fox", {}, "slow fox"}), - createColumn>({"quick fox", {}, "slow fox", {}}), - createConstColumn(4, metadata)}, - nullptr, - true)); -} -CATCH - -TEST_F(TestFullText, MatchBooleanProtocolQuery) -try -{ - tipb::FTSBooleanQuery boolean_query; - auto * required = boolean_query.add_nodes(); - required->set_occur(tipb::FTSBooleanOccurMust); - required->mutable_term()->set_term_type(tipb::FTSBooleanTermWord); - required->mutable_term()->set_text("quick"); - auto * prohibited = boolean_query.add_nodes(); - prohibited->set_occur(tipb::FTSBooleanOccurMustNot); - prohibited->mutable_term()->set_term_type(tipb::FTSBooleanTermWord); - prohibited->mutable_term()->set_text("slow"); - - const String metadata = serializeFTSBooleanQuery(boolean_query); - ASSERT_COLUMN_EQ( - createColumn({1, 0, 0}), - executeFunction( - "fts_match_boolean_expression", - {createConstColumn(3, "+quick -slow"), - createColumn({"quick brown", "slow fox", "brown fox"}), - createConstColumn(3, metadata)}, - nullptr, - true)); - - tipb::FTSBooleanQuery prohibited_query; - auto * prohibited_only = prohibited_query.add_nodes(); - prohibited_only->set_occur(tipb::FTSBooleanOccurMustNot); - prohibited_only->mutable_term()->set_term_type(tipb::FTSBooleanTermWord); - prohibited_only->mutable_term()->set_text("slow"); - const String prohibited_metadata = serializeFTSBooleanQuery(prohibited_query); - ASSERT_COLUMN_EQ( - createColumn({0, 0, 0}), - executeFunction( - "fts_match_boolean_expression", - {createConstColumn(3, "-slow"), - createColumn({"quick brown", "slow fox", "brown fox"}), - createConstColumn(3, prohibited_metadata)}, - nullptr, - true)); -} -CATCH - -TEST_F(TestFullText, MatchBooleanRejectsInvalidMetadata) -try -{ - tipb::FTSBooleanQuery boolean_query; - auto * required = boolean_query.add_nodes(); - required->set_occur(tipb::FTSBooleanOccurMust); - required->mutable_term()->set_term_type(tipb::FTSBooleanTermWord); - required->mutable_term()->set_text("quick"); - - tipb::FTSMatchBooleanMetadata metadata; - metadata.set_version(2); - *metadata.mutable_boolean_query() = boolean_query; - ASSERT_THROW( - executeFunction( - "fts_match_boolean_expression", - {createConstColumn(1, "+quick"), - createColumn({"quick brown"}), - createConstColumn(1, metadata.SerializeAsString())}, - nullptr, - true), - Exception); - - metadata.set_version(1); - metadata.clear_boolean_query(); - ASSERT_THROW( - executeFunction( - "fts_match_boolean_expression", - {createConstColumn(1, "+quick"), - createColumn({"quick brown"}), - createConstColumn(1, metadata.SerializeAsString())}, - nullptr, - true), - Exception); -} -CATCH - -TEST_F(TestFullText, MatchBooleanStandardAnalyzerProtocolSettings) -try -{ - const auto make_metadata = [](const String & term, UInt32 min_token_size, UInt32 max_token_size, bool enable_stopword) { - tipb::FTSBooleanQuery boolean_query; - boolean_query.set_query_tokenizer("STANDARD_V1"); - boolean_query.set_innodb_ft_min_token_size(min_token_size); - boolean_query.set_innodb_ft_max_token_size(max_token_size); - boolean_query.set_innodb_ft_enable_stopword(enable_stopword); - auto * required = boolean_query.add_nodes(); - required->set_occur(tipb::FTSBooleanOccurMust); - required->mutable_term()->set_term_type(tipb::FTSBooleanTermWord); - required->mutable_term()->set_text(term); - return serializeFTSBooleanQuery(boolean_query); - }; - - const auto short_term_metadata = make_metadata("go", 1, 84, true); - ASSERT_COLUMN_EQ( - createColumn({1, 0}), - executeFunction( - "fts_match_boolean_expression", - {createConstColumn(2, "+go"), - createColumn({"go", "good"}), - createConstColumn(2, short_term_metadata)}, - nullptr, - true)); - - const auto stopword_metadata = make_metadata("the", 1, 84, false); - ASSERT_COLUMN_EQ( - createColumn({1, 0}), - executeFunction( - "fts_match_boolean_expression", - {createConstColumn(2, "+the"), - createColumn({"the", "there"}), - createConstColumn(2, stopword_metadata)}, - nullptr, - true)); - - const auto collated_stopword_metadata = make_metadata("thé", 3, 84, true); - ASSERT_COLUMN_EQ( - createColumn({0}), - executeFunction( - "fts_match_boolean_expression", - {createConstColumn(1, "+thé"), - createColumn({"thé"}), - createConstColumn(1, collated_stopword_metadata)}, - TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_GENERAL_CI), - true)); - - const auto max_size_metadata = make_metadata("extraordinary", 1, 10, false); - ASSERT_COLUMN_EQ( - createColumn({0}), - executeFunction( - "fts_match_boolean_expression", - {createConstColumn(1, "+extraordinary"), - createColumn({"extraordinary"}), - createConstColumn(1, max_size_metadata)}, - nullptr, - true)); -} -CATCH - -TEST_F(TestFullText, MatchBooleanNgramProtocolQuery) -try -{ - tipb::FTSBooleanQuery boolean_query; - boolean_query.set_query_tokenizer("NGRAM_V1"); - boolean_query.set_ngram_token_size(2); - auto * required = boolean_query.add_nodes(); - required->set_occur(tipb::FTSBooleanOccurMust); - required->mutable_term()->set_term_type(tipb::FTSBooleanTermWord); - required->mutable_term()->set_text("数据库"); - auto * prohibited = boolean_query.add_nodes(); - prohibited->set_occur(tipb::FTSBooleanOccurMustNot); - prohibited->mutable_term()->set_term_type(tipb::FTSBooleanTermWord); - prohibited->mutable_term()->set_text("mysql"); - - const String metadata = serializeFTSBooleanQuery(boolean_query); - ASSERT_COLUMN_EQ( - createColumn({1, 0, 1, 0}), - executeFunction( - "fts_match_boolean_expression", - {createConstColumn(4, "+数据库 -mysql"), - createColumn({"数据库系统", "MySQL 数据库", "数据库", "数据科学"}), - createConstColumn(4, metadata)}, - nullptr, - true)); - - const auto ci_collator = TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_GENERAL_CI); - tipb::FTSBooleanQuery case_query; - case_query.set_query_tokenizer("NGRAM_V1"); - case_query.set_ngram_token_size(2); - auto * case_term = case_query.add_nodes(); - case_term->set_occur(tipb::FTSBooleanOccurMust); - case_term->mutable_term()->set_term_type(tipb::FTSBooleanTermWord); - case_term->mutable_term()->set_text("mysql"); - const String case_metadata = serializeFTSBooleanQuery(case_query); - ASSERT_COLUMN_EQ( - createColumn({1, 0}), - executeFunction( - "fts_match_boolean_expression", - {createConstColumn(2, "+mysql"), createColumn({"MySQL", "PostgreSQL"}), createConstColumn(2, case_metadata)}, - ci_collator, - true)); -} -CATCH - -TEST_F(TestFullText, MatchBooleanNgramPrefix) -try -{ - const auto make_metadata = [](UInt32 token_size, const String & prefix) { - tipb::FTSBooleanQuery boolean_query; - boolean_query.set_query_tokenizer("NGRAM_V1"); - boolean_query.set_ngram_token_size(token_size); - auto * node = boolean_query.add_nodes(); - node->set_occur(tipb::FTSBooleanOccurMust); - node->mutable_term()->set_term_type(tipb::FTSBooleanTermPrefix); - node->mutable_term()->set_text(prefix); - return serializeFTSBooleanQuery(boolean_query); - }; - - const auto query = createConstColumn(4, "+caf*"); - const auto documents = createColumn({"CAFE", "café", "cafe", "decaf"}); - const auto metadata = createConstColumn(4, make_metadata(2, "caf")); - const auto utf8mb4_bin = TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_BIN); - ASSERT_COLUMN_EQ( - createColumn({0, 1, 1, 1}), - executeFunction("fts_match_boolean_expression", {query, documents, metadata}, utf8mb4_bin, true)); - - const auto utf8mb4_general_ci = TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_GENERAL_CI); - ASSERT_COLUMN_EQ( - createColumn({1, 1, 1, 1}), - executeFunction("fts_match_boolean_expression", {query, documents, metadata}, utf8mb4_general_ci, true)); - - const auto short_query = createConstColumn(4, "+c*"); - const auto short_metadata = createConstColumn(4, make_metadata(2, "c")); - ASSERT_COLUMN_EQ( - createColumn({1, 1, 1, 1}), - executeFunction("fts_match_boolean_expression", {short_query, documents, short_metadata}, utf8mb4_general_ci, true)); -} -CATCH - -TEST_F(TestFullText, MatchBooleanCollationMatrix) -try -{ - const auto query = createConstColumn(3, "+cafe"); - const auto documents = createColumn({"CAFE", "café", "cafe"}); - const auto prefix_query = createConstColumn(3, "+caf*"); - const auto make_metadata = [](tipb::FTSBooleanTermType term_type, const String & term) { - tipb::FTSBooleanQuery boolean_query; - boolean_query.set_query_tokenizer("STANDARD_V1"); - auto * node = boolean_query.add_nodes(); - node->set_occur(tipb::FTSBooleanOccurMust); - node->mutable_term()->set_term_type(term_type); - node->mutable_term()->set_text(term); - return serializeFTSBooleanQuery(boolean_query); - }; - const auto word_metadata = createConstColumn( - 3, make_metadata(tipb::FTSBooleanTermWord, "cafe")); - const auto prefix_metadata = createConstColumn( - 3, make_metadata(tipb::FTSBooleanTermPrefix, "caf")); - - const auto utf8mb4_bin = TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_BIN); - ASSERT_COLUMN_EQ( - createColumn({0, 0, 1}), - executeFunction("fts_match_boolean_expression", {query, documents, word_metadata}, utf8mb4_bin, true)); - ASSERT_COLUMN_EQ( - createColumn({0, 1, 1}), - executeFunction("fts_match_boolean_expression", {prefix_query, documents, prefix_metadata}, utf8mb4_bin, true)); - - const auto utf8mb4_0900_bin = TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_0900_BIN); - ASSERT_COLUMN_EQ( - createColumn({0, 0, 1}), - executeFunction("fts_match_boolean_expression", {query, documents, word_metadata}, utf8mb4_0900_bin, true)); - ASSERT_COLUMN_EQ( - createColumn({0, 1, 1}), - executeFunction("fts_match_boolean_expression", {prefix_query, documents, prefix_metadata}, utf8mb4_0900_bin, true)); - - const auto utf8mb4_general_ci = TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_GENERAL_CI); - ASSERT_COLUMN_EQ( - createColumn({1, 1, 1}), - executeFunction("fts_match_boolean_expression", {query, documents, word_metadata}, utf8mb4_general_ci, true)); - ASSERT_COLUMN_EQ( - createColumn({1, 1, 1}), - executeFunction("fts_match_boolean_expression", {prefix_query, documents, prefix_metadata}, utf8mb4_general_ci, true)); - - const auto utf8mb4_unicode_ci = TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_UNICODE_CI); - ASSERT_COLUMN_EQ( - createColumn({1, 1, 1}), - executeFunction("fts_match_boolean_expression", {query, documents, word_metadata}, utf8mb4_unicode_ci, true)); - ASSERT_COLUMN_EQ( - createColumn({1, 1, 1}), - executeFunction("fts_match_boolean_expression", {prefix_query, documents, prefix_metadata}, utf8mb4_unicode_ci, true)); - - const auto utf8mb4_0900_ai_ci = TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_0900_AI_CI); - ASSERT_COLUMN_EQ( - createColumn({1, 1, 1}), - executeFunction("fts_match_boolean_expression", {query, documents, word_metadata}, utf8mb4_0900_ai_ci, true)); - ASSERT_COLUMN_EQ( - createColumn({1, 1, 1}), - executeFunction("fts_match_boolean_expression", {prefix_query, documents, prefix_metadata}, utf8mb4_0900_ai_ci, true)); -} -CATCH - -} // namespace DB::tests diff --git a/dbms/src/Functions/tests/gtest_local_match_against.cpp b/dbms/src/Functions/tests/gtest_local_match_against.cpp new file mode 100644 index 00000000000..405dd55a6c0 --- /dev/null +++ b/dbms/src/Functions/tests/gtest_local_match_against.cpp @@ -0,0 +1,555 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include +#include + +namespace DB::tests +{ +class TestLocalMatchAgainst : public DB::tests::FunctionTest +{ +}; + +namespace +{ +String serializeLocalMatchAgainstBooleanQuery(const tipb::LocalMatchAgainstBooleanQuery & query) +{ + auto encoded_query = query; + if (encoded_query.version() == 0) + encoded_query.set_version(1); // The tests exercise the currently supported Local MATCH semantic protocol. + if (encoded_query.parser() == tipb::LocalMatchAgainstParser::LocalMatchAgainstParserInvalid) + encoded_query.set_parser(tipb::LocalMatchAgainstParser::LocalMatchAgainstParserStandard); + if (encoded_query.stopword_mode() == tipb::LocalMatchAgainstStopwordMode::LocalMatchAgainstStopwordModeInvalid) + encoded_query.set_stopword_mode(tipb::LocalMatchAgainstStopwordMode::LocalMatchAgainstStopwordModeBuiltin); + if (encoded_query.stopword_collation().empty()) + encoded_query.set_stopword_collation("utf8mb4_bin"); + return encoded_query.SerializeAsString(); +} +} // namespace + +TEST_F(TestLocalMatchAgainst, MatchBooleanMultiColumnNullable) +try +{ + tipb::LocalMatchAgainstBooleanQuery boolean_query; + auto * required = boolean_query.add_nodes(); + required->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + required->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + required->set_text("quick"); + + const String metadata = serializeLocalMatchAgainstBooleanQuery(boolean_query); + ASSERT_COLUMN_EQ( + createColumn>({1, 1, 0, 0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(4, "+quick"), + createColumn>({{}, "quick fox", {}, "slow fox"}), + createColumn>({"quick fox", {}, "slow fox", {}}), + createConstColumn(4, metadata)}, + nullptr, + true)); +} +CATCH + +TEST_F(TestLocalMatchAgainst, MatchBooleanProtocolQuery) +try +{ + tipb::LocalMatchAgainstBooleanQuery boolean_query; + auto * required = boolean_query.add_nodes(); + required->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + required->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + required->set_text("quick"); + auto * prohibited = boolean_query.add_nodes(); + prohibited->set_occur(tipb::LocalMatchAgainstBooleanOccurMustNot); + prohibited->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + prohibited->set_text("slow"); + + const String metadata = serializeLocalMatchAgainstBooleanQuery(boolean_query); + ASSERT_COLUMN_EQ( + createColumn({1, 0, 0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(3, "+quick -slow"), + createColumn({"quick brown", "slow fox", "brown fox"}), + createConstColumn(3, metadata)}, + nullptr, + true)); + + tipb::LocalMatchAgainstBooleanQuery prohibited_query; + auto * prohibited_only = prohibited_query.add_nodes(); + prohibited_only->set_occur(tipb::LocalMatchAgainstBooleanOccurMustNot); + prohibited_only->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + prohibited_only->set_text("slow"); + const String prohibited_metadata = serializeLocalMatchAgainstBooleanQuery(prohibited_query); + ASSERT_COLUMN_EQ( + createColumn({0, 0, 0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(3, "-slow"), + createColumn({"quick brown", "slow fox", "brown fox"}), + createConstColumn(3, prohibited_metadata)}, + nullptr, + true)); +} +CATCH + +TEST_F(TestLocalMatchAgainst, MatchBooleanPhrasePositionsWithStopwordGaps) +try +{ + tipb::LocalMatchAgainstBooleanQuery boolean_query; + auto * required = boolean_query.add_nodes(); + required->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + required->set_term_type(tipb::LocalMatchAgainstBooleanTermPhrase); + required->set_text("quick the fox"); + + const String metadata = serializeLocalMatchAgainstBooleanQuery(boolean_query); + ASSERT_COLUMN_EQ( + createColumn({1, 0, 0, 1}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(4, "+\"quick the fox\""), + createColumn( + {"quick the fox", "quick fox", "quick slow the fox", "quick the fox quick the fox"}), + createConstColumn(4, metadata)}, + nullptr, + true)); +} +CATCH + +TEST_F(TestLocalMatchAgainst, MatchBooleanConstantDocumentColumn) +try +{ + tipb::LocalMatchAgainstBooleanQuery boolean_query; + auto * required = boolean_query.add_nodes(); + required->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + required->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + required->set_text("quick"); + + ASSERT_COLUMN_EQ( + createColumn({1, 1, 1}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(3, "+quick"), + createConstColumn(3, "quick fox"), + createConstColumn(3, serializeLocalMatchAgainstBooleanQuery(boolean_query))}, + nullptr, + true)); +} +CATCH + +TEST_F(TestLocalMatchAgainst, MatchBooleanMultiColumnStateAndPhraseBoundary) +try +{ + const auto make_query = [](tipb::LocalMatchAgainstBooleanTermType term_type) { + tipb::LocalMatchAgainstBooleanQuery boolean_query; + auto * required = boolean_query.add_nodes(); + required->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + required->set_term_type(term_type); + required->set_text("quick fox"); + return serializeLocalMatchAgainstBooleanQuery(boolean_query); + }; + + ASSERT_COLUMN_EQ( + createColumn({1, 0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(2, "+quick +fox"), + createColumn({"quick", "quick"}), + createColumn({"fox", "slow"}), + createConstColumn(2, make_query(tipb::LocalMatchAgainstBooleanTermWord))}, + nullptr, + true)); + + ASSERT_COLUMN_EQ( + createColumn({0, 1}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(2, "+\"quick fox\""), + createColumn({"quick", "quick fox"}), + createColumn({"fox", "slow"}), + createConstColumn(2, make_query(tipb::LocalMatchAgainstBooleanTermPhrase))}, + nullptr, + true)); +} +CATCH + +TEST_F(TestLocalMatchAgainst, MatchBooleanRejectsInvalidProtocol) +try +{ + tipb::LocalMatchAgainstBooleanQuery boolean_query; + boolean_query.set_version(2); + boolean_query.set_parser(tipb::LocalMatchAgainstParser::LocalMatchAgainstParserStandard); + boolean_query.set_stopword_mode(tipb::LocalMatchAgainstStopwordMode::LocalMatchAgainstStopwordModeBuiltin); + boolean_query.set_stopword_collation("utf8mb4_bin"); + auto * required = boolean_query.add_nodes(); + required->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + required->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + required->set_text("quick"); + + ASSERT_THROW( + executeFunction( + "local_match_against_boolean", + {createConstColumn(1, "+quick"), + createColumn({"quick brown"}), + createConstColumn(1, boolean_query.SerializeAsString())}, + nullptr, + true), + Exception); + + boolean_query.set_version(1); + boolean_query.set_parser(tipb::LocalMatchAgainstParser::LocalMatchAgainstParserInvalid); + ASSERT_THROW( + executeFunction( + "local_match_against_boolean", + {createConstColumn(1, "+quick"), + createColumn({"quick brown"}), + createConstColumn(1, boolean_query.SerializeAsString())}, + nullptr, + true), + Exception); + + boolean_query.set_parser(tipb::LocalMatchAgainstParser::LocalMatchAgainstParserStandard); + boolean_query.set_stopword_mode(tipb::LocalMatchAgainstStopwordMode::LocalMatchAgainstStopwordModeInvalid); + ASSERT_THROW( + executeFunction( + "local_match_against_boolean", + {createConstColumn(1, "+quick"), + createColumn({"quick brown"}), + createConstColumn(1, boolean_query.SerializeAsString())}, + nullptr, + true), + Exception); +} +CATCH + +TEST_F(TestLocalMatchAgainst, MatchBooleanStandardAnalyzerProtocolSettings) +try +{ + const auto make_metadata = [](const String & term, + UInt32 min_token_size, + UInt32 max_token_size, + bool enable_stopword, + const String & stopword_collation) { + tipb::LocalMatchAgainstBooleanQuery boolean_query; + boolean_query.set_parser(tipb::LocalMatchAgainstParser::LocalMatchAgainstParserStandard); + boolean_query.set_stopword_collation(stopword_collation); + boolean_query.set_innodb_ft_min_token_size(min_token_size); + boolean_query.set_innodb_ft_max_token_size(max_token_size); + boolean_query.set_stopword_mode( + enable_stopword ? tipb::LocalMatchAgainstStopwordMode::LocalMatchAgainstStopwordModeBuiltin + : tipb::LocalMatchAgainstStopwordMode::LocalMatchAgainstStopwordModeDisabled); + auto * required = boolean_query.add_nodes(); + required->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + required->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + required->set_text(term); + return serializeLocalMatchAgainstBooleanQuery(boolean_query); + }; + + const auto short_term_metadata = make_metadata("go", 1, 84, true, "utf8mb4_bin"); + ASSERT_COLUMN_EQ( + createColumn({1, 0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(2, "+go"), + createColumn({"go", "good"}), + createConstColumn(2, short_term_metadata)}, + nullptr, + true)); + + const auto stopword_metadata = make_metadata("the", 1, 84, false, "utf8mb4_bin"); + ASSERT_COLUMN_EQ( + createColumn({1, 0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(2, "+the"), + createColumn({"the", "there"}), + createConstColumn(2, stopword_metadata)}, + nullptr, + true)); + + const auto collated_stopword_metadata = make_metadata("thé", 3, 84, true, "utf8mb4_general_ci"); + ASSERT_COLUMN_EQ( + createColumn({0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(1, "+thé"), + createColumn({"thé"}), + createConstColumn(1, collated_stopword_metadata)}, + TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_GENERAL_CI), + true)); + + const auto binary_server_metadata = make_metadata("The", 3, 84, true, "utf8mb4_bin"); + ASSERT_COLUMN_EQ( + createColumn({1}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(1, "+The"), + createColumn({"The"}), + createConstColumn(1, binary_server_metadata)}, + TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_GENERAL_CI), + true)); + + const auto max_size_metadata = make_metadata("extraordinary", 1, 10, false, "utf8mb4_bin"); + ASSERT_COLUMN_EQ( + createColumn({0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(1, "+extraordinary"), + createColumn({"extraordinary"}), + createConstColumn(1, max_size_metadata)}, + nullptr, + true)); +} +CATCH + +TEST_F(TestLocalMatchAgainst, MatchBooleanNgramProtocolQuery) +try +{ + tipb::LocalMatchAgainstBooleanQuery boolean_query; + boolean_query.set_parser(tipb::LocalMatchAgainstParser::LocalMatchAgainstParserNgram); + boolean_query.set_ngram_token_size(2); + auto * required = boolean_query.add_nodes(); + required->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + required->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + required->set_text("数据库"); + auto * prohibited = boolean_query.add_nodes(); + prohibited->set_occur(tipb::LocalMatchAgainstBooleanOccurMustNot); + prohibited->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + prohibited->set_text("mysql"); + + const String metadata = serializeLocalMatchAgainstBooleanQuery(boolean_query); + ASSERT_COLUMN_EQ( + createColumn({1, 0, 1, 0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(4, "+数据库 -mysql"), + createColumn({"数据库系统", "MySQL 数据库", "数据库", "数据科学"}), + createConstColumn(4, metadata)}, + nullptr, + true)); + + const auto ci_collator = TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_GENERAL_CI); + tipb::LocalMatchAgainstBooleanQuery case_query; + case_query.set_parser(tipb::LocalMatchAgainstParser::LocalMatchAgainstParserNgram); + case_query.set_ngram_token_size(2); + auto * case_term = case_query.add_nodes(); + case_term->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + case_term->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + case_term->set_text("mysql"); + const String case_metadata = serializeLocalMatchAgainstBooleanQuery(case_query); + ASSERT_COLUMN_EQ( + createColumn({1, 0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(2, "+mysql"), + createColumn({"MySQL", "PostgreSQL"}), + createConstColumn(2, case_metadata)}, + ci_collator, + true)); +} +CATCH + +TEST_F(TestLocalMatchAgainst, MatchBooleanNgramStopwordSettings) +try +{ + const auto make_metadata = [](UInt32 token_size, bool enable_stopword) { + tipb::LocalMatchAgainstBooleanQuery boolean_query; + boolean_query.set_parser(tipb::LocalMatchAgainstParser::LocalMatchAgainstParserNgram); + boolean_query.set_ngram_token_size(token_size); + boolean_query.set_stopword_mode( + enable_stopword ? tipb::LocalMatchAgainstStopwordMode::LocalMatchAgainstStopwordModeBuiltin + : tipb::LocalMatchAgainstStopwordMode::LocalMatchAgainstStopwordModeDisabled); + auto * required = boolean_query.add_nodes(); + required->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + required->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + required->set_text("the"); + return serializeLocalMatchAgainstBooleanQuery(boolean_query); + }; + + const auto query = createConstColumn(3, "+the"); + const auto documents = createColumn({"the", "other", "there"}); + + // A stopword longer than ngram_token_size is ignored by the NGRAM parser. + const auto size_two_metadata = createConstColumn(3, make_metadata(2, true)); + ASSERT_COLUMN_EQ( + createColumn({1, 1, 1}), + executeFunction("local_match_against_boolean", {query, documents, size_two_metadata}, nullptr, true)); + + // At size 3, the gram "the" is removed from both query and document. + const auto enabled_metadata = createConstColumn(3, make_metadata(3, true)); + ASSERT_COLUMN_EQ( + createColumn({0, 0, 0}), + executeFunction("local_match_against_boolean", {query, documents, enabled_metadata}, nullptr, true)); + + tipb::LocalMatchAgainstBooleanQuery filtered_prefix; + filtered_prefix.set_parser(tipb::LocalMatchAgainstParser::LocalMatchAgainstParserNgram); + filtered_prefix.set_ngram_token_size(2); + filtered_prefix.set_stopword_mode(tipb::LocalMatchAgainstStopwordMode::LocalMatchAgainstStopwordModeBuiltin); + auto * prefix_node = filtered_prefix.add_nodes(); + prefix_node->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + prefix_node->set_term_type(tipb::LocalMatchAgainstBooleanTermPrefix); + prefix_node->set_text("caf"); + ASSERT_COLUMN_EQ( + createColumn({0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(1, "+caf*"), + createColumn({"cafe"}), + createConstColumn(1, serializeLocalMatchAgainstBooleanQuery(filtered_prefix))}, + nullptr, + true)); + + // Explicit OFF keeps that gram and restores the matches. + const auto disabled_metadata = createConstColumn(3, make_metadata(3, false)); + ASSERT_COLUMN_EQ( + createColumn({1, 1, 1}), + executeFunction("local_match_against_boolean", {query, documents, disabled_metadata}, nullptr, true)); + + // Stopword matching follows collation_server independently from the + // MATCH column collation. + tipb::LocalMatchAgainstBooleanQuery collated_query; + collated_query.set_parser(tipb::LocalMatchAgainstParser::LocalMatchAgainstParserNgram); + collated_query.set_ngram_token_size(2); + collated_query.set_stopword_mode(tipb::LocalMatchAgainstStopwordMode::LocalMatchAgainstStopwordModeBuiltin); + collated_query.set_stopword_collation("utf8mb4_general_ci"); + auto * collated_required = collated_query.add_nodes(); + collated_required->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + collated_required->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + collated_required->set_text("xá"); + const auto collated_metadata = createConstColumn(1, serializeLocalMatchAgainstBooleanQuery(collated_query)); + const auto document = createColumn({"xá"}); + ASSERT_COLUMN_EQ( + createColumn({0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(1, "+xá"), document, collated_metadata}, + TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_GENERAL_CI), + true)); + + // The MATCH column collation remains accent-insensitive, but the binary + // server collation does not consider "á" to contain stopword "a". + collated_query.set_stopword_collation("utf8mb4_bin"); + const auto binary_stopword_metadata + = createConstColumn(1, serializeLocalMatchAgainstBooleanQuery(collated_query)); + ASSERT_COLUMN_EQ( + createColumn({1}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(1, "+xá"), document, binary_stopword_metadata}, + TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_GENERAL_CI), + true)); +} +CATCH + +TEST_F(TestLocalMatchAgainst, MatchBooleanNgramPrefix) +try +{ + const auto make_metadata = [](UInt32 token_size, const String & prefix) { + tipb::LocalMatchAgainstBooleanQuery boolean_query; + boolean_query.set_parser(tipb::LocalMatchAgainstParser::LocalMatchAgainstParserNgram); + boolean_query.set_ngram_token_size(token_size); + boolean_query.set_stopword_mode(tipb::LocalMatchAgainstStopwordMode::LocalMatchAgainstStopwordModeDisabled); + auto * node = boolean_query.add_nodes(); + node->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + node->set_term_type(tipb::LocalMatchAgainstBooleanTermPrefix); + node->set_text(prefix); + return serializeLocalMatchAgainstBooleanQuery(boolean_query); + }; + + const auto query = createConstColumn(4, "+caf*"); + const auto documents = createColumn({"CAFE", "café", "cafe", "decaf"}); + const auto metadata = createConstColumn(4, make_metadata(2, "caf")); + const auto utf8mb4_bin = TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_BIN); + ASSERT_COLUMN_EQ( + createColumn({0, 1, 1, 1}), + executeFunction("local_match_against_boolean", {query, documents, metadata}, utf8mb4_bin, true)); + + const auto utf8mb4_general_ci = TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_GENERAL_CI); + ASSERT_COLUMN_EQ( + createColumn({1, 1, 1, 1}), + executeFunction("local_match_against_boolean", {query, documents, metadata}, utf8mb4_general_ci, true)); + + const auto short_query = createConstColumn(4, "+c*"); + const auto short_metadata = createConstColumn(4, make_metadata(2, "c")); + ASSERT_COLUMN_EQ( + createColumn({1, 1, 1, 1}), + executeFunction("local_match_against_boolean", {short_query, documents, short_metadata}, utf8mb4_general_ci, true)); +} +CATCH + +TEST_F(TestLocalMatchAgainst, MatchBooleanCollationMatrix) +try +{ + const auto query = createConstColumn(3, "+cafe"); + const auto documents = createColumn({"CAFE", "café", "cafe"}); + const auto prefix_query = createConstColumn(3, "+caf*"); + const auto make_metadata = [](tipb::LocalMatchAgainstBooleanTermType term_type, const String & term) { + tipb::LocalMatchAgainstBooleanQuery boolean_query; + boolean_query.set_parser(tipb::LocalMatchAgainstParser::LocalMatchAgainstParserStandard); + auto * node = boolean_query.add_nodes(); + node->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + node->set_term_type(term_type); + node->set_text(term); + return serializeLocalMatchAgainstBooleanQuery(boolean_query); + }; + const auto word_metadata = createConstColumn( + 3, make_metadata(tipb::LocalMatchAgainstBooleanTermWord, "cafe")); + const auto prefix_metadata = createConstColumn( + 3, make_metadata(tipb::LocalMatchAgainstBooleanTermPrefix, "caf")); + + const auto utf8mb4_bin = TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_BIN); + ASSERT_COLUMN_EQ( + createColumn({0, 0, 1}), + executeFunction("local_match_against_boolean", {query, documents, word_metadata}, utf8mb4_bin, true)); + ASSERT_COLUMN_EQ( + createColumn({0, 1, 1}), + executeFunction("local_match_against_boolean", {prefix_query, documents, prefix_metadata}, utf8mb4_bin, true)); + + const auto utf8mb4_0900_bin = TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_0900_BIN); + ASSERT_COLUMN_EQ( + createColumn({0, 0, 1}), + executeFunction("local_match_against_boolean", {query, documents, word_metadata}, utf8mb4_0900_bin, true)); + ASSERT_COLUMN_EQ( + createColumn({0, 1, 1}), + executeFunction("local_match_against_boolean", {prefix_query, documents, prefix_metadata}, utf8mb4_0900_bin, true)); + + const auto utf8mb4_general_ci = TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_GENERAL_CI); + ASSERT_COLUMN_EQ( + createColumn({1, 1, 1}), + executeFunction("local_match_against_boolean", {query, documents, word_metadata}, utf8mb4_general_ci, true)); + ASSERT_COLUMN_EQ( + createColumn({1, 1, 1}), + executeFunction("local_match_against_boolean", {prefix_query, documents, prefix_metadata}, utf8mb4_general_ci, true)); + + const auto utf8mb4_unicode_ci = TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_UNICODE_CI); + ASSERT_COLUMN_EQ( + createColumn({1, 1, 1}), + executeFunction("local_match_against_boolean", {query, documents, word_metadata}, utf8mb4_unicode_ci, true)); + ASSERT_COLUMN_EQ( + createColumn({1, 1, 1}), + executeFunction("local_match_against_boolean", {prefix_query, documents, prefix_metadata}, utf8mb4_unicode_ci, true)); + + const auto utf8mb4_0900_ai_ci = TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_0900_AI_CI); + ASSERT_COLUMN_EQ( + createColumn({1, 1, 1}), + executeFunction("local_match_against_boolean", {query, documents, word_metadata}, utf8mb4_0900_ai_ci, true)); + ASSERT_COLUMN_EQ( + createColumn({1, 1, 1}), + executeFunction("local_match_against_boolean", {prefix_query, documents, prefix_metadata}, utf8mb4_0900_ai_ci, true)); +} +CATCH + +} // namespace DB::tests From f7d3ee2622924a9d54292f4efcf433c924bdb3ad Mon Sep 17 00:00:00 2001 From: Zhigao Tong Date: Fri, 9 Oct 2026 09:07:51 +0800 Subject: [PATCH 16/32] tiflash: short-circuit Local MATCH evaluation --- .../Functions/FunctionsLocalMatchAgainst.cpp | 95 ++++++++++++++++--- .../tests/gtest_local_match_against.cpp | 77 +++++++++++++++ 2 files changed, 161 insertions(+), 11 deletions(-) diff --git a/dbms/src/Functions/FunctionsLocalMatchAgainst.cpp b/dbms/src/Functions/FunctionsLocalMatchAgainst.cpp index 2c6d34daad5..1bff20aa4a7 100644 --- a/dbms/src/Functions/FunctionsLocalMatchAgainst.cpp +++ b/dbms/src/Functions/FunctionsLocalMatchAgainst.cpp @@ -143,6 +143,8 @@ struct AnalyzerScratch struct CompiledBooleanQuery { std::vector clauses; + size_t must_clause_count = 0; + bool has_must_not = false; bool matches_nothing = false; }; @@ -531,15 +533,49 @@ void resetClauseMatchStates(std::vector & states) } } -bool updateBooleanMatchStatesForColumn( +struct BooleanMatchProgress +{ + size_t matched_must_clauses = 0; + bool matched_should_clause = false; +}; + +enum class ColumnMatchResult +{ + Continue, + Accepted, + Rejected, +}; + +ColumnMatchResult updateBooleanMatchStatesForColumn( const CompiledBooleanQuery & query, const FullTextColumn & column, const TiDB::TiDBCollatorPtr & collator, - std::vector & states) + std::vector & states, + BooleanMatchProgress & progress) { const auto & clauses = query.clauses; if (query.matches_nothing || clauses.empty()) - return false; + return ColumnMatchResult::Continue; + + const auto markClauseMatched = [&](size_t clause_index) { + states[clause_index].matched = true; + switch (clauses[clause_index].modifier) + { + case BooleanClause::Modifier::Must: + ++progress.matched_must_clauses; + break; + case BooleanClause::Modifier::Should: + progress.matched_should_clause = true; + break; + case BooleanClause::Modifier::MustNot: + return ColumnMatchResult::Rejected; + } + if (!query.has_must_not + && (query.must_clause_count != 0 ? progress.matched_must_clauses == query.must_clause_count + : progress.matched_should_clause)) + return ColumnMatchResult::Accepted; + return ColumnMatchResult::Continue; + }; // Walk this MATCH column's analyzed tokens once and update every clause's // state. Phrase clauses are checked only when their final term is @@ -557,7 +593,11 @@ bool updateBooleanMatchStatesForColumn( if (clause.phrase) { if (matchesPhraseEndingAt(clause, column, token, collator)) - state.matched = true; + { + const auto result = markClauseMatched(clause_index); + if (result != ColumnMatchResult::Continue) + return result; + } continue; } @@ -568,11 +608,15 @@ bool updateBooleanMatchStatesForColumn( --state.remaining_terms; } state.matched = state.remaining_terms == 0; - if (state.matched && clause.modifier == BooleanClause::Modifier::MustNot) - return true; + if (state.matched) + { + const auto result = markClauseMatched(clause_index); + if (result != ColumnMatchResult::Continue) + return result; + } } } - return false; + return ColumnMatchResult::Continue; } bool evaluateBooleanMatchResult( @@ -763,6 +807,14 @@ CompiledBooleanQuery compileBooleanQuery( compiled.clauses.push_back(std::move(clause)); } + for (const auto & clause : compiled.clauses) + { + if (clause.modifier == BooleanClause::Modifier::Must) + ++compiled.must_clause_count; + else if (clause.modifier == BooleanClause::Modifier::MustNot) + compiled.has_must_not = true; + } + // A BOOLEAN MODE query containing only prohibited terms has no positive // branch. TiDB's local evaluator treats it as matching no rows. if (std::none_of(compiled.clauses.begin(), compiled.clauses.end(), [](const BooleanClause & clause) { @@ -951,6 +1003,16 @@ class FunctionLocalMatchAgainstBoolean final : public IFunction enable_stopword, collator, stopword_collator); + if (compiled_query.matches_nothing) + { + ColumnPtr result_column; + if (null_map) + result_column = ColumnNullable::create(std::move(output), std::move(null_map)); + else + result_column = std::move(output); + block.getByPosition(result).column = std::move(result_column); + return; + } std::vector document_columns; document_columns.reserve(document_argument_end - 1); for (size_t arg = 1; arg < document_argument_end; ++arg) @@ -963,7 +1025,9 @@ class FunctionLocalMatchAgainstBoolean final : public IFunction for (size_t row = 0; row < rows; ++row) { resetClauseMatchStates(clause_states); - bool rejected_by_prohibited_clause = false; + BooleanMatchProgress match_progress; + bool result_known = false; + bool row_matches = false; for (const auto & document_column : document_columns) { // A NULL MATCH column contributes no tokens. This is @@ -985,13 +1049,22 @@ class FunctionLocalMatchAgainstBoolean final : public IFunction min_token_size, max_token_size, enable_stopword); - if (updateBooleanMatchStatesForColumn(compiled_query, analyzed_column, collator, clause_states)) + const auto match_result + = updateBooleanMatchStatesForColumn( + compiled_query, analyzed_column, collator, clause_states, match_progress); + if (match_result == ColumnMatchResult::Rejected) + { + result_known = true; + break; + } + if (match_result == ColumnMatchResult::Accepted) { - rejected_by_prohibited_clause = true; + result_known = true; + row_matches = true; break; } } - output_data[row] = rejected_by_prohibited_clause ? 0 : matchBooleanScore(compiled_query, clause_states); + output_data[row] = result_known ? (row_matches ? 1 : 0) : matchBooleanScore(compiled_query, clause_states); } ColumnPtr result_column; if (null_map) diff --git a/dbms/src/Functions/tests/gtest_local_match_against.cpp b/dbms/src/Functions/tests/gtest_local_match_against.cpp index 405dd55a6c0..0e82a937c5e 100644 --- a/dbms/src/Functions/tests/gtest_local_match_against.cpp +++ b/dbms/src/Functions/tests/gtest_local_match_against.cpp @@ -186,6 +186,83 @@ try } CATCH +TEST_F(TestLocalMatchAgainst, MatchBooleanDecidesPositiveMatchEarly) +try +{ + tipb::LocalMatchAgainstBooleanQuery required_query; + auto * required_quick = required_query.add_nodes(); + required_quick->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + required_quick->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + required_quick->set_text("quick"); + auto * required_fox = required_query.add_nodes(); + required_fox->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + required_fox->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + required_fox->set_text("fox"); + + ASSERT_COLUMN_EQ( + createColumn({1, 0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(2, "+quick +fox"), + createColumn({"quick", "quick"}), + createColumn({"fox", "quiet"}), + createConstColumn(2, serializeLocalMatchAgainstBooleanQuery(required_query))}, + nullptr, + true)); + + tipb::LocalMatchAgainstBooleanQuery optional_query; + auto * optional_quick = optional_query.add_nodes(); + optional_quick->set_occur(tipb::LocalMatchAgainstBooleanOccurShould); + optional_quick->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + optional_quick->set_text("quick"); + auto * optional_fox = optional_query.add_nodes(); + optional_fox->set_occur(tipb::LocalMatchAgainstBooleanOccurShould); + optional_fox->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + optional_fox->set_text("fox"); + + ASSERT_COLUMN_EQ( + createColumn({1, 0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(2, "quick fox"), + createColumn({"quick", "quiet"}), + createConstColumn(2, serializeLocalMatchAgainstBooleanQuery(optional_query))}, + nullptr, + true)); +} +CATCH + +TEST_F(TestLocalMatchAgainst, MatchBooleanStillChecksProhibitedTermsAfterPositiveMatch) +try +{ + tipb::LocalMatchAgainstBooleanQuery boolean_query; + auto * required_quick = boolean_query.add_nodes(); + required_quick->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + required_quick->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + required_quick->set_text("quick"); + auto * required_fox = boolean_query.add_nodes(); + required_fox->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + required_fox->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + required_fox->set_text("fox"); + auto * prohibited_slow = boolean_query.add_nodes(); + prohibited_slow->set_occur(tipb::LocalMatchAgainstBooleanOccurMustNot); + prohibited_slow->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + prohibited_slow->set_text("slow"); + + ASSERT_COLUMN_EQ( + createColumn({1, 0, 0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(3, "+quick +fox -slow"), + createColumn({"quick", "quick", "quick"}), + createColumn({"fox", "fox", "quiet"}), + createColumn({"calm", "slow", "slow"}), + createConstColumn(3, serializeLocalMatchAgainstBooleanQuery(boolean_query))}, + nullptr, + true)); +} +CATCH + TEST_F(TestLocalMatchAgainst, MatchBooleanRejectsInvalidProtocol) try { From 6281c5659f69f5fbcff260f54299edc3ff28b2ec Mon Sep 17 00:00:00 2001 From: Zhigao Tong Date: Fri, 9 Oct 2026 10:41:57 +0800 Subject: [PATCH 17/32] tiflash: sync Local MATCH signature ID 5250 --- contrib/tipb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/tipb b/contrib/tipb index f7dbf3d45cd..f46c2bcf8ac 160000 --- a/contrib/tipb +++ b/contrib/tipb @@ -1 +1 @@ -Subproject commit f7dbf3d45cd28bd47b41d9b80033f127a3a98de9 +Subproject commit f46c2bcf8ac205db6fc2781537c93ead09f59a3b From cc51474ae11ee4c8ad590b01b074c6c85c58cb57 Mon Sep 17 00:00:00 2001 From: Zhigao Tong Date: Fri, 9 Oct 2026 17:09:22 +0800 Subject: [PATCH 18/32] tiflash: align Local MATCH split-word and Unicode semantics --- .../Functions/FunctionsLocalMatchAgainst.cpp | 15 +- .../Functions/LocalMatchAgainstTokenChars.h | 800 ++++++++++++++++++ .../tests/gtest_local_match_against.cpp | 114 +++ 3 files changed, 923 insertions(+), 6 deletions(-) create mode 100644 dbms/src/Functions/LocalMatchAgainstTokenChars.h diff --git a/dbms/src/Functions/FunctionsLocalMatchAgainst.cpp b/dbms/src/Functions/FunctionsLocalMatchAgainst.cpp index 1bff20aa4a7..e543ca079eb 100644 --- a/dbms/src/Functions/FunctionsLocalMatchAgainst.cpp +++ b/dbms/src/Functions/FunctionsLocalMatchAgainst.cpp @@ -16,7 +16,6 @@ #include #include #include -#include #include #include #include @@ -25,8 +24,8 @@ #include #include #include +#include #include -#include #include #include @@ -163,15 +162,14 @@ void lowercaseTokenInPlace(String & token) bool isFullTextToken(UInt32 code_point) { - return isAlphaNumericASCII(static_cast(code_point)) || code_point == '_' || Poco::Unicode::isAlpha(code_point) - || Poco::Unicode::isDigit(code_point); + return LocalMatchAgainst::isTokenChar(code_point); } std::pair decodeCodePoint(std::string_view text, size_t offset) { const auto decoded = UTF8::utf8Decode(text.data() + offset, text.size() - offset); if (decoded.second == 0 || decoded.first == UTF8::UTF8_Error || decoded.second > text.size() - offset) - return {static_cast(text[offset]), 1}; + return {0xFFFD, 1}; // Go's utf8.DecodeRuneInString also consumes one invalid byte as RuneError. return {decoded.first, decoded.second}; } @@ -607,7 +605,12 @@ ColumnMatchResult updateBooleanMatchStatesForColumn( state.matched_terms[term_index] = 1; --state.remaining_terms; } - state.matched = state.remaining_terms == 0; + // STANDARD may split one SQL term into several words. Required + // terms intersect those words, but optional/prohibited terms use + // their union, exactly like TiDB's combineBooleanTermNodes. + state.matched = clause.modifier == BooleanClause::Modifier::Must + ? state.remaining_terms == 0 + : state.remaining_terms < clause.terms.size(); if (state.matched) { const auto result = markClauseMatched(clause_index); diff --git a/dbms/src/Functions/LocalMatchAgainstTokenChars.h b/dbms/src/Functions/LocalMatchAgainstTokenChars.h new file mode 100644 index 00000000000..9fddb572e1b --- /dev/null +++ b/dbms/src/Functions/LocalMatchAgainstTokenChars.h @@ -0,0 +1,800 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include + +namespace DB::LocalMatchAgainst +{ +// Generated from Go's Unicode 15.0.0 categories L and N. These ranges are +// identical to matchagainst/token_chars.go in TiDB and fix Local MATCH protocol +// version 1 semantics independently of Poco/Go upgrades. Never change an +// existing protocol version's classification; deploy TiFlash support for a +// new version before TiDB emits it. ASCII and '_' use the fast path below. +struct TokenCharRange +{ + std::uint32_t first; + std::uint32_t last; +}; +inline constexpr std::array token_char_ranges{{ + {0xAA, 0xAA}, + {0xB2, 0xB3}, + {0xB5, 0xB5}, + {0xB9, 0xBA}, + {0xBC, 0xBE}, + {0xC0, 0xD6}, + {0xD8, 0xF6}, + {0xF8, 0x2C1}, + {0x2C6, 0x2D1}, + {0x2E0, 0x2E4}, + {0x2EC, 0x2EC}, + {0x2EE, 0x2EE}, + {0x370, 0x374}, + {0x376, 0x377}, + {0x37A, 0x37D}, + {0x37F, 0x37F}, + {0x386, 0x386}, + {0x388, 0x38A}, + {0x38C, 0x38C}, + {0x38E, 0x3A1}, + {0x3A3, 0x3F5}, + {0x3F7, 0x481}, + {0x48A, 0x52F}, + {0x531, 0x556}, + {0x559, 0x559}, + {0x560, 0x588}, + {0x5D0, 0x5EA}, + {0x5EF, 0x5F2}, + {0x620, 0x64A}, + {0x660, 0x669}, + {0x66E, 0x66F}, + {0x671, 0x6D3}, + {0x6D5, 0x6D5}, + {0x6E5, 0x6E6}, + {0x6EE, 0x6FC}, + {0x6FF, 0x6FF}, + {0x710, 0x710}, + {0x712, 0x72F}, + {0x74D, 0x7A5}, + {0x7B1, 0x7B1}, + {0x7C0, 0x7EA}, + {0x7F4, 0x7F5}, + {0x7FA, 0x7FA}, + {0x800, 0x815}, + {0x81A, 0x81A}, + {0x824, 0x824}, + {0x828, 0x828}, + {0x840, 0x858}, + {0x860, 0x86A}, + {0x870, 0x887}, + {0x889, 0x88E}, + {0x8A0, 0x8C9}, + {0x904, 0x939}, + {0x93D, 0x93D}, + {0x950, 0x950}, + {0x958, 0x961}, + {0x966, 0x96F}, + {0x971, 0x980}, + {0x985, 0x98C}, + {0x98F, 0x990}, + {0x993, 0x9A8}, + {0x9AA, 0x9B0}, + {0x9B2, 0x9B2}, + {0x9B6, 0x9B9}, + {0x9BD, 0x9BD}, + {0x9CE, 0x9CE}, + {0x9DC, 0x9DD}, + {0x9DF, 0x9E1}, + {0x9E6, 0x9F1}, + {0x9F4, 0x9F9}, + {0x9FC, 0x9FC}, + {0xA05, 0xA0A}, + {0xA0F, 0xA10}, + {0xA13, 0xA28}, + {0xA2A, 0xA30}, + {0xA32, 0xA33}, + {0xA35, 0xA36}, + {0xA38, 0xA39}, + {0xA59, 0xA5C}, + {0xA5E, 0xA5E}, + {0xA66, 0xA6F}, + {0xA72, 0xA74}, + {0xA85, 0xA8D}, + {0xA8F, 0xA91}, + {0xA93, 0xAA8}, + {0xAAA, 0xAB0}, + {0xAB2, 0xAB3}, + {0xAB5, 0xAB9}, + {0xABD, 0xABD}, + {0xAD0, 0xAD0}, + {0xAE0, 0xAE1}, + {0xAE6, 0xAEF}, + {0xAF9, 0xAF9}, + {0xB05, 0xB0C}, + {0xB0F, 0xB10}, + {0xB13, 0xB28}, + {0xB2A, 0xB30}, + {0xB32, 0xB33}, + {0xB35, 0xB39}, + {0xB3D, 0xB3D}, + {0xB5C, 0xB5D}, + {0xB5F, 0xB61}, + {0xB66, 0xB6F}, + {0xB71, 0xB77}, + {0xB83, 0xB83}, + {0xB85, 0xB8A}, + {0xB8E, 0xB90}, + {0xB92, 0xB95}, + {0xB99, 0xB9A}, + {0xB9C, 0xB9C}, + {0xB9E, 0xB9F}, + {0xBA3, 0xBA4}, + {0xBA8, 0xBAA}, + {0xBAE, 0xBB9}, + {0xBD0, 0xBD0}, + {0xBE6, 0xBF2}, + {0xC05, 0xC0C}, + {0xC0E, 0xC10}, + {0xC12, 0xC28}, + {0xC2A, 0xC39}, + {0xC3D, 0xC3D}, + {0xC58, 0xC5A}, + {0xC5D, 0xC5D}, + {0xC60, 0xC61}, + {0xC66, 0xC6F}, + {0xC78, 0xC7E}, + {0xC80, 0xC80}, + {0xC85, 0xC8C}, + {0xC8E, 0xC90}, + {0xC92, 0xCA8}, + {0xCAA, 0xCB3}, + {0xCB5, 0xCB9}, + {0xCBD, 0xCBD}, + {0xCDD, 0xCDE}, + {0xCE0, 0xCE1}, + {0xCE6, 0xCEF}, + {0xCF1, 0xCF2}, + {0xD04, 0xD0C}, + {0xD0E, 0xD10}, + {0xD12, 0xD3A}, + {0xD3D, 0xD3D}, + {0xD4E, 0xD4E}, + {0xD54, 0xD56}, + {0xD58, 0xD61}, + {0xD66, 0xD78}, + {0xD7A, 0xD7F}, + {0xD85, 0xD96}, + {0xD9A, 0xDB1}, + {0xDB3, 0xDBB}, + {0xDBD, 0xDBD}, + {0xDC0, 0xDC6}, + {0xDE6, 0xDEF}, + {0xE01, 0xE30}, + {0xE32, 0xE33}, + {0xE40, 0xE46}, + {0xE50, 0xE59}, + {0xE81, 0xE82}, + {0xE84, 0xE84}, + {0xE86, 0xE8A}, + {0xE8C, 0xEA3}, + {0xEA5, 0xEA5}, + {0xEA7, 0xEB0}, + {0xEB2, 0xEB3}, + {0xEBD, 0xEBD}, + {0xEC0, 0xEC4}, + {0xEC6, 0xEC6}, + {0xED0, 0xED9}, + {0xEDC, 0xEDF}, + {0xF00, 0xF00}, + {0xF20, 0xF33}, + {0xF40, 0xF47}, + {0xF49, 0xF6C}, + {0xF88, 0xF8C}, + {0x1000, 0x102A}, + {0x103F, 0x1049}, + {0x1050, 0x1055}, + {0x105A, 0x105D}, + {0x1061, 0x1061}, + {0x1065, 0x1066}, + {0x106E, 0x1070}, + {0x1075, 0x1081}, + {0x108E, 0x108E}, + {0x1090, 0x1099}, + {0x10A0, 0x10C5}, + {0x10C7, 0x10C7}, + {0x10CD, 0x10CD}, + {0x10D0, 0x10FA}, + {0x10FC, 0x1248}, + {0x124A, 0x124D}, + {0x1250, 0x1256}, + {0x1258, 0x1258}, + {0x125A, 0x125D}, + {0x1260, 0x1288}, + {0x128A, 0x128D}, + {0x1290, 0x12B0}, + {0x12B2, 0x12B5}, + {0x12B8, 0x12BE}, + {0x12C0, 0x12C0}, + {0x12C2, 0x12C5}, + {0x12C8, 0x12D6}, + {0x12D8, 0x1310}, + {0x1312, 0x1315}, + {0x1318, 0x135A}, + {0x1369, 0x137C}, + {0x1380, 0x138F}, + {0x13A0, 0x13F5}, + {0x13F8, 0x13FD}, + {0x1401, 0x166C}, + {0x166F, 0x167F}, + {0x1681, 0x169A}, + {0x16A0, 0x16EA}, + {0x16EE, 0x16F8}, + {0x1700, 0x1711}, + {0x171F, 0x1731}, + {0x1740, 0x1751}, + {0x1760, 0x176C}, + {0x176E, 0x1770}, + {0x1780, 0x17B3}, + {0x17D7, 0x17D7}, + {0x17DC, 0x17DC}, + {0x17E0, 0x17E9}, + {0x17F0, 0x17F9}, + {0x1810, 0x1819}, + {0x1820, 0x1878}, + {0x1880, 0x1884}, + {0x1887, 0x18A8}, + {0x18AA, 0x18AA}, + {0x18B0, 0x18F5}, + {0x1900, 0x191E}, + {0x1946, 0x196D}, + {0x1970, 0x1974}, + {0x1980, 0x19AB}, + {0x19B0, 0x19C9}, + {0x19D0, 0x19DA}, + {0x1A00, 0x1A16}, + {0x1A20, 0x1A54}, + {0x1A80, 0x1A89}, + {0x1A90, 0x1A99}, + {0x1AA7, 0x1AA7}, + {0x1B05, 0x1B33}, + {0x1B45, 0x1B4C}, + {0x1B50, 0x1B59}, + {0x1B83, 0x1BA0}, + {0x1BAE, 0x1BE5}, + {0x1C00, 0x1C23}, + {0x1C40, 0x1C49}, + {0x1C4D, 0x1C7D}, + {0x1C80, 0x1C88}, + {0x1C90, 0x1CBA}, + {0x1CBD, 0x1CBF}, + {0x1CE9, 0x1CEC}, + {0x1CEE, 0x1CF3}, + {0x1CF5, 0x1CF6}, + {0x1CFA, 0x1CFA}, + {0x1D00, 0x1DBF}, + {0x1E00, 0x1F15}, + {0x1F18, 0x1F1D}, + {0x1F20, 0x1F45}, + {0x1F48, 0x1F4D}, + {0x1F50, 0x1F57}, + {0x1F59, 0x1F59}, + {0x1F5B, 0x1F5B}, + {0x1F5D, 0x1F5D}, + {0x1F5F, 0x1F7D}, + {0x1F80, 0x1FB4}, + {0x1FB6, 0x1FBC}, + {0x1FBE, 0x1FBE}, + {0x1FC2, 0x1FC4}, + {0x1FC6, 0x1FCC}, + {0x1FD0, 0x1FD3}, + {0x1FD6, 0x1FDB}, + {0x1FE0, 0x1FEC}, + {0x1FF2, 0x1FF4}, + {0x1FF6, 0x1FFC}, + {0x2070, 0x2071}, + {0x2074, 0x2079}, + {0x207F, 0x2089}, + {0x2090, 0x209C}, + {0x2102, 0x2102}, + {0x2107, 0x2107}, + {0x210A, 0x2113}, + {0x2115, 0x2115}, + {0x2119, 0x211D}, + {0x2124, 0x2124}, + {0x2126, 0x2126}, + {0x2128, 0x2128}, + {0x212A, 0x212D}, + {0x212F, 0x2139}, + {0x213C, 0x213F}, + {0x2145, 0x2149}, + {0x214E, 0x214E}, + {0x2150, 0x2189}, + {0x2460, 0x249B}, + {0x24EA, 0x24FF}, + {0x2776, 0x2793}, + {0x2C00, 0x2CE4}, + {0x2CEB, 0x2CEE}, + {0x2CF2, 0x2CF3}, + {0x2CFD, 0x2CFD}, + {0x2D00, 0x2D25}, + {0x2D27, 0x2D27}, + {0x2D2D, 0x2D2D}, + {0x2D30, 0x2D67}, + {0x2D6F, 0x2D6F}, + {0x2D80, 0x2D96}, + {0x2DA0, 0x2DA6}, + {0x2DA8, 0x2DAE}, + {0x2DB0, 0x2DB6}, + {0x2DB8, 0x2DBE}, + {0x2DC0, 0x2DC6}, + {0x2DC8, 0x2DCE}, + {0x2DD0, 0x2DD6}, + {0x2DD8, 0x2DDE}, + {0x2E2F, 0x2E2F}, + {0x3005, 0x3007}, + {0x3021, 0x3029}, + {0x3031, 0x3035}, + {0x3038, 0x303C}, + {0x3041, 0x3096}, + {0x309D, 0x309F}, + {0x30A1, 0x30FA}, + {0x30FC, 0x30FF}, + {0x3105, 0x312F}, + {0x3131, 0x318E}, + {0x3192, 0x3195}, + {0x31A0, 0x31BF}, + {0x31F0, 0x31FF}, + {0x3220, 0x3229}, + {0x3248, 0x324F}, + {0x3251, 0x325F}, + {0x3280, 0x3289}, + {0x32B1, 0x32BF}, + {0x3400, 0x4DBF}, + {0x4E00, 0xA48C}, + {0xA4D0, 0xA4FD}, + {0xA500, 0xA60C}, + {0xA610, 0xA62B}, + {0xA640, 0xA66E}, + {0xA67F, 0xA69D}, + {0xA6A0, 0xA6EF}, + {0xA717, 0xA71F}, + {0xA722, 0xA788}, + {0xA78B, 0xA7CA}, + {0xA7D0, 0xA7D1}, + {0xA7D3, 0xA7D3}, + {0xA7D5, 0xA7D9}, + {0xA7F2, 0xA801}, + {0xA803, 0xA805}, + {0xA807, 0xA80A}, + {0xA80C, 0xA822}, + {0xA830, 0xA835}, + {0xA840, 0xA873}, + {0xA882, 0xA8B3}, + {0xA8D0, 0xA8D9}, + {0xA8F2, 0xA8F7}, + {0xA8FB, 0xA8FB}, + {0xA8FD, 0xA8FE}, + {0xA900, 0xA925}, + {0xA930, 0xA946}, + {0xA960, 0xA97C}, + {0xA984, 0xA9B2}, + {0xA9CF, 0xA9D9}, + {0xA9E0, 0xA9E4}, + {0xA9E6, 0xA9FE}, + {0xAA00, 0xAA28}, + {0xAA40, 0xAA42}, + {0xAA44, 0xAA4B}, + {0xAA50, 0xAA59}, + {0xAA60, 0xAA76}, + {0xAA7A, 0xAA7A}, + {0xAA7E, 0xAAAF}, + {0xAAB1, 0xAAB1}, + {0xAAB5, 0xAAB6}, + {0xAAB9, 0xAABD}, + {0xAAC0, 0xAAC0}, + {0xAAC2, 0xAAC2}, + {0xAADB, 0xAADD}, + {0xAAE0, 0xAAEA}, + {0xAAF2, 0xAAF4}, + {0xAB01, 0xAB06}, + {0xAB09, 0xAB0E}, + {0xAB11, 0xAB16}, + {0xAB20, 0xAB26}, + {0xAB28, 0xAB2E}, + {0xAB30, 0xAB5A}, + {0xAB5C, 0xAB69}, + {0xAB70, 0xABE2}, + {0xABF0, 0xABF9}, + {0xAC00, 0xD7A3}, + {0xD7B0, 0xD7C6}, + {0xD7CB, 0xD7FB}, + {0xF900, 0xFA6D}, + {0xFA70, 0xFAD9}, + {0xFB00, 0xFB06}, + {0xFB13, 0xFB17}, + {0xFB1D, 0xFB1D}, + {0xFB1F, 0xFB28}, + {0xFB2A, 0xFB36}, + {0xFB38, 0xFB3C}, + {0xFB3E, 0xFB3E}, + {0xFB40, 0xFB41}, + {0xFB43, 0xFB44}, + {0xFB46, 0xFBB1}, + {0xFBD3, 0xFD3D}, + {0xFD50, 0xFD8F}, + {0xFD92, 0xFDC7}, + {0xFDF0, 0xFDFB}, + {0xFE70, 0xFE74}, + {0xFE76, 0xFEFC}, + {0xFF10, 0xFF19}, + {0xFF21, 0xFF3A}, + {0xFF41, 0xFF5A}, + {0xFF66, 0xFFBE}, + {0xFFC2, 0xFFC7}, + {0xFFCA, 0xFFCF}, + {0xFFD2, 0xFFD7}, + {0xFFDA, 0xFFDC}, + {0x10000, 0x1000B}, + {0x1000D, 0x10026}, + {0x10028, 0x1003A}, + {0x1003C, 0x1003D}, + {0x1003F, 0x1004D}, + {0x10050, 0x1005D}, + {0x10080, 0x100FA}, + {0x10107, 0x10133}, + {0x10140, 0x10178}, + {0x1018A, 0x1018B}, + {0x10280, 0x1029C}, + {0x102A0, 0x102D0}, + {0x102E1, 0x102FB}, + {0x10300, 0x10323}, + {0x1032D, 0x1034A}, + {0x10350, 0x10375}, + {0x10380, 0x1039D}, + {0x103A0, 0x103C3}, + {0x103C8, 0x103CF}, + {0x103D1, 0x103D5}, + {0x10400, 0x1049D}, + {0x104A0, 0x104A9}, + {0x104B0, 0x104D3}, + {0x104D8, 0x104FB}, + {0x10500, 0x10527}, + {0x10530, 0x10563}, + {0x10570, 0x1057A}, + {0x1057C, 0x1058A}, + {0x1058C, 0x10592}, + {0x10594, 0x10595}, + {0x10597, 0x105A1}, + {0x105A3, 0x105B1}, + {0x105B3, 0x105B9}, + {0x105BB, 0x105BC}, + {0x10600, 0x10736}, + {0x10740, 0x10755}, + {0x10760, 0x10767}, + {0x10780, 0x10785}, + {0x10787, 0x107B0}, + {0x107B2, 0x107BA}, + {0x10800, 0x10805}, + {0x10808, 0x10808}, + {0x1080A, 0x10835}, + {0x10837, 0x10838}, + {0x1083C, 0x1083C}, + {0x1083F, 0x10855}, + {0x10858, 0x10876}, + {0x10879, 0x1089E}, + {0x108A7, 0x108AF}, + {0x108E0, 0x108F2}, + {0x108F4, 0x108F5}, + {0x108FB, 0x1091B}, + {0x10920, 0x10939}, + {0x10980, 0x109B7}, + {0x109BC, 0x109CF}, + {0x109D2, 0x10A00}, + {0x10A10, 0x10A13}, + {0x10A15, 0x10A17}, + {0x10A19, 0x10A35}, + {0x10A40, 0x10A48}, + {0x10A60, 0x10A7E}, + {0x10A80, 0x10A9F}, + {0x10AC0, 0x10AC7}, + {0x10AC9, 0x10AE4}, + {0x10AEB, 0x10AEF}, + {0x10B00, 0x10B35}, + {0x10B40, 0x10B55}, + {0x10B58, 0x10B72}, + {0x10B78, 0x10B91}, + {0x10BA9, 0x10BAF}, + {0x10C00, 0x10C48}, + {0x10C80, 0x10CB2}, + {0x10CC0, 0x10CF2}, + {0x10CFA, 0x10D23}, + {0x10D30, 0x10D39}, + {0x10E60, 0x10E7E}, + {0x10E80, 0x10EA9}, + {0x10EB0, 0x10EB1}, + {0x10F00, 0x10F27}, + {0x10F30, 0x10F45}, + {0x10F51, 0x10F54}, + {0x10F70, 0x10F81}, + {0x10FB0, 0x10FCB}, + {0x10FE0, 0x10FF6}, + {0x11003, 0x11037}, + {0x11052, 0x1106F}, + {0x11071, 0x11072}, + {0x11075, 0x11075}, + {0x11083, 0x110AF}, + {0x110D0, 0x110E8}, + {0x110F0, 0x110F9}, + {0x11103, 0x11126}, + {0x11136, 0x1113F}, + {0x11144, 0x11144}, + {0x11147, 0x11147}, + {0x11150, 0x11172}, + {0x11176, 0x11176}, + {0x11183, 0x111B2}, + {0x111C1, 0x111C4}, + {0x111D0, 0x111DA}, + {0x111DC, 0x111DC}, + {0x111E1, 0x111F4}, + {0x11200, 0x11211}, + {0x11213, 0x1122B}, + {0x1123F, 0x11240}, + {0x11280, 0x11286}, + {0x11288, 0x11288}, + {0x1128A, 0x1128D}, + {0x1128F, 0x1129D}, + {0x1129F, 0x112A8}, + {0x112B0, 0x112DE}, + {0x112F0, 0x112F9}, + {0x11305, 0x1130C}, + {0x1130F, 0x11310}, + {0x11313, 0x11328}, + {0x1132A, 0x11330}, + {0x11332, 0x11333}, + {0x11335, 0x11339}, + {0x1133D, 0x1133D}, + {0x11350, 0x11350}, + {0x1135D, 0x11361}, + {0x11400, 0x11434}, + {0x11447, 0x1144A}, + {0x11450, 0x11459}, + {0x1145F, 0x11461}, + {0x11480, 0x114AF}, + {0x114C4, 0x114C5}, + {0x114C7, 0x114C7}, + {0x114D0, 0x114D9}, + {0x11580, 0x115AE}, + {0x115D8, 0x115DB}, + {0x11600, 0x1162F}, + {0x11644, 0x11644}, + {0x11650, 0x11659}, + {0x11680, 0x116AA}, + {0x116B8, 0x116B8}, + {0x116C0, 0x116C9}, + {0x11700, 0x1171A}, + {0x11730, 0x1173B}, + {0x11740, 0x11746}, + {0x11800, 0x1182B}, + {0x118A0, 0x118F2}, + {0x118FF, 0x11906}, + {0x11909, 0x11909}, + {0x1190C, 0x11913}, + {0x11915, 0x11916}, + {0x11918, 0x1192F}, + {0x1193F, 0x1193F}, + {0x11941, 0x11941}, + {0x11950, 0x11959}, + {0x119A0, 0x119A7}, + {0x119AA, 0x119D0}, + {0x119E1, 0x119E1}, + {0x119E3, 0x119E3}, + {0x11A00, 0x11A00}, + {0x11A0B, 0x11A32}, + {0x11A3A, 0x11A3A}, + {0x11A50, 0x11A50}, + {0x11A5C, 0x11A89}, + {0x11A9D, 0x11A9D}, + {0x11AB0, 0x11AF8}, + {0x11C00, 0x11C08}, + {0x11C0A, 0x11C2E}, + {0x11C40, 0x11C40}, + {0x11C50, 0x11C6C}, + {0x11C72, 0x11C8F}, + {0x11D00, 0x11D06}, + {0x11D08, 0x11D09}, + {0x11D0B, 0x11D30}, + {0x11D46, 0x11D46}, + {0x11D50, 0x11D59}, + {0x11D60, 0x11D65}, + {0x11D67, 0x11D68}, + {0x11D6A, 0x11D89}, + {0x11D98, 0x11D98}, + {0x11DA0, 0x11DA9}, + {0x11EE0, 0x11EF2}, + {0x11F02, 0x11F02}, + {0x11F04, 0x11F10}, + {0x11F12, 0x11F33}, + {0x11F50, 0x11F59}, + {0x11FB0, 0x11FB0}, + {0x11FC0, 0x11FD4}, + {0x12000, 0x12399}, + {0x12400, 0x1246E}, + {0x12480, 0x12543}, + {0x12F90, 0x12FF0}, + {0x13000, 0x1342F}, + {0x13441, 0x13446}, + {0x14400, 0x14646}, + {0x16800, 0x16A38}, + {0x16A40, 0x16A5E}, + {0x16A60, 0x16A69}, + {0x16A70, 0x16ABE}, + {0x16AC0, 0x16AC9}, + {0x16AD0, 0x16AED}, + {0x16B00, 0x16B2F}, + {0x16B40, 0x16B43}, + {0x16B50, 0x16B59}, + {0x16B5B, 0x16B61}, + {0x16B63, 0x16B77}, + {0x16B7D, 0x16B8F}, + {0x16E40, 0x16E96}, + {0x16F00, 0x16F4A}, + {0x16F50, 0x16F50}, + {0x16F93, 0x16F9F}, + {0x16FE0, 0x16FE1}, + {0x16FE3, 0x16FE3}, + {0x17000, 0x187F7}, + {0x18800, 0x18CD5}, + {0x18D00, 0x18D08}, + {0x1AFF0, 0x1AFF3}, + {0x1AFF5, 0x1AFFB}, + {0x1AFFD, 0x1AFFE}, + {0x1B000, 0x1B122}, + {0x1B132, 0x1B132}, + {0x1B150, 0x1B152}, + {0x1B155, 0x1B155}, + {0x1B164, 0x1B167}, + {0x1B170, 0x1B2FB}, + {0x1BC00, 0x1BC6A}, + {0x1BC70, 0x1BC7C}, + {0x1BC80, 0x1BC88}, + {0x1BC90, 0x1BC99}, + {0x1D2C0, 0x1D2D3}, + {0x1D2E0, 0x1D2F3}, + {0x1D360, 0x1D378}, + {0x1D400, 0x1D454}, + {0x1D456, 0x1D49C}, + {0x1D49E, 0x1D49F}, + {0x1D4A2, 0x1D4A2}, + {0x1D4A5, 0x1D4A6}, + {0x1D4A9, 0x1D4AC}, + {0x1D4AE, 0x1D4B9}, + {0x1D4BB, 0x1D4BB}, + {0x1D4BD, 0x1D4C3}, + {0x1D4C5, 0x1D505}, + {0x1D507, 0x1D50A}, + {0x1D50D, 0x1D514}, + {0x1D516, 0x1D51C}, + {0x1D51E, 0x1D539}, + {0x1D53B, 0x1D53E}, + {0x1D540, 0x1D544}, + {0x1D546, 0x1D546}, + {0x1D54A, 0x1D550}, + {0x1D552, 0x1D6A5}, + {0x1D6A8, 0x1D6C0}, + {0x1D6C2, 0x1D6DA}, + {0x1D6DC, 0x1D6FA}, + {0x1D6FC, 0x1D714}, + {0x1D716, 0x1D734}, + {0x1D736, 0x1D74E}, + {0x1D750, 0x1D76E}, + {0x1D770, 0x1D788}, + {0x1D78A, 0x1D7A8}, + {0x1D7AA, 0x1D7C2}, + {0x1D7C4, 0x1D7CB}, + {0x1D7CE, 0x1D7FF}, + {0x1DF00, 0x1DF1E}, + {0x1DF25, 0x1DF2A}, + {0x1E030, 0x1E06D}, + {0x1E100, 0x1E12C}, + {0x1E137, 0x1E13D}, + {0x1E140, 0x1E149}, + {0x1E14E, 0x1E14E}, + {0x1E290, 0x1E2AD}, + {0x1E2C0, 0x1E2EB}, + {0x1E2F0, 0x1E2F9}, + {0x1E4D0, 0x1E4EB}, + {0x1E4F0, 0x1E4F9}, + {0x1E7E0, 0x1E7E6}, + {0x1E7E8, 0x1E7EB}, + {0x1E7ED, 0x1E7EE}, + {0x1E7F0, 0x1E7FE}, + {0x1E800, 0x1E8C4}, + {0x1E8C7, 0x1E8CF}, + {0x1E900, 0x1E943}, + {0x1E94B, 0x1E94B}, + {0x1E950, 0x1E959}, + {0x1EC71, 0x1ECAB}, + {0x1ECAD, 0x1ECAF}, + {0x1ECB1, 0x1ECB4}, + {0x1ED01, 0x1ED2D}, + {0x1ED2F, 0x1ED3D}, + {0x1EE00, 0x1EE03}, + {0x1EE05, 0x1EE1F}, + {0x1EE21, 0x1EE22}, + {0x1EE24, 0x1EE24}, + {0x1EE27, 0x1EE27}, + {0x1EE29, 0x1EE32}, + {0x1EE34, 0x1EE37}, + {0x1EE39, 0x1EE39}, + {0x1EE3B, 0x1EE3B}, + {0x1EE42, 0x1EE42}, + {0x1EE47, 0x1EE47}, + {0x1EE49, 0x1EE49}, + {0x1EE4B, 0x1EE4B}, + {0x1EE4D, 0x1EE4F}, + {0x1EE51, 0x1EE52}, + {0x1EE54, 0x1EE54}, + {0x1EE57, 0x1EE57}, + {0x1EE59, 0x1EE59}, + {0x1EE5B, 0x1EE5B}, + {0x1EE5D, 0x1EE5D}, + {0x1EE5F, 0x1EE5F}, + {0x1EE61, 0x1EE62}, + {0x1EE64, 0x1EE64}, + {0x1EE67, 0x1EE6A}, + {0x1EE6C, 0x1EE72}, + {0x1EE74, 0x1EE77}, + {0x1EE79, 0x1EE7C}, + {0x1EE7E, 0x1EE7E}, + {0x1EE80, 0x1EE89}, + {0x1EE8B, 0x1EE9B}, + {0x1EEA1, 0x1EEA3}, + {0x1EEA5, 0x1EEA9}, + {0x1EEAB, 0x1EEBB}, + {0x1F100, 0x1F10C}, + {0x1FBF0, 0x1FBF9}, + {0x20000, 0x2A6DF}, + {0x2A700, 0x2B739}, + {0x2B740, 0x2B81D}, + {0x2B820, 0x2CEA1}, + {0x2CEB0, 0x2EBE0}, + {0x2F800, 0x2FA1D}, + {0x30000, 0x3134A}, + {0x31350, 0x323AF}, +}}; + +inline bool isTokenChar(std::uint32_t code_point) +{ + if (code_point < 128) + return (code_point >= 'a' && code_point <= 'z') || (code_point >= 'A' && code_point <= 'Z') + || (code_point >= '0' && code_point <= '9') || code_point == '_'; + std::size_t lo = 0; + std::size_t hi = token_char_ranges.size(); + while (lo < hi) + { + const std::size_t mid = lo + (hi - lo) / 2; + const auto & span = token_char_ranges[mid]; + if (code_point < span.first) + hi = mid; + else if (code_point > span.last) + lo = mid + 1; + else + return true; + } + return false; +} +} // namespace DB::LocalMatchAgainst diff --git a/dbms/src/Functions/tests/gtest_local_match_against.cpp b/dbms/src/Functions/tests/gtest_local_match_against.cpp index 0e82a937c5e..c17078f9905 100644 --- a/dbms/src/Functions/tests/gtest_local_match_against.cpp +++ b/dbms/src/Functions/tests/gtest_local_match_against.cpp @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include #include #include #include @@ -629,4 +630,117 @@ try } CATCH +TEST_F(TestLocalMatchAgainst, MatchBooleanSplitWordModifiers) +try +{ + const auto collator = TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_BIN); + const auto documents = createColumn({"foo only", "bar only", "foo bar", "baz foo", "baz bar", "baz qux"}); + for (const auto occur : {tipb::LocalMatchAgainstBooleanOccurShould, + tipb::LocalMatchAgainstBooleanOccurMust, + tipb::LocalMatchAgainstBooleanOccurMustNot}) + { + tipb::LocalMatchAgainstBooleanQuery query; + auto * node = query.add_nodes(); + node->set_occur(occur); + node->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + node->set_text("foo.bar"); + if (occur == tipb::LocalMatchAgainstBooleanOccurMustNot) + { + auto * positive = query.add_nodes(); + positive->set_occur(tipb::LocalMatchAgainstBooleanOccurShould); + positive->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + positive->set_text("baz"); + } + const auto expected = occur == tipb::LocalMatchAgainstBooleanOccurShould + ? createColumn({1, 1, 1, 1, 1, 0}) + : occur == tipb::LocalMatchAgainstBooleanOccurMust + ? createColumn({0, 0, 1, 0, 0, 0}) + : createColumn({0, 0, 0, 0, 0, 1}); + const auto metadata = createConstColumn(6, serializeLocalMatchAgainstBooleanQuery(query)); + ASSERT_COLUMN_EQ( + expected, + executeFunction( + "local_match_against_boolean", + {createConstColumn(6, "foo.bar"), documents, metadata}, + collator, + true)); + } + // The intersection for a required split word may span MATCH columns. + tipb::LocalMatchAgainstBooleanQuery query; + auto * node = query.add_nodes(); + node->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + node->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + node->set_text("foo.bar"); + ASSERT_COLUMN_EQ( + createColumn({1, 0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(2, "+foo.bar"), + createColumn({"foo", "foo"}), + createColumn({"bar", "qux"}), + createConstColumn(2, serializeLocalMatchAgainstBooleanQuery(query))}, + collator, + true)); +} +CATCH + +TEST_F(TestLocalMatchAgainst, MatchBooleanUnicodeTokenBoundaries) +try +{ + const auto collator = TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_BIN); + const auto documents = createColumn( + {"foo🙃bar", "foo👁bar", "foo bar", "foobar", "foo𞤀bar", "foo𝟙bar", "foo\U0002EBF0bar", "foo\xff" "bar"}); + for (const auto size : {0U, 2U, 3U}) + { + tipb::LocalMatchAgainstBooleanQuery query; + query.set_parser(size == 0 ? tipb::LocalMatchAgainstParserStandard : tipb::LocalMatchAgainstParserNgram); + query.set_ngram_token_size(size); + query.set_stopword_mode(tipb::LocalMatchAgainstStopwordModeDisabled); + auto * node = query.add_nodes(); + node->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + node->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + node->set_text(size == 0 ? "foo" : "foob"); + ASSERT_COLUMN_EQ( + size == 0 ? createColumn({1, 1, 1, 0, 0, 0, 1, 1}) + : createColumn({0, 0, 0, 1, 0, 0, 0, 0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(8, node->text()), + documents, + createConstColumn(8, serializeLocalMatchAgainstBooleanQuery(query))}, + collator, + true)); + node->set_text("𞤀bar"); + ASSERT_COLUMN_EQ( + size == 0 ? createColumn({0, 1, 0}) : createColumn({1, 1, 0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(3, "+𞤀bar"), + createColumn({"foo𞤀bar", "𞤀bar", "foo bar"}), + createConstColumn(3, serializeLocalMatchAgainstBooleanQuery(query))}, + collator, + true)); + } +} +CATCH + +TEST(LocalMatchAgainstTokenChars, ProtocolV1Classification) +{ + for (const UInt32 code_point : {0x1F643U, 0x1F441U, 0x301U, 0xFFFDU, 0x2EBF0U, 0x110000U}) + EXPECT_FALSE(LocalMatchAgainst::isTokenChar(code_point)) << code_point; + for (const UInt32 code_point : {0x5FU, 0x4E2DU, 0xE9U, 0x1D7D9U, 0x1E900U}) + EXPECT_TRUE(LocalMatchAgainst::isTokenChar(code_point)) << code_point; + UInt32 previous = 127; + for (const auto & span : LocalMatchAgainst::token_char_ranges) + { + ASSERT_GT(span.first, previous); + ASSERT_LE(span.first, span.last); + for (UInt32 code_point = previous + 1; code_point < span.first; ++code_point) + ASSERT_FALSE(LocalMatchAgainst::isTokenChar(code_point)) << code_point; + for (UInt32 code_point = span.first; code_point <= span.last; ++code_point) + ASSERT_TRUE(LocalMatchAgainst::isTokenChar(code_point)) << code_point; + previous = span.last; + } +} + } // namespace DB::tests From 30160fecc872279ec589c958274c1f747858003d Mon Sep 17 00:00:00 2001 From: Zhigao Tong Date: Fri, 9 Oct 2026 18:46:32 +0800 Subject: [PATCH 19/32] tiflash: verify Local MATCH Unicode classification fingerprint --- dbms/src/Functions/LocalMatchAgainstTokenChars.h | 9 +++++---- .../Functions/tests/gtest_local_match_against.cpp | 14 ++++++++++++++ 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/dbms/src/Functions/LocalMatchAgainstTokenChars.h b/dbms/src/Functions/LocalMatchAgainstTokenChars.h index 9fddb572e1b..d326bf37725 100644 --- a/dbms/src/Functions/LocalMatchAgainstTokenChars.h +++ b/dbms/src/Functions/LocalMatchAgainstTokenChars.h @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +// Generated by TiDB pkg/expression/matchagainst/generator; DO NOT EDIT. #pragma once #include @@ -20,10 +21,10 @@ namespace DB::LocalMatchAgainst { -// Generated from Go's Unicode 15.0.0 categories L and N. These ranges are -// identical to matchagainst/token_chars.go in TiDB and fix Local MATCH protocol -// version 1 semantics independently of Poco/Go upgrades. Never change an -// existing protocol version's classification; deploy TiFlash support for a +// Generated from Go's Unicode 15.0.0 categories L and N. TiDB uses its standard +// library and strictly tests that Unicode version and the protocol-v1 checksum. +// Regenerate/check this header using the paired TiDB generator. Never change +// an existing protocol version's classification; deploy TiFlash support for a // new version before TiDB emits it. ASCII and '_' use the fast path below. struct TokenCharRange { diff --git a/dbms/src/Functions/tests/gtest_local_match_against.cpp b/dbms/src/Functions/tests/gtest_local_match_against.cpp index c17078f9905..20577ce8ed1 100644 --- a/dbms/src/Functions/tests/gtest_local_match_against.cpp +++ b/dbms/src/Functions/tests/gtest_local_match_against.cpp @@ -741,6 +741,20 @@ TEST(LocalMatchAgainstTokenChars, ProtocolV1Classification) ASSERT_TRUE(LocalMatchAgainst::isTokenChar(code_point)) << code_point; previous = span.last; } + for (UInt32 code_point = previous + 1; code_point <= 0x10FFFFU; ++code_point) + ASSERT_FALSE(LocalMatchAgainst::isTokenChar(code_point)) << code_point; + + // FNV-1a over one 0/1 byte per code point, U+0000..U+10FFFF. The paired + // TiDB tests hash Go's Unicode 15.0.0 IsLetter/IsNumber/'_' classification + // identically. This fixed regression checksum must not change in v1, + // even when the header is regenerated or either toolchain is upgraded. + std::uint64_t fingerprint = 14695981039346656037ULL; + for (UInt32 code_point = 0; code_point <= 0x10FFFFU; ++code_point) + { + fingerprint ^= static_cast(LocalMatchAgainst::isTokenChar(code_point)); + fingerprint *= 1099511628211ULL; + } + EXPECT_EQ(0x71f51f3810b3b529ULL, fingerprint); } } // namespace DB::tests From 184f58001ee4b6eff99851db37b590b415fd32fe Mon Sep 17 00:00:00 2001 From: Zhigao Tong Date: Sat, 10 Oct 2026 11:01:11 +0800 Subject: [PATCH 20/32] fix: align Local MATCH semantics and cache phrase verification --- dbms/CMakeLists.txt | 23 ++ .../Functions/FunctionsLocalMatchAgainst.cpp | 179 ++++++-- .../Functions/LocalMatchAgainstTokenChars.h | 2 +- .../tests/bench_local_match_against.cpp | 131 ++++++ .../tests/gtest_local_match_against.cpp | 390 ++++++++++++++++-- 5 files changed, 659 insertions(+), 66 deletions(-) create mode 100644 dbms/src/Functions/tests/bench_local_match_against.cpp diff --git a/dbms/CMakeLists.txt b/dbms/CMakeLists.txt index dec2aa38795..868534f3dba 100644 --- a/dbms/CMakeLists.txt +++ b/dbms/CMakeLists.txt @@ -404,6 +404,29 @@ if (ENABLE_TESTS) endif() add_check(bench_dbms) + + # Allow Local MATCH smoke/Release benchmarks without building unrelated + # vector-index benchmarks and their external dataset dependencies. + add_executable(bench_local_match_against EXCLUDE_FROM_ALL + ${TiFlash_SOURCE_DIR}/dbms/src/Functions/tests/bench_local_match_against.cpp + ${TiFlash_SOURCE_DIR}/dbms/src/Server/StorageConfigParser.cpp + ${TiFlash_SOURCE_DIR}/dbms/src/Server/UserConfigParser.cpp + ${TiFlash_SOURCE_DIR}/dbms/src/Server/RaftConfigParser.cpp + ${TiFlash_SOURCE_DIR}/dbms/src/AggregateFunctions/AggregateFunctionSum.cpp) + target_include_directories(bench_local_match_against BEFORE PRIVATE + ${SPARCEHASH_INCLUDE_DIR} ${benchmark_SOURCE_DIR}/include) + target_compile_definitions(bench_local_match_against PUBLIC DBMS_PUBLIC_GTEST) + target_link_libraries(bench_local_match_against + gtest + benchmark + dbms + test_util_bench_main + tiflash_functions + server_for_test + delta_merge + tiflash_aggregate_functions + kvstore) + add_check(bench_local_match_against) endif () # dbms diff --git a/dbms/src/Functions/FunctionsLocalMatchAgainst.cpp b/dbms/src/Functions/FunctionsLocalMatchAgainst.cpp index e543ca079eb..c5eb4ca8a9c 100644 --- a/dbms/src/Functions/FunctionsLocalMatchAgainst.cpp +++ b/dbms/src/Functions/FunctionsLocalMatchAgainst.cpp @@ -30,6 +30,7 @@ #include #include +#include #include #include #include @@ -51,11 +52,14 @@ constexpr size_t default_min_token_size = 3; constexpr size_t default_max_token_size = 84; constexpr size_t default_ngram_token_size = 2; // This is the Local MATCH semantic protocol version, not the TiDB/TiFlash -// product version. Version 1 fixes the Boolean AST interpretation, analyzer +// product version. Version 2 fixes the Boolean AST interpretation, analyzer // behavior, and built-in stopword set. TiFlash must support a version before // TiDB emits it; unknown versions are rejected, and existing versions must // never be reinterpreted with changed semantics. -constexpr UInt32 local_match_against_protocol_version = 1; +// Version 2 returns zero for NULL searches and uses MySQL's multibyte NGRAM +// document scanner. STANDARD and the NGRAM Boolean lexer use BMP word +// characters; filtered phrases also verify the unfiltered document stream. +constexpr UInt32 local_match_against_protocol_version = 2; struct FullTextToken { @@ -130,6 +134,7 @@ struct BooleanClause bool prefix = false; std::vector terms; std::vector offsets; + std::unique_ptr verification; std::vector> prefix_matchers; }; @@ -145,6 +150,7 @@ struct CompiledBooleanQuery size_t must_clause_count = 0; bool has_must_not = false; bool matches_nothing = false; + bool needs_verification = false; }; void lowercaseTokenInPlace(String & token) @@ -162,7 +168,7 @@ void lowercaseTokenInPlace(String & token) bool isFullTextToken(UInt32 code_point) { - return LocalMatchAgainst::isTokenChar(code_point); + return code_point <= 0xFFFF && LocalMatchAgainst::isTokenChar(code_point); } std::pair decodeCodePoint(std::string_view text, size_t offset) @@ -177,14 +183,19 @@ void tokenizeTextInto( std::string_view text, FullTextColumn & result, const TiDB::TiDBCollatorPtr & collator = nullptr, - bool preserve_case = false) + bool preserve_case = false, + bool ngram_document = false) { result.clear(); size_t position = 0; for (size_t i = 0; i < text.size();) { const auto [code_point, length] = decodeCodePoint(text, i); - if (!isFullTextToken(code_point)) + // MySQL ngram_parse retains any valid multibyte character but stops + // at malformed UTF-8. ASCII non-word characters still split runs. + if (ngram_document && length == 1 && code_point >= 0x80) + break; + if (!(ngram_document && length > 1) && !isFullTextToken(code_point)) { i += length; continue; @@ -195,7 +206,9 @@ void tokenizeTextInto( while (i < text.size()) { const auto [code_point, length] = decodeCodePoint(text, i); - if (!isFullTextToken(code_point)) + if (ngram_document && length == 1 && code_point >= 0x80) + break; + if (!(ngram_document && length > 1) && !isFullTextToken(code_point)) break; ++code_points; i += length; @@ -343,7 +356,7 @@ void analyzeNgramTextInto( return; const bool preserve_case = enable_stopword && static_cast(stopword_collator); - tokenizeTextInto(text, scratch.raw_tokens, collator, preserve_case); + tokenizeTextInto(text, scratch.raw_tokens, collator, preserve_case, true); size_t next_position_base = 0; auto & char_boundaries = scratch.char_boundaries; for (auto & token : scratch.raw_tokens) @@ -360,10 +373,10 @@ void analyzeNgramTextInto( } const size_t char_count = token.code_points; - const size_t base_position = std::max(token.position, next_position_base); + const size_t base_position = next_position_base; if (char_count < ngram_token_size) { - next_position_base = std::max(next_position_base, token.position + 1); + // A short document run emits no grams and contributes no position. continue; } @@ -457,11 +470,10 @@ struct ClauseMatchState std::vector matched_terms; size_t remaining_terms = 0; bool matched = false; + bool phrase_verification_failed = false; }; -void initializeClauseMatchStates( - const std::vector & clauses, - std::vector & states) +void initializeClauseMatchStates(const std::vector & clauses, std::vector & states) { states.resize(clauses.size()); for (size_t i = 0; i < clauses.size(); ++i) @@ -514,11 +526,9 @@ void analyzeColumnInto( bool enable_stopword = true) { if (use_ngram) - analyzeNgramTextInto( - document, output, scratch, ngram_token_size, collator, stopword_collator, enable_stopword); + analyzeNgramTextInto(document, output, scratch, ngram_token_size, collator, stopword_collator, enable_stopword); else - analyzeTextInto( - document, output, collator, stopword_collator, min_token_size, max_token_size, enable_stopword); + analyzeTextInto(document, output, collator, stopword_collator, min_token_size, max_token_size, enable_stopword); } void resetClauseMatchStates(std::vector & states) @@ -547,6 +557,7 @@ enum class ColumnMatchResult ColumnMatchResult updateBooleanMatchStatesForColumn( const CompiledBooleanQuery & query, const FullTextColumn & column, + const FullTextColumn & unfiltered_column, const TiDB::TiDBCollatorPtr & collator, std::vector & states, BooleanMatchProgress & progress) @@ -555,6 +566,12 @@ ColumnMatchResult updateBooleanMatchStatesForColumn( if (query.matches_nothing || clauses.empty()) return ColumnMatchResult::Continue; + // Raw phrase verification searches the entire column, so a failed result + // needs no recheck at later anchors. Unlike positive clause matches, this + // negative cache belongs only to this column, not other MATCH columns. + for (auto & state : states) + state.phrase_verification_failed = false; + const auto markClauseMatched = [&](size_t clause_index) { states[clause_index].matched = true; switch (clauses[clause_index].modifier) @@ -585,13 +602,21 @@ ColumnMatchResult updateBooleanMatchStatesForColumn( { const auto & clause = clauses[clause_index]; auto & state = states[clause_index]; - if (state.matched || clause.terms.empty()) + if (state.matched || state.phrase_verification_failed || clause.terms.empty()) continue; if (clause.phrase) { if (matchesPhraseEndingAt(clause, column, token, collator)) { + if (clause.verification + && !std::any_of(unfiltered_column.begin(), unfiltered_column.end(), [&](const auto & end) { + return matchesPhraseEndingAt(*clause.verification, unfiltered_column, end, collator); + })) + { + state.phrase_verification_failed = true; + continue; + } const auto result = markClauseMatched(clause_index); if (result != ColumnMatchResult::Continue) return result; @@ -622,9 +647,7 @@ ColumnMatchResult updateBooleanMatchStatesForColumn( return ColumnMatchResult::Continue; } -bool evaluateBooleanMatchResult( - const CompiledBooleanQuery & query, - const std::vector & states) +bool evaluateBooleanMatchResult(const CompiledBooleanQuery & query, const std::vector & states) { if (query.matches_nothing || query.clauses.empty()) return false; @@ -680,6 +703,7 @@ CompiledBooleanQuery compileBooleanQuery( for (const auto & node : query.nodes()) { BooleanClause clause; + size_t first_query_position = 0; switch (node.occur()) { case tipb::LocalMatchAgainstBooleanOccur::LocalMatchAgainstBooleanOccurMust: @@ -705,6 +729,7 @@ CompiledBooleanQuery compileBooleanQuery( { clause.phrase = true; const size_t first_position = terms.empty() ? 0 : terms.front().position; + first_query_position = first_position; for (const auto & token : terms) { clause.terms.push_back(token.text); @@ -733,6 +758,7 @@ CompiledBooleanQuery compileBooleanQuery( // "caf*" ("ca", "af" at token size 2) never match. clause.phrase = true; const size_t first_position = terms.front().position; + first_query_position = first_position; for (const auto & token : terms) { clause.terms.push_back(token.text); @@ -786,7 +812,30 @@ CompiledBooleanQuery compileBooleanQuery( { clause.phrase = true; const auto terms = analyze_query(node.text()); + if (use_ngram) + { + const auto words = tokenizeText(node.text(), collator); + bool seen_indexed_word = false; + if (std::any_of(words.begin(), words.end(), [&](const auto & word) { + if (word.code_points >= ngram_token_size) + { + seen_indexed_word = seen_indexed_word || !analyze_query(word.text).empty(); + return false; + } + if (!enable_stopword || seen_indexed_word) + return true; + std::vector boundaries{0}; + for (size_t offset = 0; offset < word.text.size();) + { + offset += decodeCodePoint(word.text, offset).second; + boundaries.push_back(offset); + } + return !containsNgramStopword(word.text, boundaries, 0, word.code_points, stopword_collator); + })) + break; // A short query unigram cannot occur in the document's fixed-size ngrams. + } const size_t first_position = terms.empty() ? 0 : terms.front().position; + first_query_position = first_position; for (const auto & token : terms) { clause.terms.push_back(token.text); @@ -799,6 +848,32 @@ CompiledBooleanQuery compileBooleanQuery( return compiled; } + if (clause.phrase && !clause.terms.empty()) + { + auto raw_terms = use_ngram + ? analyzeNgramText(node.text(), ngram_token_size, collator, nullptr, false) + : analyzeText(node.text(), collator, nullptr, 0, std::numeric_limits::max(), false); + // InnoDB discards leading filtered tokens, but retains every + // following word in the original phrase used for verification. + raw_terms.filterInPlace([&](const auto & token) { return token.position >= first_query_position; }); + if (raw_terms.size() != clause.terms.size()) + { + // InnoDB verifies phrases against the original document, not + // just the index tokens: removing "the" must not turn it into + // an arbitrary positional gap. Ordinary terms/prefixes do not + // pay for this additional document stream. + clause.verification = std::make_unique(); + clause.verification->phrase = true; + const size_t first = raw_terms.front().position; + for (const auto & term : raw_terms) + { + clause.verification->terms.push_back(term.text); + clause.verification->offsets.push_back(term.position - first); + } + compiled.needs_verification = true; + } + } + if (clause.prefix && collator) { clause.prefix_matchers.reserve(clause.terms.size()); @@ -813,7 +888,18 @@ CompiledBooleanQuery compileBooleanQuery( for (const auto & clause : compiled.clauses) { if (clause.modifier == BooleanClause::Modifier::Must) + { + // An analyzer-filtered required clause cannot match any document. + // Do this after compiling ALL nodes so malformed later nodes + // retain their existing no-match handling, rather than being + // hidden by an earlier block-level shortcut. + if (clause.terms.empty()) + { + compiled.matches_nothing = true; + return compiled; + } ++compiled.must_clause_count; + } else if (clause.modifier == BooleanClause::Modifier::MustNot) compiled.has_must_not = true; } @@ -828,9 +914,7 @@ CompiledBooleanQuery compileBooleanQuery( return compiled; } -Float64 matchBooleanScore( - const CompiledBooleanQuery & query, - const std::vector & states) +Float64 matchBooleanScore(const CompiledBooleanQuery & query, const std::vector & states) { // The protocol path is the no-score MATCH ... AGAINST BOOLEAN MODE // predicate introduced by #70484/#70485. TiDB's local evaluator returns @@ -845,8 +929,7 @@ class FullTextColumnAccessor bool isNull(size_t row) const { - return nullable_column != nullptr - && nullable_column->getNullMapData()[nullable_is_constant ? 0 : row] != 0; + return nullable_column != nullptr && nullable_column->getNullMapData()[nullable_is_constant ? 0 : row] != 0; } std::string_view getString(size_t row) const @@ -953,13 +1036,19 @@ class FunctionLocalMatchAgainstBoolean final : public IFunction auto & output_data = output->getData(); const bool nullable = block.getByPosition(result).type->isNullable(); auto null_map = nullable ? ColumnUInt8::create(rows, 0) : nullptr; - auto * null_map_data = null_map ? &null_map->getData() : nullptr; const FullTextColumnAccessor query_accessor(*query_column); + tipb::LocalMatchAgainstBooleanQuery protocol_boolean_query; + if (arguments.size() <= 2 + || !decodeLocalMatchAgainstBooleanQuery( + *block.getByPosition(arguments.back()).column, + protocol_boolean_query)) + throw Exception( + "local_match_against_boolean requires valid Boolean query metadata", + ErrorCodes::ILLEGAL_COLUMN); if (query_accessor.isNull(0)) { - if (null_map_data != nullptr) - std::fill(null_map_data->begin(), null_map_data->end(), 1); + // InnoDB interprets NULL AGAINST as an empty search: non-NULL zero. ColumnPtr result_column; if (null_map) result_column = ColumnNullable::create(std::move(output), std::move(null_map)); @@ -969,14 +1058,6 @@ class FunctionLocalMatchAgainstBoolean final : public IFunction return; } size_t document_argument_end = arguments.size() - 1; - tipb::LocalMatchAgainstBooleanQuery protocol_boolean_query; - if (arguments.size() <= 2 - || !decodeLocalMatchAgainstBooleanQuery( - *block.getByPosition(arguments.back()).column, - protocol_boolean_query)) - throw Exception( - "local_match_against_boolean requires valid Boolean query metadata", - ErrorCodes::ILLEGAL_COLUMN); const bool use_ngram = protocol_boolean_query.parser() == tipb::LocalMatchAgainstParser::LocalMatchAgainstParserNgram; const size_t ngram_token_size = !use_ngram || protocol_boolean_query.ngram_token_size() == 0 @@ -988,8 +1069,7 @@ class FunctionLocalMatchAgainstBoolean final : public IFunction = has_standard_config ? protocol_boolean_query.innodb_ft_min_token_size() : default_min_token_size; const size_t max_token_size = has_standard_config ? protocol_boolean_query.innodb_ft_max_token_size() : default_max_token_size; - const bool enable_stopword - = protocol_boolean_query.stopword_mode() + const bool enable_stopword = protocol_boolean_query.stopword_mode() == tipb::LocalMatchAgainstStopwordMode::LocalMatchAgainstStopwordModeBuiltin; const auto stopword_collator = enable_stopword ? TiDB::ITiDBCollator::getCollator(protocol_boolean_query.stopword_collation()) : nullptr; @@ -1023,6 +1103,7 @@ class FunctionLocalMatchAgainstBoolean final : public IFunction std::vector clause_states; initializeClauseMatchStates(compiled_query.clauses, clause_states); FullTextColumn analyzed_column; + FullTextColumn unfiltered_column; AnalyzerScratch analyzer_scratch; for (size_t row = 0; row < rows; ++row) @@ -1052,9 +1133,25 @@ class FunctionLocalMatchAgainstBoolean final : public IFunction min_token_size, max_token_size, enable_stopword); - const auto match_result - = updateBooleanMatchStatesForColumn( - compiled_query, analyzed_column, collator, clause_states, match_progress); + if (compiled_query.needs_verification) + analyzeColumnInto( + document_column.getString(row), + unfiltered_column, + analyzer_scratch, + use_ngram, + ngram_token_size, + collator, + nullptr, + 0, + std::numeric_limits::max(), + false); + const auto match_result = updateBooleanMatchStatesForColumn( + compiled_query, + analyzed_column, + unfiltered_column, + collator, + clause_states, + match_progress); if (match_result == ColumnMatchResult::Rejected) { result_known = true; diff --git a/dbms/src/Functions/LocalMatchAgainstTokenChars.h b/dbms/src/Functions/LocalMatchAgainstTokenChars.h index d326bf37725..f1216b226e1 100644 --- a/dbms/src/Functions/LocalMatchAgainstTokenChars.h +++ b/dbms/src/Functions/LocalMatchAgainstTokenChars.h @@ -22,7 +22,7 @@ namespace DB::LocalMatchAgainst { // Generated from Go's Unicode 15.0.0 categories L and N. TiDB uses its standard -// library and strictly tests that Unicode version and the protocol-v1 checksum. +// library and strictly tests that Unicode version and the Unicode 15.0.0 checksum. // Regenerate/check this header using the paired TiDB generator. Never change // an existing protocol version's classification; deploy TiFlash support for a // new version before TiDB emits it. ASCII and '_' use the fast path below. diff --git a/dbms/src/Functions/tests/bench_local_match_against.cpp b/dbms/src/Functions/tests/bench_local_match_against.cpp new file mode 100644 index 00000000000..d327aa6c316 --- /dev/null +++ b/dbms/src/Functions/tests/bench_local_match_against.cpp @@ -0,0 +1,131 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include +#include +#include + +#include +#include + +namespace DB +{ +void registerFunctionsLocalMatchAgainst(FunctionFactory & factory); +} + +namespace DB::tests +{ +namespace +{ +String benchmarkDocument(size_t row, size_t size) +{ + // Keep identical to tests/ftse2e/benchdata in the paired TiDB checkout. + static const std::array units{ + "quick brown fox prefix 数据库 ", + "quick slow fox prefix 数据库 ", + "QUICK brown FOX PREFIX 数据库 ", + "other words unrelated sample 文档 "}; + const auto & unit = units[row % units.size()]; + String result; + result.reserve(size); + for (size_t i = 0; i < size / unit.size(); ++i) + result += unit; + result.append(size % unit.size(), ' '); + return result; +} + +void BM_LocalMatchAgainst(benchmark::State & state) +try +{ + const auto bytes = static_cast(state.range(0)); + const auto parser = state.range(1); // 0=STANDARD, 2/3=NGRAM token size. + const bool ci = state.range(2); + const bool stopwords = state.range(3); + const auto scenario = state.range(4); // Required/excluded, phrase, prefix, filtered MUST. + constexpr size_t rows = 16; + tipb::LocalMatchAgainstBooleanQuery query; + query.set_version(2); + query.set_parser(parser == 0 ? tipb::LocalMatchAgainstParserStandard : tipb::LocalMatchAgainstParserNgram); + query.set_ngram_token_size(parser == 0 ? 2 : parser); + query.set_innodb_ft_min_token_size(3); + query.set_innodb_ft_max_token_size(84); + query.set_stopword_mode( + stopwords ? tipb::LocalMatchAgainstStopwordModeBuiltin : tipb::LocalMatchAgainstStopwordModeDisabled); + query.set_stopword_collation(ci ? "utf8mb4_general_ci" : "utf8mb4_bin"); + auto * required = query.add_nodes(); + required->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + required->set_term_type( + scenario == 1 ? tipb::LocalMatchAgainstBooleanTermPhrase + : scenario == 2 ? tipb::LocalMatchAgainstBooleanTermPrefix + : tipb::LocalMatchAgainstBooleanTermWord); + required->set_text(scenario == 1 ? "quick brown fox" : scenario == 2 ? "pre" : scenario == 3 ? "the" : "quick"); + if (scenario == 0) + { + auto * prohibited = query.add_nodes(); + prohibited->set_occur(tipb::LocalMatchAgainstBooleanOccurMustNot); + prohibited->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + prohibited->set_text("slow"); + } + static const std::array searches{"+quick -slow", "+\"quick brown fox\"", "+pre*", "+the"}; + std::vector documents; + for (size_t row = 0; row < rows; ++row) + documents.push_back(benchmarkDocument(row, bytes)); + auto context = TiFlashTestEnv::getContext(); + const ColumnsWithTypeAndName arguments{ + createConstColumn(rows, searches[scenario]), + createColumn(documents), + createConstColumn(rows, query.SerializeAsString())}; + const auto collator = TiDB::ITiDBCollator::getCollator( + ci ? TiDB::ITiDBCollator::UTF8MB4_GENERAL_CI : TiDB::ITiDBCollator::UTF8MB4_BIN); + auto & factory = FunctionFactory::instance(); + if (!factory.tryGet("local_match_against_boolean", *context)) + registerFunctionsLocalMatchAgainst(factory); + auto function = factory.get("local_match_against_boolean", *context)->build(arguments, collator); + Block block(arguments); + block.insert({nullptr, function->getReturnType(), "result"}); + const ColumnNumbers argument_numbers{0, 1, 2}; + // Inputs and function setup are outside timing. Query compilation remains + // inside execution, exactly as in the production block-level scalar path. + for (auto _ : state) + { + function->execute(block, argument_numbers, 3); + benchmark::DoNotOptimize(block.getByPosition(3).column); + } + state.SetItemsProcessed(state.iterations() * rows); + state.SetBytesProcessed(state.iterations() * rows * bytes); +} +catch (const Exception & e) +{ + state.SkipWithError(e.displayText().c_str()); +} +catch (const std::exception & e) +{ + state.SkipWithError(e.what()); +} + +void benchmarkArguments(benchmark::internal::Benchmark * benchmark) +{ + for (const int bytes : {128, 4096, 262144}) + for (const int parser : {0, 2, 3}) + for (const int ci : {0, 1}) + for (const int stopwords : {0, 1}) + for (const int scenario : {0, 1, 2, 3}) + benchmark->Args({bytes, parser, ci, stopwords, scenario}); +} +BENCHMARK(BM_LocalMatchAgainst)->Apply(benchmarkArguments); +} // namespace +} // namespace DB::tests diff --git a/dbms/src/Functions/tests/gtest_local_match_against.cpp b/dbms/src/Functions/tests/gtest_local_match_against.cpp index 20577ce8ed1..d7f4042e195 100644 --- a/dbms/src/Functions/tests/gtest_local_match_against.cpp +++ b/dbms/src/Functions/tests/gtest_local_match_against.cpp @@ -27,11 +27,48 @@ class TestLocalMatchAgainst : public DB::tests::FunctionTest namespace { +// Count token comparisons rather than wall time, so a repeated full-column +// verification regression is deterministic even on a loaded CI host. +class CountingMatchCollator final : public TiDB::ITiDBCollator +{ +public: + CountingMatchCollator() + : ITiDBCollator(UTF8MB4_BIN) + , inner(getCollator(UTF8MB4_BIN)) + { + collator_type = inner->getCollatorType(); + } + + int compare(const char * lhs, size_t lhs_size, const char * rhs, size_t rhs_size) const override + { + ++comparisons; + return inner->compare(lhs, lhs_size, rhs, rhs_size); + } + StringRef convert(const char * s, size_t size, String & buffer, std::vector * lens) const override + { + return inner->convert(s, size, buffer, lens); + } + StringRef sortKeyNoTrim(const char * s, size_t size, String & buffer) const override + { + return inner->sortKeyNoTrim(s, size, buffer); + } + StringRef sortKey(const char * s, size_t size, String & buffer) const override + { + return inner->sortKey(s, size, buffer); + } + std::unique_ptr pattern() const override { return inner->pattern(); } + + mutable size_t comparisons = 0; + +private: + TiDB::TiDBCollatorPtr inner; +}; + String serializeLocalMatchAgainstBooleanQuery(const tipb::LocalMatchAgainstBooleanQuery & query) { auto encoded_query = query; if (encoded_query.version() == 0) - encoded_query.set_version(1); // The tests exercise the currently supported Local MATCH semantic protocol. + encoded_query.set_version(2); // The tests exercise the currently supported Local MATCH semantic protocol. if (encoded_query.parser() == tipb::LocalMatchAgainstParser::LocalMatchAgainstParserInvalid) encoded_query.set_parser(tipb::LocalMatchAgainstParser::LocalMatchAgainstParserStandard); if (encoded_query.stopword_mode() == tipb::LocalMatchAgainstStopwordMode::LocalMatchAgainstStopwordModeInvalid) @@ -65,6 +102,283 @@ try } CATCH +TEST_F(TestLocalMatchAgainst, FilteredPhraseVerificationComparisonGrowth) +try +{ + tipb::LocalMatchAgainstBooleanQuery query; + auto * node = query.add_nodes(); + node->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + node->set_term_type(tipb::LocalMatchAgainstBooleanTermPhrase); + node->set_text("foo a bar"); + size_t small_comparisons = 0; + for (const size_t repeats : {32, 256}) + { + String document; + for (size_t i = 0; i < repeats; ++i) + document += "foo x bar "; + CountingMatchCollator collator; + ASSERT_COLUMN_EQ( + createColumn({0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(1, "+\"foo a bar\""), + createColumn({document}), + createConstColumn(1, serializeLocalMatchAgainstBooleanQuery(query))}, + &collator, + true)); + if (repeats == 32) + small_comparisons = collator.comparisons; + else + { + EXPECT_GT(small_comparisons, 0); + // Eight times the input may add proportional token work, not + // 64 times the work from rechecking every failed anchor. + EXPECT_LE(collator.comparisons, small_comparisons * 12); + } + } +} +CATCH + +TEST_F(TestLocalMatchAgainst, FilteredPhraseVerificationCacheScope) +try +{ + String miss; + for (size_t i = 0; i < 64; ++i) + miss += "foo x bar "; + for (const auto occur : + {tipb::LocalMatchAgainstBooleanOccurMust, + tipb::LocalMatchAgainstBooleanOccurShould, + tipb::LocalMatchAgainstBooleanOccurMustNot}) + { + tipb::LocalMatchAgainstBooleanQuery query; + if (occur != tipb::LocalMatchAgainstBooleanOccurShould) + { + auto * anchor = query.add_nodes(); + anchor->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + anchor->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + anchor->set_text("foo"); + } + auto * phrase = query.add_nodes(); + phrase->set_occur(occur); + phrase->set_term_type(tipb::LocalMatchAgainstBooleanTermPhrase); + phrase->set_text("foo a bar"); + const Float64 hit = occur == tipb::LocalMatchAgainstBooleanOccurMustNot ? 0 : 1; + const Float64 no_phrase = occur == tipb::LocalMatchAgainstBooleanOccurMustNot ? 1 : 0; + ASSERT_COLUMN_EQ( + createColumn>({hit, hit, no_phrase, hit, no_phrase}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(5, "foo \"foo a bar\""), + createColumn>({miss, "foo a bar", miss, {}, miss}), + createColumn>({"foo a bar", miss, {}, "foo a bar", miss}), + createConstColumn(5, serializeLocalMatchAgainstBooleanQuery(query))}, + nullptr, + true)); + } + + tipb::LocalMatchAgainstBooleanQuery query; + auto * required = query.add_nodes(); + required->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + required->set_term_type(tipb::LocalMatchAgainstBooleanTermPhrase); + required->set_text("foo x bar"); + auto * excluded = query.add_nodes(); + excluded->set_occur(tipb::LocalMatchAgainstBooleanOccurMustNot); + excluded->set_term_type(tipb::LocalMatchAgainstBooleanTermPhrase); + excluded->set_text("foo a bar"); + ASSERT_COLUMN_EQ( + createColumn({1, 0, 1}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(3, "+\"foo x bar\" -\"foo a bar\""), + createColumn({miss, "foo x bar foo a bar", miss}), + createConstColumn(3, serializeLocalMatchAgainstBooleanQuery(query))}, + nullptr, + true)); + + // NGRAM size 3 removes "the" but retains "foo" and "zoo". + // STANDARD exercises the same cache with a filtered stopword. + for (const bool ngram : {false, true}) + { + tipb::LocalMatchAgainstBooleanQuery filtered; + filtered.set_parser(ngram ? tipb::LocalMatchAgainstParserNgram : tipb::LocalMatchAgainstParserStandard); + filtered.set_ngram_token_size(3); + auto * node = filtered.add_nodes(); + node->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + node->set_term_type(tipb::LocalMatchAgainstBooleanTermPhrase); + node->set_text("foo the zoo"); + String repetitive_miss; + for (size_t i = 0; i < 64; ++i) + repetitive_miss += "foo xyz zoo "; + ASSERT_COLUMN_EQ( + createColumn>({1, 1, 0, 1}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(4, "+\"foo the zoo\""), + createColumn>({repetitive_miss, "foo the zoo", repetitive_miss, {}}), + createColumn>({"foo the zoo", repetitive_miss, {}, "foo the zoo"}), + createConstColumn(4, serializeLocalMatchAgainstBooleanQuery(filtered))}, + nullptr, + true)); + } +} +CATCH + +TEST_F(TestLocalMatchAgainst, MatchBooleanMySQLCharacterBoundaries) +try +{ + tipb::LocalMatchAgainstBooleanQuery query; + query.set_parser(tipb::LocalMatchAgainstParserNgram); + query.set_ngram_token_size(2); + query.set_stopword_mode(tipb::LocalMatchAgainstStopwordModeDisabled); + auto * node = query.add_nodes(); + node->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + node->set_term_type(tipb::LocalMatchAgainstBooleanTermPrefix); + node->set_text("a"); + ASSERT_COLUMN_EQ( + createColumn({0, 1, 1, 1, 1, 1}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(6, "+a*"), + createColumn({"a,b", "a，b", "a🙃b", "a𞤀b", "a𝟙b", String("ab") + char(0xff) + "cd"}), + createConstColumn(6, serializeLocalMatchAgainstBooleanQuery(query))}, + nullptr, + true)); + + query.set_parser(tipb::LocalMatchAgainstParserStandard); + node->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + node->set_text("foo"); + ASSERT_COLUMN_EQ( + createColumn({1, 1, 1, 0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(4, "+foo"), + createColumn({"foo𞤀bar", "foo𝟙bar", "foo🙃bar", "foobar"}), + createConstColumn(4, serializeLocalMatchAgainstBooleanQuery(query))}, + nullptr, + true)); +} +CATCH + +TEST_F(TestLocalMatchAgainst, MatchBooleanFilteredPhraseVerification) +try +{ + for (const bool ngram : {false, true}) + for (const bool stopwords : {false, true}) + { + tipb::LocalMatchAgainstBooleanQuery query; + query.set_parser(ngram ? tipb::LocalMatchAgainstParserNgram : tipb::LocalMatchAgainstParserStandard); + query.set_ngram_token_size(2); + query.set_stopword_mode( + stopwords ? tipb::LocalMatchAgainstStopwordModeBuiltin : tipb::LocalMatchAgainstStopwordModeDisabled); + auto * node = query.add_nodes(); + node->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + node->set_term_type(tipb::LocalMatchAgainstBooleanTermPhrase); + node->set_text("quick x fox"); + ASSERT_COLUMN_EQ( + createColumn({ngram ? 0.0 : 1.0, 0, 0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(3, "+\"quick x fox\""), + createColumn({"quick x fox", "quick the fox", "quick xx fox"}), + createConstColumn(3, serializeLocalMatchAgainstBooleanQuery(query))}, + nullptr, + true)); + node->set_text("foo bar"); + ASSERT_COLUMN_EQ( + createColumn({1, 1, ngram ? 0.0 : 1.0, ngram ? 0.0 : 1.0, 0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(5, "+\"foo bar\""), + createColumn({"foo bar", "foo,bar", "foo，bar", "foo🙃bar", "foobar"}), + createConstColumn(5, serializeLocalMatchAgainstBooleanQuery(query))}, + nullptr, + true)); + } + + tipb::LocalMatchAgainstBooleanQuery query; + query.set_parser(tipb::LocalMatchAgainstParserNgram); + query.set_ngram_token_size(3); + query.set_stopword_mode(tipb::LocalMatchAgainstStopwordModeBuiltin); + auto * node = query.add_nodes(); + node->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + node->set_term_type(tipb::LocalMatchAgainstBooleanTermPhrase); + for (const String & text : {String("quick the fox"), String("quick a fox")}) + { + node->set_text(text); + ASSERT_COLUMN_EQ( + createColumn({1, 1, 1}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(3, text), + createColumn({"quick fox", "quick x fox", "fox quick"}), + createConstColumn(3, serializeLocalMatchAgainstBooleanQuery(query))}, + nullptr, + true)); + } + for (const String & text : {String("foo a zoo"), String("foo zoo")}) + { + node->set_text(text); + const Float64 expected = text == "foo zoo" ? 1.0 : 0.0; + ASSERT_COLUMN_EQ( + createColumn({expected, expected, expected, 0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(4, text), + createColumn({"foo zoo", "foo a zoo", "foo x zoo", "zoo foo"}), + createConstColumn(4, serializeLocalMatchAgainstBooleanQuery(query))}, + nullptr, + true)); + } +} +CATCH + +TEST_F(TestLocalMatchAgainst, MatchBooleanFilteredRequiredClause) +try +{ + for (const bool ngram : {false, true}) + for (const String & term : {String("the"), String("x")}) + { + tipb::LocalMatchAgainstBooleanQuery query; + query.set_parser(ngram ? tipb::LocalMatchAgainstParserNgram : tipb::LocalMatchAgainstParserStandard); + query.set_ngram_token_size(3); + auto * required = query.add_nodes(); + required->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + required->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + required->set_text(term); + auto * optional = query.add_nodes(); + optional->set_occur(tipb::LocalMatchAgainstBooleanOccurShould); + optional->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + optional->set_text("quick"); + const auto columns = ColumnsWithTypeAndName{ + createConstColumn(3, "+" + term + " quick"), + createColumn>({"the x quick", String(4096, 'x'), {}}), + createConstColumn(3, serializeLocalMatchAgainstBooleanQuery(query))}; + ASSERT_COLUMN_EQ( + createColumn>({0, 0, 0}), + executeFunction("local_match_against_boolean", columns, nullptr, true)); + + // NULL AGAINST is an empty search even for an impossible query. + auto null_search = columns; + null_search[0] = createConstColumn>(3, {}); + ASSERT_COLUMN_EQ( + createColumn>({0, 0, 0}), + executeFunction("local_match_against_boolean", null_search, nullptr, true)); + + // Envelope validation must still reject an unknown version. + query.set_version(999); + EXPECT_THROW( + executeFunction( + "local_match_against_boolean", + {columns[0], + columns[1], + createConstColumn(3, serializeLocalMatchAgainstBooleanQuery(query))}, + nullptr, + true), + Exception); + } +} +CATCH + TEST_F(TestLocalMatchAgainst, MatchBooleanProtocolQuery) try { @@ -122,8 +436,7 @@ try executeFunction( "local_match_against_boolean", {createConstColumn(4, "+\"quick the fox\""), - createColumn( - {"quick the fox", "quick fox", "quick slow the fox", "quick the fox quick the fox"}), + createColumn({"quick the fox", "quick fox", "quick slow the fox", "quick the fox quick the fox"}), createConstColumn(4, metadata)}, nullptr, true)); @@ -268,7 +581,7 @@ TEST_F(TestLocalMatchAgainst, MatchBooleanRejectsInvalidProtocol) try { tipb::LocalMatchAgainstBooleanQuery boolean_query; - boolean_query.set_version(2); + boolean_query.set_version(1); // Version 1 must not be silently reinterpreted as version 2. boolean_query.set_parser(tipb::LocalMatchAgainstParser::LocalMatchAgainstParserStandard); boolean_query.set_stopword_mode(tipb::LocalMatchAgainstStopwordMode::LocalMatchAgainstStopwordModeBuiltin); boolean_query.set_stopword_collation("utf8mb4_bin"); @@ -287,7 +600,7 @@ try true), Exception); - boolean_query.set_version(1); + boolean_query.set_version(2); boolean_query.set_parser(tipb::LocalMatchAgainstParser::LocalMatchAgainstParserInvalid); ASSERT_THROW( executeFunction( @@ -564,7 +877,11 @@ try const auto short_metadata = createConstColumn(4, make_metadata(2, "c")); ASSERT_COLUMN_EQ( createColumn({1, 1, 1, 1}), - executeFunction("local_match_against_boolean", {short_query, documents, short_metadata}, utf8mb4_general_ci, true)); + executeFunction( + "local_match_against_boolean", + {short_query, documents, short_metadata}, + utf8mb4_general_ci, + true)); } CATCH @@ -583,10 +900,10 @@ try node->set_text(term); return serializeLocalMatchAgainstBooleanQuery(boolean_query); }; - const auto word_metadata = createConstColumn( - 3, make_metadata(tipb::LocalMatchAgainstBooleanTermWord, "cafe")); - const auto prefix_metadata = createConstColumn( - 3, make_metadata(tipb::LocalMatchAgainstBooleanTermPrefix, "caf")); + const auto word_metadata + = createConstColumn(3, make_metadata(tipb::LocalMatchAgainstBooleanTermWord, "cafe")); + const auto prefix_metadata + = createConstColumn(3, make_metadata(tipb::LocalMatchAgainstBooleanTermPrefix, "caf")); const auto utf8mb4_bin = TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_BIN); ASSERT_COLUMN_EQ( @@ -602,7 +919,11 @@ try executeFunction("local_match_against_boolean", {query, documents, word_metadata}, utf8mb4_0900_bin, true)); ASSERT_COLUMN_EQ( createColumn({0, 1, 1}), - executeFunction("local_match_against_boolean", {prefix_query, documents, prefix_metadata}, utf8mb4_0900_bin, true)); + executeFunction( + "local_match_against_boolean", + {prefix_query, documents, prefix_metadata}, + utf8mb4_0900_bin, + true)); const auto utf8mb4_general_ci = TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_GENERAL_CI); ASSERT_COLUMN_EQ( @@ -610,7 +931,11 @@ try executeFunction("local_match_against_boolean", {query, documents, word_metadata}, utf8mb4_general_ci, true)); ASSERT_COLUMN_EQ( createColumn({1, 1, 1}), - executeFunction("local_match_against_boolean", {prefix_query, documents, prefix_metadata}, utf8mb4_general_ci, true)); + executeFunction( + "local_match_against_boolean", + {prefix_query, documents, prefix_metadata}, + utf8mb4_general_ci, + true)); const auto utf8mb4_unicode_ci = TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_UNICODE_CI); ASSERT_COLUMN_EQ( @@ -618,7 +943,11 @@ try executeFunction("local_match_against_boolean", {query, documents, word_metadata}, utf8mb4_unicode_ci, true)); ASSERT_COLUMN_EQ( createColumn({1, 1, 1}), - executeFunction("local_match_against_boolean", {prefix_query, documents, prefix_metadata}, utf8mb4_unicode_ci, true)); + executeFunction( + "local_match_against_boolean", + {prefix_query, documents, prefix_metadata}, + utf8mb4_unicode_ci, + true)); const auto utf8mb4_0900_ai_ci = TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_0900_AI_CI); ASSERT_COLUMN_EQ( @@ -626,7 +955,11 @@ try executeFunction("local_match_against_boolean", {query, documents, word_metadata}, utf8mb4_0900_ai_ci, true)); ASSERT_COLUMN_EQ( createColumn({1, 1, 1}), - executeFunction("local_match_against_boolean", {prefix_query, documents, prefix_metadata}, utf8mb4_0900_ai_ci, true)); + executeFunction( + "local_match_against_boolean", + {prefix_query, documents, prefix_metadata}, + utf8mb4_0900_ai_ci, + true)); } CATCH @@ -635,9 +968,10 @@ try { const auto collator = TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_BIN); const auto documents = createColumn({"foo only", "bar only", "foo bar", "baz foo", "baz bar", "baz qux"}); - for (const auto occur : {tipb::LocalMatchAgainstBooleanOccurShould, - tipb::LocalMatchAgainstBooleanOccurMust, - tipb::LocalMatchAgainstBooleanOccurMustNot}) + for (const auto occur : + {tipb::LocalMatchAgainstBooleanOccurShould, + tipb::LocalMatchAgainstBooleanOccurMust, + tipb::LocalMatchAgainstBooleanOccurMustNot}) { tipb::LocalMatchAgainstBooleanQuery query; auto * node = query.add_nodes(); @@ -653,9 +987,8 @@ try } const auto expected = occur == tipb::LocalMatchAgainstBooleanOccurShould ? createColumn({1, 1, 1, 1, 1, 0}) - : occur == tipb::LocalMatchAgainstBooleanOccurMust - ? createColumn({0, 0, 1, 0, 0, 0}) - : createColumn({0, 0, 0, 0, 0, 1}); + : occur == tipb::LocalMatchAgainstBooleanOccurMust ? createColumn({0, 0, 1, 0, 0, 0}) + : createColumn({0, 0, 0, 0, 0, 1}); const auto metadata = createConstColumn(6, serializeLocalMatchAgainstBooleanQuery(query)); ASSERT_COLUMN_EQ( expected, @@ -689,7 +1022,15 @@ try { const auto collator = TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_BIN); const auto documents = createColumn( - {"foo🙃bar", "foo👁bar", "foo bar", "foobar", "foo𞤀bar", "foo𝟙bar", "foo\U0002EBF0bar", "foo\xff" "bar"}); + {"foo🙃bar", + "foo👁bar", + "foo bar", + "foobar", + "foo𞤀bar", + "foo𝟙bar", + "foo\U0002EBF0bar", + "foo\xff" + "bar"}); for (const auto size : {0U, 2U, 3U}) { tipb::LocalMatchAgainstBooleanQuery query; @@ -701,7 +1042,7 @@ try node->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); node->set_text(size == 0 ? "foo" : "foob"); ASSERT_COLUMN_EQ( - size == 0 ? createColumn({1, 1, 1, 0, 0, 0, 1, 1}) + size == 0 ? createColumn({1, 1, 1, 0, 1, 1, 1, 1}) : createColumn({0, 0, 0, 1, 0, 0, 0, 0}), executeFunction( "local_match_against_boolean", @@ -710,9 +1051,10 @@ try createConstColumn(8, serializeLocalMatchAgainstBooleanQuery(query))}, collator, true)); - node->set_text("𞤀bar"); + // TiDB's version-2 NGRAM Boolean lexer emits "bar" for this search. + node->set_text(size == 0 ? "𞤀bar" : "bar"); ASSERT_COLUMN_EQ( - size == 0 ? createColumn({0, 1, 0}) : createColumn({1, 1, 0}), + createColumn({1, 1, 1}), executeFunction( "local_match_against_boolean", {createConstColumn(3, "+𞤀bar"), From febfdb4c900a8265d613b8b1000f079c056d1159 Mon Sep 17 00:00:00 2001 From: Zhigao Tong Date: Sat, 10 Oct 2026 19:07:46 +0800 Subject: [PATCH 21/32] perf: reduce Local MATCH analyzer work and token copies Add ASCII decoding fast paths and collation-key builtin stopword lookup. Fuse STANDARD filtering with scanning, collect NGRAM boundaries once, and borrow collated document tokens without changing query ownership or positional semantics. Verified with 25 related gtests, real TiUP E2E on macOS and h124, token sizes 2/3, and paired single-thread server benchmarks. --- .../Functions/FunctionsLocalMatchAgainst.cpp | 293 +++++++++++------- .../tests/gtest_local_match_against.cpp | 246 +++++++++++++++ 2 files changed, 424 insertions(+), 115 deletions(-) diff --git a/dbms/src/Functions/FunctionsLocalMatchAgainst.cpp b/dbms/src/Functions/FunctionsLocalMatchAgainst.cpp index c5eb4ca8a9c..080ebdef66c 100644 --- a/dbms/src/Functions/FunctionsLocalMatchAgainst.cpp +++ b/dbms/src/Functions/FunctionsLocalMatchAgainst.cpp @@ -32,6 +32,7 @@ #include #include #include +#include #include #include #include @@ -63,9 +64,15 @@ constexpr UInt32 local_match_against_protocol_version = 2; struct FullTextToken { + // Query terms and the no-collator lowercase path own their spelling. + // Document views refer only to the current input column, which outlives + // both analyzed streams and matching. Never retain them across blocks. String text; + std::string_view source; size_t position = 0; size_t code_points = 0; + + std::string_view value() const { return source.data() ? source : std::string_view(text); } }; class FullTextColumn @@ -79,12 +86,18 @@ class FullTextColumn void reserve(size_t size) { tokens.reserve(size); } - void append(std::string_view text, size_t position, size_t code_points) + void append(std::string_view text, size_t position, size_t code_points, bool borrow = false) { if (used == tokens.size()) tokens.emplace_back(); auto & token = tokens[used++]; - token.text.assign(text.data(), text.size()); + if (borrow) + token.source = text; + else + { + token.source = {}; + token.text.assign(text.data(), text.size()); + } token.position = position; token.code_points = code_points; } @@ -140,8 +153,8 @@ struct BooleanClause struct AnalyzerScratch { - FullTextColumn raw_tokens; std::vector char_boundaries; + String lowercase_run; }; struct CompiledBooleanQuery @@ -173,65 +186,62 @@ bool isFullTextToken(UInt32 code_point) std::pair decodeCodePoint(std::string_view text, size_t offset) { + // Most document bytes are ASCII. Keep the multibyte/invalid-byte + // semantics below, but avoid calling the generic decoder for ASCII. + const auto byte = static_cast(text[offset]); + if (byte < 0x80) + return {byte, 1}; const auto decoded = UTF8::utf8Decode(text.data() + offset, text.size() - offset); if (decoded.second == 0 || decoded.first == UTF8::UTF8_Error || decoded.second > text.size() - offset) return {0xFFFD, 1}; // Go's utf8.DecodeRuneInString also consumes one invalid byte as RuneError. return {decoded.first, decoded.second}; } -void tokenizeTextInto( - std::string_view text, - FullTextColumn & result, - const TiDB::TiDBCollatorPtr & collator = nullptr, - bool preserve_case = false, - bool ngram_document = false) +template +void scanStandardTokens(std::string_view text, Consumer && consume) { - result.clear(); size_t position = 0; for (size_t i = 0; i < text.size();) { const auto [code_point, length] = decodeCodePoint(text, i); - // MySQL ngram_parse retains any valid multibyte character but stops - // at malformed UTF-8. ASCII non-word characters still split runs. - if (ngram_document && length == 1 && code_point >= 0x80) - break; - if (!(ngram_document && length > 1) && !isFullTextToken(code_point)) + if (!isFullTextToken(code_point)) { i += length; continue; } const size_t token_start = i; - size_t code_points = 0; + // The first character has already been decoded and classified. + size_t code_points = 1; + i += length; while (i < text.size()) { const auto [code_point, length] = decodeCodePoint(text, i); - if (ngram_document && length == 1 && code_point >= 0x80) - break; - if (!(ngram_document && length > 1) && !isFullTextToken(code_point)) + if (!isFullTextToken(code_point)) break; ++code_points; i += length; } - // If a collation is present, retain the source spelling and let the - // collator decide case and accent equivalence during matching. This - // is important for binary collations, where lower-casing would make - // a case-sensitive MATCH unexpectedly case-insensitive. Keep the - // legacy lower-case behavior for callers without collation metadata. - result.append(text.substr(token_start, i - token_start), position++, code_points); - if (!collator && !preserve_case) - lowercaseTokenInPlace(result[result.size() - 1].text); + // Positions include source tokens subsequently removed by filtering. + consume(text.substr(token_start, i - token_start), position++, code_points); } } -FullTextColumn tokenizeText( - std::string_view text, - const TiDB::TiDBCollatorPtr & collator = nullptr, - bool preserve_case = false) +void tokenizeTextInto(std::string_view text, FullTextColumn & result, const TiDB::TiDBCollatorPtr & collator = nullptr) +{ + result.clear(); + scanStandardTokens(text, [&](std::string_view token, size_t position, size_t code_points) { + result.append(token, position, code_points); + if (!collator) + lowercaseTokenInPlace(result[result.size() - 1].text); + }); +} + +FullTextColumn tokenizeText(std::string_view text, const TiDB::TiDBCollatorPtr & collator = nullptr) { FullTextColumn result; - tokenizeTextInto(text, result, collator, preserve_case); + tokenizeTextInto(text, result, collator); return result; } @@ -244,32 +254,61 @@ const std::unordered_set & defaultStopwords() return stopwords; } -const std::unordered_map> & defaultStopwordsByLength() +// Owned collation keys are compiled once per executeImpl, then reused by +// query analysis and every document in the block. The scratch key is local +// to that execution, never shared across concurrent calls. Use the stopword +// (server) collation, not the MATCH column collation. +class BuiltinStopwordLookup { - static const auto stopwords_by_length = [] { - std::unordered_map> result; +public: + explicit BuiltinStopwordLookup(TiDB::TiDBCollatorPtr collator_) + : collator(collator_) + { for (const auto & stopword : defaultStopwords()) { - const size_t length - = UTF8::countCodePoints(reinterpret_cast(stopword.data()), stopword.size()); - result[length].push_back(stopword); + const auto key = collator->sortKeyFastPath(stopword.data(), stopword.size(), key_buffer); + // sortKeyFastPath may reference its input or key_buffer; copy it. + String owned_key(key.data, key.size); + keys.insert(owned_key); + // The built-in list is ASCII. Preserve NGRAM's code-point length + // buckets even for collations with expansions/ignorable weights. + keys_by_length[stopword.size()].insert(std::move(owned_key)); } - return result; - }(); - return stopwords_by_length; -} + } + + bool contains(std::string_view token) const { return contains(token, keys); } -bool isDefaultStopword(std::string_view token, const TiDB::TiDBCollatorPtr & stopword_collator) + bool contains(std::string_view token, size_t code_points) const + { + const auto bucket = keys_by_length.find(code_points); + return bucket != keys_by_length.end() && contains(token, bucket->second); + } + +private: + struct KeyHash + { + using is_transparent = void; + size_t operator()(std::string_view key) const { return std::hash{}(key); } + }; + using Keys = std::unordered_set>; + + bool contains(std::string_view token, const Keys & lookup) const + { + // sortKey (rather than sortKeyNoTrim) preserves PAD SPACE/NO PAD + // equality, as well as the collator's case/accent semantics. + const auto key = collator->sortKeyFastPath(token.data(), token.size(), key_buffer); + return lookup.contains(std::string_view(key.data, key.size)); + } + + TiDB::TiDBCollatorPtr collator; + Keys keys; + std::unordered_map keys_by_length; + mutable String key_buffer; +}; + +bool isDefaultStopword(std::string_view token, const BuiltinStopwordLookup * stopwords) { - const auto & stopwords = defaultStopwords(); - if (!stopword_collator) - return stopwords.contains(Poco::UTF8::toLower(String(token))); - // MySQL compares stopwords using collation_server. This is independent of - // the MATCH column collation and may be case-sensitive (for example - // utf8mb4_bin). - return std::any_of(stopwords.begin(), stopwords.end(), [&](const String & stopword) { - return stopword_collator->compare(token.data(), token.size(), stopword.data(), stopword.size()) == 0; - }); + return stopwords ? stopwords->contains(token) : defaultStopwords().contains(Poco::UTF8::toLower(String(token))); } bool containsNgramStopword( @@ -277,29 +316,18 @@ bool containsNgramStopword( const std::vector & char_boundaries, size_t ngram_start, size_t ngram_token_size, - const TiDB::TiDBCollatorPtr & stopword_collator) + const BuiltinStopwordLookup * stopwords) { const size_t ngram_end = ngram_start + ngram_token_size; - const auto & stopwords = defaultStopwords(); - const auto & stopwords_by_length = defaultStopwordsByLength(); for (size_t start = ngram_start; start < ngram_end; ++start) { for (size_t end = start + 1; end <= ngram_end; ++end) { const auto candidate = token.substr(char_boundaries[start], char_boundaries[end] - char_boundaries[start]); - if (!stopword_collator && stopwords.contains(Poco::UTF8::toLower(String(candidate)))) + if (!stopwords && defaultStopwords().contains(Poco::UTF8::toLower(String(candidate)))) + return true; + if (stopwords && stopwords->contains(candidate, end - start)) return true; - if (stopword_collator) - { - const auto found = stopwords_by_length.find(end - start); - if (found != stopwords_by_length.end() - && std::any_of(found->second.begin(), found->second.end(), [&](const String & stopword) { - return stopword_collator - ->compare(candidate.data(), candidate.size(), stopword.data(), stopword.size()) - == 0; - })) - return true; - } } } return false; @@ -309,36 +337,35 @@ void analyzeTextInto( std::string_view text, FullTextColumn & result, const TiDB::TiDBCollatorPtr & collator = nullptr, - const TiDB::TiDBCollatorPtr & stopword_collator = nullptr, + const BuiltinStopwordLookup * stopwords = nullptr, size_t min_token_size = default_min_token_size, size_t max_token_size = default_max_token_size, - bool enable_stopword = true) + bool enable_stopword = true, + bool borrow_document = false) { - const bool preserve_case = enable_stopword && static_cast(stopword_collator); - tokenizeTextInto(text, result, collator, preserve_case); - result.filterInPlace([&](const FullTextToken & token) { - return token.code_points >= min_token_size && token.code_points <= max_token_size - && (!enable_stopword || !isDefaultStopword(token.text, stopword_collator)); + result.clear(); + scanStandardTokens(text, [&](std::string_view token, size_t position, size_t code_points) { + // Filter before materializing. Stopwords use the source spelling and + // server collation, independently of the MATCH column collation. + if (code_points < min_token_size || code_points > max_token_size + || (enable_stopword && isDefaultStopword(token, stopwords))) + return; + result.append(token, position, code_points, borrow_document && collator); + if (!collator) + lowercaseTokenInPlace(result[result.size() - 1].text); }); - // When stopwords are compared with collation_server, tokenization must - // retain source case until filtering is complete. MATCH without an - // explicit column collation still uses the existing case-insensitive - // behavior afterwards. - if (!collator && preserve_case) - for (auto & token : result) - lowercaseTokenInPlace(token.text); } FullTextColumn analyzeText( std::string_view text, const TiDB::TiDBCollatorPtr & collator = nullptr, - const TiDB::TiDBCollatorPtr & stopword_collator = nullptr, + const BuiltinStopwordLookup * stopwords = nullptr, size_t min_token_size = default_min_token_size, size_t max_token_size = default_max_token_size, bool enable_stopword = true) { FullTextColumn result; - analyzeTextInto(text, result, collator, stopword_collator, min_token_size, max_token_size, enable_stopword); + analyzeTextInto(text, result, collator, stopwords, min_token_size, max_token_size, enable_stopword); return result; } @@ -348,31 +375,45 @@ void analyzeNgramTextInto( AnalyzerScratch & scratch, size_t ngram_token_size, const TiDB::TiDBCollatorPtr & collator = nullptr, - const TiDB::TiDBCollatorPtr & stopword_collator = nullptr, - bool enable_stopword = true) + const BuiltinStopwordLookup * stopwords = nullptr, + bool enable_stopword = true, + bool borrow_document = false) { result.clear(); if (ngram_token_size == 0) return; - const bool preserve_case = enable_stopword && static_cast(stopword_collator); - tokenizeTextInto(text, scratch.raw_tokens, collator, preserve_case, true); + const bool preserve_case = enable_stopword && stopwords; size_t next_position_base = 0; auto & char_boundaries = scratch.char_boundaries; - for (auto & token : scratch.raw_tokens) + for (size_t offset = 0; offset < text.size();) { + const auto [first_code_point, first_length] = decodeCodePoint(text, offset); + // Stop the document on malformed UTF-8, retain all valid multibyte + // characters, and split runs on ASCII nonwords, as MySQL does. + if (first_length == 1 && first_code_point >= 0x80) + break; + if (first_length == 1 && !isFullTextToken(first_code_point)) + { + offset += first_length; + continue; + } + const size_t run_start = offset; char_boundaries.clear(); - char_boundaries.reserve(token.code_points + 1); char_boundaries.push_back(0); - for (size_t offset = 0; offset < token.text.size();) + offset += first_length; + char_boundaries.push_back(offset - run_start); + while (offset < text.size()) { - const auto [code_point, length] = decodeCodePoint(token.text, offset); - (void)code_point; + const auto [code_point, length] = decodeCodePoint(text, offset); + if (length == 1 && (code_point >= 0x80 || !isFullTextToken(code_point))) + break; offset += length; - char_boundaries.push_back(offset); + char_boundaries.push_back(offset - run_start); } - const size_t char_count = token.code_points; + std::string_view run = text.substr(run_start, offset - run_start); + const size_t char_count = char_boundaries.size() - 1; const size_t base_position = next_position_base; if (char_count < ngram_token_size) { @@ -380,17 +421,34 @@ void analyzeNgramTextInto( continue; } + if (!collator && !preserve_case) + { + // The no-collator path lowercases a whole run before splitting. + // Unicode case mapping can change byte widths, so only that path + // needs a copy and another boundary scan. Never borrow scratch. + scratch.lowercase_run.assign(run.data(), run.size()); + lowercaseTokenInPlace(scratch.lowercase_run); + run = scratch.lowercase_run; + char_boundaries.clear(); + char_boundaries.push_back(0); + for (size_t i = 0; i < run.size();) + { + i += decodeCodePoint(run, i).second; + char_boundaries.push_back(i); + } + } + for (size_t start = 0; start + ngram_token_size <= char_count; ++start) { const size_t begin = char_boundaries[start]; const size_t end = char_boundaries[start + ngram_token_size]; - if (!enable_stopword - || !containsNgramStopword(token.text, char_boundaries, start, ngram_token_size, stopword_collator)) + if (!enable_stopword || !containsNgramStopword(run, char_boundaries, start, ngram_token_size, stopwords)) { result.append( - std::string_view(token.text).substr(begin, end - begin), + run.substr(begin, end - begin), base_position + start, - ngram_token_size); + ngram_token_size, + borrow_document && collator); if (!collator && preserve_case) lowercaseTokenInPlace(result[result.size() - 1].text); } @@ -403,12 +461,12 @@ FullTextColumn analyzeNgramText( std::string_view text, size_t ngram_token_size, const TiDB::TiDBCollatorPtr & collator = nullptr, - const TiDB::TiDBCollatorPtr & stopword_collator = nullptr, + const BuiltinStopwordLookup * stopwords = nullptr, bool enable_stopword = true) { FullTextColumn result; AnalyzerScratch scratch; - analyzeNgramTextInto(text, result, scratch, ngram_token_size, collator, stopword_collator, enable_stopword); + analyzeNgramTextInto(text, result, scratch, ngram_token_size, collator, stopwords, enable_stopword); return result; } @@ -458,11 +516,12 @@ bool matchesTermAt( const FullTextToken & token, const TiDB::TiDBCollatorPtr & collator) { + const auto value = token.value(); if (!clause.prefix) - return textEquals(token.text, clause.terms[term_index], collator); + return textEquals(value, clause.terms[term_index], collator); if (collator && term_index < clause.prefix_matchers.size()) - return clause.prefix_matchers[term_index]->match(token.text.data(), token.text.size()); - return textStartsWith(token.text, clause.terms[term_index], collator); + return clause.prefix_matchers[term_index]->match(value.data(), value.size()); + return textStartsWith(value, clause.terms[term_index], collator); } struct ClauseMatchState @@ -507,7 +566,8 @@ bool matchesPhraseEndingAt( column.end(), expected_position, [](const FullTextToken & token, size_t position) { return token.position < position; }); - if (it == column.end() || it->position != expected_position || !textEquals(it->text, clause.terms[i], collator)) + if (it == column.end() || it->position != expected_position + || !textEquals(it->value(), clause.terms[i], collator)) return false; } return true; @@ -520,15 +580,15 @@ void analyzeColumnInto( bool use_ngram, size_t ngram_token_size, const TiDB::TiDBCollatorPtr & collator = nullptr, - const TiDB::TiDBCollatorPtr & stopword_collator = nullptr, + const BuiltinStopwordLookup * stopwords = nullptr, size_t min_token_size = default_min_token_size, size_t max_token_size = default_max_token_size, bool enable_stopword = true) { if (use_ngram) - analyzeNgramTextInto(document, output, scratch, ngram_token_size, collator, stopword_collator, enable_stopword); + analyzeNgramTextInto(document, output, scratch, ngram_token_size, collator, stopwords, enable_stopword, true); else - analyzeTextInto(document, output, collator, stopword_collator, min_token_size, max_token_size, enable_stopword); + analyzeTextInto(document, output, collator, stopwords, min_token_size, max_token_size, enable_stopword, true); } void resetClauseMatchStates(std::vector & states) @@ -687,7 +747,7 @@ CompiledBooleanQuery compileBooleanQuery( size_t max_token_size, bool enable_stopword, const TiDB::TiDBCollatorPtr & collator = nullptr, - const TiDB::TiDBCollatorPtr & stopword_collator = nullptr) + const BuiltinStopwordLookup * stopwords = nullptr) { CompiledBooleanQuery compiled; if (!use_ngram && (max_token_size == 0 || min_token_size > max_token_size)) @@ -696,9 +756,8 @@ CompiledBooleanQuery compileBooleanQuery( return compiled; } auto analyze_query = [&](std::string_view text) { - return use_ngram - ? analyzeNgramText(text, ngram_token_size, collator, stopword_collator, enable_stopword) - : analyzeText(text, collator, stopword_collator, min_token_size, max_token_size, enable_stopword); + return use_ngram ? analyzeNgramText(text, ngram_token_size, collator, stopwords, enable_stopword) + : analyzeText(text, collator, stopwords, min_token_size, max_token_size, enable_stopword); }; for (const auto & node : query.nodes()) { @@ -746,7 +805,7 @@ CompiledBooleanQuery compileBooleanQuery( case tipb::LocalMatchAgainstBooleanTermType::LocalMatchAgainstBooleanTermPrefix: { const auto terms = use_ngram - ? analyzeNgramText(node.text(), ngram_token_size, collator, stopword_collator, enable_stopword) + ? analyzeNgramText(node.text(), ngram_token_size, collator, stopwords, enable_stopword) : tokenizeText(node.text(), collator); if (use_ngram) { @@ -830,7 +889,7 @@ CompiledBooleanQuery compileBooleanQuery( offset += decodeCodePoint(word.text, offset).second; boundaries.push_back(offset); } - return !containsNgramStopword(word.text, boundaries, 0, word.code_points, stopword_collator); + return !containsNgramStopword(word.text, boundaries, 0, word.code_points, stopwords); })) break; // A short query unigram cannot occur in the document's fixed-size ngrams. } @@ -1077,6 +1136,10 @@ class FunctionLocalMatchAgainstBoolean final : public IFunction throw Exception( "local_match_against_boolean requires a supported stopword collation when stopwords are enabled", ErrorCodes::ILLEGAL_COLUMN); + std::optional stopword_lookup; + if (enable_stopword) + stopword_lookup.emplace(stopword_collator); + const auto * stopwords = stopword_lookup ? &*stopword_lookup : nullptr; const CompiledBooleanQuery compiled_query = compileBooleanQuery( protocol_boolean_query, use_ngram, @@ -1085,7 +1148,7 @@ class FunctionLocalMatchAgainstBoolean final : public IFunction max_token_size, enable_stopword, collator, - stopword_collator); + stopwords); if (compiled_query.matches_nothing) { ColumnPtr result_column; @@ -1129,7 +1192,7 @@ class FunctionLocalMatchAgainstBoolean final : public IFunction use_ngram, ngram_token_size, collator, - stopword_collator, + stopwords, min_token_size, max_token_size, enable_stopword); diff --git a/dbms/src/Functions/tests/gtest_local_match_against.cpp b/dbms/src/Functions/tests/gtest_local_match_against.cpp index d7f4042e195..fcf98cbde88 100644 --- a/dbms/src/Functions/tests/gtest_local_match_against.cpp +++ b/dbms/src/Functions/tests/gtest_local_match_against.cpp @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include #include #include #include @@ -19,6 +20,9 @@ #include #include +#include +#include + namespace DB::tests { class TestLocalMatchAgainst : public DB::tests::FunctionTest @@ -1017,6 +1021,248 @@ try } CATCH +TEST_F(TestLocalMatchAgainst, BuiltinStopwordLookupMatchesComparisons) +try +{ + // Independent reference: retain the old compare-based stopword rule, + // including NGRAM's substring and source-code-point-length restriction. + constexpr std::array stopwords{ + "a", "about", "an", "are", "as", "at", "be", "by", "com", "de", "en", "for", + "from", "how", "i", "in", "is", "it", "la", "of", "on", "or", "that", "the", + "this", "to", "was", "what", "when", "where", "who", "will", "with", "und", "www"}; + std::vector candidates; + for (const auto word : stopwords) + candidates.emplace_back(word); + for (const String & word : + {"ABOUT", "The", "THE", "WWW", "WITH", "ábout", "thé", "tĥe", "ｗｉｔｈ", "ｗｗｗ", "aß", "ß", + "xá", "áx", "xＡ", "Ａx", "xáy", "aaa", "AAA", "xy", "xyz", "foo", "数据库"}) + candidates.push_back(word); + for (const auto collation : + {"binary", + "ascii_bin", + "latin1_bin", + "utf8_bin", + "utf8mb4_bin", + "utf8mb4_0900_bin", + "utf8_general_ci", + "utf8mb4_general_ci", + "utf8_unicode_ci", + "utf8mb4_unicode_ci", + "utf8mb4_0900_ai_ci"}) + { + const auto stopword_collator = TiDB::ITiDBCollator::getCollator(collation); + ASSERT_NE(stopword_collator, nullptr); + // Check the key-equality premise directly, including padding and + // combining marks which the STANDARD tokenizer may split away. + String key_buffer; + auto key_copy = [&](const String & text) { + const auto key = stopword_collator->sortKeyFastPath(text.data(), text.size(), key_buffer); + return String(key.data, key.size); + }; + for (const auto word : stopwords) + { + const String source(word); + const auto source_key = key_copy(source); + for (const auto & candidate : candidates) + for (const String & suffix : {"", " ", " ", "\u0301"}) + { + const String text = candidate + suffix; + EXPECT_EQ( + source_key == key_copy(text), + stopword_collator->compare(source.data(), source.size(), text.data(), text.size()) == 0) + << collation << " source=" << source << " candidate=" << text; + } + } + auto equals_stopword = [&](std::string_view token, size_t length, bool restrict_length) { + return std::any_of(stopwords.begin(), stopwords.end(), [&](const auto word) { + return (!restrict_length || length == word.size()) + && stopword_collator->compare(token.data(), token.size(), word.data(), word.size()) == 0; + }); + }; + for (const auto size : {0U, 1U, 2U, 3U}) + for (const auto & candidate : candidates) + { + SCOPED_TRACE(fmt::format("stopword_collation={} size={} candidate={}", collation, size, candidate)); + std::vector boundaries{0}; + for (size_t offset = 0; offset < candidate.size();) + { + offset += UTF8::utf8Decode(candidate.data() + offset, candidate.size() - offset).second; + boundaries.push_back(offset); + } + bool retained = size == 0 && !equals_stopword(candidate, boundaries.size() - 1, false); + if (size != 0) + for (size_t start = 0; start + size < boundaries.size(); ++start) + { + bool filtered = false; + for (size_t begin = start; begin < start + size; ++begin) + for (size_t end = begin + 1; end <= start + size; ++end) + filtered = filtered + || equals_stopword( + std::string_view(candidate).substr( + boundaries[begin], + boundaries[end] - boundaries[begin]), + end - begin, + true); + retained = retained || !filtered; + } + tipb::LocalMatchAgainstBooleanQuery query; + query.set_parser( + size == 0 ? tipb::LocalMatchAgainstParserStandard : tipb::LocalMatchAgainstParserNgram); + query.set_ngram_token_size(size); + query.set_innodb_ft_min_token_size(1); + query.set_innodb_ft_max_token_size(84); + query.set_stopword_collation(collation); + auto * node = query.add_nodes(); + node->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + node->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + node->set_text(candidate); + // Repeated rows exercise key-buffer reuse. Column and server + // collations are deliberately independent. + for (const auto match_collation : {"utf8mb4_bin", "utf8mb4_general_ci"}) + ASSERT_COLUMN_EQ( + createColumn({retained ? 1.0 : 0.0, retained ? 1.0 : 0.0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(2, candidate), + createColumn({candidate, candidate}), + createConstColumn(2, serializeLocalMatchAgainstBooleanQuery(query))}, + TiDB::ITiDBCollator::getCollator(match_collation), + true)); + } + } +} +CATCH + +TEST_F(TestLocalMatchAgainst, MatchBooleanASCIIAndMalformedUTF8) +try +{ + const auto collator = TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_BIN); + const auto documents = createColumn( + {"bar", + "foo bar", + "foobar", + "bar!", + String("bar\0foo", 7), + "bar\xff" + "foo", + "foo\xff" + "bar", + "foo中bar", + ""}); + for (const auto size : {0U, 2U, 3U}) + { + tipb::LocalMatchAgainstBooleanQuery query; + query.set_parser(size == 0 ? tipb::LocalMatchAgainstParserStandard : tipb::LocalMatchAgainstParserNgram); + query.set_ngram_token_size(size); + query.set_stopword_mode(tipb::LocalMatchAgainstStopwordModeDisabled); + auto * node = query.add_nodes(); + node->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + node->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + node->set_text("bar"); + ASSERT_COLUMN_EQ( + size == 0 ? createColumn({1, 1, 0, 1, 1, 1, 1, 0, 0}) + : createColumn({1, 1, 1, 1, 1, 1, 0, 1, 0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(9, "+bar"), + documents, + createConstColumn(9, serializeLocalMatchAgainstBooleanQuery(query))}, + collator, + true)); + } +} +CATCH + +TEST_F(TestLocalMatchAgainst, DocumentViewsPreserveFilteredPhrase) +try +{ + String miss; + for (size_t i = 0; i < 4096; ++i) + miss += "foo xyz zoo "; + for (const bool ngram : {false, true}) + for (const bool stopwords : {false, true}) + for (const auto collation : {"utf8mb4_bin", "utf8mb4_general_ci"}) + { + SCOPED_TRACE(fmt::format("ngram={} stopwords={} collation={}", ngram, stopwords, collation)); + tipb::LocalMatchAgainstBooleanQuery query; + query.set_parser(ngram ? tipb::LocalMatchAgainstParserNgram : tipb::LocalMatchAgainstParserStandard); + query.set_ngram_token_size(3); + query.set_stopword_mode( + stopwords ? tipb::LocalMatchAgainstStopwordModeBuiltin + : tipb::LocalMatchAgainstStopwordModeDisabled); + auto * phrase = query.add_nodes(); + phrase->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + phrase->set_term_type(tipb::LocalMatchAgainstBooleanTermPhrase); + phrase->set_text("foo the zoo"); + auto * excluded = query.add_nodes(); + excluded->set_occur(tipb::LocalMatchAgainstBooleanOccurMustNot); + excluded->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + excluded->set_text("blocked"); + ASSERT_COLUMN_EQ( + createColumn>({1, 1, 1, 0, 0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(5, "+\"foo the zoo\" -blocked"), + createColumn>({miss, "foo the zoo", {}, "", miss}), + createColumn>( + {"foo the zoo", miss, "foo the zoo", "foo the zoo blocked", miss}), + createConstColumn(5, serializeLocalMatchAgainstBooleanQuery(query))}, + TiDB::ITiDBCollator::getCollator(collation), + true)); + } +} +CATCH + +TEST_F(TestLocalMatchAgainst, NgramDocumentViewsAndPositions) +try +{ + // Overlapping grams in separate runs still have consecutive positions; + // short runs contribute none. Exercise vector growth, scratch reuse and + // successive large/small/NULL documents without borrowing scratch memory. + for (const auto size : {1U, 2U, 3U, 4U, 5U, 6U, 7U, 8U, 9U, 10U}) + for (const bool chinese : {false, true}) + for (const auto collation : {"", "utf8mb4_bin", "utf8mb4_general_ci"}) + { + SCOPED_TRACE(fmt::format("size={} chinese={} collation={}", size, chinese, collation)); + const String alphabet = chinese ? "甲乙丙丁戊己庚辛壬癸子丑" : "ABCDEFGHIJKL"; + const size_t width = chinese ? 3 : 1; + const String query_text = alphabet.substr(0, (size + 1) * width); + const String first = query_text.substr(0, size * width); + const String last = query_text.substr(width); + const String short_run(size - 1, 'z'); + const String padding(4096, 'z'); + const auto documents = createColumn>( + {padding + " " + query_text, + query_text, + first + "!" + last, + first + "!" + String(size, 'z') + "!" + last, + first + "!" + short_run + "!" + last, + query_text + "\xff" + padding, + first + "\xff" + last, + {}, + "", + query_text + "!" + padding}); + tipb::LocalMatchAgainstBooleanQuery query; + query.set_parser(tipb::LocalMatchAgainstParserNgram); + query.set_ngram_token_size(size); + query.set_stopword_mode(tipb::LocalMatchAgainstStopwordModeDisabled); + auto * node = query.add_nodes(); + node->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + node->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + node->set_text(query_text); + ASSERT_COLUMN_EQ( + createColumn>({1, 1, 1, 0, 1, 1, 0, 0, 0, 1}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(10, query_text), + documents, + createConstColumn(10, serializeLocalMatchAgainstBooleanQuery(query))}, + String(collation).empty() ? nullptr : TiDB::ITiDBCollator::getCollator(collation), + true)); + } +} +CATCH + TEST_F(TestLocalMatchAgainst, MatchBooleanUnicodeTokenBoundaries) try { From 3028a5c6b3086e9a0e1b778c12e2c7196c11f15e Mon Sep 17 00:00:00 2001 From: Zhigao Tong Date: Sat, 10 Oct 2026 21:26:27 +0800 Subject: [PATCH 22/32] perf: slide Local MATCH ngram stopword checks --- .../Functions/FunctionsLocalMatchAgainst.cpp | 83 +++++++-- .../tests/gtest_local_match_against.cpp | 164 +++++++++++++++++- 2 files changed, 232 insertions(+), 15 deletions(-) diff --git a/dbms/src/Functions/FunctionsLocalMatchAgainst.cpp b/dbms/src/Functions/FunctionsLocalMatchAgainst.cpp index 080ebdef66c..9d6340e26d5 100644 --- a/dbms/src/Functions/FunctionsLocalMatchAgainst.cpp +++ b/dbms/src/Functions/FunctionsLocalMatchAgainst.cpp @@ -27,6 +27,7 @@ #include #include #include +#include #include #include @@ -86,7 +87,9 @@ class FullTextColumn void reserve(size_t size) { tokens.reserve(size); } - void append(std::string_view text, size_t position, size_t code_points, bool borrow = false) + // This is per-token work. Keep it inline even when the surrounding + // analyzer grows: outlining it penalizes the unfiltered/OFF stream too. + ALWAYS_INLINE void append(std::string_view text, size_t position, size_t code_points, bool borrow = false) { if (used == tokens.size()) tokens.emplace_back(); @@ -311,6 +314,19 @@ bool isDefaultStopword(std::string_view token, const BuiltinStopwordLookup * sto return stopwords ? stopwords->contains(token) : defaultStopwords().contains(Poco::UTF8::toLower(String(token))); } +size_t maxBuiltinStopwordLength() +{ + // The supported built-in list is ASCII: byte and code-point lengths are + // identical. Do not examine lengths which cannot match any stopword. + static const size_t max_length = [] { + size_t result = 0; + for (const auto & word : defaultStopwords()) + result = std::max(result, word.size()); + return result; + }(); + return max_length; +} + bool containsNgramStopword( std::string_view token, const std::vector & char_boundaries, @@ -384,6 +400,7 @@ void analyzeNgramTextInto( return; const bool preserve_case = enable_stopword && stopwords; + const size_t max_stopword_length = enable_stopword ? std::min(ngram_token_size, maxBuiltinStopwordLength()) : 0; size_t next_position_base = 0; auto & char_boundaries = scratch.char_boundaries; for (size_t offset = 0; offset < text.size();) @@ -438,20 +455,64 @@ void analyzeNgramTextInto( } } - for (size_t start = 0; start + ngram_token_size <= char_count; ++start) - { + const auto append_gram = [&](size_t start) ALWAYS_INLINE { const size_t begin = char_boundaries[start]; const size_t end = char_boundaries[start + ngram_token_size]; - if (!enable_stopword || !containsNgramStopword(run, char_boundaries, start, ngram_token_size, stopwords)) + result.append( + run.substr(begin, end - begin), + base_position + start, + ngram_token_size, + borrow_document && collator); + if (!collator && preserve_case) + lowercaseTokenInPlace(result[result.size() - 1].text); + }; + + // Keep the unfiltered stream and stopwords-OFF path free of per-gram + // stopword branches and sliding-state dependencies. + if (!enable_stopword) + { + for (size_t start = 0; start + ngram_token_size <= char_count; ++start) + append_gram(start); + next_position_base = base_position + char_count - ngram_token_size + 1; + continue; + } + + // Advance candidate endings once per run, instead of enumerating all + // substrings again for every overlapping gram. Keep one past the + // greatest start of a matched stopword ending at/before the gram end. + // The gram [start, end) is filtered iff that value is greater than + // start: such a stopword lies wholly inside the current gram. Once + // start advances beyond it, it expires without a queue or cache. + size_t stopword_end = 0; + size_t filtered_until_start = 0; + for (size_t start = 0; start + ngram_token_size <= char_count; ++start) + { + while (stopword_end < start + ngram_token_size) { - result.append( - run.substr(begin, end - begin), - base_position + start, - ngram_token_size, - borrow_document && collator); - if (!collator && preserve_case) - lowercaseTokenInPlace(result[result.size() - 1].text); + ++stopword_end; + const size_t max_length = std::min(stopword_end, max_stopword_length); + for (size_t length = 1; length <= max_length; ++length) + { + const size_t candidate_start = stopword_end - length; + // Longer candidates at this ending have earlier starts + // and cannot improve a start already known to match. + if (candidate_start < filtered_until_start) + break; + const auto candidate = run.substr( + char_boundaries[candidate_start], + char_boundaries[stopword_end] - char_boundaries[candidate_start]); + const bool matched = stopwords + ? stopwords->contains(candidate, length) + : defaultStopwords().contains(Poco::UTF8::toLower(String(candidate))); + if (matched) + { + filtered_until_start = candidate_start + 1; + break; + } + } } + if (filtered_until_start <= start) + append_gram(start); } next_position_base = base_position + char_count - ngram_token_size + 1; } diff --git a/dbms/src/Functions/tests/gtest_local_match_against.cpp b/dbms/src/Functions/tests/gtest_local_match_against.cpp index fcf98cbde88..b3eef0587ea 100644 --- a/dbms/src/Functions/tests/gtest_local_match_against.cpp +++ b/dbms/src/Functions/tests/gtest_local_match_against.cpp @@ -31,6 +31,11 @@ class TestLocalMatchAgainst : public DB::tests::FunctionTest namespace { +constexpr std::array builtin_stopwords{ + "a", "about", "an", "are", "as", "at", "be", "by", "com", "de", "en", "for", + "from", "how", "i", "in", "is", "it", "la", "of", "on", "or", "that", "the", + "this", "to", "was", "what", "when", "where", "who", "will", "with", "und", "www"}; + // Count token comparisons rather than wall time, so a repeated full-column // verification regression is deterministic even on a loaded CI host. class CountingMatchCollator final : public TiDB::ITiDBCollator @@ -1026,10 +1031,7 @@ try { // Independent reference: retain the old compare-based stopword rule, // including NGRAM's substring and source-code-point-length restriction. - constexpr std::array stopwords{ - "a", "about", "an", "are", "as", "at", "be", "by", "com", "de", "en", "for", - "from", "how", "i", "in", "is", "it", "la", "of", "on", "or", "that", "the", - "this", "to", "was", "what", "when", "where", "who", "will", "with", "und", "www"}; + const auto & stopwords = builtin_stopwords; std::vector candidates; for (const auto word : stopwords) candidates.emplace_back(word); @@ -1133,6 +1135,160 @@ try } CATCH +TEST_F(TestLocalMatchAgainst, NgramSlidingStopwordWindows) +try +{ + InferredDataVector> documents{ + "", + "z", + "zzaz", + "zzazzzzzzzzzzz", + "zthe", + "zzthezz", + "zzthezzzzzzzzzz", + "zaboutzz", + "zaboutzaboutzzzzzz", + "zzazzizzz", + "za zzzzzzzzzzz", + "zzz!zzz", + "zThezz", + "zTHEzz", + "zthézz", + "ztĥezz", + "zｗｉｔｈzz", + "zｗｗｗzz", + "zázz", + "zＡzz", + "zßzz", + "z\u0301azz", + "z🙃a🙃zz", + "z数据azzz", + "ZTHEZZ", + String("za\0zzzzzzzzzz", 13), + String("zthe\xffzzzzzzzzzz"), + String("\xffzzzzzzzzzz"), + "zzazzazzzzzzzzzzzzz", + "zaboutzzzzzzzzzz", + "z", + {}, + "zzzzzzzzzzzz"}; + // Deterministic mixed runs, ending candidates and repeated rows. The + // reference below never uses the optimized lookup or sliding state. + UInt32 seed = 70485; + const std::array pieces{"z", "a", "i", "the", "about", "thé", "🙃", "!"}; + for (size_t row = 0; row < 24; ++row) + { + String text; + for (size_t i = 0; i < 12; ++i) + { + seed = seed * 1664525 + 1013904223; + text += pieces[(seed >> 16) % pieces.size()]; + } + documents.emplace_back(std::move(text)); + } + + for (const auto stopword_collation : + {"binary", + "ascii_bin", + "latin1_bin", + "utf8_bin", + "utf8mb4_bin", + "utf8mb4_0900_bin", + "utf8_general_ci", + "utf8mb4_general_ci", + "utf8_unicode_ci", + "utf8mb4_unicode_ci", + "utf8mb4_0900_ai_ci"}) + for (const auto match_collation : {"utf8mb4_bin", "utf8mb4_general_ci"}) + for (size_t size = 1; size <= 10; ++size) + { + SCOPED_TRACE(fmt::format("stopwords={} match={} size={}", stopword_collation, match_collation, size)); + const auto stopword_collator = TiDB::ITiDBCollator::getCollator(stopword_collation); + const auto match_collator = TiDB::ITiDBCollator::getCollator(match_collation); + auto prefix = match_collator->pattern(); + prefix->compile("z%", '\\'); + InferredDataVector> expected; + for (const auto & document : documents) + { + bool retained = false; + if (document) + { + const String & text = *document; + size_t run_start = 0; + std::vector boundaries{0}; + auto check_run = [&] { + for (size_t start = 0; start + size < boundaries.size() && !retained; ++start) + { + bool filtered = false; + for (size_t begin = start; begin < start + size && !filtered; ++begin) + for (size_t end = begin + 1; end <= start + size && !filtered; ++end) + { + const auto candidate = std::string_view(text).substr( + run_start + boundaries[begin], + boundaries[end] - boundaries[begin]); + filtered = std::any_of( + builtin_stopwords.begin(), + builtin_stopwords.end(), + [&](auto word) { + return word.size() == end - begin + && stopword_collator->compare( + candidate.data(), + candidate.size(), + word.data(), + word.size()) + == 0; + }); + } + const auto gram = std::string_view(text).substr( + run_start + boundaries[start], + boundaries[start + size] - boundaries[start]); + retained = !filtered && prefix->match(gram.data(), gram.size()); + } + }; + for (size_t offset = 0; offset < text.size();) + { + const auto byte = static_cast(text[offset]); + const auto decoded = UTF8::utf8Decode(text.data() + offset, text.size() - offset); + if (decoded.second == 0 || decoded.first == UTF8::UTF8_Error + || decoded.second > text.size() - offset || (byte >= 0x80 && decoded.second == 1)) + break; + if (byte < 0x80 && !LocalMatchAgainst::isTokenChar(byte)) + { + check_run(); + run_start = ++offset; + boundaries.assign(1, 0); + } + else + { + offset += decoded.second; + boundaries.push_back(offset - run_start); + } + } + check_run(); + } + expected.emplace_back(retained ? 1.0 : 0.0); + } + tipb::LocalMatchAgainstBooleanQuery query; + query.set_parser(tipb::LocalMatchAgainstParserNgram); + query.set_ngram_token_size(size); + query.set_stopword_collation(stopword_collation); + auto * node = query.add_nodes(); + node->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + node->set_term_type(tipb::LocalMatchAgainstBooleanTermPrefix); + node->set_text("z"); + ASSERT_COLUMN_EQ( + createColumn>(expected), + executeFunction( + "local_match_against_boolean", + {createConstColumn(documents.size(), "+z*"), + createColumn>(documents), + createConstColumn(documents.size(), serializeLocalMatchAgainstBooleanQuery(query))}, + match_collator, + true)); + } +} +CATCH + TEST_F(TestLocalMatchAgainst, MatchBooleanASCIIAndMalformedUTF8) try { From f82ab25ac036d5a97703ffd5bbbc7b5b39ca8646 Mon Sep 17 00:00:00 2001 From: Zhigao Tong Date: Sun, 11 Oct 2026 03:56:42 +0800 Subject: [PATCH 23/32] refactor: simplify Local MATCH cold-path setup Normalize analyzer configuration for query compilation and share term collection and result finalization. Preserve scalar row-path arguments and tokenizer/matcher behavior. Add analyzer defaults and NULL-result regressions. --- .../Functions/FunctionsLocalMatchAgainst.cpp | 182 +++++++++--------- .../tests/gtest_local_match_against.cpp | 61 ++++++ 2 files changed, 148 insertions(+), 95 deletions(-) diff --git a/dbms/src/Functions/FunctionsLocalMatchAgainst.cpp b/dbms/src/Functions/FunctionsLocalMatchAgainst.cpp index 9d6340e26d5..883a87cb047 100644 --- a/dbms/src/Functions/FunctionsLocalMatchAgainst.cpp +++ b/dbms/src/Functions/FunctionsLocalMatchAgainst.cpp @@ -169,6 +169,46 @@ struct CompiledBooleanQuery bool needs_verification = false; }; +// Normalize the already validated wire configuration once per block. Keep +// the tokenizer implementations and their filtered/unfiltered rules intact. +struct LocalMatchAnalyzerConfig +{ + bool use_ngram; + size_t ngram_token_size; + size_t min_token_size; + size_t max_token_size; + bool enable_stopword; + + explicit LocalMatchAnalyzerConfig(const tipb::LocalMatchAgainstBooleanQuery & query) + : use_ngram(query.parser() == tipb::LocalMatchAgainstParserNgram) + , ngram_token_size( + !use_ngram || query.ngram_token_size() == 0 ? default_ngram_token_size : query.ngram_token_size()) + , min_token_size( + query.innodb_ft_min_token_size() != 0 || query.innodb_ft_max_token_size() != 0 + ? query.innodb_ft_min_token_size() + : default_min_token_size) + , max_token_size( + query.innodb_ft_min_token_size() != 0 || query.innodb_ft_max_token_size() != 0 + ? query.innodb_ft_max_token_size() + : default_max_token_size) + , enable_stopword(query.stopword_mode() == tipb::LocalMatchAgainstStopwordModeBuiltin) + {} +}; + +// Query tokens own their text. This cold-path helper only collects the same +// terms and source-relative offsets; document views never enter it. +size_t collectClauseTerms(BooleanClause & clause, const FullTextColumn & tokens) +{ + const size_t first_position = tokens.empty() ? 0 : tokens.front().position; + for (const auto & token : tokens) + { + clause.terms.push_back(token.text); + if (clause.phrase) + clause.offsets.push_back(token.position - first_position); + } + return first_position; +} + void lowercaseTokenInPlace(String & token) { const bool is_ascii = std::all_of(token.begin(), token.end(), [](unsigned char c) { return c < 0x80; }); @@ -802,23 +842,26 @@ bool evaluateBooleanMatchResult(const CompiledBooleanQuery & query, const std::v CompiledBooleanQuery compileBooleanQuery( const tipb::LocalMatchAgainstBooleanQuery & query, - bool use_ngram, - size_t ngram_token_size, - size_t min_token_size, - size_t max_token_size, - bool enable_stopword, + const LocalMatchAnalyzerConfig & config, const TiDB::TiDBCollatorPtr & collator = nullptr, const BuiltinStopwordLookup * stopwords = nullptr) { CompiledBooleanQuery compiled; - if (!use_ngram && (max_token_size == 0 || min_token_size > max_token_size)) + if (!config.use_ngram && (config.max_token_size == 0 || config.min_token_size > config.max_token_size)) { compiled.matches_nothing = true; return compiled; } auto analyze_query = [&](std::string_view text) { - return use_ngram ? analyzeNgramText(text, ngram_token_size, collator, stopwords, enable_stopword) - : analyzeText(text, collator, stopwords, min_token_size, max_token_size, enable_stopword); + return config.use_ngram + ? analyzeNgramText(text, config.ngram_token_size, collator, stopwords, config.enable_stopword) + : analyzeText( + text, + collator, + stopwords, + config.min_token_size, + config.max_token_size, + config.enable_stopword); }; for (const auto & node : query.nodes()) { @@ -845,30 +888,16 @@ CompiledBooleanQuery compileBooleanQuery( case tipb::LocalMatchAgainstBooleanTermType::LocalMatchAgainstBooleanTermWord: { const auto terms = analyze_query(node.text()); - if (use_ngram) - { - clause.phrase = true; - const size_t first_position = terms.empty() ? 0 : terms.front().position; - first_query_position = first_position; - for (const auto & token : terms) - { - clause.terms.push_back(token.text); - clause.offsets.push_back(token.position - first_position); - } - } - else - { - for (const auto & token : terms) - clause.terms.push_back(token.text); - } + clause.phrase = config.use_ngram; + first_query_position = collectClauseTerms(clause, terms); break; } case tipb::LocalMatchAgainstBooleanTermType::LocalMatchAgainstBooleanTermPrefix: { - const auto terms = use_ngram - ? analyzeNgramText(node.text(), ngram_token_size, collator, stopwords, enable_stopword) + const auto terms = config.use_ngram + ? analyzeNgramText(node.text(), config.ngram_token_size, collator, stopwords, config.enable_stopword) : tokenizeText(node.text(), collator); - if (use_ngram) + if (config.use_ngram) { if (!terms.empty()) { @@ -877,13 +906,7 @@ CompiledBooleanQuery compileBooleanQuery( // only one analyzed term here makes prefixes such as // "caf*" ("ca", "af" at token size 2) never match. clause.phrase = true; - const size_t first_position = terms.front().position; - first_query_position = first_position; - for (const auto & token : terms) - { - clause.terms.push_back(token.text); - clause.offsets.push_back(token.position - first_position); - } + first_query_position = collectClauseTerms(clause, terms); break; } @@ -896,7 +919,7 @@ CompiledBooleanQuery compileBooleanQuery( const auto code_points = UTF8::countCodePoints( reinterpret_cast(source_terms.front().text.data()), source_terms.front().text.size()); - if (code_points < ngram_token_size) + if (code_points < config.ngram_token_size) { clause.prefix = true; clause.terms.push_back(source_terms.front().text); @@ -919,7 +942,7 @@ CompiledBooleanQuery compileBooleanQuery( const auto code_points = UTF8::countCodePoints( reinterpret_cast(terms.front().text.data()), terms.front().text.size()); - if (code_points > max_token_size) + if (code_points > config.max_token_size) { if (clause.modifier == BooleanClause::Modifier::Must) compiled.clauses.push_back(std::move(clause)); @@ -932,17 +955,17 @@ CompiledBooleanQuery compileBooleanQuery( { clause.phrase = true; const auto terms = analyze_query(node.text()); - if (use_ngram) + if (config.use_ngram) { const auto words = tokenizeText(node.text(), collator); bool seen_indexed_word = false; if (std::any_of(words.begin(), words.end(), [&](const auto & word) { - if (word.code_points >= ngram_token_size) + if (word.code_points >= config.ngram_token_size) { seen_indexed_word = seen_indexed_word || !analyze_query(word.text).empty(); return false; } - if (!enable_stopword || seen_indexed_word) + if (!config.enable_stopword || seen_indexed_word) return true; std::vector boundaries{0}; for (size_t offset = 0; offset < word.text.size();) @@ -954,13 +977,7 @@ CompiledBooleanQuery compileBooleanQuery( })) break; // A short query unigram cannot occur in the document's fixed-size ngrams. } - const size_t first_position = terms.empty() ? 0 : terms.front().position; - first_query_position = first_position; - for (const auto & token : terms) - { - clause.terms.push_back(token.text); - clause.offsets.push_back(token.position - first_position); - } + first_query_position = collectClauseTerms(clause, terms); break; } default: @@ -970,8 +987,8 @@ CompiledBooleanQuery compileBooleanQuery( if (clause.phrase && !clause.terms.empty()) { - auto raw_terms = use_ngram - ? analyzeNgramText(node.text(), ngram_token_size, collator, nullptr, false) + auto raw_terms = config.use_ngram + ? analyzeNgramText(node.text(), config.ngram_token_size, collator, nullptr, false) : analyzeText(node.text(), collator, nullptr, 0, std::numeric_limits::max(), false); // InnoDB discards leading filtered tokens, but retains every // following word in the original phrase used for verification. @@ -984,12 +1001,7 @@ CompiledBooleanQuery compileBooleanQuery( // pay for this additional document stream. clause.verification = std::make_unique(); clause.verification->phrase = true; - const size_t first = raw_terms.front().position; - for (const auto & term : raw_terms) - { - clause.verification->terms.push_back(term.text); - clause.verification->offsets.push_back(term.position - first); - } + collectClauseTerms(*clause.verification, raw_terms); compiled.needs_verification = true; } } @@ -1156,6 +1168,12 @@ class FunctionLocalMatchAgainstBoolean final : public IFunction auto & output_data = output->getData(); const bool nullable = block.getByPosition(result).type->isNullable(); auto null_map = nullable ? ColumnUInt8::create(rows, 0) : nullptr; + const auto finish_result = [&] { + ColumnPtr result_column = std::move(output); + if (null_map) + result_column = ColumnNullable::create(std::move(result_column), std::move(null_map)); + block.getByPosition(result).column = std::move(result_column); + }; const FullTextColumnAccessor query_accessor(*query_column); tipb::LocalMatchAgainstBooleanQuery protocol_boolean_query; @@ -1169,28 +1187,19 @@ class FunctionLocalMatchAgainstBoolean final : public IFunction if (query_accessor.isNull(0)) { // InnoDB interprets NULL AGAINST as an empty search: non-NULL zero. - ColumnPtr result_column; - if (null_map) - result_column = ColumnNullable::create(std::move(output), std::move(null_map)); - else - result_column = std::move(output); - block.getByPosition(result).column = std::move(result_column); + finish_result(); return; } - size_t document_argument_end = arguments.size() - 1; - const bool use_ngram - = protocol_boolean_query.parser() == tipb::LocalMatchAgainstParser::LocalMatchAgainstParserNgram; - const size_t ngram_token_size = !use_ngram || protocol_boolean_query.ngram_token_size() == 0 - ? default_ngram_token_size - : protocol_boolean_query.ngram_token_size(); - const bool has_standard_config = protocol_boolean_query.innodb_ft_min_token_size() != 0 - || protocol_boolean_query.innodb_ft_max_token_size() != 0; - const size_t min_token_size - = has_standard_config ? protocol_boolean_query.innodb_ft_min_token_size() : default_min_token_size; - const size_t max_token_size - = has_standard_config ? protocol_boolean_query.innodb_ft_max_token_size() : default_max_token_size; - const bool enable_stopword = protocol_boolean_query.stopword_mode() - == tipb::LocalMatchAgainstStopwordMode::LocalMatchAgainstStopwordModeBuiltin; + const size_t document_argument_end = arguments.size() - 1; + const LocalMatchAnalyzerConfig config(protocol_boolean_query); + // Keep scalar values in the row hot path. Passing the config by + // reference changes Release code generation, notably with stopwords + // disabled; the config object belongs to query compilation only. + const bool use_ngram = config.use_ngram; + const size_t ngram_token_size = config.ngram_token_size; + const size_t min_token_size = config.min_token_size; + const size_t max_token_size = config.max_token_size; + const bool enable_stopword = config.enable_stopword; const auto stopword_collator = enable_stopword ? TiDB::ITiDBCollator::getCollator(protocol_boolean_query.stopword_collation()) : nullptr; if (enable_stopword && !stopword_collator) @@ -1201,23 +1210,11 @@ class FunctionLocalMatchAgainstBoolean final : public IFunction if (enable_stopword) stopword_lookup.emplace(stopword_collator); const auto * stopwords = stopword_lookup ? &*stopword_lookup : nullptr; - const CompiledBooleanQuery compiled_query = compileBooleanQuery( - protocol_boolean_query, - use_ngram, - ngram_token_size, - min_token_size, - max_token_size, - enable_stopword, - collator, - stopwords); + const CompiledBooleanQuery compiled_query + = compileBooleanQuery(protocol_boolean_query, config, collator, stopwords); if (compiled_query.matches_nothing) { - ColumnPtr result_column; - if (null_map) - result_column = ColumnNullable::create(std::move(output), std::move(null_map)); - else - result_column = std::move(output); - block.getByPosition(result).column = std::move(result_column); + finish_result(); return; } std::vector document_columns; @@ -1290,12 +1287,7 @@ class FunctionLocalMatchAgainstBoolean final : public IFunction } output_data[row] = result_known ? (row_matches ? 1 : 0) : matchBooleanScore(compiled_query, clause_states); } - ColumnPtr result_column; - if (null_map) - result_column = ColumnNullable::create(std::move(output), std::move(null_map)); - else - result_column = std::move(output); - block.getByPosition(result).column = std::move(result_column); + finish_result(); } private: diff --git a/dbms/src/Functions/tests/gtest_local_match_against.cpp b/dbms/src/Functions/tests/gtest_local_match_against.cpp index b3eef0587ea..e0fb64f77ca 100644 --- a/dbms/src/Functions/tests/gtest_local_match_against.cpp +++ b/dbms/src/Functions/tests/gtest_local_match_against.cpp @@ -715,6 +715,67 @@ try } CATCH +TEST_F(TestLocalMatchAgainst, AnalyzerDefaultsAndResultFinalization) +try +{ + struct ConfigCase + { + UInt32 min_size; + UInt32 max_size; + InferredDataVector expected; + }; + for (const auto & config : + {ConfigCase{0, 0, {1, 0, 0, 0}}, ConfigCase{0, 4, {0, 1, 0, 0}}, ConfigCase{5, 4, {0, 0, 0, 0}}}) + { + SCOPED_TRACE(fmt::format("min={} max={}", config.min_size, config.max_size)); + tipb::LocalMatchAgainstBooleanQuery query; + query.set_innodb_ft_min_token_size(config.min_size); + query.set_innodb_ft_max_token_size(config.max_size); + query.set_stopword_mode(tipb::LocalMatchAgainstStopwordModeDisabled); + for (const auto * term : {"quick", "a"}) + { + auto * node = query.add_nodes(); + node->set_occur(tipb::LocalMatchAgainstBooleanOccurShould); + node->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + node->set_text(term); + } + const auto metadata = createConstColumn(4, serializeLocalMatchAgainstBooleanQuery(query)); + ASSERT_COLUMN_EQ( + createColumn(config.expected), + executeFunction( + "local_match_against_boolean", + {createConstColumn(4, "quick a"), createColumn({"quick", "a", "", "none"}), metadata}, + nullptr, + true)); + InferredDataVector> nullable_expected; + for (auto value : config.expected) + nullable_expected.emplace_back(value); + ASSERT_COLUMN_EQ( + createColumn>(nullable_expected), + executeFunction( + "local_match_against_boolean", + {createConstColumn(4, "quick a"), + createColumn>({"quick", "a", {}, ""}), + metadata}, + nullptr, + true)); + // NULL search must still bypass analyzer setup and return non-NULL + // zeros, even when that setup would reject the stopword collation. + query.set_stopword_mode(tipb::LocalMatchAgainstStopwordModeBuiltin); + query.set_stopword_collation("unsupported_stopword_collation"); + ASSERT_COLUMN_EQ( + createColumn>({0, 0, 0, 0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn>(4, {}), + createColumn({"quick", "a", "", "none"}), + createConstColumn(4, serializeLocalMatchAgainstBooleanQuery(query))}, + nullptr, + true)); + } +} +CATCH + TEST_F(TestLocalMatchAgainst, MatchBooleanNgramProtocolQuery) try { From 9bbf1a70d2c025fb175556a9b921f268ab490427 Mon Sep 17 00:00:00 2001 From: Zhigao Tong Date: Mon, 12 Oct 2026 15:10:49 +0800 Subject: [PATCH 24/32] test: remove Local MATCH microbenchmark target --- dbms/CMakeLists.txt | 22 --- .../tests/bench_local_match_against.cpp | 131 ------------------ 2 files changed, 153 deletions(-) delete mode 100644 dbms/src/Functions/tests/bench_local_match_against.cpp diff --git a/dbms/CMakeLists.txt b/dbms/CMakeLists.txt index 868534f3dba..db30050fd2a 100644 --- a/dbms/CMakeLists.txt +++ b/dbms/CMakeLists.txt @@ -405,28 +405,6 @@ if (ENABLE_TESTS) add_check(bench_dbms) - # Allow Local MATCH smoke/Release benchmarks without building unrelated - # vector-index benchmarks and their external dataset dependencies. - add_executable(bench_local_match_against EXCLUDE_FROM_ALL - ${TiFlash_SOURCE_DIR}/dbms/src/Functions/tests/bench_local_match_against.cpp - ${TiFlash_SOURCE_DIR}/dbms/src/Server/StorageConfigParser.cpp - ${TiFlash_SOURCE_DIR}/dbms/src/Server/UserConfigParser.cpp - ${TiFlash_SOURCE_DIR}/dbms/src/Server/RaftConfigParser.cpp - ${TiFlash_SOURCE_DIR}/dbms/src/AggregateFunctions/AggregateFunctionSum.cpp) - target_include_directories(bench_local_match_against BEFORE PRIVATE - ${SPARCEHASH_INCLUDE_DIR} ${benchmark_SOURCE_DIR}/include) - target_compile_definitions(bench_local_match_against PUBLIC DBMS_PUBLIC_GTEST) - target_link_libraries(bench_local_match_against - gtest - benchmark - dbms - test_util_bench_main - tiflash_functions - server_for_test - delta_merge - tiflash_aggregate_functions - kvstore) - add_check(bench_local_match_against) endif () # dbms diff --git a/dbms/src/Functions/tests/bench_local_match_against.cpp b/dbms/src/Functions/tests/bench_local_match_against.cpp deleted file mode 100644 index d327aa6c316..00000000000 --- a/dbms/src/Functions/tests/bench_local_match_against.cpp +++ /dev/null @@ -1,131 +0,0 @@ -// Copyright 2026 PingCAP, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include -#include -#include -#include -#include -#include - -#include -#include - -namespace DB -{ -void registerFunctionsLocalMatchAgainst(FunctionFactory & factory); -} - -namespace DB::tests -{ -namespace -{ -String benchmarkDocument(size_t row, size_t size) -{ - // Keep identical to tests/ftse2e/benchdata in the paired TiDB checkout. - static const std::array units{ - "quick brown fox prefix 数据库 ", - "quick slow fox prefix 数据库 ", - "QUICK brown FOX PREFIX 数据库 ", - "other words unrelated sample 文档 "}; - const auto & unit = units[row % units.size()]; - String result; - result.reserve(size); - for (size_t i = 0; i < size / unit.size(); ++i) - result += unit; - result.append(size % unit.size(), ' '); - return result; -} - -void BM_LocalMatchAgainst(benchmark::State & state) -try -{ - const auto bytes = static_cast(state.range(0)); - const auto parser = state.range(1); // 0=STANDARD, 2/3=NGRAM token size. - const bool ci = state.range(2); - const bool stopwords = state.range(3); - const auto scenario = state.range(4); // Required/excluded, phrase, prefix, filtered MUST. - constexpr size_t rows = 16; - tipb::LocalMatchAgainstBooleanQuery query; - query.set_version(2); - query.set_parser(parser == 0 ? tipb::LocalMatchAgainstParserStandard : tipb::LocalMatchAgainstParserNgram); - query.set_ngram_token_size(parser == 0 ? 2 : parser); - query.set_innodb_ft_min_token_size(3); - query.set_innodb_ft_max_token_size(84); - query.set_stopword_mode( - stopwords ? tipb::LocalMatchAgainstStopwordModeBuiltin : tipb::LocalMatchAgainstStopwordModeDisabled); - query.set_stopword_collation(ci ? "utf8mb4_general_ci" : "utf8mb4_bin"); - auto * required = query.add_nodes(); - required->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); - required->set_term_type( - scenario == 1 ? tipb::LocalMatchAgainstBooleanTermPhrase - : scenario == 2 ? tipb::LocalMatchAgainstBooleanTermPrefix - : tipb::LocalMatchAgainstBooleanTermWord); - required->set_text(scenario == 1 ? "quick brown fox" : scenario == 2 ? "pre" : scenario == 3 ? "the" : "quick"); - if (scenario == 0) - { - auto * prohibited = query.add_nodes(); - prohibited->set_occur(tipb::LocalMatchAgainstBooleanOccurMustNot); - prohibited->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); - prohibited->set_text("slow"); - } - static const std::array searches{"+quick -slow", "+\"quick brown fox\"", "+pre*", "+the"}; - std::vector documents; - for (size_t row = 0; row < rows; ++row) - documents.push_back(benchmarkDocument(row, bytes)); - auto context = TiFlashTestEnv::getContext(); - const ColumnsWithTypeAndName arguments{ - createConstColumn(rows, searches[scenario]), - createColumn(documents), - createConstColumn(rows, query.SerializeAsString())}; - const auto collator = TiDB::ITiDBCollator::getCollator( - ci ? TiDB::ITiDBCollator::UTF8MB4_GENERAL_CI : TiDB::ITiDBCollator::UTF8MB4_BIN); - auto & factory = FunctionFactory::instance(); - if (!factory.tryGet("local_match_against_boolean", *context)) - registerFunctionsLocalMatchAgainst(factory); - auto function = factory.get("local_match_against_boolean", *context)->build(arguments, collator); - Block block(arguments); - block.insert({nullptr, function->getReturnType(), "result"}); - const ColumnNumbers argument_numbers{0, 1, 2}; - // Inputs and function setup are outside timing. Query compilation remains - // inside execution, exactly as in the production block-level scalar path. - for (auto _ : state) - { - function->execute(block, argument_numbers, 3); - benchmark::DoNotOptimize(block.getByPosition(3).column); - } - state.SetItemsProcessed(state.iterations() * rows); - state.SetBytesProcessed(state.iterations() * rows * bytes); -} -catch (const Exception & e) -{ - state.SkipWithError(e.displayText().c_str()); -} -catch (const std::exception & e) -{ - state.SkipWithError(e.what()); -} - -void benchmarkArguments(benchmark::internal::Benchmark * benchmark) -{ - for (const int bytes : {128, 4096, 262144}) - for (const int parser : {0, 2, 3}) - for (const int ci : {0, 1}) - for (const int stopwords : {0, 1}) - for (const int scenario : {0, 1, 2, 3}) - benchmark->Args({bytes, parser, ci, stopwords, scenario}); -} -BENCHMARK(BM_LocalMatchAgainst)->Apply(benchmarkArguments); -} // namespace -} // namespace DB::tests From b0ea7e60129fea2c7c34f63ced8af1089fdab377 Mon Sep 17 00:00:00 2001 From: Zhigao Tong Date: Mon, 12 Oct 2026 15:51:46 +0800 Subject: [PATCH 25/32] refactor: remove legacy Local MATCH scan integration --- dbms/CMakeLists.txt | 1 - .../DAGExpressionAnalyzerHelper.cpp | 8 +- .../Flash/Planner/Plans/PhysicalTableScan.cpp | 25 +- .../Flash/Planner/Plans/PhysicalTableScan.h | 7 +- .../Functions/FunctionsLocalMatchAgainst.cpp | 8 +- dbms/src/TiDB/Schema/TiDB.cpp | 19 +- .../2026-09-18-match-against-tiflash.md | 279 ------------------ 7 files changed, 24 insertions(+), 323 deletions(-) delete mode 100644 docs/design/2026-09-18-match-against-tiflash.md diff --git a/dbms/CMakeLists.txt b/dbms/CMakeLists.txt index db30050fd2a..dec2aa38795 100644 --- a/dbms/CMakeLists.txt +++ b/dbms/CMakeLists.txt @@ -404,7 +404,6 @@ if (ENABLE_TESTS) endif() add_check(bench_dbms) - endif () # dbms diff --git a/dbms/src/Flash/Coprocessor/DAGExpressionAnalyzerHelper.cpp b/dbms/src/Flash/Coprocessor/DAGExpressionAnalyzerHelper.cpp index a64f3bea5c8..45ce61a1df0 100644 --- a/dbms/src/Flash/Coprocessor/DAGExpressionAnalyzerHelper.cpp +++ b/dbms/src/Flash/Coprocessor/DAGExpressionAnalyzerHelper.cpp @@ -513,7 +513,7 @@ String DAGExpressionAnalyzerHelper::buildDefaultFunction( const tipb::Expr & expr, const ExpressionActionsPtr & actions) { - String func_name = getFunctionName(expr); + const String & func_name = getFunctionName(expr); Names argument_names; for (const auto & child : expr.children()) { @@ -522,10 +522,8 @@ String DAGExpressionAnalyzerHelper::buildDefaultFunction( } if (expr.sig() == tipb::ScalarFuncSig::LocalMatchAgainstBoolean) { - // Local MATCH metadata is carried in the generic scalar-function metadata slot - // and must not be confused with SQL arguments. The runtime function - // interface uses a private variant with a required final constant. - func_name = "local_match_against_boolean"; + // Expr.val carries Local MATCH metadata, not an SQL argument. Adapt it + // to the runtime function's required final constant argument. const auto metadata_expr = constructStringLiteralTiExpr(expr.val()); argument_names.push_back(analyzer->getActions(metadata_expr, actions)); } diff --git a/dbms/src/Flash/Planner/Plans/PhysicalTableScan.cpp b/dbms/src/Flash/Planner/Plans/PhysicalTableScan.cpp index 39b4c6432da..364cd2e94a9 100644 --- a/dbms/src/Flash/Planner/Plans/PhysicalTableScan.cpp +++ b/dbms/src/Flash/Planner/Plans/PhysicalTableScan.cpp @@ -14,11 +14,8 @@ #include #include -#include -#include #include #include -#include #include #include #include @@ -116,7 +113,7 @@ void PhysicalTableScan::buildBlockInputStreamImpl(DAGPipeline & pipeline, Contex DAGStorageInterpreter storage_interpreter(context, tidb_table_scan, filter_conditions, max_streams); storage_interpreter.execute(pipeline); } - buildProjection(pipeline, context); + buildProjection(pipeline); } void PhysicalTableScan::buildPipeline( @@ -139,7 +136,7 @@ void PhysicalTableScan::buildPipeline( DAGStorageInterpreter storage_interpreter(context, tidb_table_scan, filter_conditions, context.getMaxStreams()); storage_interpreter.execute(exec_context, pipeline_exec_builder); } - buildProjection(exec_context, pipeline_exec_builder, context); + buildProjection(exec_context, pipeline_exec_builder); PhysicalPlanNode::buildPipeline(builder, context, exec_context); } @@ -154,7 +151,7 @@ void PhysicalTableScan::buildPipelineExecGroupImpl( group_builder = std::move(pipeline_exec_builder); } -void PhysicalTableScan::buildProjection(DAGPipeline & pipeline, Context & /*context*/) +void PhysicalTableScan::buildProjection(DAGPipeline & pipeline) { const auto & schema_project_cols = buildTableScanProjectionCols( tidb_table_scan.getLogicalTableID(), @@ -168,8 +165,7 @@ void PhysicalTableScan::buildProjection(DAGPipeline & pipeline, Context & /*cont void PhysicalTableScan::buildProjection( PipelineExecutorContext & exec_context, - PipelineExecGroupBuilder & group_builder, - Context & /*context*/) + PipelineExecGroupBuilder & group_builder) { auto header = group_builder.getCurrentHeader(); const auto & schema_project_cols @@ -194,11 +190,14 @@ const Block & PhysicalTableScan::getSampleBlock() const bool PhysicalTableScan::setFilterConditions(const String & filter_executor_id, const tipb::Selection & selection) { - if (!hasFilterConditions()) - filter_conditions = FilterConditions::filterConditionsFrom(filter_executor_id, selection); - else - for (const auto & condition : selection.conditions()) - *filter_conditions.conditions.Add() = condition; + /// Since there is at most one selection on the table scan, setFilterConditions() will only be called at most once. + /// So in this case hasFilterConditions() is always false. + if (unlikely(hasFilterConditions())) + { + return false; + } + + filter_conditions = FilterConditions::filterConditionsFrom(filter_executor_id, selection); return true; } diff --git a/dbms/src/Flash/Planner/Plans/PhysicalTableScan.h b/dbms/src/Flash/Planner/Plans/PhysicalTableScan.h index eff25aa37bd..284b75d0c73 100644 --- a/dbms/src/Flash/Planner/Plans/PhysicalTableScan.h +++ b/dbms/src/Flash/Planner/Plans/PhysicalTableScan.h @@ -59,11 +59,8 @@ class PhysicalTableScan : public PhysicalLeaf Context & /*context*/, size_t /*concurrency*/) override; - void buildProjection(DAGPipeline & pipeline, Context & context); - void buildProjection( - PipelineExecutorContext & exec_context, - PipelineExecGroupBuilder & group_builder, - Context & context); + void buildProjection(DAGPipeline & pipeline); + void buildProjection(PipelineExecutorContext & exec_context, PipelineExecGroupBuilder & group_builder); private: FilterConditions filter_conditions; diff --git a/dbms/src/Functions/FunctionsLocalMatchAgainst.cpp b/dbms/src/Functions/FunctionsLocalMatchAgainst.cpp index 883a87cb047..5a844d6f763 100644 --- a/dbms/src/Functions/FunctionsLocalMatchAgainst.cpp +++ b/dbms/src/Functions/FunctionsLocalMatchAgainst.cpp @@ -1046,11 +1046,9 @@ CompiledBooleanQuery compileBooleanQuery( return compiled; } -Float64 matchBooleanScore(const CompiledBooleanQuery & query, const std::vector & states) +Float64 matchBooleanValue(const CompiledBooleanQuery & query, const std::vector & states) { - // The protocol path is the no-score MATCH ... AGAINST BOOLEAN MODE - // predicate introduced by #70484/#70485. TiDB's local evaluator returns - // a boolean 0/1 result, so do not expose term-frequency counts here. + // Local MATCH returns a Boolean 0/1 value, not a relevance score. return evaluateBooleanMatchResult(query, states) ? 1 : 0; } @@ -1285,7 +1283,7 @@ class FunctionLocalMatchAgainstBoolean final : public IFunction break; } } - output_data[row] = result_known ? (row_matches ? 1 : 0) : matchBooleanScore(compiled_query, clause_states); + output_data[row] = result_known ? (row_matches ? 1 : 0) : matchBooleanValue(compiled_query, clause_states); } finish_result(); } diff --git a/dbms/src/TiDB/Schema/TiDB.cpp b/dbms/src/TiDB/Schema/TiDB.cpp index f55b2b7f29a..4402215c6dc 100644 --- a/dbms/src/TiDB/Schema/TiDB.cpp +++ b/dbms/src/TiDB/Schema/TiDB.cpp @@ -1363,21 +1363,10 @@ tipb::FieldType columnInfoToFieldType(const ColumnInfo & ci) ret.set_decimal(ci.decimal); if (!ci.collate.isEmpty()) { - if (ci.collate.isInteger()) - { - // Table-scan ColumnInfo comes directly from tipb and carries the - // signed collation ID used by TiDB's new-collation protocol. - // Preserve it for expression field types instead of treating it - // as a collation name. - ret.set_collate(ci.collate.convert()); - } - else - { - auto collator_name = ci.collate.convert(); - TiDBCollatorPtr collator = ITiDBCollator::getCollator(collator_name); - RUNTIME_CHECK_MSG(collator, "cannot find collator: {}", collator_name); - ret.set_collate(collator->getCollatorId()); - } + auto collator_name = ci.collate.convert(); + TiDBCollatorPtr collator = ITiDBCollator::getCollator(collator_name); + RUNTIME_CHECK_MSG(collator, "cannot find collator: {}", collator_name); + ret.set_collate(collator->getCollatorId()); } for (const auto & elem : ci.elems) { diff --git a/docs/design/2026-09-18-match-against-tiflash.md b/docs/design/2026-09-18-match-against-tiflash.md deleted file mode 100644 index fae8220581c..00000000000 --- a/docs/design/2026-09-18-match-against-tiflash.md +++ /dev/null @@ -1,279 +0,0 @@ -# TiFlash MATCH AGAINST pushdown - -## Implementation status - -This document describes the implementation currently on the `match_against` -branch. Validation was performed with the release-8.5 TiDB/TiFlash branches -and TiUP playground v8.5.8. It includes Boolean pushdown, STANDARD_V1 and -NGRAM_V1 analysis, nullable MATCH-column handling, and collation-aware -fallback behavior. - -The delivery target is the release-8.5 family. The supported product surface -is single- or multi-column MATCH in BOOLEAN MODE, provided the TiDB planner can -resolve one matching public FULLTEXT index: - -```sql -MATCH(col) AGAINST('+tidb -mysql' IN BOOLEAN MODE) -MATCH(title, body) AGAINST('+tidb -mysql' IN BOOLEAN MODE) -``` - -For multi-column queries, the ordered MATCH column list must exactly match the -columns of one public composite FULLTEXT index on the same table. Independent -single-column indexes do not qualify for native TiFlash pushdown. - -The implementation is a snapshot-local scan evaluator. TiFlash does not read -or maintain a physical FULLTEXT inverted index in this phase. TiDB still -requires a public FULLTEXT index as planner metadata and as the source of the -parser configuration. TiFlash is used when a table has an available replica; -without a TiFlash replica, TiDB uses its local evaluator through the normal -TiKV read path. - -## Introduction - -This document describes the TiFlash-side implementation for evaluating TiDB -`MATCH ... AGAINST` and `FTS_MATCH_WORD` expressions on the read snapshot. -The implementation is intentionally limited to the existing release-8.5 -TiDB-to-TiFlash protocol. TiKV, TICI, and TiDB-side changes are out of scope -for this TiFlash change. - -## Motivation or Background - -The TiDB planner can serialize full-text scalar signatures and the -`FTSQueryTypeWithScore` table-scan metadata into tipb, but the release-8.5 -TiFlash branch did not recognize those signatures. A snapshot-local evaluator -allows TiFlash to execute the predicate and materialize `_FTS_SCORE` after -reading the row, without depending on an asynchronously maintained external -index. - -## Detailed Design - -### Protocol and planner path - -The `contrib/tipb` submodule is pinned to the FTS protocol commit -`26695da26e` on the `release-8.5-match-against` line. The protocol carries -`ScalarFuncSig_FTSMatchWord`, `ScalarFuncSig_FTSMatchExpression`, -`FTSQueryInfo`, `FTSBooleanQuery`, and `used_columnar_indexes`. `FTSBooleanQuery` -also carries `query_tokenizer` and `ngram_token_size`, so TiFlash uses the -same analyzer selected by the FULLTEXT index and TiDB session configuration. - -TiFlash maps the two scalar signatures as follows: - -| tipb signature | TiFlash function | semantics | -| --- | --- | --- | -| `FTSMatchWord` | `fts_match_word` | Legacy one-column full-text evaluator | -| `FTSMatchExpression` | `fts_match_expression` | Boolean-aware evaluator over MATCH columns | - -For the #70484/#70485 boolean predicate, TiDB serializes a `FTSQueryInfo` -into the table scan. For native pushdown, the MATCH columns must resolve in -order to one public composite FULLTEXT index on the same table; this prevents -the planner from substituting unrelated per-column indexes. `TiDBTableScan` -extracts the query and `PhysicalTableScan` builds an `fts_match_expression` -filter containing: - -1. the original query text; -2. MATCH column references and their TiDB field types; -3. the encoded `FTSBooleanQuery` containing required and prohibited terms. - -The filter is attached to the TiFlash table-scan pipeline rather than added -as a root TiDB Selection. The expression carries the first MATCH column's -protocol collation to the TiFlash function. The score path still supports the -existing `_FTS_SCORE` placeholder, but score semantics are outside the -boolean-mode delivery target. - -The expression pipeline runs on the same read snapshot as the row data. The -FTS query is evaluated by the scan executor after rows are read; it is not -used by the DeltaMerge rough-set index. Consequently, current logs may report -`FTSMatchExpression is not supported` for rough-set pruning while the MATCH -predicate itself is still evaluated successfully by the TiFlash scan. - -### Analyzer and matcher - -The evaluator implements the STANDARD_V1 and NGRAM_V1 Boolean matching subset -used by #70484/#70485: - -- Both analyzers recognize Unicode letter/number token runs with `_` preserved; -- STANDARD_V1 uses Unicode lower-casing when no protocol collator is available, - a default token length range of 3..84, and the default InnoDB stopword set; -- NGRAM_V1 emits fixed-size Unicode code-point ngrams. The token size is read - from `ngram_token_size`, whose release-8.5 default is 2; -- Boolean `+` required terms, `-` prohibited terms, quoted phrases, and a - trailing `*` prefix; -- phrase positions are retained across analyzer filtering, so removed - stopwords do not close phrase gaps; -- NULL MATCH columns contribute no tokens and do not nullify the whole row. - -The predicate path consumes the numeric result as a Boolean value (`0` means -no match, a positive value means match). Queries containing Boolean syntax -use the Boolean matcher; plain queries retain the existing token-overlap -behavior for compatibility with the current scalar signature. The -implementation still returns a deterministic positive term-frequency value -for the existing `_FTS_SCORE` placeholder path, but that score is not part of -the #70484/#70485 acceptance scope and is not claimed to be MySQL/InnoDB -relevance. Nullable query arguments preserve NULL. - -### Collation handling - -When a protocol collation is available, TiFlash preserves the source spelling -of tokens and compares terms through the TiDB collator. This makes matching -case and accent behavior follow the column collation. Prefix matching reuses -the same collator-aware comparison path. - -The local TiDB fallback is a separate implementation, but it now receives the -MATCH column collation in its analyzer configuration and uses the same -collator-aware token, prefix, and phrase comparisons. The fallback therefore -preserves the tested `utf8mb4_bin`, `utf8mb4_general_ci`, and -`utf8mb4_0900_ai_ci` behavior when TiFlash is unavailable. - -### Nullable columns - -`FunctionsFullText.cpp` handles nullable string columns by treating a NULL -MATCH column as an empty document. The table-scan expression must nevertheless -declare a nullable result type: otherwise the DAG analyzer inserts a cast from -`Nullable(Float64)` to `Float64` and a NULL row fails with -`Cannot convert NULL value to non-Nullable type`. - -`PhysicalTableScan` now clears `ColumnFlagNotNull` when any MATCH column is -nullable. This keeps the planner metadata aligned with the function return -type and lets a nullable column be evaluated as a normal no-match row in a -WHERE predicate. - -### Consistency boundary - -This is a scan evaluator, not a DeltaMerge native full-text index. It therefore -does not introduce a second mutable index and cannot return rows that are -invisible at the read timestamp. The index id in `FTSQueryInfo` is used only to -identify the query contract in this phase; it is not read by a separate index -reader. The score is calculated from the query and row text, so it does not -provide corpus-wide IDF normalization. - -## Test Design - -### Functional Tests - -`dbms/src/Functions/tests/gtest_fulltext.cpp` covers required/prohibited terms, -phrases with stopword gaps, prefixes, word boundaries, nullable input, NULL -MATCH columns, collation behavior, and unsupported score modifiers. The -`MatchExpressionCollationMatrix` case covers `utf8mb4_bin`, -`utf8mb4_0900_bin`, `utf8mb4_general_ci`, `utf8mb4_unicode_ci`, and -`utf8mb4_0900_ai_ci` for both ordinary terms and prefix terms. - -On 2026-09-23, the `gtests_dbms --gtest_filter='TestFullText.*'` suite passed -all 14 tests. This includes regressions for parsing `run*` as a prefix query -and matching `+run*` against `RUNNER` under a case-insensitive collation. The -fix was in Boolean query parsing: the scanner stopped before `*` but did not -attach it to the preceding term. TiUP SQL E2E was not rerun after this parser -change; the E2E results below are from the earlier validation run. The -multi-column planner resolver and TiFlash evaluator have targeted unit -coverage, but multi-column MATCH has not yet been verified through TiUP SQL. - -### Local E2E validation - -The local E2E test used TiUP playground v8.5.8 with locally built TiDB and -TiFlash binaries. The TiFlash replica was waited to `AVAILABLE=1` and -`PROGRESS=1` before querying. - -The main query: - -```sql -SELECT id, body -FROM articles -WHERE MATCH(body) AGAINST('+tidb -mysql' IN BOOLEAN MODE) -ORDER BY id; -``` - -returned row `2` with the data set used for the release-8.5 verification. Its -plan was: - -```text -TableReader -└─ExchangeSender mpp[tiflash] - └─TableFullScan mpp[tiflash] -``` - -There was no root `Selection(match_against(...))`. `EXPLAIN ANALYZE` showed -five rows scanned and one row output by the TiFlash task. - -The currently supported TiDB collations were tested on TiFlash: - -- `ascii_bin` -- `latin1_bin` -- `utf8_bin` -- `utf8_general_ci` -- `utf8_unicode_ci` -- `utf8mb4_bin` -- `utf8mb4_general_ci` -- `utf8mb4_unicode_ci` -- `utf8mb4_0900_bin` -- `utf8mb4_0900_ai_ci` - -The matrix covered exact terms, `*` prefixes, case behavior, and -`cafe`/`café` accent behavior. Binary collations matched only the exact case -and spelling; CI/AI collations matched the expected case or accent variants. - -The nullable regression test used a `utf8mb4_bin NULL` column containing -`quick runner`, `NULL`, and `QUICK runner`. The query returned only -`quick runner`, used `mpp[tiflash]`, and no longer raised the NULL conversion -error. - -The no-replica test returned successfully through: - -```text -Selection -└─TableReader - └─TableFullScan cop[tikv] -``` - -This confirms the TiDB fallback path remains available when TiFlash cannot be -used and returns the same result as the native path after -`tidb_enable_local_match_against` is enabled. With the session variable left -`OFF`, TiDB intentionally returns the existing error requiring either a -TiFlash FULLTEXT replica or local MATCH AGAINST evaluation. - -The NGRAM E2E used a public FULLTEXT index with `WITH PARSER NGRAM`, -`ngram_token_size = 2`, and the query `+数据库 -mysql`. TiFlash pushdown and -TiDB fallback returned identical rows under both `utf8mb4_bin` and -`utf8mb4_general_ci`; the binary collation remained case-sensitive while the -CI collation matched `mysql` and `MySQL` equivalently. - -### Compatibility Tests - -The normal local TiFlash read path and region-retry remote-read path use the -same expression evaluator. TICI/Disaggregated execution is intentionally not -part of this phase. - -### Benchmark Tests - -No benchmark is included in this phase. The scan evaluator is expected to be -slower than an inverted index and should be treated as compute pushdown, not -index-accelerated search. - -## Impacts & Risks - -The implementation adds per-row tokenization and matching CPU cost. Its score -is deterministic but is not a native MySQL relevance score. Custom stopword -lists and non-default `AGAINST` modifiers are not represented in the current -wire contract and must not be treated as supported by this phase. NGRAM_V1 is -supported for the configured `ngram_token_size`; it is still a scan evaluator, -not an inverted-index implementation. - -The scan evaluator currently does not perform inverted-index lookup or -rough-set pruning for FTS. Large tables may therefore scan many rows even -when the final MATCH result is small. STANDARD_V1 Chinese/CJK behavior follows -its default minimum token length, while NGRAM_V1 handles CJK through fixed-size -code-point windows. This phase does not claim general linguistic segmentation. - -## Investigation & Alternatives - -An asynchronous external index was rejected because it cannot provide the -required snapshot consistency. A native DeltaMerge full-text index remains a -long-term performance design, but it requires a larger write-path, snapshot, -and index-reader change set. - -## Remaining work - -Before product release, multi-column MATCH still needs TiUP SQL E2E coverage; -the paired TiDB branch and TiFlash branch also need CI coverage, release-8.5 -build verification, and code review. If exact -corpus-wide ranking or index-level performance is required, a future phase -must define a relevance-score contract and add a native DeltaMerge full-text -index. From 96cc7fb8a7b15f37c37ddf528b8f30a8b4377c5e Mon Sep 17 00:00:00 2001 From: Zhigao Tong Date: Mon, 12 Oct 2026 16:14:44 +0800 Subject: [PATCH 26/32] test: cover Local MATCH DAG and block correctness --- .../tests/gtest_local_match_against.cpp | 275 ++++++++++++++++++ 1 file changed, 275 insertions(+) diff --git a/dbms/src/Functions/tests/gtest_local_match_against.cpp b/dbms/src/Functions/tests/gtest_local_match_against.cpp index e0fb64f77ca..81e03bfdcfc 100644 --- a/dbms/src/Functions/tests/gtest_local_match_against.cpp +++ b/dbms/src/Functions/tests/gtest_local_match_against.cpp @@ -13,6 +13,8 @@ // limitations under the License. #include +#include +#include #include #include #include @@ -86,8 +88,281 @@ String serializeLocalMatchAgainstBooleanQuery(const tipb::LocalMatchAgainstBoole encoded_query.set_stopword_collation("utf8mb4_bin"); return encoded_query.SerializeAsString(); } + +tipb::LocalMatchAgainstBooleanQuery makeRequiredLocalMatchQuery( + const String & text, + tipb::LocalMatchAgainstBooleanTermType term_type = tipb::LocalMatchAgainstBooleanTermWord) +{ + tipb::LocalMatchAgainstBooleanQuery query; + query.set_innodb_ft_min_token_size(3); + query.set_innodb_ft_max_token_size(84); + auto * node = query.add_nodes(); + node->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + node->set_term_type(term_type); + node->set_text(text); + return query; +} + +// Construct the wire expression directly: metadata belongs to Expr.val, not +// SQL children. ColumnRef offsets and signed collations use the real DAG path. +tipb::Expr makeLocalMatchDAGExpr( + const String & search, + const tipb::LocalMatchAgainstBooleanQuery & query, + const NamesAndTypes & inputs, + Int32 collation, + bool null_search = false) +{ + tipb::Expr expr; + expr.set_tp(tipb::ExprType::ScalarFunc); + expr.set_sig(tipb::ScalarFuncSig::LocalMatchAgainstBoolean); + expr.set_val(serializeLocalMatchAgainstBooleanQuery(query)); + auto * result_type = expr.mutable_field_type(); + result_type->set_tp(TiDB::TypeDouble); + result_type->set_collate(-collation); + const bool nullable = null_search + || std::any_of(inputs.begin(), inputs.end(), [](const auto & input) { return input.type->isNullable(); }); + result_type->set_flag(nullable ? 0 : TiDB::ColumnFlagNotNull); + auto search_expr = null_search ? constructNULLLiteralTiExpr() : constructStringLiteralTiExpr(search); + search_expr.mutable_field_type()->set_tp(TiDB::TypeString); + search_expr.mutable_field_type()->set_collate(-collation); + *expr.add_children() = std::move(search_expr); + for (size_t i = 0; i < inputs.size(); ++i) + { + auto column = constructInt64LiteralTiExpr(static_cast(i)); + column.set_tp(tipb::ExprType::ColumnRef); + column.mutable_field_type()->set_tp(TiDB::TypeString); + column.mutable_field_type()->set_flag(inputs[i].type->isNullable() ? 0 : TiDB::ColumnFlagNotNull); + column.mutable_field_type()->set_collate(-collation); + *expr.add_children() = std::move(column); + } + return expr; +} + +std::pair buildLocalMatchDAGActions( + Context & context, + const NamesAndTypes & inputs, + const std::vector & expressions) +{ + DAGExpressionAnalyzer analyzer(inputs, context); + ExpressionActionsChain chain; + auto & step = analyzer.initAndGetLastStep(chain); + Names results; + for (const auto & expr : expressions) + { + results.push_back(analyzer.getActions(expr, step.actions)); + step.required_output.push_back(results.back()); + } + chain.finalize(); + return {step.actions, results}; +} } // namespace +TEST_F(TestLocalMatchAgainst, MatchBooleanDAGCollationAndMultiColumn) +try +{ + const ColumnsWithTypeAndName columns{ + createColumn>({"CAFE", {}, "cafe", "cafeteria", {}, "cafe"}, "title"), + createColumn>({"", "café", "mysql", "", {}, "MYSQL"}, "body")}; + const NamesAndTypes inputs{{"title", columns[0].type}, {"body", columns[1].type}}; + for (const auto * collation : + {"utf8mb4_bin", "utf8mb4_0900_bin", "utf8mb4_general_ci", "utf8mb4_unicode_ci", "utf8mb4_0900_ai_ci"}) + { + const auto collator = TiDB::ITiDBCollator::getCollator(collation); + ASSERT_NE(collator, nullptr); + const bool binary = std::string_view(collation).ends_with("_bin"); + for (const bool prefix : {false, true}) + { + SCOPED_TRACE(fmt::format("collation={} prefix={}", collation, prefix)); + auto query = makeRequiredLocalMatchQuery( + prefix ? "caf" : "cafe", + prefix ? tipb::LocalMatchAgainstBooleanTermPrefix : tipb::LocalMatchAgainstBooleanTermWord); + auto * prohibited = query.add_nodes(); + prohibited->set_occur(tipb::LocalMatchAgainstBooleanOccurMustNot); + prohibited->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + prohibited->set_text("mysql"); + const auto expr = makeLocalMatchDAGExpr( + prefix ? "+caf* -mysql" : "+cafe -mysql", + query, + inputs, + collator->getCollatorId()); + ASSERT_EQ(expr.children_size(), 3); // Query + two MATCH columns; no metadata SQL child. + const auto [actions, names] = buildLocalMatchDAGActions(*context, inputs, {expr}); + Block block(columns); + actions->execute(block); + const auto expected = prefix ? (binary ? createColumn>({0, 1, 0, 1, 0, 1}) + : createColumn>({1, 1, 0, 1, 0, 0})) + : (binary ? createColumn>({0, 0, 0, 0, 0, 1}) + : createColumn>({1, 1, 0, 0, 0, 0})); + ASSERT_COLUMN_EQ(expected, block.getByName(names[0])); + } + } +} +CATCH + +TEST_F(TestLocalMatchAgainst, MatchBooleanDAGMetadataIsolation) +try +{ + const auto documents + = createColumn({"the", "the 中文", "中文", "中文词", "中文测试", "", "中 文", "foo", "thé"}, "body"); + const NamesAndTypes inputs{{"body", documents.type}}; + const auto binary = TiDB::ITiDBCollator::UTF8MB4_BIN; + std::vector expressions; + ColumnsWithTypeAndName expected; + const auto append = [&](const String & search, + const tipb::LocalMatchAgainstBooleanQuery & query, + Int32 collation, + const ColumnWithTypeAndName & result) { + expressions.push_back(makeLocalMatchDAGExpr(search, query, inputs, collation)); + expected.push_back(result); + }; + auto query = makeRequiredLocalMatchQuery("the"); + append("+the", query, binary, createColumn({0, 0, 0, 0, 0, 0, 0, 0, 0})); + query.set_stopword_mode(tipb::LocalMatchAgainstStopwordModeDisabled); + append("+the", query, binary, createColumn({1, 1, 0, 0, 0, 0, 0, 0, 0})); + + query = makeRequiredLocalMatchQuery("中文"); + query.set_parser(tipb::LocalMatchAgainstParserNgram); + query.set_stopword_mode(tipb::LocalMatchAgainstStopwordModeDisabled); + query.set_ngram_token_size(2); + append("+中文", query, binary, createColumn({0, 1, 1, 1, 1, 0, 0, 0, 0})); + query.set_ngram_token_size(3); + append("+中文", query, binary, createColumn({0, 0, 0, 0, 0, 0, 0, 0, 0})); + + query = makeRequiredLocalMatchQuery("foo"); + append("+foo", query, binary, createColumn({0, 0, 0, 0, 0, 0, 0, 1, 0})); + query.set_innodb_ft_min_token_size(4); + append("+foo", query, binary, createColumn({0, 0, 0, 0, 0, 0, 0, 0, 0})); + + query = makeRequiredLocalMatchQuery("thé"); + query.set_stopword_collation("utf8mb4_bin"); + append("+thé", query, TiDB::ITiDBCollator::UTF8MB4_GENERAL_CI, createColumn({0, 0, 0, 0, 0, 0, 0, 0, 1})); + query.set_stopword_collation("utf8mb4_general_ci"); + append("+thé", query, TiDB::ITiDBCollator::UTF8MB4_GENERAL_CI, createColumn({0, 0, 0, 0, 0, 0, 0, 0, 0})); + + const auto [actions, names] = buildLocalMatchDAGActions(*context, inputs, expressions); + ASSERT_EQ(names.size(), expected.size()); + for (size_t i = 0; i < names.size(); i += 2) + ASSERT_NE(names[i], names[i + 1]) << "Different metadata must not alias the same action"; + Block block({documents}); + actions->execute(block); + for (size_t i = 0; i < names.size(); ++i) + { + SCOPED_TRACE(i); + ASSERT_COLUMN_EQ(expected[i], block.getByName(names[i])); + } +} +CATCH + +TEST_F(TestLocalMatchAgainst, MatchBooleanDAGReusesActionsAcrossBlocks) +try +{ + struct BlockCase + { + ColumnsWithTypeAndName columns; + ColumnWithTypeAndName expected; + }; + for (const bool ngram : {false, true}) + { + SCOPED_TRACE(ngram); + auto query = makeRequiredLocalMatchQuery( + ngram ? "中文测试" : "quick the fox", + tipb::LocalMatchAgainstBooleanTermPhrase); + if (ngram) + { + query.set_parser(tipb::LocalMatchAgainstParserNgram); + query.set_ngram_token_size(2); + query.set_stopword_mode(tipb::LocalMatchAgainstStopwordModeDisabled); + } + auto * prohibited = query.add_nodes(); + prohibited->set_occur(tipb::LocalMatchAgainstBooleanOccurMustNot); + prohibited->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + prohibited->set_text("mysql"); + const auto first = ngram + ? BlockCase{{createColumn>({"中文测试", "中文", "中文测试", "", {}}, "title"), createColumn>({{}, "测试", "mysql", "中文测试", {}}, "body")}, createColumn>({1, 0, 0, 1, 0})} + : BlockCase{ + {createColumn>( + {"QUICK the fox", "quick the fox", "quick", {}, "quick the fox"}, + "title"), + createColumn>({{}, "mysql", "the fox", {}, "MYSQL"}, "body")}, + createColumn>({1, 0, 0, 0, 0})}; + const auto second = ngram + ? BlockCase{{createColumn>({"中文 test", {}, "中文测试", "中 文测试"}, "title"), createColumn>({"", {}, "", "中文测试"}, "body")}, createColumn>({0, 0, 1, 1})} + : BlockCase{ + {createColumn>( + {"slow the fox", "quick the fox", {}, "quick fox", "quick the fox"}, + "title"), + createColumn>({"", {}, "quick the fox", "", {}}, "body")}, + createColumn>({0, 1, 1, 0, 1})}; + const BlockCase nulls{ + {createColumn>({{}, {}}, "title"), createColumn>({{}, {}}, "body")}, + createColumn>({0, 0})}; + const BlockCase empty{ + {createColumn>({}, "title"), createColumn>({}, "body")}, + createColumn>({})}; + const NamesAndTypes inputs{{"title", first.columns[0].type}, {"body", first.columns[1].type}}; + const auto expr = makeLocalMatchDAGExpr( + ngram ? "+\"中文测试\" -mysql" : "+\"quick the fox\" -mysql", + query, + inputs, + TiDB::ITiDBCollator::UTF8MB4_GENERAL_CI); + const auto [actions, names] = buildLocalMatchDAGActions(*context, inputs, {expr}); + // The same compiled actions/function sees changing row counts, empty + // input, and old input again. No previous result or NULL map may leak. + for (size_t pass = 0; pass < 2; ++pass) + for (const auto * batch : {&first, &second, &nulls, &empty, &first}) + { + SCOPED_TRACE(fmt::format("pass={} rows={}", pass, batch->expected.column->size())); + Block block(batch->columns); + actions->execute(block); + ASSERT_COLUMN_EQ(batch->expected, block.getByName(names[0])); + } + } +} +CATCH + +TEST_F(TestLocalMatchAgainst, MatchBooleanDAGColumnWrappersAndEmptyBlocks) +try +{ + struct ColumnCase + { + ColumnWithTypeAndName document; + ColumnWithTypeAndName expected; + }; + const std::vector cases{ + {createColumn({"quick", "", "quick fox", "slow"}, "body"), createColumn({1, 0, 1, 0})}, + {createColumn>({"quick", {}, "", "slow"}, "body"), + createColumn>({1, 0, 0, 0})}, + {createColumn>({{}, {}, {}, {}}, "body"), createColumn>({0, 0, 0, 0})}, + {createConstColumn(4, "quick", "body"), createColumn({1, 1, 1, 1})}, + {createConstColumn>(4, "quick", "body"), createColumn>({1, 1, 1, 1})}, + {createConstColumn>(4, {}, "body"), createColumn>({0, 0, 0, 0})}, + {createConstColumn(4, "", "body"), createColumn({0, 0, 0, 0})}}; + const auto query = makeRequiredLocalMatchQuery("quick"); + for (size_t i = 0; i < cases.size(); ++i) + for (const bool null_search : {false, true}) + { + SCOPED_TRACE(fmt::format("case={} null_search={}", i, null_search)); + const auto & column_case = cases[i]; + const NamesAndTypes inputs{{"body", column_case.document.type}}; + const auto expr + = makeLocalMatchDAGExpr("+quick", query, inputs, TiDB::ITiDBCollator::UTF8MB4_BIN, null_search); + const auto [actions, names] = buildLocalMatchDAGActions(*context, inputs, {expr}); + const auto expected = null_search ? createColumn>({0, 0, 0, 0}) : column_case.expected; + for (const size_t rows : {4U, 0U, 4U}) + { + SCOPED_TRACE(rows); + auto document = column_case.document; + document.column = document.column->cloneResized(rows); + auto expected_rows = expected; + expected_rows.column = expected_rows.column->cloneResized(rows); + Block block({document}); + actions->execute(block); + ASSERT_COLUMN_EQ(expected_rows, block.getByName(names[0])); + } + } +} +CATCH + TEST_F(TestLocalMatchAgainst, MatchBooleanMultiColumnNullable) try { From 3e73ec07a5c0f66a347564113973209d499280c6 Mon Sep 17 00:00:00 2001 From: "TONG,Zhigao" Date: Tue, 13 Oct 2026 01:21:17 +0800 Subject: [PATCH 27/32] format --- .../Functions/FunctionsLocalMatchAgainst.cpp | 12 +- .../Functions/LocalMatchAgainstTokenChars.h | 893 +++--------------- .../tests/gtest_local_match_against.cpp | 29 +- 3 files changed, 170 insertions(+), 764 deletions(-) diff --git a/dbms/src/Functions/FunctionsLocalMatchAgainst.cpp b/dbms/src/Functions/FunctionsLocalMatchAgainst.cpp index 5a844d6f763..ef2db5073e6 100644 --- a/dbms/src/Functions/FunctionsLocalMatchAgainst.cpp +++ b/dbms/src/Functions/FunctionsLocalMatchAgainst.cpp @@ -856,12 +856,12 @@ CompiledBooleanQuery compileBooleanQuery( return config.use_ngram ? analyzeNgramText(text, config.ngram_token_size, collator, stopwords, config.enable_stopword) : analyzeText( - text, - collator, - stopwords, - config.min_token_size, - config.max_token_size, - config.enable_stopword); + text, + collator, + stopwords, + config.min_token_size, + config.max_token_size, + config.enable_stopword); }; for (const auto & node : query.nodes()) { diff --git a/dbms/src/Functions/LocalMatchAgainstTokenChars.h b/dbms/src/Functions/LocalMatchAgainstTokenChars.h index f1216b226e1..c97dc312f74 100644 --- a/dbms/src/Functions/LocalMatchAgainstTokenChars.h +++ b/dbms/src/Functions/LocalMatchAgainstTokenChars.h @@ -32,750 +32,155 @@ struct TokenCharRange std::uint32_t last; }; inline constexpr std::array token_char_ranges{{ - {0xAA, 0xAA}, - {0xB2, 0xB3}, - {0xB5, 0xB5}, - {0xB9, 0xBA}, - {0xBC, 0xBE}, - {0xC0, 0xD6}, - {0xD8, 0xF6}, - {0xF8, 0x2C1}, - {0x2C6, 0x2D1}, - {0x2E0, 0x2E4}, - {0x2EC, 0x2EC}, - {0x2EE, 0x2EE}, - {0x370, 0x374}, - {0x376, 0x377}, - {0x37A, 0x37D}, - {0x37F, 0x37F}, - {0x386, 0x386}, - {0x388, 0x38A}, - {0x38C, 0x38C}, - {0x38E, 0x3A1}, - {0x3A3, 0x3F5}, - {0x3F7, 0x481}, - {0x48A, 0x52F}, - {0x531, 0x556}, - {0x559, 0x559}, - {0x560, 0x588}, - {0x5D0, 0x5EA}, - {0x5EF, 0x5F2}, - {0x620, 0x64A}, - {0x660, 0x669}, - {0x66E, 0x66F}, - {0x671, 0x6D3}, - {0x6D5, 0x6D5}, - {0x6E5, 0x6E6}, - {0x6EE, 0x6FC}, - {0x6FF, 0x6FF}, - {0x710, 0x710}, - {0x712, 0x72F}, - {0x74D, 0x7A5}, - {0x7B1, 0x7B1}, - {0x7C0, 0x7EA}, - {0x7F4, 0x7F5}, - {0x7FA, 0x7FA}, - {0x800, 0x815}, - {0x81A, 0x81A}, - {0x824, 0x824}, - {0x828, 0x828}, - {0x840, 0x858}, - {0x860, 0x86A}, - {0x870, 0x887}, - {0x889, 0x88E}, - {0x8A0, 0x8C9}, - {0x904, 0x939}, - {0x93D, 0x93D}, - {0x950, 0x950}, - {0x958, 0x961}, - {0x966, 0x96F}, - {0x971, 0x980}, - {0x985, 0x98C}, - {0x98F, 0x990}, - {0x993, 0x9A8}, - {0x9AA, 0x9B0}, - {0x9B2, 0x9B2}, - {0x9B6, 0x9B9}, - {0x9BD, 0x9BD}, - {0x9CE, 0x9CE}, - {0x9DC, 0x9DD}, - {0x9DF, 0x9E1}, - {0x9E6, 0x9F1}, - {0x9F4, 0x9F9}, - {0x9FC, 0x9FC}, - {0xA05, 0xA0A}, - {0xA0F, 0xA10}, - {0xA13, 0xA28}, - {0xA2A, 0xA30}, - {0xA32, 0xA33}, - {0xA35, 0xA36}, - {0xA38, 0xA39}, - {0xA59, 0xA5C}, - {0xA5E, 0xA5E}, - {0xA66, 0xA6F}, - {0xA72, 0xA74}, - {0xA85, 0xA8D}, - {0xA8F, 0xA91}, - {0xA93, 0xAA8}, - {0xAAA, 0xAB0}, - {0xAB2, 0xAB3}, - {0xAB5, 0xAB9}, - {0xABD, 0xABD}, - {0xAD0, 0xAD0}, - {0xAE0, 0xAE1}, - {0xAE6, 0xAEF}, - {0xAF9, 0xAF9}, - {0xB05, 0xB0C}, - {0xB0F, 0xB10}, - {0xB13, 0xB28}, - {0xB2A, 0xB30}, - {0xB32, 0xB33}, - {0xB35, 0xB39}, - {0xB3D, 0xB3D}, - {0xB5C, 0xB5D}, - {0xB5F, 0xB61}, - {0xB66, 0xB6F}, - {0xB71, 0xB77}, - {0xB83, 0xB83}, - {0xB85, 0xB8A}, - {0xB8E, 0xB90}, - {0xB92, 0xB95}, - {0xB99, 0xB9A}, - {0xB9C, 0xB9C}, - {0xB9E, 0xB9F}, - {0xBA3, 0xBA4}, - {0xBA8, 0xBAA}, - {0xBAE, 0xBB9}, - {0xBD0, 0xBD0}, - {0xBE6, 0xBF2}, - {0xC05, 0xC0C}, - {0xC0E, 0xC10}, - {0xC12, 0xC28}, - {0xC2A, 0xC39}, - {0xC3D, 0xC3D}, - {0xC58, 0xC5A}, - {0xC5D, 0xC5D}, - {0xC60, 0xC61}, - {0xC66, 0xC6F}, - {0xC78, 0xC7E}, - {0xC80, 0xC80}, - {0xC85, 0xC8C}, - {0xC8E, 0xC90}, - {0xC92, 0xCA8}, - {0xCAA, 0xCB3}, - {0xCB5, 0xCB9}, - {0xCBD, 0xCBD}, - {0xCDD, 0xCDE}, - {0xCE0, 0xCE1}, - {0xCE6, 0xCEF}, - {0xCF1, 0xCF2}, - {0xD04, 0xD0C}, - {0xD0E, 0xD10}, - {0xD12, 0xD3A}, - {0xD3D, 0xD3D}, - {0xD4E, 0xD4E}, - {0xD54, 0xD56}, - {0xD58, 0xD61}, - {0xD66, 0xD78}, - {0xD7A, 0xD7F}, - {0xD85, 0xD96}, - {0xD9A, 0xDB1}, - {0xDB3, 0xDBB}, - {0xDBD, 0xDBD}, - {0xDC0, 0xDC6}, - {0xDE6, 0xDEF}, - {0xE01, 0xE30}, - {0xE32, 0xE33}, - {0xE40, 0xE46}, - {0xE50, 0xE59}, - {0xE81, 0xE82}, - {0xE84, 0xE84}, - {0xE86, 0xE8A}, - {0xE8C, 0xEA3}, - {0xEA5, 0xEA5}, - {0xEA7, 0xEB0}, - {0xEB2, 0xEB3}, - {0xEBD, 0xEBD}, - {0xEC0, 0xEC4}, - {0xEC6, 0xEC6}, - {0xED0, 0xED9}, - {0xEDC, 0xEDF}, - {0xF00, 0xF00}, - {0xF20, 0xF33}, - {0xF40, 0xF47}, - {0xF49, 0xF6C}, - {0xF88, 0xF8C}, - {0x1000, 0x102A}, - {0x103F, 0x1049}, - {0x1050, 0x1055}, - {0x105A, 0x105D}, - {0x1061, 0x1061}, - {0x1065, 0x1066}, - {0x106E, 0x1070}, - {0x1075, 0x1081}, - {0x108E, 0x108E}, - {0x1090, 0x1099}, - {0x10A0, 0x10C5}, - {0x10C7, 0x10C7}, - {0x10CD, 0x10CD}, - {0x10D0, 0x10FA}, - {0x10FC, 0x1248}, - {0x124A, 0x124D}, - {0x1250, 0x1256}, - {0x1258, 0x1258}, - {0x125A, 0x125D}, - {0x1260, 0x1288}, - {0x128A, 0x128D}, - {0x1290, 0x12B0}, - {0x12B2, 0x12B5}, - {0x12B8, 0x12BE}, - {0x12C0, 0x12C0}, - {0x12C2, 0x12C5}, - {0x12C8, 0x12D6}, - {0x12D8, 0x1310}, - {0x1312, 0x1315}, - {0x1318, 0x135A}, - {0x1369, 0x137C}, - {0x1380, 0x138F}, - {0x13A0, 0x13F5}, - {0x13F8, 0x13FD}, - {0x1401, 0x166C}, - {0x166F, 0x167F}, - {0x1681, 0x169A}, - {0x16A0, 0x16EA}, - {0x16EE, 0x16F8}, - {0x1700, 0x1711}, - {0x171F, 0x1731}, - {0x1740, 0x1751}, - {0x1760, 0x176C}, - {0x176E, 0x1770}, - {0x1780, 0x17B3}, - {0x17D7, 0x17D7}, - {0x17DC, 0x17DC}, - {0x17E0, 0x17E9}, - {0x17F0, 0x17F9}, - {0x1810, 0x1819}, - {0x1820, 0x1878}, - {0x1880, 0x1884}, - {0x1887, 0x18A8}, - {0x18AA, 0x18AA}, - {0x18B0, 0x18F5}, - {0x1900, 0x191E}, - {0x1946, 0x196D}, - {0x1970, 0x1974}, - {0x1980, 0x19AB}, - {0x19B0, 0x19C9}, - {0x19D0, 0x19DA}, - {0x1A00, 0x1A16}, - {0x1A20, 0x1A54}, - {0x1A80, 0x1A89}, - {0x1A90, 0x1A99}, - {0x1AA7, 0x1AA7}, - {0x1B05, 0x1B33}, - {0x1B45, 0x1B4C}, - {0x1B50, 0x1B59}, - {0x1B83, 0x1BA0}, - {0x1BAE, 0x1BE5}, - {0x1C00, 0x1C23}, - {0x1C40, 0x1C49}, - {0x1C4D, 0x1C7D}, - {0x1C80, 0x1C88}, - {0x1C90, 0x1CBA}, - {0x1CBD, 0x1CBF}, - {0x1CE9, 0x1CEC}, - {0x1CEE, 0x1CF3}, - {0x1CF5, 0x1CF6}, - {0x1CFA, 0x1CFA}, - {0x1D00, 0x1DBF}, - {0x1E00, 0x1F15}, - {0x1F18, 0x1F1D}, - {0x1F20, 0x1F45}, - {0x1F48, 0x1F4D}, - {0x1F50, 0x1F57}, - {0x1F59, 0x1F59}, - {0x1F5B, 0x1F5B}, - {0x1F5D, 0x1F5D}, - {0x1F5F, 0x1F7D}, - {0x1F80, 0x1FB4}, - {0x1FB6, 0x1FBC}, - {0x1FBE, 0x1FBE}, - {0x1FC2, 0x1FC4}, - {0x1FC6, 0x1FCC}, - {0x1FD0, 0x1FD3}, - {0x1FD6, 0x1FDB}, - {0x1FE0, 0x1FEC}, - {0x1FF2, 0x1FF4}, - {0x1FF6, 0x1FFC}, - {0x2070, 0x2071}, - {0x2074, 0x2079}, - {0x207F, 0x2089}, - {0x2090, 0x209C}, - {0x2102, 0x2102}, - {0x2107, 0x2107}, - {0x210A, 0x2113}, - {0x2115, 0x2115}, - {0x2119, 0x211D}, - {0x2124, 0x2124}, - {0x2126, 0x2126}, - {0x2128, 0x2128}, - {0x212A, 0x212D}, - {0x212F, 0x2139}, - {0x213C, 0x213F}, - {0x2145, 0x2149}, - {0x214E, 0x214E}, - {0x2150, 0x2189}, - {0x2460, 0x249B}, - {0x24EA, 0x24FF}, - {0x2776, 0x2793}, - {0x2C00, 0x2CE4}, - {0x2CEB, 0x2CEE}, - {0x2CF2, 0x2CF3}, - {0x2CFD, 0x2CFD}, - {0x2D00, 0x2D25}, - {0x2D27, 0x2D27}, - {0x2D2D, 0x2D2D}, - {0x2D30, 0x2D67}, - {0x2D6F, 0x2D6F}, - {0x2D80, 0x2D96}, - {0x2DA0, 0x2DA6}, - {0x2DA8, 0x2DAE}, - {0x2DB0, 0x2DB6}, - {0x2DB8, 0x2DBE}, - {0x2DC0, 0x2DC6}, - {0x2DC8, 0x2DCE}, - {0x2DD0, 0x2DD6}, - {0x2DD8, 0x2DDE}, - {0x2E2F, 0x2E2F}, - {0x3005, 0x3007}, - {0x3021, 0x3029}, - {0x3031, 0x3035}, - {0x3038, 0x303C}, - {0x3041, 0x3096}, - {0x309D, 0x309F}, - {0x30A1, 0x30FA}, - {0x30FC, 0x30FF}, - {0x3105, 0x312F}, - {0x3131, 0x318E}, - {0x3192, 0x3195}, - {0x31A0, 0x31BF}, - {0x31F0, 0x31FF}, - {0x3220, 0x3229}, - {0x3248, 0x324F}, - {0x3251, 0x325F}, - {0x3280, 0x3289}, - {0x32B1, 0x32BF}, - {0x3400, 0x4DBF}, - {0x4E00, 0xA48C}, - {0xA4D0, 0xA4FD}, - {0xA500, 0xA60C}, - {0xA610, 0xA62B}, - {0xA640, 0xA66E}, - {0xA67F, 0xA69D}, - {0xA6A0, 0xA6EF}, - {0xA717, 0xA71F}, - {0xA722, 0xA788}, - {0xA78B, 0xA7CA}, - {0xA7D0, 0xA7D1}, - {0xA7D3, 0xA7D3}, - {0xA7D5, 0xA7D9}, - {0xA7F2, 0xA801}, - {0xA803, 0xA805}, - {0xA807, 0xA80A}, - {0xA80C, 0xA822}, - {0xA830, 0xA835}, - {0xA840, 0xA873}, - {0xA882, 0xA8B3}, - {0xA8D0, 0xA8D9}, - {0xA8F2, 0xA8F7}, - {0xA8FB, 0xA8FB}, - {0xA8FD, 0xA8FE}, - {0xA900, 0xA925}, - {0xA930, 0xA946}, - {0xA960, 0xA97C}, - {0xA984, 0xA9B2}, - {0xA9CF, 0xA9D9}, - {0xA9E0, 0xA9E4}, - {0xA9E6, 0xA9FE}, - {0xAA00, 0xAA28}, - {0xAA40, 0xAA42}, - {0xAA44, 0xAA4B}, - {0xAA50, 0xAA59}, - {0xAA60, 0xAA76}, - {0xAA7A, 0xAA7A}, - {0xAA7E, 0xAAAF}, - {0xAAB1, 0xAAB1}, - {0xAAB5, 0xAAB6}, - {0xAAB9, 0xAABD}, - {0xAAC0, 0xAAC0}, - {0xAAC2, 0xAAC2}, - {0xAADB, 0xAADD}, - {0xAAE0, 0xAAEA}, - {0xAAF2, 0xAAF4}, - {0xAB01, 0xAB06}, - {0xAB09, 0xAB0E}, - {0xAB11, 0xAB16}, - {0xAB20, 0xAB26}, - {0xAB28, 0xAB2E}, - {0xAB30, 0xAB5A}, - {0xAB5C, 0xAB69}, - {0xAB70, 0xABE2}, - {0xABF0, 0xABF9}, - {0xAC00, 0xD7A3}, - {0xD7B0, 0xD7C6}, - {0xD7CB, 0xD7FB}, - {0xF900, 0xFA6D}, - {0xFA70, 0xFAD9}, - {0xFB00, 0xFB06}, - {0xFB13, 0xFB17}, - {0xFB1D, 0xFB1D}, - {0xFB1F, 0xFB28}, - {0xFB2A, 0xFB36}, - {0xFB38, 0xFB3C}, - {0xFB3E, 0xFB3E}, - {0xFB40, 0xFB41}, - {0xFB43, 0xFB44}, - {0xFB46, 0xFBB1}, - {0xFBD3, 0xFD3D}, - {0xFD50, 0xFD8F}, - {0xFD92, 0xFDC7}, - {0xFDF0, 0xFDFB}, - {0xFE70, 0xFE74}, - {0xFE76, 0xFEFC}, - {0xFF10, 0xFF19}, - {0xFF21, 0xFF3A}, - {0xFF41, 0xFF5A}, - {0xFF66, 0xFFBE}, - {0xFFC2, 0xFFC7}, - {0xFFCA, 0xFFCF}, - {0xFFD2, 0xFFD7}, - {0xFFDA, 0xFFDC}, - {0x10000, 0x1000B}, - {0x1000D, 0x10026}, - {0x10028, 0x1003A}, - {0x1003C, 0x1003D}, - {0x1003F, 0x1004D}, - {0x10050, 0x1005D}, - {0x10080, 0x100FA}, - {0x10107, 0x10133}, - {0x10140, 0x10178}, - {0x1018A, 0x1018B}, - {0x10280, 0x1029C}, - {0x102A0, 0x102D0}, - {0x102E1, 0x102FB}, - {0x10300, 0x10323}, - {0x1032D, 0x1034A}, - {0x10350, 0x10375}, - {0x10380, 0x1039D}, - {0x103A0, 0x103C3}, - {0x103C8, 0x103CF}, - {0x103D1, 0x103D5}, - {0x10400, 0x1049D}, - {0x104A0, 0x104A9}, - {0x104B0, 0x104D3}, - {0x104D8, 0x104FB}, - {0x10500, 0x10527}, - {0x10530, 0x10563}, - {0x10570, 0x1057A}, - {0x1057C, 0x1058A}, - {0x1058C, 0x10592}, - {0x10594, 0x10595}, - {0x10597, 0x105A1}, - {0x105A3, 0x105B1}, - {0x105B3, 0x105B9}, - {0x105BB, 0x105BC}, - {0x10600, 0x10736}, - {0x10740, 0x10755}, - {0x10760, 0x10767}, - {0x10780, 0x10785}, - {0x10787, 0x107B0}, - {0x107B2, 0x107BA}, - {0x10800, 0x10805}, - {0x10808, 0x10808}, - {0x1080A, 0x10835}, - {0x10837, 0x10838}, - {0x1083C, 0x1083C}, - {0x1083F, 0x10855}, - {0x10858, 0x10876}, - {0x10879, 0x1089E}, - {0x108A7, 0x108AF}, - {0x108E0, 0x108F2}, - {0x108F4, 0x108F5}, - {0x108FB, 0x1091B}, - {0x10920, 0x10939}, - {0x10980, 0x109B7}, - {0x109BC, 0x109CF}, - {0x109D2, 0x10A00}, - {0x10A10, 0x10A13}, - {0x10A15, 0x10A17}, - {0x10A19, 0x10A35}, - {0x10A40, 0x10A48}, - {0x10A60, 0x10A7E}, - {0x10A80, 0x10A9F}, - {0x10AC0, 0x10AC7}, - {0x10AC9, 0x10AE4}, - {0x10AEB, 0x10AEF}, - {0x10B00, 0x10B35}, - {0x10B40, 0x10B55}, - {0x10B58, 0x10B72}, - {0x10B78, 0x10B91}, - {0x10BA9, 0x10BAF}, - {0x10C00, 0x10C48}, - {0x10C80, 0x10CB2}, - {0x10CC0, 0x10CF2}, - {0x10CFA, 0x10D23}, - {0x10D30, 0x10D39}, - {0x10E60, 0x10E7E}, - {0x10E80, 0x10EA9}, - {0x10EB0, 0x10EB1}, - {0x10F00, 0x10F27}, - {0x10F30, 0x10F45}, - {0x10F51, 0x10F54}, - {0x10F70, 0x10F81}, - {0x10FB0, 0x10FCB}, - {0x10FE0, 0x10FF6}, - {0x11003, 0x11037}, - {0x11052, 0x1106F}, - {0x11071, 0x11072}, - {0x11075, 0x11075}, - {0x11083, 0x110AF}, - {0x110D0, 0x110E8}, - {0x110F0, 0x110F9}, - {0x11103, 0x11126}, - {0x11136, 0x1113F}, - {0x11144, 0x11144}, - {0x11147, 0x11147}, - {0x11150, 0x11172}, - {0x11176, 0x11176}, - {0x11183, 0x111B2}, - {0x111C1, 0x111C4}, - {0x111D0, 0x111DA}, - {0x111DC, 0x111DC}, - {0x111E1, 0x111F4}, - {0x11200, 0x11211}, - {0x11213, 0x1122B}, - {0x1123F, 0x11240}, - {0x11280, 0x11286}, - {0x11288, 0x11288}, - {0x1128A, 0x1128D}, - {0x1128F, 0x1129D}, - {0x1129F, 0x112A8}, - {0x112B0, 0x112DE}, - {0x112F0, 0x112F9}, - {0x11305, 0x1130C}, - {0x1130F, 0x11310}, - {0x11313, 0x11328}, - {0x1132A, 0x11330}, - {0x11332, 0x11333}, - {0x11335, 0x11339}, - {0x1133D, 0x1133D}, - {0x11350, 0x11350}, - {0x1135D, 0x11361}, - {0x11400, 0x11434}, - {0x11447, 0x1144A}, - {0x11450, 0x11459}, - {0x1145F, 0x11461}, - {0x11480, 0x114AF}, - {0x114C4, 0x114C5}, - {0x114C7, 0x114C7}, - {0x114D0, 0x114D9}, - {0x11580, 0x115AE}, - {0x115D8, 0x115DB}, - {0x11600, 0x1162F}, - {0x11644, 0x11644}, - {0x11650, 0x11659}, - {0x11680, 0x116AA}, - {0x116B8, 0x116B8}, - {0x116C0, 0x116C9}, - {0x11700, 0x1171A}, - {0x11730, 0x1173B}, - {0x11740, 0x11746}, - {0x11800, 0x1182B}, - {0x118A0, 0x118F2}, - {0x118FF, 0x11906}, - {0x11909, 0x11909}, - {0x1190C, 0x11913}, - {0x11915, 0x11916}, - {0x11918, 0x1192F}, - {0x1193F, 0x1193F}, - {0x11941, 0x11941}, - {0x11950, 0x11959}, - {0x119A0, 0x119A7}, - {0x119AA, 0x119D0}, - {0x119E1, 0x119E1}, - {0x119E3, 0x119E3}, - {0x11A00, 0x11A00}, - {0x11A0B, 0x11A32}, - {0x11A3A, 0x11A3A}, - {0x11A50, 0x11A50}, - {0x11A5C, 0x11A89}, - {0x11A9D, 0x11A9D}, - {0x11AB0, 0x11AF8}, - {0x11C00, 0x11C08}, - {0x11C0A, 0x11C2E}, - {0x11C40, 0x11C40}, - {0x11C50, 0x11C6C}, - {0x11C72, 0x11C8F}, - {0x11D00, 0x11D06}, - {0x11D08, 0x11D09}, - {0x11D0B, 0x11D30}, - {0x11D46, 0x11D46}, - {0x11D50, 0x11D59}, - {0x11D60, 0x11D65}, - {0x11D67, 0x11D68}, - {0x11D6A, 0x11D89}, - {0x11D98, 0x11D98}, - {0x11DA0, 0x11DA9}, - {0x11EE0, 0x11EF2}, - {0x11F02, 0x11F02}, - {0x11F04, 0x11F10}, - {0x11F12, 0x11F33}, - {0x11F50, 0x11F59}, - {0x11FB0, 0x11FB0}, - {0x11FC0, 0x11FD4}, - {0x12000, 0x12399}, - {0x12400, 0x1246E}, - {0x12480, 0x12543}, - {0x12F90, 0x12FF0}, - {0x13000, 0x1342F}, - {0x13441, 0x13446}, - {0x14400, 0x14646}, - {0x16800, 0x16A38}, - {0x16A40, 0x16A5E}, - {0x16A60, 0x16A69}, - {0x16A70, 0x16ABE}, - {0x16AC0, 0x16AC9}, - {0x16AD0, 0x16AED}, - {0x16B00, 0x16B2F}, - {0x16B40, 0x16B43}, - {0x16B50, 0x16B59}, - {0x16B5B, 0x16B61}, - {0x16B63, 0x16B77}, - {0x16B7D, 0x16B8F}, - {0x16E40, 0x16E96}, - {0x16F00, 0x16F4A}, - {0x16F50, 0x16F50}, - {0x16F93, 0x16F9F}, - {0x16FE0, 0x16FE1}, - {0x16FE3, 0x16FE3}, - {0x17000, 0x187F7}, - {0x18800, 0x18CD5}, - {0x18D00, 0x18D08}, - {0x1AFF0, 0x1AFF3}, - {0x1AFF5, 0x1AFFB}, - {0x1AFFD, 0x1AFFE}, - {0x1B000, 0x1B122}, - {0x1B132, 0x1B132}, - {0x1B150, 0x1B152}, - {0x1B155, 0x1B155}, - {0x1B164, 0x1B167}, - {0x1B170, 0x1B2FB}, - {0x1BC00, 0x1BC6A}, - {0x1BC70, 0x1BC7C}, - {0x1BC80, 0x1BC88}, - {0x1BC90, 0x1BC99}, - {0x1D2C0, 0x1D2D3}, - {0x1D2E0, 0x1D2F3}, - {0x1D360, 0x1D378}, - {0x1D400, 0x1D454}, - {0x1D456, 0x1D49C}, - {0x1D49E, 0x1D49F}, - {0x1D4A2, 0x1D4A2}, - {0x1D4A5, 0x1D4A6}, - {0x1D4A9, 0x1D4AC}, - {0x1D4AE, 0x1D4B9}, - {0x1D4BB, 0x1D4BB}, - {0x1D4BD, 0x1D4C3}, - {0x1D4C5, 0x1D505}, - {0x1D507, 0x1D50A}, - {0x1D50D, 0x1D514}, - {0x1D516, 0x1D51C}, - {0x1D51E, 0x1D539}, - {0x1D53B, 0x1D53E}, - {0x1D540, 0x1D544}, - {0x1D546, 0x1D546}, - {0x1D54A, 0x1D550}, - {0x1D552, 0x1D6A5}, - {0x1D6A8, 0x1D6C0}, - {0x1D6C2, 0x1D6DA}, - {0x1D6DC, 0x1D6FA}, - {0x1D6FC, 0x1D714}, - {0x1D716, 0x1D734}, - {0x1D736, 0x1D74E}, - {0x1D750, 0x1D76E}, - {0x1D770, 0x1D788}, - {0x1D78A, 0x1D7A8}, - {0x1D7AA, 0x1D7C2}, - {0x1D7C4, 0x1D7CB}, - {0x1D7CE, 0x1D7FF}, - {0x1DF00, 0x1DF1E}, - {0x1DF25, 0x1DF2A}, - {0x1E030, 0x1E06D}, - {0x1E100, 0x1E12C}, - {0x1E137, 0x1E13D}, - {0x1E140, 0x1E149}, - {0x1E14E, 0x1E14E}, - {0x1E290, 0x1E2AD}, - {0x1E2C0, 0x1E2EB}, - {0x1E2F0, 0x1E2F9}, - {0x1E4D0, 0x1E4EB}, - {0x1E4F0, 0x1E4F9}, - {0x1E7E0, 0x1E7E6}, - {0x1E7E8, 0x1E7EB}, - {0x1E7ED, 0x1E7EE}, - {0x1E7F0, 0x1E7FE}, - {0x1E800, 0x1E8C4}, - {0x1E8C7, 0x1E8CF}, - {0x1E900, 0x1E943}, - {0x1E94B, 0x1E94B}, - {0x1E950, 0x1E959}, - {0x1EC71, 0x1ECAB}, - {0x1ECAD, 0x1ECAF}, - {0x1ECB1, 0x1ECB4}, - {0x1ED01, 0x1ED2D}, - {0x1ED2F, 0x1ED3D}, - {0x1EE00, 0x1EE03}, - {0x1EE05, 0x1EE1F}, - {0x1EE21, 0x1EE22}, - {0x1EE24, 0x1EE24}, - {0x1EE27, 0x1EE27}, - {0x1EE29, 0x1EE32}, - {0x1EE34, 0x1EE37}, - {0x1EE39, 0x1EE39}, - {0x1EE3B, 0x1EE3B}, - {0x1EE42, 0x1EE42}, - {0x1EE47, 0x1EE47}, - {0x1EE49, 0x1EE49}, - {0x1EE4B, 0x1EE4B}, - {0x1EE4D, 0x1EE4F}, - {0x1EE51, 0x1EE52}, - {0x1EE54, 0x1EE54}, - {0x1EE57, 0x1EE57}, - {0x1EE59, 0x1EE59}, - {0x1EE5B, 0x1EE5B}, - {0x1EE5D, 0x1EE5D}, - {0x1EE5F, 0x1EE5F}, - {0x1EE61, 0x1EE62}, - {0x1EE64, 0x1EE64}, - {0x1EE67, 0x1EE6A}, - {0x1EE6C, 0x1EE72}, - {0x1EE74, 0x1EE77}, - {0x1EE79, 0x1EE7C}, - {0x1EE7E, 0x1EE7E}, - {0x1EE80, 0x1EE89}, - {0x1EE8B, 0x1EE9B}, - {0x1EEA1, 0x1EEA3}, - {0x1EEA5, 0x1EEA9}, - {0x1EEAB, 0x1EEBB}, - {0x1F100, 0x1F10C}, - {0x1FBF0, 0x1FBF9}, - {0x20000, 0x2A6DF}, - {0x2A700, 0x2B739}, - {0x2B740, 0x2B81D}, - {0x2B820, 0x2CEA1}, - {0x2CEB0, 0x2EBE0}, - {0x2F800, 0x2FA1D}, - {0x30000, 0x3134A}, - {0x31350, 0x323AF}, + {0xAA, 0xAA}, {0xB2, 0xB3}, {0xB5, 0xB5}, {0xB9, 0xBA}, {0xBC, 0xBE}, + {0xC0, 0xD6}, {0xD8, 0xF6}, {0xF8, 0x2C1}, {0x2C6, 0x2D1}, {0x2E0, 0x2E4}, + {0x2EC, 0x2EC}, {0x2EE, 0x2EE}, {0x370, 0x374}, {0x376, 0x377}, {0x37A, 0x37D}, + {0x37F, 0x37F}, {0x386, 0x386}, {0x388, 0x38A}, {0x38C, 0x38C}, {0x38E, 0x3A1}, + {0x3A3, 0x3F5}, {0x3F7, 0x481}, {0x48A, 0x52F}, {0x531, 0x556}, {0x559, 0x559}, + {0x560, 0x588}, {0x5D0, 0x5EA}, {0x5EF, 0x5F2}, {0x620, 0x64A}, {0x660, 0x669}, + {0x66E, 0x66F}, {0x671, 0x6D3}, {0x6D5, 0x6D5}, {0x6E5, 0x6E6}, {0x6EE, 0x6FC}, + {0x6FF, 0x6FF}, {0x710, 0x710}, {0x712, 0x72F}, {0x74D, 0x7A5}, {0x7B1, 0x7B1}, + {0x7C0, 0x7EA}, {0x7F4, 0x7F5}, {0x7FA, 0x7FA}, {0x800, 0x815}, {0x81A, 0x81A}, + {0x824, 0x824}, {0x828, 0x828}, {0x840, 0x858}, {0x860, 0x86A}, {0x870, 0x887}, + {0x889, 0x88E}, {0x8A0, 0x8C9}, {0x904, 0x939}, {0x93D, 0x93D}, {0x950, 0x950}, + {0x958, 0x961}, {0x966, 0x96F}, {0x971, 0x980}, {0x985, 0x98C}, {0x98F, 0x990}, + {0x993, 0x9A8}, {0x9AA, 0x9B0}, {0x9B2, 0x9B2}, {0x9B6, 0x9B9}, {0x9BD, 0x9BD}, + {0x9CE, 0x9CE}, {0x9DC, 0x9DD}, {0x9DF, 0x9E1}, {0x9E6, 0x9F1}, {0x9F4, 0x9F9}, + {0x9FC, 0x9FC}, {0xA05, 0xA0A}, {0xA0F, 0xA10}, {0xA13, 0xA28}, {0xA2A, 0xA30}, + {0xA32, 0xA33}, {0xA35, 0xA36}, {0xA38, 0xA39}, {0xA59, 0xA5C}, {0xA5E, 0xA5E}, + {0xA66, 0xA6F}, {0xA72, 0xA74}, {0xA85, 0xA8D}, {0xA8F, 0xA91}, {0xA93, 0xAA8}, + {0xAAA, 0xAB0}, {0xAB2, 0xAB3}, {0xAB5, 0xAB9}, {0xABD, 0xABD}, {0xAD0, 0xAD0}, + {0xAE0, 0xAE1}, {0xAE6, 0xAEF}, {0xAF9, 0xAF9}, {0xB05, 0xB0C}, {0xB0F, 0xB10}, + {0xB13, 0xB28}, {0xB2A, 0xB30}, {0xB32, 0xB33}, {0xB35, 0xB39}, {0xB3D, 0xB3D}, + {0xB5C, 0xB5D}, {0xB5F, 0xB61}, {0xB66, 0xB6F}, {0xB71, 0xB77}, {0xB83, 0xB83}, + {0xB85, 0xB8A}, {0xB8E, 0xB90}, {0xB92, 0xB95}, {0xB99, 0xB9A}, {0xB9C, 0xB9C}, + {0xB9E, 0xB9F}, {0xBA3, 0xBA4}, {0xBA8, 0xBAA}, {0xBAE, 0xBB9}, {0xBD0, 0xBD0}, + {0xBE6, 0xBF2}, {0xC05, 0xC0C}, {0xC0E, 0xC10}, {0xC12, 0xC28}, {0xC2A, 0xC39}, + {0xC3D, 0xC3D}, {0xC58, 0xC5A}, {0xC5D, 0xC5D}, {0xC60, 0xC61}, {0xC66, 0xC6F}, + {0xC78, 0xC7E}, {0xC80, 0xC80}, {0xC85, 0xC8C}, {0xC8E, 0xC90}, {0xC92, 0xCA8}, + {0xCAA, 0xCB3}, {0xCB5, 0xCB9}, {0xCBD, 0xCBD}, {0xCDD, 0xCDE}, {0xCE0, 0xCE1}, + {0xCE6, 0xCEF}, {0xCF1, 0xCF2}, {0xD04, 0xD0C}, {0xD0E, 0xD10}, {0xD12, 0xD3A}, + {0xD3D, 0xD3D}, {0xD4E, 0xD4E}, {0xD54, 0xD56}, {0xD58, 0xD61}, {0xD66, 0xD78}, + {0xD7A, 0xD7F}, {0xD85, 0xD96}, {0xD9A, 0xDB1}, {0xDB3, 0xDBB}, {0xDBD, 0xDBD}, + {0xDC0, 0xDC6}, {0xDE6, 0xDEF}, {0xE01, 0xE30}, {0xE32, 0xE33}, {0xE40, 0xE46}, + {0xE50, 0xE59}, {0xE81, 0xE82}, {0xE84, 0xE84}, {0xE86, 0xE8A}, {0xE8C, 0xEA3}, + {0xEA5, 0xEA5}, {0xEA7, 0xEB0}, {0xEB2, 0xEB3}, {0xEBD, 0xEBD}, {0xEC0, 0xEC4}, + {0xEC6, 0xEC6}, {0xED0, 0xED9}, {0xEDC, 0xEDF}, {0xF00, 0xF00}, {0xF20, 0xF33}, + {0xF40, 0xF47}, {0xF49, 0xF6C}, {0xF88, 0xF8C}, {0x1000, 0x102A}, {0x103F, 0x1049}, + {0x1050, 0x1055}, {0x105A, 0x105D}, {0x1061, 0x1061}, {0x1065, 0x1066}, {0x106E, 0x1070}, + {0x1075, 0x1081}, {0x108E, 0x108E}, {0x1090, 0x1099}, {0x10A0, 0x10C5}, {0x10C7, 0x10C7}, + {0x10CD, 0x10CD}, {0x10D0, 0x10FA}, {0x10FC, 0x1248}, {0x124A, 0x124D}, {0x1250, 0x1256}, + {0x1258, 0x1258}, {0x125A, 0x125D}, {0x1260, 0x1288}, {0x128A, 0x128D}, {0x1290, 0x12B0}, + {0x12B2, 0x12B5}, {0x12B8, 0x12BE}, {0x12C0, 0x12C0}, {0x12C2, 0x12C5}, {0x12C8, 0x12D6}, + {0x12D8, 0x1310}, {0x1312, 0x1315}, {0x1318, 0x135A}, {0x1369, 0x137C}, {0x1380, 0x138F}, + {0x13A0, 0x13F5}, {0x13F8, 0x13FD}, {0x1401, 0x166C}, {0x166F, 0x167F}, {0x1681, 0x169A}, + {0x16A0, 0x16EA}, {0x16EE, 0x16F8}, {0x1700, 0x1711}, {0x171F, 0x1731}, {0x1740, 0x1751}, + {0x1760, 0x176C}, {0x176E, 0x1770}, {0x1780, 0x17B3}, {0x17D7, 0x17D7}, {0x17DC, 0x17DC}, + {0x17E0, 0x17E9}, {0x17F0, 0x17F9}, {0x1810, 0x1819}, {0x1820, 0x1878}, {0x1880, 0x1884}, + {0x1887, 0x18A8}, {0x18AA, 0x18AA}, {0x18B0, 0x18F5}, {0x1900, 0x191E}, {0x1946, 0x196D}, + {0x1970, 0x1974}, {0x1980, 0x19AB}, {0x19B0, 0x19C9}, {0x19D0, 0x19DA}, {0x1A00, 0x1A16}, + {0x1A20, 0x1A54}, {0x1A80, 0x1A89}, {0x1A90, 0x1A99}, {0x1AA7, 0x1AA7}, {0x1B05, 0x1B33}, + {0x1B45, 0x1B4C}, {0x1B50, 0x1B59}, {0x1B83, 0x1BA0}, {0x1BAE, 0x1BE5}, {0x1C00, 0x1C23}, + {0x1C40, 0x1C49}, {0x1C4D, 0x1C7D}, {0x1C80, 0x1C88}, {0x1C90, 0x1CBA}, {0x1CBD, 0x1CBF}, + {0x1CE9, 0x1CEC}, {0x1CEE, 0x1CF3}, {0x1CF5, 0x1CF6}, {0x1CFA, 0x1CFA}, {0x1D00, 0x1DBF}, + {0x1E00, 0x1F15}, {0x1F18, 0x1F1D}, {0x1F20, 0x1F45}, {0x1F48, 0x1F4D}, {0x1F50, 0x1F57}, + {0x1F59, 0x1F59}, {0x1F5B, 0x1F5B}, {0x1F5D, 0x1F5D}, {0x1F5F, 0x1F7D}, {0x1F80, 0x1FB4}, + {0x1FB6, 0x1FBC}, {0x1FBE, 0x1FBE}, {0x1FC2, 0x1FC4}, {0x1FC6, 0x1FCC}, {0x1FD0, 0x1FD3}, + {0x1FD6, 0x1FDB}, {0x1FE0, 0x1FEC}, {0x1FF2, 0x1FF4}, {0x1FF6, 0x1FFC}, {0x2070, 0x2071}, + {0x2074, 0x2079}, {0x207F, 0x2089}, {0x2090, 0x209C}, {0x2102, 0x2102}, {0x2107, 0x2107}, + {0x210A, 0x2113}, {0x2115, 0x2115}, {0x2119, 0x211D}, {0x2124, 0x2124}, {0x2126, 0x2126}, + {0x2128, 0x2128}, {0x212A, 0x212D}, {0x212F, 0x2139}, {0x213C, 0x213F}, {0x2145, 0x2149}, + {0x214E, 0x214E}, {0x2150, 0x2189}, {0x2460, 0x249B}, {0x24EA, 0x24FF}, {0x2776, 0x2793}, + {0x2C00, 0x2CE4}, {0x2CEB, 0x2CEE}, {0x2CF2, 0x2CF3}, {0x2CFD, 0x2CFD}, {0x2D00, 0x2D25}, + {0x2D27, 0x2D27}, {0x2D2D, 0x2D2D}, {0x2D30, 0x2D67}, {0x2D6F, 0x2D6F}, {0x2D80, 0x2D96}, + {0x2DA0, 0x2DA6}, {0x2DA8, 0x2DAE}, {0x2DB0, 0x2DB6}, {0x2DB8, 0x2DBE}, {0x2DC0, 0x2DC6}, + {0x2DC8, 0x2DCE}, {0x2DD0, 0x2DD6}, {0x2DD8, 0x2DDE}, {0x2E2F, 0x2E2F}, {0x3005, 0x3007}, + {0x3021, 0x3029}, {0x3031, 0x3035}, {0x3038, 0x303C}, {0x3041, 0x3096}, {0x309D, 0x309F}, + {0x30A1, 0x30FA}, {0x30FC, 0x30FF}, {0x3105, 0x312F}, {0x3131, 0x318E}, {0x3192, 0x3195}, + {0x31A0, 0x31BF}, {0x31F0, 0x31FF}, {0x3220, 0x3229}, {0x3248, 0x324F}, {0x3251, 0x325F}, + {0x3280, 0x3289}, {0x32B1, 0x32BF}, {0x3400, 0x4DBF}, {0x4E00, 0xA48C}, {0xA4D0, 0xA4FD}, + {0xA500, 0xA60C}, {0xA610, 0xA62B}, {0xA640, 0xA66E}, {0xA67F, 0xA69D}, {0xA6A0, 0xA6EF}, + {0xA717, 0xA71F}, {0xA722, 0xA788}, {0xA78B, 0xA7CA}, {0xA7D0, 0xA7D1}, {0xA7D3, 0xA7D3}, + {0xA7D5, 0xA7D9}, {0xA7F2, 0xA801}, {0xA803, 0xA805}, {0xA807, 0xA80A}, {0xA80C, 0xA822}, + {0xA830, 0xA835}, {0xA840, 0xA873}, {0xA882, 0xA8B3}, {0xA8D0, 0xA8D9}, {0xA8F2, 0xA8F7}, + {0xA8FB, 0xA8FB}, {0xA8FD, 0xA8FE}, {0xA900, 0xA925}, {0xA930, 0xA946}, {0xA960, 0xA97C}, + {0xA984, 0xA9B2}, {0xA9CF, 0xA9D9}, {0xA9E0, 0xA9E4}, {0xA9E6, 0xA9FE}, {0xAA00, 0xAA28}, + {0xAA40, 0xAA42}, {0xAA44, 0xAA4B}, {0xAA50, 0xAA59}, {0xAA60, 0xAA76}, {0xAA7A, 0xAA7A}, + {0xAA7E, 0xAAAF}, {0xAAB1, 0xAAB1}, {0xAAB5, 0xAAB6}, {0xAAB9, 0xAABD}, {0xAAC0, 0xAAC0}, + {0xAAC2, 0xAAC2}, {0xAADB, 0xAADD}, {0xAAE0, 0xAAEA}, {0xAAF2, 0xAAF4}, {0xAB01, 0xAB06}, + {0xAB09, 0xAB0E}, {0xAB11, 0xAB16}, {0xAB20, 0xAB26}, {0xAB28, 0xAB2E}, {0xAB30, 0xAB5A}, + {0xAB5C, 0xAB69}, {0xAB70, 0xABE2}, {0xABF0, 0xABF9}, {0xAC00, 0xD7A3}, {0xD7B0, 0xD7C6}, + {0xD7CB, 0xD7FB}, {0xF900, 0xFA6D}, {0xFA70, 0xFAD9}, {0xFB00, 0xFB06}, {0xFB13, 0xFB17}, + {0xFB1D, 0xFB1D}, {0xFB1F, 0xFB28}, {0xFB2A, 0xFB36}, {0xFB38, 0xFB3C}, {0xFB3E, 0xFB3E}, + {0xFB40, 0xFB41}, {0xFB43, 0xFB44}, {0xFB46, 0xFBB1}, {0xFBD3, 0xFD3D}, {0xFD50, 0xFD8F}, + {0xFD92, 0xFDC7}, {0xFDF0, 0xFDFB}, {0xFE70, 0xFE74}, {0xFE76, 0xFEFC}, {0xFF10, 0xFF19}, + {0xFF21, 0xFF3A}, {0xFF41, 0xFF5A}, {0xFF66, 0xFFBE}, {0xFFC2, 0xFFC7}, {0xFFCA, 0xFFCF}, + {0xFFD2, 0xFFD7}, {0xFFDA, 0xFFDC}, {0x10000, 0x1000B}, {0x1000D, 0x10026}, {0x10028, 0x1003A}, + {0x1003C, 0x1003D}, {0x1003F, 0x1004D}, {0x10050, 0x1005D}, {0x10080, 0x100FA}, {0x10107, 0x10133}, + {0x10140, 0x10178}, {0x1018A, 0x1018B}, {0x10280, 0x1029C}, {0x102A0, 0x102D0}, {0x102E1, 0x102FB}, + {0x10300, 0x10323}, {0x1032D, 0x1034A}, {0x10350, 0x10375}, {0x10380, 0x1039D}, {0x103A0, 0x103C3}, + {0x103C8, 0x103CF}, {0x103D1, 0x103D5}, {0x10400, 0x1049D}, {0x104A0, 0x104A9}, {0x104B0, 0x104D3}, + {0x104D8, 0x104FB}, {0x10500, 0x10527}, {0x10530, 0x10563}, {0x10570, 0x1057A}, {0x1057C, 0x1058A}, + {0x1058C, 0x10592}, {0x10594, 0x10595}, {0x10597, 0x105A1}, {0x105A3, 0x105B1}, {0x105B3, 0x105B9}, + {0x105BB, 0x105BC}, {0x10600, 0x10736}, {0x10740, 0x10755}, {0x10760, 0x10767}, {0x10780, 0x10785}, + {0x10787, 0x107B0}, {0x107B2, 0x107BA}, {0x10800, 0x10805}, {0x10808, 0x10808}, {0x1080A, 0x10835}, + {0x10837, 0x10838}, {0x1083C, 0x1083C}, {0x1083F, 0x10855}, {0x10858, 0x10876}, {0x10879, 0x1089E}, + {0x108A7, 0x108AF}, {0x108E0, 0x108F2}, {0x108F4, 0x108F5}, {0x108FB, 0x1091B}, {0x10920, 0x10939}, + {0x10980, 0x109B7}, {0x109BC, 0x109CF}, {0x109D2, 0x10A00}, {0x10A10, 0x10A13}, {0x10A15, 0x10A17}, + {0x10A19, 0x10A35}, {0x10A40, 0x10A48}, {0x10A60, 0x10A7E}, {0x10A80, 0x10A9F}, {0x10AC0, 0x10AC7}, + {0x10AC9, 0x10AE4}, {0x10AEB, 0x10AEF}, {0x10B00, 0x10B35}, {0x10B40, 0x10B55}, {0x10B58, 0x10B72}, + {0x10B78, 0x10B91}, {0x10BA9, 0x10BAF}, {0x10C00, 0x10C48}, {0x10C80, 0x10CB2}, {0x10CC0, 0x10CF2}, + {0x10CFA, 0x10D23}, {0x10D30, 0x10D39}, {0x10E60, 0x10E7E}, {0x10E80, 0x10EA9}, {0x10EB0, 0x10EB1}, + {0x10F00, 0x10F27}, {0x10F30, 0x10F45}, {0x10F51, 0x10F54}, {0x10F70, 0x10F81}, {0x10FB0, 0x10FCB}, + {0x10FE0, 0x10FF6}, {0x11003, 0x11037}, {0x11052, 0x1106F}, {0x11071, 0x11072}, {0x11075, 0x11075}, + {0x11083, 0x110AF}, {0x110D0, 0x110E8}, {0x110F0, 0x110F9}, {0x11103, 0x11126}, {0x11136, 0x1113F}, + {0x11144, 0x11144}, {0x11147, 0x11147}, {0x11150, 0x11172}, {0x11176, 0x11176}, {0x11183, 0x111B2}, + {0x111C1, 0x111C4}, {0x111D0, 0x111DA}, {0x111DC, 0x111DC}, {0x111E1, 0x111F4}, {0x11200, 0x11211}, + {0x11213, 0x1122B}, {0x1123F, 0x11240}, {0x11280, 0x11286}, {0x11288, 0x11288}, {0x1128A, 0x1128D}, + {0x1128F, 0x1129D}, {0x1129F, 0x112A8}, {0x112B0, 0x112DE}, {0x112F0, 0x112F9}, {0x11305, 0x1130C}, + {0x1130F, 0x11310}, {0x11313, 0x11328}, {0x1132A, 0x11330}, {0x11332, 0x11333}, {0x11335, 0x11339}, + {0x1133D, 0x1133D}, {0x11350, 0x11350}, {0x1135D, 0x11361}, {0x11400, 0x11434}, {0x11447, 0x1144A}, + {0x11450, 0x11459}, {0x1145F, 0x11461}, {0x11480, 0x114AF}, {0x114C4, 0x114C5}, {0x114C7, 0x114C7}, + {0x114D0, 0x114D9}, {0x11580, 0x115AE}, {0x115D8, 0x115DB}, {0x11600, 0x1162F}, {0x11644, 0x11644}, + {0x11650, 0x11659}, {0x11680, 0x116AA}, {0x116B8, 0x116B8}, {0x116C0, 0x116C9}, {0x11700, 0x1171A}, + {0x11730, 0x1173B}, {0x11740, 0x11746}, {0x11800, 0x1182B}, {0x118A0, 0x118F2}, {0x118FF, 0x11906}, + {0x11909, 0x11909}, {0x1190C, 0x11913}, {0x11915, 0x11916}, {0x11918, 0x1192F}, {0x1193F, 0x1193F}, + {0x11941, 0x11941}, {0x11950, 0x11959}, {0x119A0, 0x119A7}, {0x119AA, 0x119D0}, {0x119E1, 0x119E1}, + {0x119E3, 0x119E3}, {0x11A00, 0x11A00}, {0x11A0B, 0x11A32}, {0x11A3A, 0x11A3A}, {0x11A50, 0x11A50}, + {0x11A5C, 0x11A89}, {0x11A9D, 0x11A9D}, {0x11AB0, 0x11AF8}, {0x11C00, 0x11C08}, {0x11C0A, 0x11C2E}, + {0x11C40, 0x11C40}, {0x11C50, 0x11C6C}, {0x11C72, 0x11C8F}, {0x11D00, 0x11D06}, {0x11D08, 0x11D09}, + {0x11D0B, 0x11D30}, {0x11D46, 0x11D46}, {0x11D50, 0x11D59}, {0x11D60, 0x11D65}, {0x11D67, 0x11D68}, + {0x11D6A, 0x11D89}, {0x11D98, 0x11D98}, {0x11DA0, 0x11DA9}, {0x11EE0, 0x11EF2}, {0x11F02, 0x11F02}, + {0x11F04, 0x11F10}, {0x11F12, 0x11F33}, {0x11F50, 0x11F59}, {0x11FB0, 0x11FB0}, {0x11FC0, 0x11FD4}, + {0x12000, 0x12399}, {0x12400, 0x1246E}, {0x12480, 0x12543}, {0x12F90, 0x12FF0}, {0x13000, 0x1342F}, + {0x13441, 0x13446}, {0x14400, 0x14646}, {0x16800, 0x16A38}, {0x16A40, 0x16A5E}, {0x16A60, 0x16A69}, + {0x16A70, 0x16ABE}, {0x16AC0, 0x16AC9}, {0x16AD0, 0x16AED}, {0x16B00, 0x16B2F}, {0x16B40, 0x16B43}, + {0x16B50, 0x16B59}, {0x16B5B, 0x16B61}, {0x16B63, 0x16B77}, {0x16B7D, 0x16B8F}, {0x16E40, 0x16E96}, + {0x16F00, 0x16F4A}, {0x16F50, 0x16F50}, {0x16F93, 0x16F9F}, {0x16FE0, 0x16FE1}, {0x16FE3, 0x16FE3}, + {0x17000, 0x187F7}, {0x18800, 0x18CD5}, {0x18D00, 0x18D08}, {0x1AFF0, 0x1AFF3}, {0x1AFF5, 0x1AFFB}, + {0x1AFFD, 0x1AFFE}, {0x1B000, 0x1B122}, {0x1B132, 0x1B132}, {0x1B150, 0x1B152}, {0x1B155, 0x1B155}, + {0x1B164, 0x1B167}, {0x1B170, 0x1B2FB}, {0x1BC00, 0x1BC6A}, {0x1BC70, 0x1BC7C}, {0x1BC80, 0x1BC88}, + {0x1BC90, 0x1BC99}, {0x1D2C0, 0x1D2D3}, {0x1D2E0, 0x1D2F3}, {0x1D360, 0x1D378}, {0x1D400, 0x1D454}, + {0x1D456, 0x1D49C}, {0x1D49E, 0x1D49F}, {0x1D4A2, 0x1D4A2}, {0x1D4A5, 0x1D4A6}, {0x1D4A9, 0x1D4AC}, + {0x1D4AE, 0x1D4B9}, {0x1D4BB, 0x1D4BB}, {0x1D4BD, 0x1D4C3}, {0x1D4C5, 0x1D505}, {0x1D507, 0x1D50A}, + {0x1D50D, 0x1D514}, {0x1D516, 0x1D51C}, {0x1D51E, 0x1D539}, {0x1D53B, 0x1D53E}, {0x1D540, 0x1D544}, + {0x1D546, 0x1D546}, {0x1D54A, 0x1D550}, {0x1D552, 0x1D6A5}, {0x1D6A8, 0x1D6C0}, {0x1D6C2, 0x1D6DA}, + {0x1D6DC, 0x1D6FA}, {0x1D6FC, 0x1D714}, {0x1D716, 0x1D734}, {0x1D736, 0x1D74E}, {0x1D750, 0x1D76E}, + {0x1D770, 0x1D788}, {0x1D78A, 0x1D7A8}, {0x1D7AA, 0x1D7C2}, {0x1D7C4, 0x1D7CB}, {0x1D7CE, 0x1D7FF}, + {0x1DF00, 0x1DF1E}, {0x1DF25, 0x1DF2A}, {0x1E030, 0x1E06D}, {0x1E100, 0x1E12C}, {0x1E137, 0x1E13D}, + {0x1E140, 0x1E149}, {0x1E14E, 0x1E14E}, {0x1E290, 0x1E2AD}, {0x1E2C0, 0x1E2EB}, {0x1E2F0, 0x1E2F9}, + {0x1E4D0, 0x1E4EB}, {0x1E4F0, 0x1E4F9}, {0x1E7E0, 0x1E7E6}, {0x1E7E8, 0x1E7EB}, {0x1E7ED, 0x1E7EE}, + {0x1E7F0, 0x1E7FE}, {0x1E800, 0x1E8C4}, {0x1E8C7, 0x1E8CF}, {0x1E900, 0x1E943}, {0x1E94B, 0x1E94B}, + {0x1E950, 0x1E959}, {0x1EC71, 0x1ECAB}, {0x1ECAD, 0x1ECAF}, {0x1ECB1, 0x1ECB4}, {0x1ED01, 0x1ED2D}, + {0x1ED2F, 0x1ED3D}, {0x1EE00, 0x1EE03}, {0x1EE05, 0x1EE1F}, {0x1EE21, 0x1EE22}, {0x1EE24, 0x1EE24}, + {0x1EE27, 0x1EE27}, {0x1EE29, 0x1EE32}, {0x1EE34, 0x1EE37}, {0x1EE39, 0x1EE39}, {0x1EE3B, 0x1EE3B}, + {0x1EE42, 0x1EE42}, {0x1EE47, 0x1EE47}, {0x1EE49, 0x1EE49}, {0x1EE4B, 0x1EE4B}, {0x1EE4D, 0x1EE4F}, + {0x1EE51, 0x1EE52}, {0x1EE54, 0x1EE54}, {0x1EE57, 0x1EE57}, {0x1EE59, 0x1EE59}, {0x1EE5B, 0x1EE5B}, + {0x1EE5D, 0x1EE5D}, {0x1EE5F, 0x1EE5F}, {0x1EE61, 0x1EE62}, {0x1EE64, 0x1EE64}, {0x1EE67, 0x1EE6A}, + {0x1EE6C, 0x1EE72}, {0x1EE74, 0x1EE77}, {0x1EE79, 0x1EE7C}, {0x1EE7E, 0x1EE7E}, {0x1EE80, 0x1EE89}, + {0x1EE8B, 0x1EE9B}, {0x1EEA1, 0x1EEA3}, {0x1EEA5, 0x1EEA9}, {0x1EEAB, 0x1EEBB}, {0x1F100, 0x1F10C}, + {0x1FBF0, 0x1FBF9}, {0x20000, 0x2A6DF}, {0x2A700, 0x2B739}, {0x2B740, 0x2B81D}, {0x2B820, 0x2CEA1}, + {0x2CEB0, 0x2EBE0}, {0x2F800, 0x2FA1D}, {0x30000, 0x3134A}, {0x31350, 0x323AF}, }}; inline bool isTokenChar(std::uint32_t code_point) diff --git a/dbms/src/Functions/tests/gtest_local_match_against.cpp b/dbms/src/Functions/tests/gtest_local_match_against.cpp index 81e03bfdcfc..d14e62b2388 100644 --- a/dbms/src/Functions/tests/gtest_local_match_against.cpp +++ b/dbms/src/Functions/tests/gtest_local_match_against.cpp @@ -189,10 +189,11 @@ try const auto [actions, names] = buildLocalMatchDAGActions(*context, inputs, {expr}); Block block(columns); actions->execute(block); - const auto expected = prefix ? (binary ? createColumn>({0, 1, 0, 1, 0, 1}) - : createColumn>({1, 1, 0, 1, 0, 0})) - : (binary ? createColumn>({0, 0, 0, 0, 0, 1}) - : createColumn>({1, 1, 0, 0, 0, 0})); + const auto expected = prefix + ? (binary ? createColumn>({0, 1, 0, 1, 0, 1}) + : createColumn>({1, 1, 0, 1, 0, 0})) + : (binary ? createColumn>({0, 0, 0, 0, 0, 1}) + : createColumn>({1, 1, 0, 0, 0, 0})); ASSERT_COLUMN_EQ(expected, block.getByName(names[0])); } } @@ -280,19 +281,19 @@ try const auto first = ngram ? BlockCase{{createColumn>({"中文测试", "中文", "中文测试", "", {}}, "title"), createColumn>({{}, "测试", "mysql", "中文测试", {}}, "body")}, createColumn>({1, 0, 0, 1, 0})} : BlockCase{ - {createColumn>( - {"QUICK the fox", "quick the fox", "quick", {}, "quick the fox"}, - "title"), - createColumn>({{}, "mysql", "the fox", {}, "MYSQL"}, "body")}, - createColumn>({1, 0, 0, 0, 0})}; + {createColumn>( + {"QUICK the fox", "quick the fox", "quick", {}, "quick the fox"}, + "title"), + createColumn>({{}, "mysql", "the fox", {}, "MYSQL"}, "body")}, + createColumn>({1, 0, 0, 0, 0})}; const auto second = ngram ? BlockCase{{createColumn>({"中文 test", {}, "中文测试", "中 文测试"}, "title"), createColumn>({"", {}, "", "中文测试"}, "body")}, createColumn>({0, 0, 1, 1})} : BlockCase{ - {createColumn>( - {"slow the fox", "quick the fox", {}, "quick fox", "quick the fox"}, - "title"), - createColumn>({"", {}, "quick the fox", "", {}}, "body")}, - createColumn>({0, 1, 1, 0, 1})}; + {createColumn>( + {"slow the fox", "quick the fox", {}, "quick fox", "quick the fox"}, + "title"), + createColumn>({"", {}, "quick the fox", "", {}}, "body")}, + createColumn>({0, 1, 1, 0, 1})}; const BlockCase nulls{ {createColumn>({{}, {}}, "title"), createColumn>({{}, {}}, "body")}, createColumn>({0, 0})}; From cd0b66dc1639393a349e433b981ac96029984296 Mon Sep 17 00:00:00 2001 From: Zhigao Tong Date: Tue, 13 Oct 2026 09:25:20 +0800 Subject: [PATCH 28/32] fix: resolve Local MATCH clang-tidy errors --- .../Coprocessor/DAGExpressionAnalyzerHelper.cpp | 2 +- dbms/src/Functions/FunctionsLocalMatchAgainst.cpp | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/dbms/src/Flash/Coprocessor/DAGExpressionAnalyzerHelper.cpp b/dbms/src/Flash/Coprocessor/DAGExpressionAnalyzerHelper.cpp index 45ce61a1df0..82800b83869 100644 --- a/dbms/src/Flash/Coprocessor/DAGExpressionAnalyzerHelper.cpp +++ b/dbms/src/Flash/Coprocessor/DAGExpressionAnalyzerHelper.cpp @@ -328,7 +328,7 @@ String DAGExpressionAnalyzerHelper::buildSingleParamJsonRelatedFunctions( function_json_unquote) { bool valid_check - = !(isScalarFunctionExpr(input_expr) && input_expr.sig() == tipb::ScalarFuncSig::CastJsonAsString); + = !isScalarFunctionExpr(input_expr) || input_expr.sig() != tipb::ScalarFuncSig::CastJsonAsString; function_json_unquote->setNeedValidCheck(valid_check); } else if (auto * function_cast_json_as_string = dynamic_cast(function_impl); diff --git a/dbms/src/Functions/FunctionsLocalMatchAgainst.cpp b/dbms/src/Functions/FunctionsLocalMatchAgainst.cpp index ef2db5073e6..343e77f9df3 100644 --- a/dbms/src/Functions/FunctionsLocalMatchAgainst.cpp +++ b/dbms/src/Functions/FunctionsLocalMatchAgainst.cpp @@ -73,7 +73,7 @@ struct FullTextToken size_t position = 0; size_t code_points = 0; - std::string_view value() const { return source.data() ? source : std::string_view(text); } + std::string_view value() const { return source.data() ? source : std::string_view{text}; } }; class FullTextColumn @@ -733,7 +733,7 @@ ColumnMatchResult updateBooleanMatchStatesForColumn( for (auto & state : states) state.phrase_verification_failed = false; - const auto markClauseMatched = [&](size_t clause_index) { + const auto mark_clause_matched = [&](size_t clause_index) { states[clause_index].matched = true; switch (clauses[clause_index].modifier) { @@ -778,7 +778,7 @@ ColumnMatchResult updateBooleanMatchStatesForColumn( state.phrase_verification_failed = true; continue; } - const auto result = markClauseMatched(clause_index); + const auto result = mark_clause_matched(clause_index); if (result != ColumnMatchResult::Continue) return result; } @@ -799,7 +799,7 @@ ColumnMatchResult updateBooleanMatchStatesForColumn( : state.remaining_terms < clause.terms.size(); if (state.matched) { - const auto result = markClauseMatched(clause_index); + const auto result = mark_clause_matched(clause_index); if (result != ColumnMatchResult::Continue) return result; } @@ -1198,7 +1198,7 @@ class FunctionLocalMatchAgainstBoolean final : public IFunction const size_t min_token_size = config.min_token_size; const size_t max_token_size = config.max_token_size; const bool enable_stopword = config.enable_stopword; - const auto stopword_collator + const auto * const stopword_collator = enable_stopword ? TiDB::ITiDBCollator::getCollator(protocol_boolean_query.stopword_collation()) : nullptr; if (enable_stopword && !stopword_collator) throw Exception( @@ -1289,7 +1289,7 @@ class FunctionLocalMatchAgainstBoolean final : public IFunction } private: - TiDB::TiDBCollatorPtr collator; + TiDB::TiDBCollatorPtr collator = nullptr; }; } // namespace From ade972a4837b93331679b1cb956f12ade815cddb Mon Sep 17 00:00:00 2001 From: Zhigao Tong Date: Tue, 13 Oct 2026 10:18:36 +0800 Subject: [PATCH 29/32] fix: accept untyped NULL Local MATCH searches --- dbms/src/Functions/FunctionsLocalMatchAgainst.cpp | 12 ++++++++---- .../Functions/tests/gtest_local_match_against.cpp | 2 +- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/dbms/src/Functions/FunctionsLocalMatchAgainst.cpp b/dbms/src/Functions/FunctionsLocalMatchAgainst.cpp index 343e77f9df3..e36e551ce68 100644 --- a/dbms/src/Functions/FunctionsLocalMatchAgainst.cpp +++ b/dbms/src/Functions/FunctionsLocalMatchAgainst.cpp @@ -1141,9 +1141,14 @@ class FunctionLocalMatchAgainstBoolean final : public IFunction if (arguments.size() < 3) throw Exception("local_match_against_boolean requires a query, at least one column, and metadata"); bool nullable = false; - for (const auto & argument : arguments) + for (size_t i = 0; i < arguments.size(); ++i) { - if (!removeNullable(argument)->isString()) + const auto & argument = arguments[i]; + const auto nested_type = removeNullable(argument); + // An untyped SQL NULL search arrives as Nullable(Nothing). + // Documents and metadata still require string types. + const bool null_query = i == 0 && nested_type->getTypeId() == TypeIndex::Nothing; + if (!nested_type->isString() && !null_query) throw Exception( "Illegal type " + argument->getName() + " of argument of function " + getName(), ErrorCodes::ILLEGAL_COLUMN); @@ -1173,7 +1178,6 @@ class FunctionLocalMatchAgainstBoolean final : public IFunction block.getByPosition(result).column = std::move(result_column); }; - const FullTextColumnAccessor query_accessor(*query_column); tipb::LocalMatchAgainstBooleanQuery protocol_boolean_query; if (arguments.size() <= 2 || !decodeLocalMatchAgainstBooleanQuery( @@ -1182,7 +1186,7 @@ class FunctionLocalMatchAgainstBoolean final : public IFunction throw Exception( "local_match_against_boolean requires valid Boolean query metadata", ErrorCodes::ILLEGAL_COLUMN); - if (query_accessor.isNull(0)) + if (query_column->isNullAt(0)) { // InnoDB interprets NULL AGAINST as an empty search: non-NULL zero. finish_result(); diff --git a/dbms/src/Functions/tests/gtest_local_match_against.cpp b/dbms/src/Functions/tests/gtest_local_match_against.cpp index d14e62b2388..d48f6ef8b99 100644 --- a/dbms/src/Functions/tests/gtest_local_match_against.cpp +++ b/dbms/src/Functions/tests/gtest_local_match_against.cpp @@ -123,7 +123,6 @@ tipb::Expr makeLocalMatchDAGExpr( || std::any_of(inputs.begin(), inputs.end(), [](const auto & input) { return input.type->isNullable(); }); result_type->set_flag(nullable ? 0 : TiDB::ColumnFlagNotNull); auto search_expr = null_search ? constructNULLLiteralTiExpr() : constructStringLiteralTiExpr(search); - search_expr.mutable_field_type()->set_tp(TiDB::TypeString); search_expr.mutable_field_type()->set_collate(-collation); *expr.add_children() = std::move(search_expr); for (size_t i = 0; i < inputs.size(); ++i) @@ -347,6 +346,7 @@ try const NamesAndTypes inputs{{"body", column_case.document.type}}; const auto expr = makeLocalMatchDAGExpr("+quick", query, inputs, TiDB::ITiDBCollator::UTF8MB4_BIN, null_search); + ASSERT_EQ(expr.children(0).field_type().tp(), null_search ? TiDB::TypeNull : TiDB::TypeString); const auto [actions, names] = buildLocalMatchDAGActions(*context, inputs, {expr}); const auto expected = null_search ? createColumn>({0, 0, 0, 0}) : column_case.expected; for (const size_t rows : {4U, 0U, 4U}) From c55c74cbdecdc4222028686e866485a6816d2b1e Mon Sep 17 00:00:00 2001 From: Zhigao Tong Date: Tue, 13 Oct 2026 15:15:40 +0800 Subject: [PATCH 30/32] fix: update image references to require environment variables for Local MATCH services Signed-off-by: Zhigao Tong --- tests/docker/cluster.yaml | 6 +++--- tests/docker/cluster_disable_new_collation.yaml | 6 +++--- tests/docker/cluster_new_collation.yaml | 6 +++--- tests/docker/cluster_tidb_fail_point.yaml | 6 +++--- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/docker/cluster.yaml b/tests/docker/cluster.yaml index dd2f1dbf33f..0f727a7cf9c 100644 --- a/tests/docker/cluster.yaml +++ b/tests/docker/cluster.yaml @@ -16,7 +16,7 @@ version: '2.3' services: pd0: - image: hub.pingcap.net/qa/pd:${PD_BRANCH:-master} + image: ${PD_IMAGE:?PD_IMAGE must be set} security_opt: - seccomp:unconfined volumes: @@ -35,7 +35,7 @@ services: - --log-file=/log/pd.log restart: on-failure tikv0: - image: hub.pingcap.net/qa/tikv:${TIKV_BRANCH:-master} + image: ${TIKV_IMAGE:?TIKV_IMAGE must be set} security_opt: - seccomp:unconfined volumes: @@ -55,7 +55,7 @@ services: - "pd0" restart: on-failure tidb0: - image: hub.pingcap.net/qa/tidb:${TIDB_BRANCH:-master} + image: ${TIDB_IMAGE:?TIDB_IMAGE must be set} security_opt: - seccomp:unconfined volumes: diff --git a/tests/docker/cluster_disable_new_collation.yaml b/tests/docker/cluster_disable_new_collation.yaml index 8dd13e2c0a8..ef79caf2995 100644 --- a/tests/docker/cluster_disable_new_collation.yaml +++ b/tests/docker/cluster_disable_new_collation.yaml @@ -16,7 +16,7 @@ version: '2.3' services: pd0: - image: hub.pingcap.net/qa/pd:${PD_BRANCH:-master} + image: ${PD_IMAGE:?PD_IMAGE must be set} security_opt: - seccomp:unconfined volumes: @@ -35,7 +35,7 @@ services: - --log-file=/log/pd.log restart: on-failure tikv0: - image: hub.pingcap.net/qa/tikv:${TIKV_BRANCH:-master} + image: ${TIKV_IMAGE:?TIKV_IMAGE must be set} security_opt: - seccomp:unconfined volumes: @@ -55,7 +55,7 @@ services: - "pd0" restart: on-failure tidb0: - image: hub.pingcap.net/qa/tidb:${TIDB_BRANCH:-master} + image: ${TIDB_IMAGE:?TIDB_IMAGE must be set} security_opt: - seccomp:unconfined volumes: diff --git a/tests/docker/cluster_new_collation.yaml b/tests/docker/cluster_new_collation.yaml index 153b40f327b..17a2ebf207d 100644 --- a/tests/docker/cluster_new_collation.yaml +++ b/tests/docker/cluster_new_collation.yaml @@ -16,7 +16,7 @@ version: '2.3' services: pd0: - image: hub.pingcap.net/qa/pd:${PD_BRANCH:-master} + image: ${PD_IMAGE:?PD_IMAGE must be set} security_opt: - seccomp:unconfined volumes: @@ -35,7 +35,7 @@ services: - --log-file=/log/pd.log restart: on-failure tikv0: - image: hub.pingcap.net/qa/tikv:${TIKV_BRANCH:-master} + image: ${TIKV_IMAGE:?TIKV_IMAGE must be set} security_opt: - seccomp:unconfined volumes: @@ -55,7 +55,7 @@ services: - "pd0" restart: on-failure tidb0: - image: hub.pingcap.net/qa/tidb:${TIDB_BRANCH:-master} + image: ${TIDB_IMAGE:?TIDB_IMAGE must be set} security_opt: - seccomp:unconfined volumes: diff --git a/tests/docker/cluster_tidb_fail_point.yaml b/tests/docker/cluster_tidb_fail_point.yaml index faec3c48898..c5817670086 100644 --- a/tests/docker/cluster_tidb_fail_point.yaml +++ b/tests/docker/cluster_tidb_fail_point.yaml @@ -16,7 +16,7 @@ version: '2.3' services: pd0: - image: hub.pingcap.net/qa/pd:${PD_BRANCH:-master} + image: ${PD_IMAGE:?PD_IMAGE must be set} security_opt: - seccomp:unconfined volumes: @@ -35,7 +35,7 @@ services: - --log-file=/log/pd.log restart: on-failure tikv0: - image: hub.pingcap.net/qa/tikv:${TIKV_BRANCH:-master} + image: ${TIKV_IMAGE:?TIKV_IMAGE must be set} security_opt: - seccomp:unconfined volumes: @@ -55,7 +55,7 @@ services: - "pd0" restart: on-failure tidb0: - image: hub.pingcap.net/qa/tidb:${TIDB_BRANCH:-master}-failpoint + image: ${TIDB_IMAGE:?TIDB_IMAGE must be set}-failpoint security_opt: - seccomp:unconfined environment: From 7c71fbe9aff8eec40c33c5878a7d03b265a8359f Mon Sep 17 00:00:00 2001 From: Zhigao Tong Date: Wed, 14 Oct 2026 16:53:34 +0800 Subject: [PATCH 31/32] deps: use official Local MATCH tipb protocol --- contrib/tipb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/tipb b/contrib/tipb index f46c2bcf8ac..4044aac591d 160000 --- a/contrib/tipb +++ b/contrib/tipb @@ -1 +1 @@ -Subproject commit f46c2bcf8ac205db6fc2781537c93ead09f59a3b +Subproject commit 4044aac591d778250cad3488dfd6e6f574411f1a From 71904a207d0f4b11f59880b8b15cfc277735b8b0 Mon Sep 17 00:00:00 2001 From: Zhigao Tong Date: Thu, 15 Oct 2026 15:48:17 +0800 Subject: [PATCH 32/32] fix: reject invalid Local MATCH Boolean node metadata --- .../Functions/FunctionsLocalMatchAgainst.cpp | 19 ++++- .../Functions/LocalMatchAgainstTokenChars.h | 5 ++ .../tests/gtest_local_match_against.cpp | 77 +++++++++++++++++++ 3 files changed, 97 insertions(+), 4 deletions(-) diff --git a/dbms/src/Functions/FunctionsLocalMatchAgainst.cpp b/dbms/src/Functions/FunctionsLocalMatchAgainst.cpp index e36e551ce68..39f827553b2 100644 --- a/dbms/src/Functions/FunctionsLocalMatchAgainst.cpp +++ b/dbms/src/Functions/FunctionsLocalMatchAgainst.cpp @@ -879,8 +879,7 @@ CompiledBooleanQuery compileBooleanQuery( clause.modifier = BooleanClause::Modifier::Should; break; default: - compiled.matches_nothing = true; - return compiled; + throw Exception("Invalid Local MATCH Boolean node occurrence", ErrorCodes::ILLEGAL_COLUMN); } switch (node.term_type()) @@ -981,8 +980,7 @@ CompiledBooleanQuery compileBooleanQuery( break; } default: - compiled.matches_nothing = true; - return compiled; + throw Exception("Invalid Local MATCH Boolean node term type", ErrorCodes::ILLEGAL_COLUMN); } if (clause.phrase && !clause.terms.empty()) @@ -1116,6 +1114,19 @@ bool decodeLocalMatchAgainstBooleanQuery(const IColumn & column, tipb::LocalMatc if (query.stopword_mode() != tipb::LocalMatchAgainstStopwordMode::LocalMatchAgainstStopwordModeDisabled && query.stopword_mode() != tipb::LocalMatchAgainstStopwordMode::LocalMatchAgainstStopwordModeBuiltin) return false; + // Validate every node before NULL-search or analyzer no-match shortcuts. + // Missing/unknown proto2 enum values read as Invalid, not as valid clauses. + for (const auto & node : query.nodes()) + { + if (node.occur() != tipb::LocalMatchAgainstBooleanOccurMust + && node.occur() != tipb::LocalMatchAgainstBooleanOccurMustNot + && node.occur() != tipb::LocalMatchAgainstBooleanOccurShould) + return false; + if (node.term_type() != tipb::LocalMatchAgainstBooleanTermWord + && node.term_type() != tipb::LocalMatchAgainstBooleanTermPrefix + && node.term_type() != tipb::LocalMatchAgainstBooleanTermPhrase) + return false; + } return true; } diff --git a/dbms/src/Functions/LocalMatchAgainstTokenChars.h b/dbms/src/Functions/LocalMatchAgainstTokenChars.h index c97dc312f74..ec2d3987c7c 100644 --- a/dbms/src/Functions/LocalMatchAgainstTokenChars.h +++ b/dbms/src/Functions/LocalMatchAgainstTokenChars.h @@ -23,6 +23,11 @@ namespace DB::LocalMatchAgainst { // Generated from Go's Unicode 15.0.0 categories L and N. TiDB uses its standard // library and strictly tests that Unicode version and the Unicode 15.0.0 checksum. +// Equivalent Go predicate: +// word := r <= unicode.MaxRune && +// (unicode.IsLetter(r) || unicode.IsNumber(r) || r == '_') +// This helper covers all Unicode code points; the Boolean lexer separately +// restricts word characters to the BMP (r <= 0xFFFF). // Regenerate/check this header using the paired TiDB generator. Never change // an existing protocol version's classification; deploy TiFlash support for a // new version before TiDB emits it. ASCII and '_' use the fast path below. diff --git a/dbms/src/Functions/tests/gtest_local_match_against.cpp b/dbms/src/Functions/tests/gtest_local_match_against.cpp index d48f6ef8b99..c90d9968248 100644 --- a/dbms/src/Functions/tests/gtest_local_match_against.cpp +++ b/dbms/src/Functions/tests/gtest_local_match_against.cpp @@ -25,6 +25,11 @@ #include #include +namespace DB::ErrorCodes +{ +extern const int ILLEGAL_COLUMN; +} + namespace DB::tests { class TestLocalMatchAgainst : public DB::tests::FunctionTest @@ -911,6 +916,78 @@ try } CATCH +TEST_F(TestLocalMatchAgainst, MatchBooleanRejectsInvalidNodeEnums) +try +{ + enum class InvalidEnumEncoding + { + Missing, + Invalid, + Unknown, + }; + for (const bool invalid_occur : {false, true}) + for (const auto encoding : + {InvalidEnumEncoding::Missing, InvalidEnumEncoding::Invalid, InvalidEnumEncoding::Unknown}) + { + auto query = makeRequiredLocalMatchQuery("quick"); + query.set_version(2); + query.set_parser(tipb::LocalMatchAgainstParserStandard); + query.set_stopword_mode(tipb::LocalMatchAgainstStopwordModeDisabled); + // Put the malformed node after a valid one: all nodes must be checked. + auto * node = query.add_nodes(); + node->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + node->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + node->set_text("fox"); + if (invalid_occur) + { + node->clear_occur(); + if (encoding == InvalidEnumEncoding::Invalid) + node->set_occur(tipb::LocalMatchAgainstBooleanOccurInvalid); + } + else + { + node->clear_term_type(); + if (encoding == InvalidEnumEncoding::Invalid) + node->set_term_type(tipb::LocalMatchAgainstBooleanTermInvalid); + } + // Encode unknown values on the wire; generated enum setters may + // assert rather than allow an out-of-range value in debug builds. + if (encoding == InvalidEnumEncoding::Unknown) + ASSERT_TRUE(node->MergeFromString(invalid_occur ? "\x08\x7f" : "\x10\x7f")); + + for (const bool empty_analyzer : {false, true}) + for (const bool null_search : {false, true}) + { + SCOPED_TRACE(fmt::format( + "invalid_occur={} encoding={} empty_analyzer={} null_search={}", + invalid_occur, + static_cast(encoding), + empty_analyzer, + null_search)); + query.set_innodb_ft_max_token_size(empty_analyzer ? 0 : 84); + auto search = createConstColumn(1, "+quick"); + if (null_search) + search = createOnlyNullColumnConst(1); + try + { + executeFunction( + "local_match_against_boolean", + {search, + createColumn({"quick fox"}), + createConstColumn(1, query.SerializeAsString())}, + nullptr, + true); + FAIL() << "Invalid Local MATCH node metadata must be rejected"; + } + catch (const Exception & error) + { + ASSERT_EQ(ErrorCodes::ILLEGAL_COLUMN, error.code()); + } + } + } +} +CATCH + TEST_F(TestLocalMatchAgainst, MatchBooleanStandardAnalyzerProtocolSettings) try {