diff --git a/contrib/tipb b/contrib/tipb index 0607513e7fa..4044aac591d 160000 --- a/contrib/tipb +++ b/contrib/tipb @@ -1 +1 @@ -Subproject commit 0607513e7fa40f3b564f0b269da76c1c8dc90032 +Subproject commit 4044aac591d778250cad3488dfd6e6f574411f1a diff --git a/dbms/src/Flash/Coprocessor/DAGExpressionAnalyzerHelper.cpp b/dbms/src/Flash/Coprocessor/DAGExpressionAnalyzerHelper.cpp index 14f5e55c4bb..82800b83869 100644 --- a/dbms/src/Flash/Coprocessor/DAGExpressionAnalyzerHelper.cpp +++ b/dbms/src/Flash/Coprocessor/DAGExpressionAnalyzerHelper.cpp @@ -328,7 +328,7 @@ String DAGExpressionAnalyzerHelper::buildSingleParamJsonRelatedFunctions( function_json_unquote) { bool valid_check - = !(isScalarFunctionExpr(input_expr) && input_expr.sig() == tipb::ScalarFuncSig::CastJsonAsString); + = !isScalarFunctionExpr(input_expr) || input_expr.sig() != tipb::ScalarFuncSig::CastJsonAsString; function_json_unquote->setNeedValidCheck(valid_check); } else if (auto * function_cast_json_as_string = dynamic_cast(function_impl); @@ -520,6 +520,13 @@ String DAGExpressionAnalyzerHelper::buildDefaultFunction( String name = analyzer->getActions(child, actions); argument_names.push_back(name); } + if (expr.sig() == tipb::ScalarFuncSig::LocalMatchAgainstBoolean) + { + // Expr.val carries Local MATCH metadata, not an SQL argument. Adapt it + // to the runtime function's required final constant argument. + const auto metadata_expr = constructStringLiteralTiExpr(expr.val()); + argument_names.push_back(analyzer->getActions(metadata_expr, actions)); + } return analyzer->applyFunction(func_name, argument_names, actions, getCollatorFromExpr(expr)); } diff --git a/dbms/src/Flash/Coprocessor/DAGUtils.cpp b/dbms/src/Flash/Coprocessor/DAGUtils.cpp index efacecc9b93..79abe601636 100644 --- a/dbms/src/Flash/Coprocessor/DAGUtils.cpp +++ b/dbms/src/Flash/Coprocessor/DAGUtils.cpp @@ -455,6 +455,7 @@ const std::unordered_map scalar_func_map({ {tipb::ScalarFuncSig::RegexpInStrSig, "regexp_instr"}, {tipb::ScalarFuncSig::RegexpReplaceSig, "regexp_replace"}, {tipb::ScalarFuncSig::RegexpSubstrSig, "regexp_substr"}, + {tipb::ScalarFuncSig::LocalMatchAgainstBoolean, "local_match_against_boolean"}, {tipb::ScalarFuncSig::JsonExtractSig, "json_extract"}, {tipb::ScalarFuncSig::JsonUnquoteSig, "json_unquote"}, diff --git a/dbms/src/Functions/FunctionsLocalMatchAgainst.cpp b/dbms/src/Functions/FunctionsLocalMatchAgainst.cpp new file mode 100644 index 00000000000..39f827553b2 --- /dev/null +++ b/dbms/src/Functions/FunctionsLocalMatchAgainst.cpp @@ -0,0 +1,1315 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ +namespace ErrorCodes +{ +extern const int ILLEGAL_COLUMN; +} + +namespace +{ +constexpr size_t default_min_token_size = 3; +constexpr size_t default_max_token_size = 84; +constexpr size_t default_ngram_token_size = 2; +// This is the Local MATCH semantic protocol version, not the TiDB/TiFlash +// product version. Version 2 fixes the Boolean AST interpretation, analyzer +// behavior, and built-in stopword set. TiFlash must support a version before +// TiDB emits it; unknown versions are rejected, and existing versions must +// never be reinterpreted with changed semantics. +// Version 2 returns zero for NULL searches and uses MySQL's multibyte NGRAM +// document scanner. STANDARD and the NGRAM Boolean lexer use BMP word +// characters; filtered phrases also verify the unfiltered document stream. +constexpr UInt32 local_match_against_protocol_version = 2; + +struct FullTextToken +{ + // Query terms and the no-collator lowercase path own their spelling. + // Document views refer only to the current input column, which outlives + // both analyzed streams and matching. Never retain them across blocks. + String text; + std::string_view source; + size_t position = 0; + size_t code_points = 0; + + std::string_view value() const { return source.data() ? source : std::string_view{text}; } +}; + +class FullTextColumn +{ +public: + using Container = std::vector; + using iterator = Container::iterator; + using const_iterator = Container::const_iterator; + + void clear() { used = 0; } + + void reserve(size_t size) { tokens.reserve(size); } + + // This is per-token work. Keep it inline even when the surrounding + // analyzer grows: outlining it penalizes the unfiltered/OFF stream too. + ALWAYS_INLINE void append(std::string_view text, size_t position, size_t code_points, bool borrow = false) + { + if (used == tokens.size()) + tokens.emplace_back(); + auto & token = tokens[used++]; + if (borrow) + token.source = text; + else + { + token.source = {}; + token.text.assign(text.data(), text.size()); + } + token.position = position; + token.code_points = code_points; + } + + template + void filterInPlace(Predicate predicate) + { + size_t write = 0; + for (size_t read = 0; read < used; ++read) + { + if (!predicate(tokens[read])) + continue; + if (write != read) + std::swap(tokens[write], tokens[read]); + ++write; + } + used = write; + } + + bool empty() const { return used == 0; } + size_t size() const { return used; } + FullTextToken & front() { return tokens.front(); } + const FullTextToken & front() const { return tokens.front(); } + FullTextToken & operator[](size_t index) { return tokens[index]; } + const FullTextToken & operator[](size_t index) const { return tokens[index]; } + iterator begin() { return tokens.begin(); } + iterator end() { return tokens.begin() + used; } + const_iterator begin() const { return tokens.begin(); } + const_iterator end() const { return tokens.begin() + used; } + +private: + Container tokens; + size_t used = 0; +}; + +struct BooleanClause +{ + enum class Modifier + { + Should, + Must, + MustNot, + }; + + Modifier modifier = Modifier::Should; + bool phrase = false; + bool prefix = false; + std::vector terms; + std::vector offsets; + std::unique_ptr verification; + std::vector> prefix_matchers; +}; + +struct AnalyzerScratch +{ + std::vector char_boundaries; + String lowercase_run; +}; + +struct CompiledBooleanQuery +{ + std::vector clauses; + size_t must_clause_count = 0; + bool has_must_not = false; + bool matches_nothing = false; + bool needs_verification = false; +}; + +// Normalize the already validated wire configuration once per block. Keep +// the tokenizer implementations and their filtered/unfiltered rules intact. +struct LocalMatchAnalyzerConfig +{ + bool use_ngram; + size_t ngram_token_size; + size_t min_token_size; + size_t max_token_size; + bool enable_stopword; + + explicit LocalMatchAnalyzerConfig(const tipb::LocalMatchAgainstBooleanQuery & query) + : use_ngram(query.parser() == tipb::LocalMatchAgainstParserNgram) + , ngram_token_size( + !use_ngram || query.ngram_token_size() == 0 ? default_ngram_token_size : query.ngram_token_size()) + , min_token_size( + query.innodb_ft_min_token_size() != 0 || query.innodb_ft_max_token_size() != 0 + ? query.innodb_ft_min_token_size() + : default_min_token_size) + , max_token_size( + query.innodb_ft_min_token_size() != 0 || query.innodb_ft_max_token_size() != 0 + ? query.innodb_ft_max_token_size() + : default_max_token_size) + , enable_stopword(query.stopword_mode() == tipb::LocalMatchAgainstStopwordModeBuiltin) + {} +}; + +// Query tokens own their text. This cold-path helper only collects the same +// terms and source-relative offsets; document views never enter it. +size_t collectClauseTerms(BooleanClause & clause, const FullTextColumn & tokens) +{ + const size_t first_position = tokens.empty() ? 0 : tokens.front().position; + for (const auto & token : tokens) + { + clause.terms.push_back(token.text); + if (clause.phrase) + clause.offsets.push_back(token.position - first_position); + } + return first_position; +} + +void lowercaseTokenInPlace(String & token) +{ + const bool is_ascii = std::all_of(token.begin(), token.end(), [](unsigned char c) { return c < 0x80; }); + if (is_ascii) + { + for (char & c : token) + if (c >= 'A' && c <= 'Z') + c = static_cast(c + ('a' - 'A')); + return; + } + Poco::UTF8::toLowerInPlace(token); +} + +bool isFullTextToken(UInt32 code_point) +{ + return code_point <= 0xFFFF && LocalMatchAgainst::isTokenChar(code_point); +} + +std::pair decodeCodePoint(std::string_view text, size_t offset) +{ + // Most document bytes are ASCII. Keep the multibyte/invalid-byte + // semantics below, but avoid calling the generic decoder for ASCII. + const auto byte = static_cast(text[offset]); + if (byte < 0x80) + return {byte, 1}; + const auto decoded = UTF8::utf8Decode(text.data() + offset, text.size() - offset); + if (decoded.second == 0 || decoded.first == UTF8::UTF8_Error || decoded.second > text.size() - offset) + return {0xFFFD, 1}; // Go's utf8.DecodeRuneInString also consumes one invalid byte as RuneError. + return {decoded.first, decoded.second}; +} + +template +void scanStandardTokens(std::string_view text, Consumer && consume) +{ + size_t position = 0; + for (size_t i = 0; i < text.size();) + { + const auto [code_point, length] = decodeCodePoint(text, i); + if (!isFullTextToken(code_point)) + { + i += length; + continue; + } + + const size_t token_start = i; + // The first character has already been decoded and classified. + size_t code_points = 1; + i += length; + while (i < text.size()) + { + const auto [code_point, length] = decodeCodePoint(text, i); + if (!isFullTextToken(code_point)) + break; + ++code_points; + i += length; + } + + // Positions include source tokens subsequently removed by filtering. + consume(text.substr(token_start, i - token_start), position++, code_points); + } +} + +void tokenizeTextInto(std::string_view text, FullTextColumn & result, const TiDB::TiDBCollatorPtr & collator = nullptr) +{ + result.clear(); + scanStandardTokens(text, [&](std::string_view token, size_t position, size_t code_points) { + result.append(token, position, code_points); + if (!collator) + lowercaseTokenInPlace(result[result.size() - 1].text); + }); +} + +FullTextColumn tokenizeText(std::string_view text, const TiDB::TiDBCollatorPtr & collator = nullptr) +{ + FullTextColumn result; + tokenizeTextInto(text, result, collator); + return result; +} + +const std::unordered_set & defaultStopwords() +{ + static const std::unordered_set stopwords{ + "a", "about", "an", "are", "as", "at", "be", "by", "com", "de", "en", "for", + "from", "how", "i", "in", "is", "it", "la", "of", "on", "or", "that", "the", + "this", "to", "was", "what", "when", "where", "who", "will", "with", "und", "www"}; + return stopwords; +} + +// Owned collation keys are compiled once per executeImpl, then reused by +// query analysis and every document in the block. The scratch key is local +// to that execution, never shared across concurrent calls. Use the stopword +// (server) collation, not the MATCH column collation. +class BuiltinStopwordLookup +{ +public: + explicit BuiltinStopwordLookup(TiDB::TiDBCollatorPtr collator_) + : collator(collator_) + { + for (const auto & stopword : defaultStopwords()) + { + const auto key = collator->sortKeyFastPath(stopword.data(), stopword.size(), key_buffer); + // sortKeyFastPath may reference its input or key_buffer; copy it. + String owned_key(key.data, key.size); + keys.insert(owned_key); + // The built-in list is ASCII. Preserve NGRAM's code-point length + // buckets even for collations with expansions/ignorable weights. + keys_by_length[stopword.size()].insert(std::move(owned_key)); + } + } + + bool contains(std::string_view token) const { return contains(token, keys); } + + bool contains(std::string_view token, size_t code_points) const + { + const auto bucket = keys_by_length.find(code_points); + return bucket != keys_by_length.end() && contains(token, bucket->second); + } + +private: + struct KeyHash + { + using is_transparent = void; + size_t operator()(std::string_view key) const { return std::hash{}(key); } + }; + using Keys = std::unordered_set>; + + bool contains(std::string_view token, const Keys & lookup) const + { + // sortKey (rather than sortKeyNoTrim) preserves PAD SPACE/NO PAD + // equality, as well as the collator's case/accent semantics. + const auto key = collator->sortKeyFastPath(token.data(), token.size(), key_buffer); + return lookup.contains(std::string_view(key.data, key.size)); + } + + TiDB::TiDBCollatorPtr collator; + Keys keys; + std::unordered_map keys_by_length; + mutable String key_buffer; +}; + +bool isDefaultStopword(std::string_view token, const BuiltinStopwordLookup * stopwords) +{ + return stopwords ? stopwords->contains(token) : defaultStopwords().contains(Poco::UTF8::toLower(String(token))); +} + +size_t maxBuiltinStopwordLength() +{ + // The supported built-in list is ASCII: byte and code-point lengths are + // identical. Do not examine lengths which cannot match any stopword. + static const size_t max_length = [] { + size_t result = 0; + for (const auto & word : defaultStopwords()) + result = std::max(result, word.size()); + return result; + }(); + return max_length; +} + +bool containsNgramStopword( + std::string_view token, + const std::vector & char_boundaries, + size_t ngram_start, + size_t ngram_token_size, + const BuiltinStopwordLookup * stopwords) +{ + const size_t ngram_end = ngram_start + ngram_token_size; + for (size_t start = ngram_start; start < ngram_end; ++start) + { + for (size_t end = start + 1; end <= ngram_end; ++end) + { + const auto candidate = token.substr(char_boundaries[start], char_boundaries[end] - char_boundaries[start]); + if (!stopwords && defaultStopwords().contains(Poco::UTF8::toLower(String(candidate)))) + return true; + if (stopwords && stopwords->contains(candidate, end - start)) + return true; + } + } + return false; +} + +void analyzeTextInto( + std::string_view text, + FullTextColumn & result, + const TiDB::TiDBCollatorPtr & collator = nullptr, + const BuiltinStopwordLookup * stopwords = nullptr, + size_t min_token_size = default_min_token_size, + size_t max_token_size = default_max_token_size, + bool enable_stopword = true, + bool borrow_document = false) +{ + result.clear(); + scanStandardTokens(text, [&](std::string_view token, size_t position, size_t code_points) { + // Filter before materializing. Stopwords use the source spelling and + // server collation, independently of the MATCH column collation. + if (code_points < min_token_size || code_points > max_token_size + || (enable_stopword && isDefaultStopword(token, stopwords))) + return; + result.append(token, position, code_points, borrow_document && collator); + if (!collator) + lowercaseTokenInPlace(result[result.size() - 1].text); + }); +} + +FullTextColumn analyzeText( + std::string_view text, + const TiDB::TiDBCollatorPtr & collator = nullptr, + const BuiltinStopwordLookup * stopwords = nullptr, + size_t min_token_size = default_min_token_size, + size_t max_token_size = default_max_token_size, + bool enable_stopword = true) +{ + FullTextColumn result; + analyzeTextInto(text, result, collator, stopwords, min_token_size, max_token_size, enable_stopword); + return result; +} + +void analyzeNgramTextInto( + std::string_view text, + FullTextColumn & result, + AnalyzerScratch & scratch, + size_t ngram_token_size, + const TiDB::TiDBCollatorPtr & collator = nullptr, + const BuiltinStopwordLookup * stopwords = nullptr, + bool enable_stopword = true, + bool borrow_document = false) +{ + result.clear(); + if (ngram_token_size == 0) + return; + + const bool preserve_case = enable_stopword && stopwords; + const size_t max_stopword_length = enable_stopword ? std::min(ngram_token_size, maxBuiltinStopwordLength()) : 0; + size_t next_position_base = 0; + auto & char_boundaries = scratch.char_boundaries; + for (size_t offset = 0; offset < text.size();) + { + const auto [first_code_point, first_length] = decodeCodePoint(text, offset); + // Stop the document on malformed UTF-8, retain all valid multibyte + // characters, and split runs on ASCII nonwords, as MySQL does. + if (first_length == 1 && first_code_point >= 0x80) + break; + if (first_length == 1 && !isFullTextToken(first_code_point)) + { + offset += first_length; + continue; + } + const size_t run_start = offset; + char_boundaries.clear(); + char_boundaries.push_back(0); + offset += first_length; + char_boundaries.push_back(offset - run_start); + while (offset < text.size()) + { + const auto [code_point, length] = decodeCodePoint(text, offset); + if (length == 1 && (code_point >= 0x80 || !isFullTextToken(code_point))) + break; + offset += length; + char_boundaries.push_back(offset - run_start); + } + + std::string_view run = text.substr(run_start, offset - run_start); + const size_t char_count = char_boundaries.size() - 1; + const size_t base_position = next_position_base; + if (char_count < ngram_token_size) + { + // A short document run emits no grams and contributes no position. + continue; + } + + if (!collator && !preserve_case) + { + // The no-collator path lowercases a whole run before splitting. + // Unicode case mapping can change byte widths, so only that path + // needs a copy and another boundary scan. Never borrow scratch. + scratch.lowercase_run.assign(run.data(), run.size()); + lowercaseTokenInPlace(scratch.lowercase_run); + run = scratch.lowercase_run; + char_boundaries.clear(); + char_boundaries.push_back(0); + for (size_t i = 0; i < run.size();) + { + i += decodeCodePoint(run, i).second; + char_boundaries.push_back(i); + } + } + + const auto append_gram = [&](size_t start) ALWAYS_INLINE { + const size_t begin = char_boundaries[start]; + const size_t end = char_boundaries[start + ngram_token_size]; + result.append( + run.substr(begin, end - begin), + base_position + start, + ngram_token_size, + borrow_document && collator); + if (!collator && preserve_case) + lowercaseTokenInPlace(result[result.size() - 1].text); + }; + + // Keep the unfiltered stream and stopwords-OFF path free of per-gram + // stopword branches and sliding-state dependencies. + if (!enable_stopword) + { + for (size_t start = 0; start + ngram_token_size <= char_count; ++start) + append_gram(start); + next_position_base = base_position + char_count - ngram_token_size + 1; + continue; + } + + // Advance candidate endings once per run, instead of enumerating all + // substrings again for every overlapping gram. Keep one past the + // greatest start of a matched stopword ending at/before the gram end. + // The gram [start, end) is filtered iff that value is greater than + // start: such a stopword lies wholly inside the current gram. Once + // start advances beyond it, it expires without a queue or cache. + size_t stopword_end = 0; + size_t filtered_until_start = 0; + for (size_t start = 0; start + ngram_token_size <= char_count; ++start) + { + while (stopword_end < start + ngram_token_size) + { + ++stopword_end; + const size_t max_length = std::min(stopword_end, max_stopword_length); + for (size_t length = 1; length <= max_length; ++length) + { + const size_t candidate_start = stopword_end - length; + // Longer candidates at this ending have earlier starts + // and cannot improve a start already known to match. + if (candidate_start < filtered_until_start) + break; + const auto candidate = run.substr( + char_boundaries[candidate_start], + char_boundaries[stopword_end] - char_boundaries[candidate_start]); + const bool matched = stopwords + ? stopwords->contains(candidate, length) + : defaultStopwords().contains(Poco::UTF8::toLower(String(candidate))); + if (matched) + { + filtered_until_start = candidate_start + 1; + break; + } + } + } + if (filtered_until_start <= start) + append_gram(start); + } + next_position_base = base_position + char_count - ngram_token_size + 1; + } +} + +FullTextColumn analyzeNgramText( + std::string_view text, + size_t ngram_token_size, + const TiDB::TiDBCollatorPtr & collator = nullptr, + const BuiltinStopwordLookup * stopwords = nullptr, + bool enable_stopword = true) +{ + FullTextColumn result; + AnalyzerScratch scratch; + analyzeNgramTextInto(text, result, scratch, ngram_token_size, collator, stopwords, enable_stopword); + return result; +} + +bool textEquals(std::string_view lhs, std::string_view rhs, const TiDB::TiDBCollatorPtr & collator) +{ + if (!collator) + return lhs == rhs; + return collator->compare(lhs.data(), lhs.size(), rhs.data(), rhs.size()) == 0; +} + +String makePrefixPattern(std::string_view prefix) +{ + // Reuse the same collation-aware pattern implementation as LIKE. Escape + // token characters that have LIKE meaning before appending the wildcard. + String pattern; + pattern.reserve(prefix.size() + 1); + for (const char c : prefix) + { + if (c == '\\' || c == '%' || c == '_') + pattern.push_back('\\'); + pattern.push_back(c); + } + pattern.push_back('%'); + return pattern; +} + +std::unique_ptr compilePrefixMatcher( + std::string_view prefix, + const TiDB::TiDBCollatorPtr & collator) +{ + auto matcher = collator->pattern(); + matcher->compile(makePrefixPattern(prefix), '\\'); + return matcher; +} + +bool textStartsWith(std::string_view value, std::string_view prefix, const TiDB::TiDBCollatorPtr & collator) +{ + if (!collator) + return value.starts_with(prefix); + const auto matcher = compilePrefixMatcher(prefix, collator); + return matcher->match(value.data(), value.size()); +} + +bool matchesTermAt( + const BooleanClause & clause, + size_t term_index, + const FullTextToken & token, + const TiDB::TiDBCollatorPtr & collator) +{ + const auto value = token.value(); + if (!clause.prefix) + return textEquals(value, clause.terms[term_index], collator); + if (collator && term_index < clause.prefix_matchers.size()) + return clause.prefix_matchers[term_index]->match(value.data(), value.size()); + return textStartsWith(value, clause.terms[term_index], collator); +} + +struct ClauseMatchState +{ + std::vector matched_terms; + size_t remaining_terms = 0; + bool matched = false; + bool phrase_verification_failed = false; +}; + +void initializeClauseMatchStates(const std::vector & clauses, std::vector & states) +{ + states.resize(clauses.size()); + for (size_t i = 0; i < clauses.size(); ++i) + if (!clauses[i].phrase) + { + states[i].matched_terms.resize(clauses[i].terms.size()); + states[i].remaining_terms = clauses[i].terms.size(); + } +} + +bool matchesPhraseEndingAt( + const BooleanClause & clause, + const FullTextColumn & column, + const FullTextToken & end, + const TiDB::TiDBCollatorPtr & collator) +{ + if (clause.terms.empty() || clause.terms.size() != clause.offsets.size()) + return false; + + const size_t last_term = clause.terms.size() - 1; + const size_t last_offset = clause.offsets[last_term]; + if (end.position < last_offset || !matchesTermAt(clause, last_term, end, collator)) + return false; + + const size_t start_position = end.position - last_offset; + for (size_t i = 0; i < last_term; ++i) + { + const size_t expected_position = start_position + clause.offsets[i]; + const auto it = std::lower_bound( + column.begin(), + column.end(), + expected_position, + [](const FullTextToken & token, size_t position) { return token.position < position; }); + if (it == column.end() || it->position != expected_position + || !textEquals(it->value(), clause.terms[i], collator)) + return false; + } + return true; +} + +void analyzeColumnInto( + std::string_view document, + FullTextColumn & output, + AnalyzerScratch & scratch, + bool use_ngram, + size_t ngram_token_size, + const TiDB::TiDBCollatorPtr & collator = nullptr, + const BuiltinStopwordLookup * stopwords = nullptr, + size_t min_token_size = default_min_token_size, + size_t max_token_size = default_max_token_size, + bool enable_stopword = true) +{ + if (use_ngram) + analyzeNgramTextInto(document, output, scratch, ngram_token_size, collator, stopwords, enable_stopword, true); + else + analyzeTextInto(document, output, collator, stopwords, min_token_size, max_token_size, enable_stopword, true); +} + +void resetClauseMatchStates(std::vector & states) +{ + for (auto & state : states) + { + state.matched = false; + state.remaining_terms = state.matched_terms.size(); + std::fill(state.matched_terms.begin(), state.matched_terms.end(), 0); + } +} + +struct BooleanMatchProgress +{ + size_t matched_must_clauses = 0; + bool matched_should_clause = false; +}; + +enum class ColumnMatchResult +{ + Continue, + Accepted, + Rejected, +}; + +ColumnMatchResult updateBooleanMatchStatesForColumn( + const CompiledBooleanQuery & query, + const FullTextColumn & column, + const FullTextColumn & unfiltered_column, + const TiDB::TiDBCollatorPtr & collator, + std::vector & states, + BooleanMatchProgress & progress) +{ + const auto & clauses = query.clauses; + if (query.matches_nothing || clauses.empty()) + return ColumnMatchResult::Continue; + + // Raw phrase verification searches the entire column, so a failed result + // needs no recheck at later anchors. Unlike positive clause matches, this + // negative cache belongs only to this column, not other MATCH columns. + for (auto & state : states) + state.phrase_verification_failed = false; + + const auto mark_clause_matched = [&](size_t clause_index) { + states[clause_index].matched = true; + switch (clauses[clause_index].modifier) + { + case BooleanClause::Modifier::Must: + ++progress.matched_must_clauses; + break; + case BooleanClause::Modifier::Should: + progress.matched_should_clause = true; + break; + case BooleanClause::Modifier::MustNot: + return ColumnMatchResult::Rejected; + } + if (!query.has_must_not + && (query.must_clause_count != 0 ? progress.matched_must_clauses == query.must_clause_count + : progress.matched_should_clause)) + return ColumnMatchResult::Accepted; + return ColumnMatchResult::Continue; + }; + + // Walk this MATCH column's analyzed tokens once and update every clause's + // state. Phrase clauses are checked only when their final term is + // encountered; positional lookups preserve stopword gaps and phrases do + // not cross MATCH columns. + for (const auto & token : column) + { + for (size_t clause_index = 0; clause_index < clauses.size(); ++clause_index) + { + const auto & clause = clauses[clause_index]; + auto & state = states[clause_index]; + if (state.matched || state.phrase_verification_failed || clause.terms.empty()) + continue; + + if (clause.phrase) + { + if (matchesPhraseEndingAt(clause, column, token, collator)) + { + if (clause.verification + && !std::any_of(unfiltered_column.begin(), unfiltered_column.end(), [&](const auto & end) { + return matchesPhraseEndingAt(*clause.verification, unfiltered_column, end, collator); + })) + { + state.phrase_verification_failed = true; + continue; + } + const auto result = mark_clause_matched(clause_index); + if (result != ColumnMatchResult::Continue) + return result; + } + continue; + } + + for (size_t term_index = 0; term_index < clause.terms.size(); ++term_index) + if (!state.matched_terms[term_index] && matchesTermAt(clause, term_index, token, collator)) + { + state.matched_terms[term_index] = 1; + --state.remaining_terms; + } + // STANDARD may split one SQL term into several words. Required + // terms intersect those words, but optional/prohibited terms use + // their union, exactly like TiDB's combineBooleanTermNodes. + state.matched = clause.modifier == BooleanClause::Modifier::Must + ? state.remaining_terms == 0 + : state.remaining_terms < clause.terms.size(); + if (state.matched) + { + const auto result = mark_clause_matched(clause_index); + if (result != ColumnMatchResult::Continue) + return result; + } + } + } + return ColumnMatchResult::Continue; +} + +bool evaluateBooleanMatchResult(const CompiledBooleanQuery & query, const std::vector & states) +{ + if (query.matches_nothing || query.clauses.empty()) + return false; + bool has_positive = false; + bool has_must = false; + bool positive_match = false; + for (size_t i = 0; i < query.clauses.size(); ++i) + { + const auto & clause = query.clauses[i]; + const bool matched = states[i].matched; + switch (clause.modifier) + { + case BooleanClause::Modifier::Must: + has_must = true; + if (!matched) + return false; + break; + case BooleanClause::Modifier::MustNot: + if (matched) + return false; + break; + case BooleanClause::Modifier::Should: + has_positive = true; + positive_match = positive_match || matched; + break; + } + } + + return has_must || !has_positive || positive_match; +} + +CompiledBooleanQuery compileBooleanQuery( + const tipb::LocalMatchAgainstBooleanQuery & query, + const LocalMatchAnalyzerConfig & config, + const TiDB::TiDBCollatorPtr & collator = nullptr, + const BuiltinStopwordLookup * stopwords = nullptr) +{ + CompiledBooleanQuery compiled; + if (!config.use_ngram && (config.max_token_size == 0 || config.min_token_size > config.max_token_size)) + { + compiled.matches_nothing = true; + return compiled; + } + auto analyze_query = [&](std::string_view text) { + return config.use_ngram + ? analyzeNgramText(text, config.ngram_token_size, collator, stopwords, config.enable_stopword) + : analyzeText( + text, + collator, + stopwords, + config.min_token_size, + config.max_token_size, + config.enable_stopword); + }; + for (const auto & node : query.nodes()) + { + BooleanClause clause; + size_t first_query_position = 0; + switch (node.occur()) + { + case tipb::LocalMatchAgainstBooleanOccur::LocalMatchAgainstBooleanOccurMust: + clause.modifier = BooleanClause::Modifier::Must; + break; + case tipb::LocalMatchAgainstBooleanOccur::LocalMatchAgainstBooleanOccurMustNot: + clause.modifier = BooleanClause::Modifier::MustNot; + break; + case tipb::LocalMatchAgainstBooleanOccur::LocalMatchAgainstBooleanOccurShould: + clause.modifier = BooleanClause::Modifier::Should; + break; + default: + throw Exception("Invalid Local MATCH Boolean node occurrence", ErrorCodes::ILLEGAL_COLUMN); + } + + switch (node.term_type()) + { + case tipb::LocalMatchAgainstBooleanTermType::LocalMatchAgainstBooleanTermWord: + { + const auto terms = analyze_query(node.text()); + clause.phrase = config.use_ngram; + first_query_position = collectClauseTerms(clause, terms); + break; + } + case tipb::LocalMatchAgainstBooleanTermType::LocalMatchAgainstBooleanTermPrefix: + { + const auto terms = config.use_ngram + ? analyzeNgramText(node.text(), config.ngram_token_size, collator, stopwords, config.enable_stopword) + : tokenizeText(node.text(), collator); + if (config.use_ngram) + { + if (!terms.empty()) + { + // TiDB expands an NGRAM prefix at least as long as one + // gram into a positional phrase of query ngrams. Matching + // only one analyzed term here makes prefixes such as + // "caf*" ("ca", "af" at token size 2) never match. + clause.phrase = true; + first_query_position = collectClauseTerms(clause, terms); + break; + } + + // A query shorter than the configured gram size is retained + // by TiDB as a prefix over document ngrams instead of being + // discarded by the analyzer. + const auto source_terms = tokenizeText(node.text(), collator); + if (source_terms.size() == 1) + { + const auto code_points = UTF8::countCodePoints( + reinterpret_cast(source_terms.front().text.data()), + source_terms.front().text.size()); + if (code_points < config.ngram_token_size) + { + clause.prefix = true; + clause.terms.push_back(source_terms.front().text); + break; + } + } + + if (clause.modifier == BooleanClause::Modifier::Must) + compiled.clauses.push_back(std::move(clause)); + continue; + } + + clause.prefix = true; + if (terms.size() != 1) + { + if (clause.modifier == BooleanClause::Modifier::Must) + compiled.clauses.push_back(std::move(clause)); + continue; + } + const auto code_points = UTF8::countCodePoints( + reinterpret_cast(terms.front().text.data()), + terms.front().text.size()); + if (code_points > config.max_token_size) + { + if (clause.modifier == BooleanClause::Modifier::Must) + compiled.clauses.push_back(std::move(clause)); + continue; + } + clause.terms.push_back(terms.front().text); + break; + } + case tipb::LocalMatchAgainstBooleanTermType::LocalMatchAgainstBooleanTermPhrase: + { + clause.phrase = true; + const auto terms = analyze_query(node.text()); + if (config.use_ngram) + { + const auto words = tokenizeText(node.text(), collator); + bool seen_indexed_word = false; + if (std::any_of(words.begin(), words.end(), [&](const auto & word) { + if (word.code_points >= config.ngram_token_size) + { + seen_indexed_word = seen_indexed_word || !analyze_query(word.text).empty(); + return false; + } + if (!config.enable_stopword || seen_indexed_word) + return true; + std::vector boundaries{0}; + for (size_t offset = 0; offset < word.text.size();) + { + offset += decodeCodePoint(word.text, offset).second; + boundaries.push_back(offset); + } + return !containsNgramStopword(word.text, boundaries, 0, word.code_points, stopwords); + })) + break; // A short query unigram cannot occur in the document's fixed-size ngrams. + } + first_query_position = collectClauseTerms(clause, terms); + break; + } + default: + throw Exception("Invalid Local MATCH Boolean node term type", ErrorCodes::ILLEGAL_COLUMN); + } + + if (clause.phrase && !clause.terms.empty()) + { + auto raw_terms = config.use_ngram + ? analyzeNgramText(node.text(), config.ngram_token_size, collator, nullptr, false) + : analyzeText(node.text(), collator, nullptr, 0, std::numeric_limits::max(), false); + // InnoDB discards leading filtered tokens, but retains every + // following word in the original phrase used for verification. + raw_terms.filterInPlace([&](const auto & token) { return token.position >= first_query_position; }); + if (raw_terms.size() != clause.terms.size()) + { + // InnoDB verifies phrases against the original document, not + // just the index tokens: removing "the" must not turn it into + // an arbitrary positional gap. Ordinary terms/prefixes do not + // pay for this additional document stream. + clause.verification = std::make_unique(); + clause.verification->phrase = true; + collectClauseTerms(*clause.verification, raw_terms); + compiled.needs_verification = true; + } + } + + if (clause.prefix && collator) + { + clause.prefix_matchers.reserve(clause.terms.size()); + for (const auto & term : clause.terms) + clause.prefix_matchers.push_back(compilePrefixMatcher(term, collator)); + } + + if (!clause.terms.empty() || clause.modifier == BooleanClause::Modifier::Must) + compiled.clauses.push_back(std::move(clause)); + } + + for (const auto & clause : compiled.clauses) + { + if (clause.modifier == BooleanClause::Modifier::Must) + { + // An analyzer-filtered required clause cannot match any document. + // Do this after compiling ALL nodes so malformed later nodes + // retain their existing no-match handling, rather than being + // hidden by an earlier block-level shortcut. + if (clause.terms.empty()) + { + compiled.matches_nothing = true; + return compiled; + } + ++compiled.must_clause_count; + } + else if (clause.modifier == BooleanClause::Modifier::MustNot) + compiled.has_must_not = true; + } + + // A BOOLEAN MODE query containing only prohibited terms has no positive + // branch. TiDB's local evaluator treats it as matching no rows. + if (std::none_of(compiled.clauses.begin(), compiled.clauses.end(), [](const BooleanClause & clause) { + return clause.modifier != BooleanClause::Modifier::MustNot; + })) + compiled.matches_nothing = true; + + return compiled; +} + +Float64 matchBooleanValue(const CompiledBooleanQuery & query, const std::vector & states) +{ + // Local MATCH returns a Boolean 0/1 value, not a relevance score. + return evaluateBooleanMatchResult(query, states) ? 1 : 0; +} + +class FullTextColumnAccessor +{ +public: + explicit FullTextColumnAccessor(const IColumn & column) { initialize(column, false); } + + bool isNull(size_t row) const + { + return nullable_column != nullptr && nullable_column->getNullMapData()[nullable_is_constant ? 0 : row] != 0; + } + + std::string_view getString(size_t row) const + { + const auto & chars = string_column->getChars(); + const auto & offsets = string_column->getOffsets(); + const size_t string_row = string_is_constant ? 0 : row; + const size_t begin = string_row == 0 ? 0 : offsets[string_row - 1]; + const size_t end = offsets[string_row]; + return {reinterpret_cast(&chars[begin]), end - begin - 1}; + } + +private: + void initialize(const IColumn & column, bool row_is_constant) + { + if (const auto * column_const = typeid_cast(&column)) + { + string_is_constant = true; + initialize(column_const->getDataColumn(), true); + return; + } + if (const auto * column_nullable = typeid_cast(&column)) + { + nullable_column = column_nullable; + nullable_is_constant = row_is_constant; + initialize(column_nullable->getNestedColumn(), row_is_constant); + return; + } + string_column = checkAndGetColumn(&column); + if (string_column == nullptr) + throw Exception("Full-text arguments must be string columns", ErrorCodes::ILLEGAL_COLUMN); + } + + const ColumnString * string_column = nullptr; + const ColumnNullable * nullable_column = nullptr; + bool string_is_constant = false; + bool nullable_is_constant = false; +}; + +bool decodeLocalMatchAgainstBooleanQuery(const IColumn & column, tipb::LocalMatchAgainstBooleanQuery & query) +{ + const auto * constant = typeid_cast(&column); + if (constant == nullptr) + return false; + const auto encoded = constant->getValue(); + // Expr.val directly contains the versioned Local MATCH query. Validate the + // semantic version before interpreting parser or stopword fields; never + // guess a default for a version this TiFlash binary does not understand. + if (!query.ParseFromString(encoded) || query.version() != local_match_against_protocol_version) + return false; + if (query.parser() != tipb::LocalMatchAgainstParser::LocalMatchAgainstParserStandard + && query.parser() != tipb::LocalMatchAgainstParser::LocalMatchAgainstParserNgram) + return false; + if (query.stopword_mode() != tipb::LocalMatchAgainstStopwordMode::LocalMatchAgainstStopwordModeDisabled + && query.stopword_mode() != tipb::LocalMatchAgainstStopwordMode::LocalMatchAgainstStopwordModeBuiltin) + return false; + // Validate every node before NULL-search or analyzer no-match shortcuts. + // Missing/unknown proto2 enum values read as Invalid, not as valid clauses. + for (const auto & node : query.nodes()) + { + if (node.occur() != tipb::LocalMatchAgainstBooleanOccurMust + && node.occur() != tipb::LocalMatchAgainstBooleanOccurMustNot + && node.occur() != tipb::LocalMatchAgainstBooleanOccurShould) + return false; + if (node.term_type() != tipb::LocalMatchAgainstBooleanTermWord + && node.term_type() != tipb::LocalMatchAgainstBooleanTermPrefix + && node.term_type() != tipb::LocalMatchAgainstBooleanTermPhrase) + return false; + } + return true; +} + +class FunctionLocalMatchAgainstBoolean final : public IFunction +{ +public: + static constexpr auto name = "local_match_against_boolean"; + static FunctionPtr create(const Context &) { return std::make_shared(); } + + String getName() const override { return name; } + size_t getNumberOfArguments() const override { return 0; } + bool isVariadic() const override { return true; } + bool useDefaultImplementationForConstants() const override { return false; } + // NULL MATCH columns are empty documents, not NULL MATCH results. The + // implementation below handles nullable columns per row, so the generic + // IFunction NULL wrapper must not short-circuit the whole expression. + bool useDefaultImplementationForNulls() const override { return false; } + ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {0}; } + void setCollator(const TiDB::TiDBCollatorPtr & collator_) override { collator = collator_; } + + DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override + { + if (arguments.size() < 3) + throw Exception("local_match_against_boolean requires a query, at least one column, and metadata"); + bool nullable = false; + for (size_t i = 0; i < arguments.size(); ++i) + { + const auto & argument = arguments[i]; + const auto nested_type = removeNullable(argument); + // An untyped SQL NULL search arrives as Nullable(Nothing). + // Documents and metadata still require string types. + const bool null_query = i == 0 && nested_type->getTypeId() == TypeIndex::Nothing; + if (!nested_type->isString() && !null_query) + throw Exception( + "Illegal type " + argument->getName() + " of argument of function " + getName(), + ErrorCodes::ILLEGAL_COLUMN); + nullable = nullable || argument->isNullable(); + } + DataTypePtr result_type = std::make_shared(); + return nullable ? std::make_shared(result_type) : result_type; + } + + void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result) const override + { + const auto * query_column = typeid_cast(&*block.getByPosition(arguments[0]).column); + if (query_column == nullptr) + throw Exception( + "The query argument of local_match_against_boolean must be constant", + ErrorCodes::ILLEGAL_COLUMN); + + const size_t rows = block.getByPosition(arguments[1]).column->size(); + auto output = ColumnFloat64::create(rows, 0); + auto & output_data = output->getData(); + const bool nullable = block.getByPosition(result).type->isNullable(); + auto null_map = nullable ? ColumnUInt8::create(rows, 0) : nullptr; + const auto finish_result = [&] { + ColumnPtr result_column = std::move(output); + if (null_map) + result_column = ColumnNullable::create(std::move(result_column), std::move(null_map)); + block.getByPosition(result).column = std::move(result_column); + }; + + tipb::LocalMatchAgainstBooleanQuery protocol_boolean_query; + if (arguments.size() <= 2 + || !decodeLocalMatchAgainstBooleanQuery( + *block.getByPosition(arguments.back()).column, + protocol_boolean_query)) + throw Exception( + "local_match_against_boolean requires valid Boolean query metadata", + ErrorCodes::ILLEGAL_COLUMN); + if (query_column->isNullAt(0)) + { + // InnoDB interprets NULL AGAINST as an empty search: non-NULL zero. + finish_result(); + return; + } + const size_t document_argument_end = arguments.size() - 1; + const LocalMatchAnalyzerConfig config(protocol_boolean_query); + // Keep scalar values in the row hot path. Passing the config by + // reference changes Release code generation, notably with stopwords + // disabled; the config object belongs to query compilation only. + const bool use_ngram = config.use_ngram; + const size_t ngram_token_size = config.ngram_token_size; + const size_t min_token_size = config.min_token_size; + const size_t max_token_size = config.max_token_size; + const bool enable_stopword = config.enable_stopword; + const auto * const stopword_collator + = enable_stopword ? TiDB::ITiDBCollator::getCollator(protocol_boolean_query.stopword_collation()) : nullptr; + if (enable_stopword && !stopword_collator) + throw Exception( + "local_match_against_boolean requires a supported stopword collation when stopwords are enabled", + ErrorCodes::ILLEGAL_COLUMN); + std::optional stopword_lookup; + if (enable_stopword) + stopword_lookup.emplace(stopword_collator); + const auto * stopwords = stopword_lookup ? &*stopword_lookup : nullptr; + const CompiledBooleanQuery compiled_query + = compileBooleanQuery(protocol_boolean_query, config, collator, stopwords); + if (compiled_query.matches_nothing) + { + finish_result(); + return; + } + std::vector document_columns; + document_columns.reserve(document_argument_end - 1); + for (size_t arg = 1; arg < document_argument_end; ++arg) + document_columns.emplace_back(*block.getByPosition(arguments[arg]).column); + std::vector clause_states; + initializeClauseMatchStates(compiled_query.clauses, clause_states); + FullTextColumn analyzed_column; + FullTextColumn unfiltered_column; + AnalyzerScratch analyzer_scratch; + + for (size_t row = 0; row < rows; ++row) + { + resetClauseMatchStates(clause_states); + BooleanMatchProgress match_progress; + bool result_known = false; + bool row_matches = false; + for (const auto & document_column : document_columns) + { + // A NULL MATCH column contributes no tokens. This is + // intentional: #70485 relies on a row with a NULL body + // still matching a required term from another MATCH column. + if (document_column.isNull(row)) + { + analyzed_column.clear(); + continue; + } + analyzeColumnInto( + document_column.getString(row), + analyzed_column, + analyzer_scratch, + use_ngram, + ngram_token_size, + collator, + stopwords, + min_token_size, + max_token_size, + enable_stopword); + if (compiled_query.needs_verification) + analyzeColumnInto( + document_column.getString(row), + unfiltered_column, + analyzer_scratch, + use_ngram, + ngram_token_size, + collator, + nullptr, + 0, + std::numeric_limits::max(), + false); + const auto match_result = updateBooleanMatchStatesForColumn( + compiled_query, + analyzed_column, + unfiltered_column, + collator, + clause_states, + match_progress); + if (match_result == ColumnMatchResult::Rejected) + { + result_known = true; + break; + } + if (match_result == ColumnMatchResult::Accepted) + { + result_known = true; + row_matches = true; + break; + } + } + output_data[row] = result_known ? (row_matches ? 1 : 0) : matchBooleanValue(compiled_query, clause_states); + } + finish_result(); + } + +private: + TiDB::TiDBCollatorPtr collator = nullptr; +}; +} // namespace + +void registerFunctionsLocalMatchAgainst(FunctionFactory & factory) +{ + factory.registerFunction(FunctionFactory::CaseInsensitive); +} +} // namespace DB diff --git a/dbms/src/Functions/FunctionsLocalMatchAgainst.h b/dbms/src/Functions/FunctionsLocalMatchAgainst.h new file mode 100644 index 00000000000..064c503718f --- /dev/null +++ b/dbms/src/Functions/FunctionsLocalMatchAgainst.h @@ -0,0 +1,22 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include + +namespace DB +{ +void registerFunctionsLocalMatchAgainst(FunctionFactory & factory); +} diff --git a/dbms/src/Functions/LocalMatchAgainstTokenChars.h b/dbms/src/Functions/LocalMatchAgainstTokenChars.h new file mode 100644 index 00000000000..ec2d3987c7c --- /dev/null +++ b/dbms/src/Functions/LocalMatchAgainstTokenChars.h @@ -0,0 +1,211 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Generated by TiDB pkg/expression/matchagainst/generator; DO NOT EDIT. +#pragma once + +#include +#include +#include + +namespace DB::LocalMatchAgainst +{ +// Generated from Go's Unicode 15.0.0 categories L and N. TiDB uses its standard +// library and strictly tests that Unicode version and the Unicode 15.0.0 checksum. +// Equivalent Go predicate: +// word := r <= unicode.MaxRune && +// (unicode.IsLetter(r) || unicode.IsNumber(r) || r == '_') +// This helper covers all Unicode code points; the Boolean lexer separately +// restricts word characters to the BMP (r <= 0xFFFF). +// Regenerate/check this header using the paired TiDB generator. Never change +// an existing protocol version's classification; deploy TiFlash support for a +// new version before TiDB emits it. ASCII and '_' use the fast path below. +struct TokenCharRange +{ + std::uint32_t first; + std::uint32_t last; +}; +inline constexpr std::array token_char_ranges{{ + {0xAA, 0xAA}, {0xB2, 0xB3}, {0xB5, 0xB5}, {0xB9, 0xBA}, {0xBC, 0xBE}, + {0xC0, 0xD6}, {0xD8, 0xF6}, {0xF8, 0x2C1}, {0x2C6, 0x2D1}, {0x2E0, 0x2E4}, + {0x2EC, 0x2EC}, {0x2EE, 0x2EE}, {0x370, 0x374}, {0x376, 0x377}, {0x37A, 0x37D}, + {0x37F, 0x37F}, {0x386, 0x386}, {0x388, 0x38A}, {0x38C, 0x38C}, {0x38E, 0x3A1}, + {0x3A3, 0x3F5}, {0x3F7, 0x481}, {0x48A, 0x52F}, {0x531, 0x556}, {0x559, 0x559}, + {0x560, 0x588}, {0x5D0, 0x5EA}, {0x5EF, 0x5F2}, {0x620, 0x64A}, {0x660, 0x669}, + {0x66E, 0x66F}, {0x671, 0x6D3}, {0x6D5, 0x6D5}, {0x6E5, 0x6E6}, {0x6EE, 0x6FC}, + {0x6FF, 0x6FF}, {0x710, 0x710}, {0x712, 0x72F}, {0x74D, 0x7A5}, {0x7B1, 0x7B1}, + {0x7C0, 0x7EA}, {0x7F4, 0x7F5}, {0x7FA, 0x7FA}, {0x800, 0x815}, {0x81A, 0x81A}, + {0x824, 0x824}, {0x828, 0x828}, {0x840, 0x858}, {0x860, 0x86A}, {0x870, 0x887}, + {0x889, 0x88E}, {0x8A0, 0x8C9}, {0x904, 0x939}, {0x93D, 0x93D}, {0x950, 0x950}, + {0x958, 0x961}, {0x966, 0x96F}, {0x971, 0x980}, {0x985, 0x98C}, {0x98F, 0x990}, + {0x993, 0x9A8}, {0x9AA, 0x9B0}, {0x9B2, 0x9B2}, {0x9B6, 0x9B9}, {0x9BD, 0x9BD}, + {0x9CE, 0x9CE}, {0x9DC, 0x9DD}, {0x9DF, 0x9E1}, {0x9E6, 0x9F1}, {0x9F4, 0x9F9}, + {0x9FC, 0x9FC}, {0xA05, 0xA0A}, {0xA0F, 0xA10}, {0xA13, 0xA28}, {0xA2A, 0xA30}, + {0xA32, 0xA33}, {0xA35, 0xA36}, {0xA38, 0xA39}, {0xA59, 0xA5C}, {0xA5E, 0xA5E}, + {0xA66, 0xA6F}, {0xA72, 0xA74}, {0xA85, 0xA8D}, {0xA8F, 0xA91}, {0xA93, 0xAA8}, + {0xAAA, 0xAB0}, {0xAB2, 0xAB3}, {0xAB5, 0xAB9}, {0xABD, 0xABD}, {0xAD0, 0xAD0}, + {0xAE0, 0xAE1}, {0xAE6, 0xAEF}, {0xAF9, 0xAF9}, {0xB05, 0xB0C}, {0xB0F, 0xB10}, + {0xB13, 0xB28}, {0xB2A, 0xB30}, {0xB32, 0xB33}, {0xB35, 0xB39}, {0xB3D, 0xB3D}, + {0xB5C, 0xB5D}, {0xB5F, 0xB61}, {0xB66, 0xB6F}, {0xB71, 0xB77}, {0xB83, 0xB83}, + {0xB85, 0xB8A}, {0xB8E, 0xB90}, {0xB92, 0xB95}, {0xB99, 0xB9A}, {0xB9C, 0xB9C}, + {0xB9E, 0xB9F}, {0xBA3, 0xBA4}, {0xBA8, 0xBAA}, {0xBAE, 0xBB9}, {0xBD0, 0xBD0}, + {0xBE6, 0xBF2}, {0xC05, 0xC0C}, {0xC0E, 0xC10}, {0xC12, 0xC28}, {0xC2A, 0xC39}, + {0xC3D, 0xC3D}, {0xC58, 0xC5A}, {0xC5D, 0xC5D}, {0xC60, 0xC61}, {0xC66, 0xC6F}, + {0xC78, 0xC7E}, {0xC80, 0xC80}, {0xC85, 0xC8C}, {0xC8E, 0xC90}, {0xC92, 0xCA8}, + {0xCAA, 0xCB3}, {0xCB5, 0xCB9}, {0xCBD, 0xCBD}, {0xCDD, 0xCDE}, {0xCE0, 0xCE1}, + {0xCE6, 0xCEF}, {0xCF1, 0xCF2}, {0xD04, 0xD0C}, {0xD0E, 0xD10}, {0xD12, 0xD3A}, + {0xD3D, 0xD3D}, {0xD4E, 0xD4E}, {0xD54, 0xD56}, {0xD58, 0xD61}, {0xD66, 0xD78}, + {0xD7A, 0xD7F}, {0xD85, 0xD96}, {0xD9A, 0xDB1}, {0xDB3, 0xDBB}, {0xDBD, 0xDBD}, + {0xDC0, 0xDC6}, {0xDE6, 0xDEF}, {0xE01, 0xE30}, {0xE32, 0xE33}, {0xE40, 0xE46}, + {0xE50, 0xE59}, {0xE81, 0xE82}, {0xE84, 0xE84}, {0xE86, 0xE8A}, {0xE8C, 0xEA3}, + {0xEA5, 0xEA5}, {0xEA7, 0xEB0}, {0xEB2, 0xEB3}, {0xEBD, 0xEBD}, {0xEC0, 0xEC4}, + {0xEC6, 0xEC6}, {0xED0, 0xED9}, {0xEDC, 0xEDF}, {0xF00, 0xF00}, {0xF20, 0xF33}, + {0xF40, 0xF47}, {0xF49, 0xF6C}, {0xF88, 0xF8C}, {0x1000, 0x102A}, {0x103F, 0x1049}, + {0x1050, 0x1055}, {0x105A, 0x105D}, {0x1061, 0x1061}, {0x1065, 0x1066}, {0x106E, 0x1070}, + {0x1075, 0x1081}, {0x108E, 0x108E}, {0x1090, 0x1099}, {0x10A0, 0x10C5}, {0x10C7, 0x10C7}, + {0x10CD, 0x10CD}, {0x10D0, 0x10FA}, {0x10FC, 0x1248}, {0x124A, 0x124D}, {0x1250, 0x1256}, + {0x1258, 0x1258}, {0x125A, 0x125D}, {0x1260, 0x1288}, {0x128A, 0x128D}, {0x1290, 0x12B0}, + {0x12B2, 0x12B5}, {0x12B8, 0x12BE}, {0x12C0, 0x12C0}, {0x12C2, 0x12C5}, {0x12C8, 0x12D6}, + {0x12D8, 0x1310}, {0x1312, 0x1315}, {0x1318, 0x135A}, {0x1369, 0x137C}, {0x1380, 0x138F}, + {0x13A0, 0x13F5}, {0x13F8, 0x13FD}, {0x1401, 0x166C}, {0x166F, 0x167F}, {0x1681, 0x169A}, + {0x16A0, 0x16EA}, {0x16EE, 0x16F8}, {0x1700, 0x1711}, {0x171F, 0x1731}, {0x1740, 0x1751}, + {0x1760, 0x176C}, {0x176E, 0x1770}, {0x1780, 0x17B3}, {0x17D7, 0x17D7}, {0x17DC, 0x17DC}, + {0x17E0, 0x17E9}, {0x17F0, 0x17F9}, {0x1810, 0x1819}, {0x1820, 0x1878}, {0x1880, 0x1884}, + {0x1887, 0x18A8}, {0x18AA, 0x18AA}, {0x18B0, 0x18F5}, {0x1900, 0x191E}, {0x1946, 0x196D}, + {0x1970, 0x1974}, {0x1980, 0x19AB}, {0x19B0, 0x19C9}, {0x19D0, 0x19DA}, {0x1A00, 0x1A16}, + {0x1A20, 0x1A54}, {0x1A80, 0x1A89}, {0x1A90, 0x1A99}, {0x1AA7, 0x1AA7}, {0x1B05, 0x1B33}, + {0x1B45, 0x1B4C}, {0x1B50, 0x1B59}, {0x1B83, 0x1BA0}, {0x1BAE, 0x1BE5}, {0x1C00, 0x1C23}, + {0x1C40, 0x1C49}, {0x1C4D, 0x1C7D}, {0x1C80, 0x1C88}, {0x1C90, 0x1CBA}, {0x1CBD, 0x1CBF}, + {0x1CE9, 0x1CEC}, {0x1CEE, 0x1CF3}, {0x1CF5, 0x1CF6}, {0x1CFA, 0x1CFA}, {0x1D00, 0x1DBF}, + {0x1E00, 0x1F15}, {0x1F18, 0x1F1D}, {0x1F20, 0x1F45}, {0x1F48, 0x1F4D}, {0x1F50, 0x1F57}, + {0x1F59, 0x1F59}, {0x1F5B, 0x1F5B}, {0x1F5D, 0x1F5D}, {0x1F5F, 0x1F7D}, {0x1F80, 0x1FB4}, + {0x1FB6, 0x1FBC}, {0x1FBE, 0x1FBE}, {0x1FC2, 0x1FC4}, {0x1FC6, 0x1FCC}, {0x1FD0, 0x1FD3}, + {0x1FD6, 0x1FDB}, {0x1FE0, 0x1FEC}, {0x1FF2, 0x1FF4}, {0x1FF6, 0x1FFC}, {0x2070, 0x2071}, + {0x2074, 0x2079}, {0x207F, 0x2089}, {0x2090, 0x209C}, {0x2102, 0x2102}, {0x2107, 0x2107}, + {0x210A, 0x2113}, {0x2115, 0x2115}, {0x2119, 0x211D}, {0x2124, 0x2124}, {0x2126, 0x2126}, + {0x2128, 0x2128}, {0x212A, 0x212D}, {0x212F, 0x2139}, {0x213C, 0x213F}, {0x2145, 0x2149}, + {0x214E, 0x214E}, {0x2150, 0x2189}, {0x2460, 0x249B}, {0x24EA, 0x24FF}, {0x2776, 0x2793}, + {0x2C00, 0x2CE4}, {0x2CEB, 0x2CEE}, {0x2CF2, 0x2CF3}, {0x2CFD, 0x2CFD}, {0x2D00, 0x2D25}, + {0x2D27, 0x2D27}, {0x2D2D, 0x2D2D}, {0x2D30, 0x2D67}, {0x2D6F, 0x2D6F}, {0x2D80, 0x2D96}, + {0x2DA0, 0x2DA6}, {0x2DA8, 0x2DAE}, {0x2DB0, 0x2DB6}, {0x2DB8, 0x2DBE}, {0x2DC0, 0x2DC6}, + {0x2DC8, 0x2DCE}, {0x2DD0, 0x2DD6}, {0x2DD8, 0x2DDE}, {0x2E2F, 0x2E2F}, {0x3005, 0x3007}, + {0x3021, 0x3029}, {0x3031, 0x3035}, {0x3038, 0x303C}, {0x3041, 0x3096}, {0x309D, 0x309F}, + {0x30A1, 0x30FA}, {0x30FC, 0x30FF}, {0x3105, 0x312F}, {0x3131, 0x318E}, {0x3192, 0x3195}, + {0x31A0, 0x31BF}, {0x31F0, 0x31FF}, {0x3220, 0x3229}, {0x3248, 0x324F}, {0x3251, 0x325F}, + {0x3280, 0x3289}, {0x32B1, 0x32BF}, {0x3400, 0x4DBF}, {0x4E00, 0xA48C}, {0xA4D0, 0xA4FD}, + {0xA500, 0xA60C}, {0xA610, 0xA62B}, {0xA640, 0xA66E}, {0xA67F, 0xA69D}, {0xA6A0, 0xA6EF}, + {0xA717, 0xA71F}, {0xA722, 0xA788}, {0xA78B, 0xA7CA}, {0xA7D0, 0xA7D1}, {0xA7D3, 0xA7D3}, + {0xA7D5, 0xA7D9}, {0xA7F2, 0xA801}, {0xA803, 0xA805}, {0xA807, 0xA80A}, {0xA80C, 0xA822}, + {0xA830, 0xA835}, {0xA840, 0xA873}, {0xA882, 0xA8B3}, {0xA8D0, 0xA8D9}, {0xA8F2, 0xA8F7}, + {0xA8FB, 0xA8FB}, {0xA8FD, 0xA8FE}, {0xA900, 0xA925}, {0xA930, 0xA946}, {0xA960, 0xA97C}, + {0xA984, 0xA9B2}, {0xA9CF, 0xA9D9}, {0xA9E0, 0xA9E4}, {0xA9E6, 0xA9FE}, {0xAA00, 0xAA28}, + {0xAA40, 0xAA42}, {0xAA44, 0xAA4B}, {0xAA50, 0xAA59}, {0xAA60, 0xAA76}, {0xAA7A, 0xAA7A}, + {0xAA7E, 0xAAAF}, {0xAAB1, 0xAAB1}, {0xAAB5, 0xAAB6}, {0xAAB9, 0xAABD}, {0xAAC0, 0xAAC0}, + {0xAAC2, 0xAAC2}, {0xAADB, 0xAADD}, {0xAAE0, 0xAAEA}, {0xAAF2, 0xAAF4}, {0xAB01, 0xAB06}, + {0xAB09, 0xAB0E}, {0xAB11, 0xAB16}, {0xAB20, 0xAB26}, {0xAB28, 0xAB2E}, {0xAB30, 0xAB5A}, + {0xAB5C, 0xAB69}, {0xAB70, 0xABE2}, {0xABF0, 0xABF9}, {0xAC00, 0xD7A3}, {0xD7B0, 0xD7C6}, + {0xD7CB, 0xD7FB}, {0xF900, 0xFA6D}, {0xFA70, 0xFAD9}, {0xFB00, 0xFB06}, {0xFB13, 0xFB17}, + {0xFB1D, 0xFB1D}, {0xFB1F, 0xFB28}, {0xFB2A, 0xFB36}, {0xFB38, 0xFB3C}, {0xFB3E, 0xFB3E}, + {0xFB40, 0xFB41}, {0xFB43, 0xFB44}, {0xFB46, 0xFBB1}, {0xFBD3, 0xFD3D}, {0xFD50, 0xFD8F}, + {0xFD92, 0xFDC7}, {0xFDF0, 0xFDFB}, {0xFE70, 0xFE74}, {0xFE76, 0xFEFC}, {0xFF10, 0xFF19}, + {0xFF21, 0xFF3A}, {0xFF41, 0xFF5A}, {0xFF66, 0xFFBE}, {0xFFC2, 0xFFC7}, {0xFFCA, 0xFFCF}, + {0xFFD2, 0xFFD7}, {0xFFDA, 0xFFDC}, {0x10000, 0x1000B}, {0x1000D, 0x10026}, {0x10028, 0x1003A}, + {0x1003C, 0x1003D}, {0x1003F, 0x1004D}, {0x10050, 0x1005D}, {0x10080, 0x100FA}, {0x10107, 0x10133}, + {0x10140, 0x10178}, {0x1018A, 0x1018B}, {0x10280, 0x1029C}, {0x102A0, 0x102D0}, {0x102E1, 0x102FB}, + {0x10300, 0x10323}, {0x1032D, 0x1034A}, {0x10350, 0x10375}, {0x10380, 0x1039D}, {0x103A0, 0x103C3}, + {0x103C8, 0x103CF}, {0x103D1, 0x103D5}, {0x10400, 0x1049D}, {0x104A0, 0x104A9}, {0x104B0, 0x104D3}, + {0x104D8, 0x104FB}, {0x10500, 0x10527}, {0x10530, 0x10563}, {0x10570, 0x1057A}, {0x1057C, 0x1058A}, + {0x1058C, 0x10592}, {0x10594, 0x10595}, {0x10597, 0x105A1}, {0x105A3, 0x105B1}, {0x105B3, 0x105B9}, + {0x105BB, 0x105BC}, {0x10600, 0x10736}, {0x10740, 0x10755}, {0x10760, 0x10767}, {0x10780, 0x10785}, + {0x10787, 0x107B0}, {0x107B2, 0x107BA}, {0x10800, 0x10805}, {0x10808, 0x10808}, {0x1080A, 0x10835}, + {0x10837, 0x10838}, {0x1083C, 0x1083C}, {0x1083F, 0x10855}, {0x10858, 0x10876}, {0x10879, 0x1089E}, + {0x108A7, 0x108AF}, {0x108E0, 0x108F2}, {0x108F4, 0x108F5}, {0x108FB, 0x1091B}, {0x10920, 0x10939}, + {0x10980, 0x109B7}, {0x109BC, 0x109CF}, {0x109D2, 0x10A00}, {0x10A10, 0x10A13}, {0x10A15, 0x10A17}, + {0x10A19, 0x10A35}, {0x10A40, 0x10A48}, {0x10A60, 0x10A7E}, {0x10A80, 0x10A9F}, {0x10AC0, 0x10AC7}, + {0x10AC9, 0x10AE4}, {0x10AEB, 0x10AEF}, {0x10B00, 0x10B35}, {0x10B40, 0x10B55}, {0x10B58, 0x10B72}, + {0x10B78, 0x10B91}, {0x10BA9, 0x10BAF}, {0x10C00, 0x10C48}, {0x10C80, 0x10CB2}, {0x10CC0, 0x10CF2}, + {0x10CFA, 0x10D23}, {0x10D30, 0x10D39}, {0x10E60, 0x10E7E}, {0x10E80, 0x10EA9}, {0x10EB0, 0x10EB1}, + {0x10F00, 0x10F27}, {0x10F30, 0x10F45}, {0x10F51, 0x10F54}, {0x10F70, 0x10F81}, {0x10FB0, 0x10FCB}, + {0x10FE0, 0x10FF6}, {0x11003, 0x11037}, {0x11052, 0x1106F}, {0x11071, 0x11072}, {0x11075, 0x11075}, + {0x11083, 0x110AF}, {0x110D0, 0x110E8}, {0x110F0, 0x110F9}, {0x11103, 0x11126}, {0x11136, 0x1113F}, + {0x11144, 0x11144}, {0x11147, 0x11147}, {0x11150, 0x11172}, {0x11176, 0x11176}, {0x11183, 0x111B2}, + {0x111C1, 0x111C4}, {0x111D0, 0x111DA}, {0x111DC, 0x111DC}, {0x111E1, 0x111F4}, {0x11200, 0x11211}, + {0x11213, 0x1122B}, {0x1123F, 0x11240}, {0x11280, 0x11286}, {0x11288, 0x11288}, {0x1128A, 0x1128D}, + {0x1128F, 0x1129D}, {0x1129F, 0x112A8}, {0x112B0, 0x112DE}, {0x112F0, 0x112F9}, {0x11305, 0x1130C}, + {0x1130F, 0x11310}, {0x11313, 0x11328}, {0x1132A, 0x11330}, {0x11332, 0x11333}, {0x11335, 0x11339}, + {0x1133D, 0x1133D}, {0x11350, 0x11350}, {0x1135D, 0x11361}, {0x11400, 0x11434}, {0x11447, 0x1144A}, + {0x11450, 0x11459}, {0x1145F, 0x11461}, {0x11480, 0x114AF}, {0x114C4, 0x114C5}, {0x114C7, 0x114C7}, + {0x114D0, 0x114D9}, {0x11580, 0x115AE}, {0x115D8, 0x115DB}, {0x11600, 0x1162F}, {0x11644, 0x11644}, + {0x11650, 0x11659}, {0x11680, 0x116AA}, {0x116B8, 0x116B8}, {0x116C0, 0x116C9}, {0x11700, 0x1171A}, + {0x11730, 0x1173B}, {0x11740, 0x11746}, {0x11800, 0x1182B}, {0x118A0, 0x118F2}, {0x118FF, 0x11906}, + {0x11909, 0x11909}, {0x1190C, 0x11913}, {0x11915, 0x11916}, {0x11918, 0x1192F}, {0x1193F, 0x1193F}, + {0x11941, 0x11941}, {0x11950, 0x11959}, {0x119A0, 0x119A7}, {0x119AA, 0x119D0}, {0x119E1, 0x119E1}, + {0x119E3, 0x119E3}, {0x11A00, 0x11A00}, {0x11A0B, 0x11A32}, {0x11A3A, 0x11A3A}, {0x11A50, 0x11A50}, + {0x11A5C, 0x11A89}, {0x11A9D, 0x11A9D}, {0x11AB0, 0x11AF8}, {0x11C00, 0x11C08}, {0x11C0A, 0x11C2E}, + {0x11C40, 0x11C40}, {0x11C50, 0x11C6C}, {0x11C72, 0x11C8F}, {0x11D00, 0x11D06}, {0x11D08, 0x11D09}, + {0x11D0B, 0x11D30}, {0x11D46, 0x11D46}, {0x11D50, 0x11D59}, {0x11D60, 0x11D65}, {0x11D67, 0x11D68}, + {0x11D6A, 0x11D89}, {0x11D98, 0x11D98}, {0x11DA0, 0x11DA9}, {0x11EE0, 0x11EF2}, {0x11F02, 0x11F02}, + {0x11F04, 0x11F10}, {0x11F12, 0x11F33}, {0x11F50, 0x11F59}, {0x11FB0, 0x11FB0}, {0x11FC0, 0x11FD4}, + {0x12000, 0x12399}, {0x12400, 0x1246E}, {0x12480, 0x12543}, {0x12F90, 0x12FF0}, {0x13000, 0x1342F}, + {0x13441, 0x13446}, {0x14400, 0x14646}, {0x16800, 0x16A38}, {0x16A40, 0x16A5E}, {0x16A60, 0x16A69}, + {0x16A70, 0x16ABE}, {0x16AC0, 0x16AC9}, {0x16AD0, 0x16AED}, {0x16B00, 0x16B2F}, {0x16B40, 0x16B43}, + {0x16B50, 0x16B59}, {0x16B5B, 0x16B61}, {0x16B63, 0x16B77}, {0x16B7D, 0x16B8F}, {0x16E40, 0x16E96}, + {0x16F00, 0x16F4A}, {0x16F50, 0x16F50}, {0x16F93, 0x16F9F}, {0x16FE0, 0x16FE1}, {0x16FE3, 0x16FE3}, + {0x17000, 0x187F7}, {0x18800, 0x18CD5}, {0x18D00, 0x18D08}, {0x1AFF0, 0x1AFF3}, {0x1AFF5, 0x1AFFB}, + {0x1AFFD, 0x1AFFE}, {0x1B000, 0x1B122}, {0x1B132, 0x1B132}, {0x1B150, 0x1B152}, {0x1B155, 0x1B155}, + {0x1B164, 0x1B167}, {0x1B170, 0x1B2FB}, {0x1BC00, 0x1BC6A}, {0x1BC70, 0x1BC7C}, {0x1BC80, 0x1BC88}, + {0x1BC90, 0x1BC99}, {0x1D2C0, 0x1D2D3}, {0x1D2E0, 0x1D2F3}, {0x1D360, 0x1D378}, {0x1D400, 0x1D454}, + {0x1D456, 0x1D49C}, {0x1D49E, 0x1D49F}, {0x1D4A2, 0x1D4A2}, {0x1D4A5, 0x1D4A6}, {0x1D4A9, 0x1D4AC}, + {0x1D4AE, 0x1D4B9}, {0x1D4BB, 0x1D4BB}, {0x1D4BD, 0x1D4C3}, {0x1D4C5, 0x1D505}, {0x1D507, 0x1D50A}, + {0x1D50D, 0x1D514}, {0x1D516, 0x1D51C}, {0x1D51E, 0x1D539}, {0x1D53B, 0x1D53E}, {0x1D540, 0x1D544}, + {0x1D546, 0x1D546}, {0x1D54A, 0x1D550}, {0x1D552, 0x1D6A5}, {0x1D6A8, 0x1D6C0}, {0x1D6C2, 0x1D6DA}, + {0x1D6DC, 0x1D6FA}, {0x1D6FC, 0x1D714}, {0x1D716, 0x1D734}, {0x1D736, 0x1D74E}, {0x1D750, 0x1D76E}, + {0x1D770, 0x1D788}, {0x1D78A, 0x1D7A8}, {0x1D7AA, 0x1D7C2}, {0x1D7C4, 0x1D7CB}, {0x1D7CE, 0x1D7FF}, + {0x1DF00, 0x1DF1E}, {0x1DF25, 0x1DF2A}, {0x1E030, 0x1E06D}, {0x1E100, 0x1E12C}, {0x1E137, 0x1E13D}, + {0x1E140, 0x1E149}, {0x1E14E, 0x1E14E}, {0x1E290, 0x1E2AD}, {0x1E2C0, 0x1E2EB}, {0x1E2F0, 0x1E2F9}, + {0x1E4D0, 0x1E4EB}, {0x1E4F0, 0x1E4F9}, {0x1E7E0, 0x1E7E6}, {0x1E7E8, 0x1E7EB}, {0x1E7ED, 0x1E7EE}, + {0x1E7F0, 0x1E7FE}, {0x1E800, 0x1E8C4}, {0x1E8C7, 0x1E8CF}, {0x1E900, 0x1E943}, {0x1E94B, 0x1E94B}, + {0x1E950, 0x1E959}, {0x1EC71, 0x1ECAB}, {0x1ECAD, 0x1ECAF}, {0x1ECB1, 0x1ECB4}, {0x1ED01, 0x1ED2D}, + {0x1ED2F, 0x1ED3D}, {0x1EE00, 0x1EE03}, {0x1EE05, 0x1EE1F}, {0x1EE21, 0x1EE22}, {0x1EE24, 0x1EE24}, + {0x1EE27, 0x1EE27}, {0x1EE29, 0x1EE32}, {0x1EE34, 0x1EE37}, {0x1EE39, 0x1EE39}, {0x1EE3B, 0x1EE3B}, + {0x1EE42, 0x1EE42}, {0x1EE47, 0x1EE47}, {0x1EE49, 0x1EE49}, {0x1EE4B, 0x1EE4B}, {0x1EE4D, 0x1EE4F}, + {0x1EE51, 0x1EE52}, {0x1EE54, 0x1EE54}, {0x1EE57, 0x1EE57}, {0x1EE59, 0x1EE59}, {0x1EE5B, 0x1EE5B}, + {0x1EE5D, 0x1EE5D}, {0x1EE5F, 0x1EE5F}, {0x1EE61, 0x1EE62}, {0x1EE64, 0x1EE64}, {0x1EE67, 0x1EE6A}, + {0x1EE6C, 0x1EE72}, {0x1EE74, 0x1EE77}, {0x1EE79, 0x1EE7C}, {0x1EE7E, 0x1EE7E}, {0x1EE80, 0x1EE89}, + {0x1EE8B, 0x1EE9B}, {0x1EEA1, 0x1EEA3}, {0x1EEA5, 0x1EEA9}, {0x1EEAB, 0x1EEBB}, {0x1F100, 0x1F10C}, + {0x1FBF0, 0x1FBF9}, {0x20000, 0x2A6DF}, {0x2A700, 0x2B739}, {0x2B740, 0x2B81D}, {0x2B820, 0x2CEA1}, + {0x2CEB0, 0x2EBE0}, {0x2F800, 0x2FA1D}, {0x30000, 0x3134A}, {0x31350, 0x323AF}, +}}; + +inline bool isTokenChar(std::uint32_t code_point) +{ + if (code_point < 128) + return (code_point >= 'a' && code_point <= 'z') || (code_point >= 'A' && code_point <= 'Z') + || (code_point >= '0' && code_point <= '9') || code_point == '_'; + std::size_t lo = 0; + std::size_t hi = token_char_ranges.size(); + while (lo < hi) + { + const std::size_t mid = lo + (hi - lo) / 2; + const auto & span = token_char_ranges[mid]; + if (code_point < span.first) + hi = mid; + else if (code_point > span.last) + lo = mid + 1; + else + return true; + } + return false; +} +} // namespace DB::LocalMatchAgainst diff --git a/dbms/src/Functions/registerFunctions.cpp b/dbms/src/Functions/registerFunctions.cpp index cb43ae7760d..7a9193c392c 100644 --- a/dbms/src/Functions/registerFunctions.cpp +++ b/dbms/src/Functions/registerFunctions.cpp @@ -51,6 +51,7 @@ void registerFunctionsRegexpSubstr(FunctionFactory &); void registerFunctionsRegexpReplace(FunctionFactory &); void registerFunctionsGrouping(FunctionFactory &); void registerFunctionsVector(FunctionFactory &); +void registerFunctionsLocalMatchAgainst(FunctionFactory &); void registerFunctions() { @@ -85,6 +86,7 @@ void registerFunctions() registerFunctionsIsIPAddr(factory); registerFunctionsGrouping(factory); registerFunctionsVector(factory); + registerFunctionsLocalMatchAgainst(factory); } } // namespace DB diff --git a/dbms/src/Functions/tests/gtest_local_match_against.cpp b/dbms/src/Functions/tests/gtest_local_match_against.cpp new file mode 100644 index 00000000000..c90d9968248 --- /dev/null +++ b/dbms/src/Functions/tests/gtest_local_match_against.cpp @@ -0,0 +1,1918 @@ +// Copyright 2026 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +namespace DB::ErrorCodes +{ +extern const int ILLEGAL_COLUMN; +} + +namespace DB::tests +{ +class TestLocalMatchAgainst : public DB::tests::FunctionTest +{ +}; + +namespace +{ +constexpr std::array builtin_stopwords{ + "a", "about", "an", "are", "as", "at", "be", "by", "com", "de", "en", "for", + "from", "how", "i", "in", "is", "it", "la", "of", "on", "or", "that", "the", + "this", "to", "was", "what", "when", "where", "who", "will", "with", "und", "www"}; + +// Count token comparisons rather than wall time, so a repeated full-column +// verification regression is deterministic even on a loaded CI host. +class CountingMatchCollator final : public TiDB::ITiDBCollator +{ +public: + CountingMatchCollator() + : ITiDBCollator(UTF8MB4_BIN) + , inner(getCollator(UTF8MB4_BIN)) + { + collator_type = inner->getCollatorType(); + } + + int compare(const char * lhs, size_t lhs_size, const char * rhs, size_t rhs_size) const override + { + ++comparisons; + return inner->compare(lhs, lhs_size, rhs, rhs_size); + } + StringRef convert(const char * s, size_t size, String & buffer, std::vector * lens) const override + { + return inner->convert(s, size, buffer, lens); + } + StringRef sortKeyNoTrim(const char * s, size_t size, String & buffer) const override + { + return inner->sortKeyNoTrim(s, size, buffer); + } + StringRef sortKey(const char * s, size_t size, String & buffer) const override + { + return inner->sortKey(s, size, buffer); + } + std::unique_ptr pattern() const override { return inner->pattern(); } + + mutable size_t comparisons = 0; + +private: + TiDB::TiDBCollatorPtr inner; +}; + +String serializeLocalMatchAgainstBooleanQuery(const tipb::LocalMatchAgainstBooleanQuery & query) +{ + auto encoded_query = query; + if (encoded_query.version() == 0) + encoded_query.set_version(2); // The tests exercise the currently supported Local MATCH semantic protocol. + if (encoded_query.parser() == tipb::LocalMatchAgainstParser::LocalMatchAgainstParserInvalid) + encoded_query.set_parser(tipb::LocalMatchAgainstParser::LocalMatchAgainstParserStandard); + if (encoded_query.stopword_mode() == tipb::LocalMatchAgainstStopwordMode::LocalMatchAgainstStopwordModeInvalid) + encoded_query.set_stopword_mode(tipb::LocalMatchAgainstStopwordMode::LocalMatchAgainstStopwordModeBuiltin); + if (encoded_query.stopword_collation().empty()) + encoded_query.set_stopword_collation("utf8mb4_bin"); + return encoded_query.SerializeAsString(); +} + +tipb::LocalMatchAgainstBooleanQuery makeRequiredLocalMatchQuery( + const String & text, + tipb::LocalMatchAgainstBooleanTermType term_type = tipb::LocalMatchAgainstBooleanTermWord) +{ + tipb::LocalMatchAgainstBooleanQuery query; + query.set_innodb_ft_min_token_size(3); + query.set_innodb_ft_max_token_size(84); + auto * node = query.add_nodes(); + node->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + node->set_term_type(term_type); + node->set_text(text); + return query; +} + +// Construct the wire expression directly: metadata belongs to Expr.val, not +// SQL children. ColumnRef offsets and signed collations use the real DAG path. +tipb::Expr makeLocalMatchDAGExpr( + const String & search, + const tipb::LocalMatchAgainstBooleanQuery & query, + const NamesAndTypes & inputs, + Int32 collation, + bool null_search = false) +{ + tipb::Expr expr; + expr.set_tp(tipb::ExprType::ScalarFunc); + expr.set_sig(tipb::ScalarFuncSig::LocalMatchAgainstBoolean); + expr.set_val(serializeLocalMatchAgainstBooleanQuery(query)); + auto * result_type = expr.mutable_field_type(); + result_type->set_tp(TiDB::TypeDouble); + result_type->set_collate(-collation); + const bool nullable = null_search + || std::any_of(inputs.begin(), inputs.end(), [](const auto & input) { return input.type->isNullable(); }); + result_type->set_flag(nullable ? 0 : TiDB::ColumnFlagNotNull); + auto search_expr = null_search ? constructNULLLiteralTiExpr() : constructStringLiteralTiExpr(search); + search_expr.mutable_field_type()->set_collate(-collation); + *expr.add_children() = std::move(search_expr); + for (size_t i = 0; i < inputs.size(); ++i) + { + auto column = constructInt64LiteralTiExpr(static_cast(i)); + column.set_tp(tipb::ExprType::ColumnRef); + column.mutable_field_type()->set_tp(TiDB::TypeString); + column.mutable_field_type()->set_flag(inputs[i].type->isNullable() ? 0 : TiDB::ColumnFlagNotNull); + column.mutable_field_type()->set_collate(-collation); + *expr.add_children() = std::move(column); + } + return expr; +} + +std::pair buildLocalMatchDAGActions( + Context & context, + const NamesAndTypes & inputs, + const std::vector & expressions) +{ + DAGExpressionAnalyzer analyzer(inputs, context); + ExpressionActionsChain chain; + auto & step = analyzer.initAndGetLastStep(chain); + Names results; + for (const auto & expr : expressions) + { + results.push_back(analyzer.getActions(expr, step.actions)); + step.required_output.push_back(results.back()); + } + chain.finalize(); + return {step.actions, results}; +} +} // namespace + +TEST_F(TestLocalMatchAgainst, MatchBooleanDAGCollationAndMultiColumn) +try +{ + const ColumnsWithTypeAndName columns{ + createColumn>({"CAFE", {}, "cafe", "cafeteria", {}, "cafe"}, "title"), + createColumn>({"", "café", "mysql", "", {}, "MYSQL"}, "body")}; + const NamesAndTypes inputs{{"title", columns[0].type}, {"body", columns[1].type}}; + for (const auto * collation : + {"utf8mb4_bin", "utf8mb4_0900_bin", "utf8mb4_general_ci", "utf8mb4_unicode_ci", "utf8mb4_0900_ai_ci"}) + { + const auto collator = TiDB::ITiDBCollator::getCollator(collation); + ASSERT_NE(collator, nullptr); + const bool binary = std::string_view(collation).ends_with("_bin"); + for (const bool prefix : {false, true}) + { + SCOPED_TRACE(fmt::format("collation={} prefix={}", collation, prefix)); + auto query = makeRequiredLocalMatchQuery( + prefix ? "caf" : "cafe", + prefix ? tipb::LocalMatchAgainstBooleanTermPrefix : tipb::LocalMatchAgainstBooleanTermWord); + auto * prohibited = query.add_nodes(); + prohibited->set_occur(tipb::LocalMatchAgainstBooleanOccurMustNot); + prohibited->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + prohibited->set_text("mysql"); + const auto expr = makeLocalMatchDAGExpr( + prefix ? "+caf* -mysql" : "+cafe -mysql", + query, + inputs, + collator->getCollatorId()); + ASSERT_EQ(expr.children_size(), 3); // Query + two MATCH columns; no metadata SQL child. + const auto [actions, names] = buildLocalMatchDAGActions(*context, inputs, {expr}); + Block block(columns); + actions->execute(block); + const auto expected = prefix + ? (binary ? createColumn>({0, 1, 0, 1, 0, 1}) + : createColumn>({1, 1, 0, 1, 0, 0})) + : (binary ? createColumn>({0, 0, 0, 0, 0, 1}) + : createColumn>({1, 1, 0, 0, 0, 0})); + ASSERT_COLUMN_EQ(expected, block.getByName(names[0])); + } + } +} +CATCH + +TEST_F(TestLocalMatchAgainst, MatchBooleanDAGMetadataIsolation) +try +{ + const auto documents + = createColumn({"the", "the 中文", "中文", "中文词", "中文测试", "", "中 文", "foo", "thé"}, "body"); + const NamesAndTypes inputs{{"body", documents.type}}; + const auto binary = TiDB::ITiDBCollator::UTF8MB4_BIN; + std::vector expressions; + ColumnsWithTypeAndName expected; + const auto append = [&](const String & search, + const tipb::LocalMatchAgainstBooleanQuery & query, + Int32 collation, + const ColumnWithTypeAndName & result) { + expressions.push_back(makeLocalMatchDAGExpr(search, query, inputs, collation)); + expected.push_back(result); + }; + auto query = makeRequiredLocalMatchQuery("the"); + append("+the", query, binary, createColumn({0, 0, 0, 0, 0, 0, 0, 0, 0})); + query.set_stopword_mode(tipb::LocalMatchAgainstStopwordModeDisabled); + append("+the", query, binary, createColumn({1, 1, 0, 0, 0, 0, 0, 0, 0})); + + query = makeRequiredLocalMatchQuery("中文"); + query.set_parser(tipb::LocalMatchAgainstParserNgram); + query.set_stopword_mode(tipb::LocalMatchAgainstStopwordModeDisabled); + query.set_ngram_token_size(2); + append("+中文", query, binary, createColumn({0, 1, 1, 1, 1, 0, 0, 0, 0})); + query.set_ngram_token_size(3); + append("+中文", query, binary, createColumn({0, 0, 0, 0, 0, 0, 0, 0, 0})); + + query = makeRequiredLocalMatchQuery("foo"); + append("+foo", query, binary, createColumn({0, 0, 0, 0, 0, 0, 0, 1, 0})); + query.set_innodb_ft_min_token_size(4); + append("+foo", query, binary, createColumn({0, 0, 0, 0, 0, 0, 0, 0, 0})); + + query = makeRequiredLocalMatchQuery("thé"); + query.set_stopword_collation("utf8mb4_bin"); + append("+thé", query, TiDB::ITiDBCollator::UTF8MB4_GENERAL_CI, createColumn({0, 0, 0, 0, 0, 0, 0, 0, 1})); + query.set_stopword_collation("utf8mb4_general_ci"); + append("+thé", query, TiDB::ITiDBCollator::UTF8MB4_GENERAL_CI, createColumn({0, 0, 0, 0, 0, 0, 0, 0, 0})); + + const auto [actions, names] = buildLocalMatchDAGActions(*context, inputs, expressions); + ASSERT_EQ(names.size(), expected.size()); + for (size_t i = 0; i < names.size(); i += 2) + ASSERT_NE(names[i], names[i + 1]) << "Different metadata must not alias the same action"; + Block block({documents}); + actions->execute(block); + for (size_t i = 0; i < names.size(); ++i) + { + SCOPED_TRACE(i); + ASSERT_COLUMN_EQ(expected[i], block.getByName(names[i])); + } +} +CATCH + +TEST_F(TestLocalMatchAgainst, MatchBooleanDAGReusesActionsAcrossBlocks) +try +{ + struct BlockCase + { + ColumnsWithTypeAndName columns; + ColumnWithTypeAndName expected; + }; + for (const bool ngram : {false, true}) + { + SCOPED_TRACE(ngram); + auto query = makeRequiredLocalMatchQuery( + ngram ? "中文测试" : "quick the fox", + tipb::LocalMatchAgainstBooleanTermPhrase); + if (ngram) + { + query.set_parser(tipb::LocalMatchAgainstParserNgram); + query.set_ngram_token_size(2); + query.set_stopword_mode(tipb::LocalMatchAgainstStopwordModeDisabled); + } + auto * prohibited = query.add_nodes(); + prohibited->set_occur(tipb::LocalMatchAgainstBooleanOccurMustNot); + prohibited->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + prohibited->set_text("mysql"); + const auto first = ngram + ? BlockCase{{createColumn>({"中文测试", "中文", "中文测试", "", {}}, "title"), createColumn>({{}, "测试", "mysql", "中文测试", {}}, "body")}, createColumn>({1, 0, 0, 1, 0})} + : BlockCase{ + {createColumn>( + {"QUICK the fox", "quick the fox", "quick", {}, "quick the fox"}, + "title"), + createColumn>({{}, "mysql", "the fox", {}, "MYSQL"}, "body")}, + createColumn>({1, 0, 0, 0, 0})}; + const auto second = ngram + ? BlockCase{{createColumn>({"中文 test", {}, "中文测试", "中 文测试"}, "title"), createColumn>({"", {}, "", "中文测试"}, "body")}, createColumn>({0, 0, 1, 1})} + : BlockCase{ + {createColumn>( + {"slow the fox", "quick the fox", {}, "quick fox", "quick the fox"}, + "title"), + createColumn>({"", {}, "quick the fox", "", {}}, "body")}, + createColumn>({0, 1, 1, 0, 1})}; + const BlockCase nulls{ + {createColumn>({{}, {}}, "title"), createColumn>({{}, {}}, "body")}, + createColumn>({0, 0})}; + const BlockCase empty{ + {createColumn>({}, "title"), createColumn>({}, "body")}, + createColumn>({})}; + const NamesAndTypes inputs{{"title", first.columns[0].type}, {"body", first.columns[1].type}}; + const auto expr = makeLocalMatchDAGExpr( + ngram ? "+\"中文测试\" -mysql" : "+\"quick the fox\" -mysql", + query, + inputs, + TiDB::ITiDBCollator::UTF8MB4_GENERAL_CI); + const auto [actions, names] = buildLocalMatchDAGActions(*context, inputs, {expr}); + // The same compiled actions/function sees changing row counts, empty + // input, and old input again. No previous result or NULL map may leak. + for (size_t pass = 0; pass < 2; ++pass) + for (const auto * batch : {&first, &second, &nulls, &empty, &first}) + { + SCOPED_TRACE(fmt::format("pass={} rows={}", pass, batch->expected.column->size())); + Block block(batch->columns); + actions->execute(block); + ASSERT_COLUMN_EQ(batch->expected, block.getByName(names[0])); + } + } +} +CATCH + +TEST_F(TestLocalMatchAgainst, MatchBooleanDAGColumnWrappersAndEmptyBlocks) +try +{ + struct ColumnCase + { + ColumnWithTypeAndName document; + ColumnWithTypeAndName expected; + }; + const std::vector cases{ + {createColumn({"quick", "", "quick fox", "slow"}, "body"), createColumn({1, 0, 1, 0})}, + {createColumn>({"quick", {}, "", "slow"}, "body"), + createColumn>({1, 0, 0, 0})}, + {createColumn>({{}, {}, {}, {}}, "body"), createColumn>({0, 0, 0, 0})}, + {createConstColumn(4, "quick", "body"), createColumn({1, 1, 1, 1})}, + {createConstColumn>(4, "quick", "body"), createColumn>({1, 1, 1, 1})}, + {createConstColumn>(4, {}, "body"), createColumn>({0, 0, 0, 0})}, + {createConstColumn(4, "", "body"), createColumn({0, 0, 0, 0})}}; + const auto query = makeRequiredLocalMatchQuery("quick"); + for (size_t i = 0; i < cases.size(); ++i) + for (const bool null_search : {false, true}) + { + SCOPED_TRACE(fmt::format("case={} null_search={}", i, null_search)); + const auto & column_case = cases[i]; + const NamesAndTypes inputs{{"body", column_case.document.type}}; + const auto expr + = makeLocalMatchDAGExpr("+quick", query, inputs, TiDB::ITiDBCollator::UTF8MB4_BIN, null_search); + ASSERT_EQ(expr.children(0).field_type().tp(), null_search ? TiDB::TypeNull : TiDB::TypeString); + const auto [actions, names] = buildLocalMatchDAGActions(*context, inputs, {expr}); + const auto expected = null_search ? createColumn>({0, 0, 0, 0}) : column_case.expected; + for (const size_t rows : {4U, 0U, 4U}) + { + SCOPED_TRACE(rows); + auto document = column_case.document; + document.column = document.column->cloneResized(rows); + auto expected_rows = expected; + expected_rows.column = expected_rows.column->cloneResized(rows); + Block block({document}); + actions->execute(block); + ASSERT_COLUMN_EQ(expected_rows, block.getByName(names[0])); + } + } +} +CATCH + +TEST_F(TestLocalMatchAgainst, MatchBooleanMultiColumnNullable) +try +{ + tipb::LocalMatchAgainstBooleanQuery boolean_query; + auto * required = boolean_query.add_nodes(); + required->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + required->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + required->set_text("quick"); + + const String metadata = serializeLocalMatchAgainstBooleanQuery(boolean_query); + ASSERT_COLUMN_EQ( + createColumn>({1, 1, 0, 0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(4, "+quick"), + createColumn>({{}, "quick fox", {}, "slow fox"}), + createColumn>({"quick fox", {}, "slow fox", {}}), + createConstColumn(4, metadata)}, + nullptr, + true)); +} +CATCH + +TEST_F(TestLocalMatchAgainst, FilteredPhraseVerificationComparisonGrowth) +try +{ + tipb::LocalMatchAgainstBooleanQuery query; + auto * node = query.add_nodes(); + node->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + node->set_term_type(tipb::LocalMatchAgainstBooleanTermPhrase); + node->set_text("foo a bar"); + size_t small_comparisons = 0; + for (const size_t repeats : {32, 256}) + { + String document; + for (size_t i = 0; i < repeats; ++i) + document += "foo x bar "; + CountingMatchCollator collator; + ASSERT_COLUMN_EQ( + createColumn({0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(1, "+\"foo a bar\""), + createColumn({document}), + createConstColumn(1, serializeLocalMatchAgainstBooleanQuery(query))}, + &collator, + true)); + if (repeats == 32) + small_comparisons = collator.comparisons; + else + { + EXPECT_GT(small_comparisons, 0); + // Eight times the input may add proportional token work, not + // 64 times the work from rechecking every failed anchor. + EXPECT_LE(collator.comparisons, small_comparisons * 12); + } + } +} +CATCH + +TEST_F(TestLocalMatchAgainst, FilteredPhraseVerificationCacheScope) +try +{ + String miss; + for (size_t i = 0; i < 64; ++i) + miss += "foo x bar "; + for (const auto occur : + {tipb::LocalMatchAgainstBooleanOccurMust, + tipb::LocalMatchAgainstBooleanOccurShould, + tipb::LocalMatchAgainstBooleanOccurMustNot}) + { + tipb::LocalMatchAgainstBooleanQuery query; + if (occur != tipb::LocalMatchAgainstBooleanOccurShould) + { + auto * anchor = query.add_nodes(); + anchor->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + anchor->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + anchor->set_text("foo"); + } + auto * phrase = query.add_nodes(); + phrase->set_occur(occur); + phrase->set_term_type(tipb::LocalMatchAgainstBooleanTermPhrase); + phrase->set_text("foo a bar"); + const Float64 hit = occur == tipb::LocalMatchAgainstBooleanOccurMustNot ? 0 : 1; + const Float64 no_phrase = occur == tipb::LocalMatchAgainstBooleanOccurMustNot ? 1 : 0; + ASSERT_COLUMN_EQ( + createColumn>({hit, hit, no_phrase, hit, no_phrase}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(5, "foo \"foo a bar\""), + createColumn>({miss, "foo a bar", miss, {}, miss}), + createColumn>({"foo a bar", miss, {}, "foo a bar", miss}), + createConstColumn(5, serializeLocalMatchAgainstBooleanQuery(query))}, + nullptr, + true)); + } + + tipb::LocalMatchAgainstBooleanQuery query; + auto * required = query.add_nodes(); + required->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + required->set_term_type(tipb::LocalMatchAgainstBooleanTermPhrase); + required->set_text("foo x bar"); + auto * excluded = query.add_nodes(); + excluded->set_occur(tipb::LocalMatchAgainstBooleanOccurMustNot); + excluded->set_term_type(tipb::LocalMatchAgainstBooleanTermPhrase); + excluded->set_text("foo a bar"); + ASSERT_COLUMN_EQ( + createColumn({1, 0, 1}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(3, "+\"foo x bar\" -\"foo a bar\""), + createColumn({miss, "foo x bar foo a bar", miss}), + createConstColumn(3, serializeLocalMatchAgainstBooleanQuery(query))}, + nullptr, + true)); + + // NGRAM size 3 removes "the" but retains "foo" and "zoo". + // STANDARD exercises the same cache with a filtered stopword. + for (const bool ngram : {false, true}) + { + tipb::LocalMatchAgainstBooleanQuery filtered; + filtered.set_parser(ngram ? tipb::LocalMatchAgainstParserNgram : tipb::LocalMatchAgainstParserStandard); + filtered.set_ngram_token_size(3); + auto * node = filtered.add_nodes(); + node->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + node->set_term_type(tipb::LocalMatchAgainstBooleanTermPhrase); + node->set_text("foo the zoo"); + String repetitive_miss; + for (size_t i = 0; i < 64; ++i) + repetitive_miss += "foo xyz zoo "; + ASSERT_COLUMN_EQ( + createColumn>({1, 1, 0, 1}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(4, "+\"foo the zoo\""), + createColumn>({repetitive_miss, "foo the zoo", repetitive_miss, {}}), + createColumn>({"foo the zoo", repetitive_miss, {}, "foo the zoo"}), + createConstColumn(4, serializeLocalMatchAgainstBooleanQuery(filtered))}, + nullptr, + true)); + } +} +CATCH + +TEST_F(TestLocalMatchAgainst, MatchBooleanMySQLCharacterBoundaries) +try +{ + tipb::LocalMatchAgainstBooleanQuery query; + query.set_parser(tipb::LocalMatchAgainstParserNgram); + query.set_ngram_token_size(2); + query.set_stopword_mode(tipb::LocalMatchAgainstStopwordModeDisabled); + auto * node = query.add_nodes(); + node->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + node->set_term_type(tipb::LocalMatchAgainstBooleanTermPrefix); + node->set_text("a"); + ASSERT_COLUMN_EQ( + createColumn({0, 1, 1, 1, 1, 1}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(6, "+a*"), + createColumn({"a,b", "a，b", "a🙃b", "a𞤀b", "a𝟙b", String("ab") + char(0xff) + "cd"}), + createConstColumn(6, serializeLocalMatchAgainstBooleanQuery(query))}, + nullptr, + true)); + + query.set_parser(tipb::LocalMatchAgainstParserStandard); + node->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + node->set_text("foo"); + ASSERT_COLUMN_EQ( + createColumn({1, 1, 1, 0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(4, "+foo"), + createColumn({"foo𞤀bar", "foo𝟙bar", "foo🙃bar", "foobar"}), + createConstColumn(4, serializeLocalMatchAgainstBooleanQuery(query))}, + nullptr, + true)); +} +CATCH + +TEST_F(TestLocalMatchAgainst, MatchBooleanFilteredPhraseVerification) +try +{ + for (const bool ngram : {false, true}) + for (const bool stopwords : {false, true}) + { + tipb::LocalMatchAgainstBooleanQuery query; + query.set_parser(ngram ? tipb::LocalMatchAgainstParserNgram : tipb::LocalMatchAgainstParserStandard); + query.set_ngram_token_size(2); + query.set_stopword_mode( + stopwords ? tipb::LocalMatchAgainstStopwordModeBuiltin : tipb::LocalMatchAgainstStopwordModeDisabled); + auto * node = query.add_nodes(); + node->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + node->set_term_type(tipb::LocalMatchAgainstBooleanTermPhrase); + node->set_text("quick x fox"); + ASSERT_COLUMN_EQ( + createColumn({ngram ? 0.0 : 1.0, 0, 0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(3, "+\"quick x fox\""), + createColumn({"quick x fox", "quick the fox", "quick xx fox"}), + createConstColumn(3, serializeLocalMatchAgainstBooleanQuery(query))}, + nullptr, + true)); + node->set_text("foo bar"); + ASSERT_COLUMN_EQ( + createColumn({1, 1, ngram ? 0.0 : 1.0, ngram ? 0.0 : 1.0, 0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(5, "+\"foo bar\""), + createColumn({"foo bar", "foo,bar", "foo，bar", "foo🙃bar", "foobar"}), + createConstColumn(5, serializeLocalMatchAgainstBooleanQuery(query))}, + nullptr, + true)); + } + + tipb::LocalMatchAgainstBooleanQuery query; + query.set_parser(tipb::LocalMatchAgainstParserNgram); + query.set_ngram_token_size(3); + query.set_stopword_mode(tipb::LocalMatchAgainstStopwordModeBuiltin); + auto * node = query.add_nodes(); + node->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + node->set_term_type(tipb::LocalMatchAgainstBooleanTermPhrase); + for (const String & text : {String("quick the fox"), String("quick a fox")}) + { + node->set_text(text); + ASSERT_COLUMN_EQ( + createColumn({1, 1, 1}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(3, text), + createColumn({"quick fox", "quick x fox", "fox quick"}), + createConstColumn(3, serializeLocalMatchAgainstBooleanQuery(query))}, + nullptr, + true)); + } + for (const String & text : {String("foo a zoo"), String("foo zoo")}) + { + node->set_text(text); + const Float64 expected = text == "foo zoo" ? 1.0 : 0.0; + ASSERT_COLUMN_EQ( + createColumn({expected, expected, expected, 0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(4, text), + createColumn({"foo zoo", "foo a zoo", "foo x zoo", "zoo foo"}), + createConstColumn(4, serializeLocalMatchAgainstBooleanQuery(query))}, + nullptr, + true)); + } +} +CATCH + +TEST_F(TestLocalMatchAgainst, MatchBooleanFilteredRequiredClause) +try +{ + for (const bool ngram : {false, true}) + for (const String & term : {String("the"), String("x")}) + { + tipb::LocalMatchAgainstBooleanQuery query; + query.set_parser(ngram ? tipb::LocalMatchAgainstParserNgram : tipb::LocalMatchAgainstParserStandard); + query.set_ngram_token_size(3); + auto * required = query.add_nodes(); + required->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + required->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + required->set_text(term); + auto * optional = query.add_nodes(); + optional->set_occur(tipb::LocalMatchAgainstBooleanOccurShould); + optional->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + optional->set_text("quick"); + const auto columns = ColumnsWithTypeAndName{ + createConstColumn(3, "+" + term + " quick"), + createColumn>({"the x quick", String(4096, 'x'), {}}), + createConstColumn(3, serializeLocalMatchAgainstBooleanQuery(query))}; + ASSERT_COLUMN_EQ( + createColumn>({0, 0, 0}), + executeFunction("local_match_against_boolean", columns, nullptr, true)); + + // NULL AGAINST is an empty search even for an impossible query. + auto null_search = columns; + null_search[0] = createConstColumn>(3, {}); + ASSERT_COLUMN_EQ( + createColumn>({0, 0, 0}), + executeFunction("local_match_against_boolean", null_search, nullptr, true)); + + // Envelope validation must still reject an unknown version. + query.set_version(999); + EXPECT_THROW( + executeFunction( + "local_match_against_boolean", + {columns[0], + columns[1], + createConstColumn(3, serializeLocalMatchAgainstBooleanQuery(query))}, + nullptr, + true), + Exception); + } +} +CATCH + +TEST_F(TestLocalMatchAgainst, MatchBooleanProtocolQuery) +try +{ + tipb::LocalMatchAgainstBooleanQuery boolean_query; + auto * required = boolean_query.add_nodes(); + required->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + required->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + required->set_text("quick"); + auto * prohibited = boolean_query.add_nodes(); + prohibited->set_occur(tipb::LocalMatchAgainstBooleanOccurMustNot); + prohibited->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + prohibited->set_text("slow"); + + const String metadata = serializeLocalMatchAgainstBooleanQuery(boolean_query); + ASSERT_COLUMN_EQ( + createColumn({1, 0, 0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(3, "+quick -slow"), + createColumn({"quick brown", "slow fox", "brown fox"}), + createConstColumn(3, metadata)}, + nullptr, + true)); + + tipb::LocalMatchAgainstBooleanQuery prohibited_query; + auto * prohibited_only = prohibited_query.add_nodes(); + prohibited_only->set_occur(tipb::LocalMatchAgainstBooleanOccurMustNot); + prohibited_only->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + prohibited_only->set_text("slow"); + const String prohibited_metadata = serializeLocalMatchAgainstBooleanQuery(prohibited_query); + ASSERT_COLUMN_EQ( + createColumn({0, 0, 0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(3, "-slow"), + createColumn({"quick brown", "slow fox", "brown fox"}), + createConstColumn(3, prohibited_metadata)}, + nullptr, + true)); +} +CATCH + +TEST_F(TestLocalMatchAgainst, MatchBooleanPhrasePositionsWithStopwordGaps) +try +{ + tipb::LocalMatchAgainstBooleanQuery boolean_query; + auto * required = boolean_query.add_nodes(); + required->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + required->set_term_type(tipb::LocalMatchAgainstBooleanTermPhrase); + required->set_text("quick the fox"); + + const String metadata = serializeLocalMatchAgainstBooleanQuery(boolean_query); + ASSERT_COLUMN_EQ( + createColumn({1, 0, 0, 1}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(4, "+\"quick the fox\""), + createColumn({"quick the fox", "quick fox", "quick slow the fox", "quick the fox quick the fox"}), + createConstColumn(4, metadata)}, + nullptr, + true)); +} +CATCH + +TEST_F(TestLocalMatchAgainst, MatchBooleanConstantDocumentColumn) +try +{ + tipb::LocalMatchAgainstBooleanQuery boolean_query; + auto * required = boolean_query.add_nodes(); + required->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + required->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + required->set_text("quick"); + + ASSERT_COLUMN_EQ( + createColumn({1, 1, 1}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(3, "+quick"), + createConstColumn(3, "quick fox"), + createConstColumn(3, serializeLocalMatchAgainstBooleanQuery(boolean_query))}, + nullptr, + true)); +} +CATCH + +TEST_F(TestLocalMatchAgainst, MatchBooleanMultiColumnStateAndPhraseBoundary) +try +{ + const auto make_query = [](tipb::LocalMatchAgainstBooleanTermType term_type) { + tipb::LocalMatchAgainstBooleanQuery boolean_query; + auto * required = boolean_query.add_nodes(); + required->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + required->set_term_type(term_type); + required->set_text("quick fox"); + return serializeLocalMatchAgainstBooleanQuery(boolean_query); + }; + + ASSERT_COLUMN_EQ( + createColumn({1, 0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(2, "+quick +fox"), + createColumn({"quick", "quick"}), + createColumn({"fox", "slow"}), + createConstColumn(2, make_query(tipb::LocalMatchAgainstBooleanTermWord))}, + nullptr, + true)); + + ASSERT_COLUMN_EQ( + createColumn({0, 1}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(2, "+\"quick fox\""), + createColumn({"quick", "quick fox"}), + createColumn({"fox", "slow"}), + createConstColumn(2, make_query(tipb::LocalMatchAgainstBooleanTermPhrase))}, + nullptr, + true)); +} +CATCH + +TEST_F(TestLocalMatchAgainst, MatchBooleanDecidesPositiveMatchEarly) +try +{ + tipb::LocalMatchAgainstBooleanQuery required_query; + auto * required_quick = required_query.add_nodes(); + required_quick->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + required_quick->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + required_quick->set_text("quick"); + auto * required_fox = required_query.add_nodes(); + required_fox->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + required_fox->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + required_fox->set_text("fox"); + + ASSERT_COLUMN_EQ( + createColumn({1, 0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(2, "+quick +fox"), + createColumn({"quick", "quick"}), + createColumn({"fox", "quiet"}), + createConstColumn(2, serializeLocalMatchAgainstBooleanQuery(required_query))}, + nullptr, + true)); + + tipb::LocalMatchAgainstBooleanQuery optional_query; + auto * optional_quick = optional_query.add_nodes(); + optional_quick->set_occur(tipb::LocalMatchAgainstBooleanOccurShould); + optional_quick->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + optional_quick->set_text("quick"); + auto * optional_fox = optional_query.add_nodes(); + optional_fox->set_occur(tipb::LocalMatchAgainstBooleanOccurShould); + optional_fox->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + optional_fox->set_text("fox"); + + ASSERT_COLUMN_EQ( + createColumn({1, 0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(2, "quick fox"), + createColumn({"quick", "quiet"}), + createConstColumn(2, serializeLocalMatchAgainstBooleanQuery(optional_query))}, + nullptr, + true)); +} +CATCH + +TEST_F(TestLocalMatchAgainst, MatchBooleanStillChecksProhibitedTermsAfterPositiveMatch) +try +{ + tipb::LocalMatchAgainstBooleanQuery boolean_query; + auto * required_quick = boolean_query.add_nodes(); + required_quick->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + required_quick->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + required_quick->set_text("quick"); + auto * required_fox = boolean_query.add_nodes(); + required_fox->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + required_fox->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + required_fox->set_text("fox"); + auto * prohibited_slow = boolean_query.add_nodes(); + prohibited_slow->set_occur(tipb::LocalMatchAgainstBooleanOccurMustNot); + prohibited_slow->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + prohibited_slow->set_text("slow"); + + ASSERT_COLUMN_EQ( + createColumn({1, 0, 0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(3, "+quick +fox -slow"), + createColumn({"quick", "quick", "quick"}), + createColumn({"fox", "fox", "quiet"}), + createColumn({"calm", "slow", "slow"}), + createConstColumn(3, serializeLocalMatchAgainstBooleanQuery(boolean_query))}, + nullptr, + true)); +} +CATCH + +TEST_F(TestLocalMatchAgainst, MatchBooleanRejectsInvalidProtocol) +try +{ + tipb::LocalMatchAgainstBooleanQuery boolean_query; + boolean_query.set_version(1); // Version 1 must not be silently reinterpreted as version 2. + boolean_query.set_parser(tipb::LocalMatchAgainstParser::LocalMatchAgainstParserStandard); + boolean_query.set_stopword_mode(tipb::LocalMatchAgainstStopwordMode::LocalMatchAgainstStopwordModeBuiltin); + boolean_query.set_stopword_collation("utf8mb4_bin"); + auto * required = boolean_query.add_nodes(); + required->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + required->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + required->set_text("quick"); + + ASSERT_THROW( + executeFunction( + "local_match_against_boolean", + {createConstColumn(1, "+quick"), + createColumn({"quick brown"}), + createConstColumn(1, boolean_query.SerializeAsString())}, + nullptr, + true), + Exception); + + boolean_query.set_version(2); + boolean_query.set_parser(tipb::LocalMatchAgainstParser::LocalMatchAgainstParserInvalid); + ASSERT_THROW( + executeFunction( + "local_match_against_boolean", + {createConstColumn(1, "+quick"), + createColumn({"quick brown"}), + createConstColumn(1, boolean_query.SerializeAsString())}, + nullptr, + true), + Exception); + + boolean_query.set_parser(tipb::LocalMatchAgainstParser::LocalMatchAgainstParserStandard); + boolean_query.set_stopword_mode(tipb::LocalMatchAgainstStopwordMode::LocalMatchAgainstStopwordModeInvalid); + ASSERT_THROW( + executeFunction( + "local_match_against_boolean", + {createConstColumn(1, "+quick"), + createColumn({"quick brown"}), + createConstColumn(1, boolean_query.SerializeAsString())}, + nullptr, + true), + Exception); +} +CATCH + +TEST_F(TestLocalMatchAgainst, MatchBooleanRejectsInvalidNodeEnums) +try +{ + enum class InvalidEnumEncoding + { + Missing, + Invalid, + Unknown, + }; + for (const bool invalid_occur : {false, true}) + for (const auto encoding : + {InvalidEnumEncoding::Missing, InvalidEnumEncoding::Invalid, InvalidEnumEncoding::Unknown}) + { + auto query = makeRequiredLocalMatchQuery("quick"); + query.set_version(2); + query.set_parser(tipb::LocalMatchAgainstParserStandard); + query.set_stopword_mode(tipb::LocalMatchAgainstStopwordModeDisabled); + // Put the malformed node after a valid one: all nodes must be checked. + auto * node = query.add_nodes(); + node->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + node->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + node->set_text("fox"); + if (invalid_occur) + { + node->clear_occur(); + if (encoding == InvalidEnumEncoding::Invalid) + node->set_occur(tipb::LocalMatchAgainstBooleanOccurInvalid); + } + else + { + node->clear_term_type(); + if (encoding == InvalidEnumEncoding::Invalid) + node->set_term_type(tipb::LocalMatchAgainstBooleanTermInvalid); + } + // Encode unknown values on the wire; generated enum setters may + // assert rather than allow an out-of-range value in debug builds. + if (encoding == InvalidEnumEncoding::Unknown) + ASSERT_TRUE(node->MergeFromString(invalid_occur ? "\x08\x7f" : "\x10\x7f")); + + for (const bool empty_analyzer : {false, true}) + for (const bool null_search : {false, true}) + { + SCOPED_TRACE(fmt::format( + "invalid_occur={} encoding={} empty_analyzer={} null_search={}", + invalid_occur, + static_cast(encoding), + empty_analyzer, + null_search)); + query.set_innodb_ft_max_token_size(empty_analyzer ? 0 : 84); + auto search = createConstColumn(1, "+quick"); + if (null_search) + search = createOnlyNullColumnConst(1); + try + { + executeFunction( + "local_match_against_boolean", + {search, + createColumn({"quick fox"}), + createConstColumn(1, query.SerializeAsString())}, + nullptr, + true); + FAIL() << "Invalid Local MATCH node metadata must be rejected"; + } + catch (const Exception & error) + { + ASSERT_EQ(ErrorCodes::ILLEGAL_COLUMN, error.code()); + } + } + } +} +CATCH + +TEST_F(TestLocalMatchAgainst, MatchBooleanStandardAnalyzerProtocolSettings) +try +{ + const auto make_metadata = [](const String & term, + UInt32 min_token_size, + UInt32 max_token_size, + bool enable_stopword, + const String & stopword_collation) { + tipb::LocalMatchAgainstBooleanQuery boolean_query; + boolean_query.set_parser(tipb::LocalMatchAgainstParser::LocalMatchAgainstParserStandard); + boolean_query.set_stopword_collation(stopword_collation); + boolean_query.set_innodb_ft_min_token_size(min_token_size); + boolean_query.set_innodb_ft_max_token_size(max_token_size); + boolean_query.set_stopword_mode( + enable_stopword ? tipb::LocalMatchAgainstStopwordMode::LocalMatchAgainstStopwordModeBuiltin + : tipb::LocalMatchAgainstStopwordMode::LocalMatchAgainstStopwordModeDisabled); + auto * required = boolean_query.add_nodes(); + required->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + required->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + required->set_text(term); + return serializeLocalMatchAgainstBooleanQuery(boolean_query); + }; + + const auto short_term_metadata = make_metadata("go", 1, 84, true, "utf8mb4_bin"); + ASSERT_COLUMN_EQ( + createColumn({1, 0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(2, "+go"), + createColumn({"go", "good"}), + createConstColumn(2, short_term_metadata)}, + nullptr, + true)); + + const auto stopword_metadata = make_metadata("the", 1, 84, false, "utf8mb4_bin"); + ASSERT_COLUMN_EQ( + createColumn({1, 0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(2, "+the"), + createColumn({"the", "there"}), + createConstColumn(2, stopword_metadata)}, + nullptr, + true)); + + const auto collated_stopword_metadata = make_metadata("thé", 3, 84, true, "utf8mb4_general_ci"); + ASSERT_COLUMN_EQ( + createColumn({0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(1, "+thé"), + createColumn({"thé"}), + createConstColumn(1, collated_stopword_metadata)}, + TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_GENERAL_CI), + true)); + + const auto binary_server_metadata = make_metadata("The", 3, 84, true, "utf8mb4_bin"); + ASSERT_COLUMN_EQ( + createColumn({1}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(1, "+The"), + createColumn({"The"}), + createConstColumn(1, binary_server_metadata)}, + TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_GENERAL_CI), + true)); + + const auto max_size_metadata = make_metadata("extraordinary", 1, 10, false, "utf8mb4_bin"); + ASSERT_COLUMN_EQ( + createColumn({0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(1, "+extraordinary"), + createColumn({"extraordinary"}), + createConstColumn(1, max_size_metadata)}, + nullptr, + true)); +} +CATCH + +TEST_F(TestLocalMatchAgainst, AnalyzerDefaultsAndResultFinalization) +try +{ + struct ConfigCase + { + UInt32 min_size; + UInt32 max_size; + InferredDataVector expected; + }; + for (const auto & config : + {ConfigCase{0, 0, {1, 0, 0, 0}}, ConfigCase{0, 4, {0, 1, 0, 0}}, ConfigCase{5, 4, {0, 0, 0, 0}}}) + { + SCOPED_TRACE(fmt::format("min={} max={}", config.min_size, config.max_size)); + tipb::LocalMatchAgainstBooleanQuery query; + query.set_innodb_ft_min_token_size(config.min_size); + query.set_innodb_ft_max_token_size(config.max_size); + query.set_stopword_mode(tipb::LocalMatchAgainstStopwordModeDisabled); + for (const auto * term : {"quick", "a"}) + { + auto * node = query.add_nodes(); + node->set_occur(tipb::LocalMatchAgainstBooleanOccurShould); + node->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + node->set_text(term); + } + const auto metadata = createConstColumn(4, serializeLocalMatchAgainstBooleanQuery(query)); + ASSERT_COLUMN_EQ( + createColumn(config.expected), + executeFunction( + "local_match_against_boolean", + {createConstColumn(4, "quick a"), createColumn({"quick", "a", "", "none"}), metadata}, + nullptr, + true)); + InferredDataVector> nullable_expected; + for (auto value : config.expected) + nullable_expected.emplace_back(value); + ASSERT_COLUMN_EQ( + createColumn>(nullable_expected), + executeFunction( + "local_match_against_boolean", + {createConstColumn(4, "quick a"), + createColumn>({"quick", "a", {}, ""}), + metadata}, + nullptr, + true)); + // NULL search must still bypass analyzer setup and return non-NULL + // zeros, even when that setup would reject the stopword collation. + query.set_stopword_mode(tipb::LocalMatchAgainstStopwordModeBuiltin); + query.set_stopword_collation("unsupported_stopword_collation"); + ASSERT_COLUMN_EQ( + createColumn>({0, 0, 0, 0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn>(4, {}), + createColumn({"quick", "a", "", "none"}), + createConstColumn(4, serializeLocalMatchAgainstBooleanQuery(query))}, + nullptr, + true)); + } +} +CATCH + +TEST_F(TestLocalMatchAgainst, MatchBooleanNgramProtocolQuery) +try +{ + tipb::LocalMatchAgainstBooleanQuery boolean_query; + boolean_query.set_parser(tipb::LocalMatchAgainstParser::LocalMatchAgainstParserNgram); + boolean_query.set_ngram_token_size(2); + auto * required = boolean_query.add_nodes(); + required->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + required->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + required->set_text("数据库"); + auto * prohibited = boolean_query.add_nodes(); + prohibited->set_occur(tipb::LocalMatchAgainstBooleanOccurMustNot); + prohibited->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + prohibited->set_text("mysql"); + + const String metadata = serializeLocalMatchAgainstBooleanQuery(boolean_query); + ASSERT_COLUMN_EQ( + createColumn({1, 0, 1, 0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(4, "+数据库 -mysql"), + createColumn({"数据库系统", "MySQL 数据库", "数据库", "数据科学"}), + createConstColumn(4, metadata)}, + nullptr, + true)); + + const auto ci_collator = TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_GENERAL_CI); + tipb::LocalMatchAgainstBooleanQuery case_query; + case_query.set_parser(tipb::LocalMatchAgainstParser::LocalMatchAgainstParserNgram); + case_query.set_ngram_token_size(2); + auto * case_term = case_query.add_nodes(); + case_term->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + case_term->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + case_term->set_text("mysql"); + const String case_metadata = serializeLocalMatchAgainstBooleanQuery(case_query); + ASSERT_COLUMN_EQ( + createColumn({1, 0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(2, "+mysql"), + createColumn({"MySQL", "PostgreSQL"}), + createConstColumn(2, case_metadata)}, + ci_collator, + true)); +} +CATCH + +TEST_F(TestLocalMatchAgainst, MatchBooleanNgramStopwordSettings) +try +{ + const auto make_metadata = [](UInt32 token_size, bool enable_stopword) { + tipb::LocalMatchAgainstBooleanQuery boolean_query; + boolean_query.set_parser(tipb::LocalMatchAgainstParser::LocalMatchAgainstParserNgram); + boolean_query.set_ngram_token_size(token_size); + boolean_query.set_stopword_mode( + enable_stopword ? tipb::LocalMatchAgainstStopwordMode::LocalMatchAgainstStopwordModeBuiltin + : tipb::LocalMatchAgainstStopwordMode::LocalMatchAgainstStopwordModeDisabled); + auto * required = boolean_query.add_nodes(); + required->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + required->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + required->set_text("the"); + return serializeLocalMatchAgainstBooleanQuery(boolean_query); + }; + + const auto query = createConstColumn(3, "+the"); + const auto documents = createColumn({"the", "other", "there"}); + + // A stopword longer than ngram_token_size is ignored by the NGRAM parser. + const auto size_two_metadata = createConstColumn(3, make_metadata(2, true)); + ASSERT_COLUMN_EQ( + createColumn({1, 1, 1}), + executeFunction("local_match_against_boolean", {query, documents, size_two_metadata}, nullptr, true)); + + // At size 3, the gram "the" is removed from both query and document. + const auto enabled_metadata = createConstColumn(3, make_metadata(3, true)); + ASSERT_COLUMN_EQ( + createColumn({0, 0, 0}), + executeFunction("local_match_against_boolean", {query, documents, enabled_metadata}, nullptr, true)); + + tipb::LocalMatchAgainstBooleanQuery filtered_prefix; + filtered_prefix.set_parser(tipb::LocalMatchAgainstParser::LocalMatchAgainstParserNgram); + filtered_prefix.set_ngram_token_size(2); + filtered_prefix.set_stopword_mode(tipb::LocalMatchAgainstStopwordMode::LocalMatchAgainstStopwordModeBuiltin); + auto * prefix_node = filtered_prefix.add_nodes(); + prefix_node->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + prefix_node->set_term_type(tipb::LocalMatchAgainstBooleanTermPrefix); + prefix_node->set_text("caf"); + ASSERT_COLUMN_EQ( + createColumn({0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(1, "+caf*"), + createColumn({"cafe"}), + createConstColumn(1, serializeLocalMatchAgainstBooleanQuery(filtered_prefix))}, + nullptr, + true)); + + // Explicit OFF keeps that gram and restores the matches. + const auto disabled_metadata = createConstColumn(3, make_metadata(3, false)); + ASSERT_COLUMN_EQ( + createColumn({1, 1, 1}), + executeFunction("local_match_against_boolean", {query, documents, disabled_metadata}, nullptr, true)); + + // Stopword matching follows collation_server independently from the + // MATCH column collation. + tipb::LocalMatchAgainstBooleanQuery collated_query; + collated_query.set_parser(tipb::LocalMatchAgainstParser::LocalMatchAgainstParserNgram); + collated_query.set_ngram_token_size(2); + collated_query.set_stopword_mode(tipb::LocalMatchAgainstStopwordMode::LocalMatchAgainstStopwordModeBuiltin); + collated_query.set_stopword_collation("utf8mb4_general_ci"); + auto * collated_required = collated_query.add_nodes(); + collated_required->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + collated_required->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + collated_required->set_text("xá"); + const auto collated_metadata = createConstColumn(1, serializeLocalMatchAgainstBooleanQuery(collated_query)); + const auto document = createColumn({"xá"}); + ASSERT_COLUMN_EQ( + createColumn({0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(1, "+xá"), document, collated_metadata}, + TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_GENERAL_CI), + true)); + + // The MATCH column collation remains accent-insensitive, but the binary + // server collation does not consider "á" to contain stopword "a". + collated_query.set_stopword_collation("utf8mb4_bin"); + const auto binary_stopword_metadata + = createConstColumn(1, serializeLocalMatchAgainstBooleanQuery(collated_query)); + ASSERT_COLUMN_EQ( + createColumn({1}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(1, "+xá"), document, binary_stopword_metadata}, + TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_GENERAL_CI), + true)); +} +CATCH + +TEST_F(TestLocalMatchAgainst, MatchBooleanNgramPrefix) +try +{ + const auto make_metadata = [](UInt32 token_size, const String & prefix) { + tipb::LocalMatchAgainstBooleanQuery boolean_query; + boolean_query.set_parser(tipb::LocalMatchAgainstParser::LocalMatchAgainstParserNgram); + boolean_query.set_ngram_token_size(token_size); + boolean_query.set_stopword_mode(tipb::LocalMatchAgainstStopwordMode::LocalMatchAgainstStopwordModeDisabled); + auto * node = boolean_query.add_nodes(); + node->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + node->set_term_type(tipb::LocalMatchAgainstBooleanTermPrefix); + node->set_text(prefix); + return serializeLocalMatchAgainstBooleanQuery(boolean_query); + }; + + const auto query = createConstColumn(4, "+caf*"); + const auto documents = createColumn({"CAFE", "café", "cafe", "decaf"}); + const auto metadata = createConstColumn(4, make_metadata(2, "caf")); + const auto utf8mb4_bin = TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_BIN); + ASSERT_COLUMN_EQ( + createColumn({0, 1, 1, 1}), + executeFunction("local_match_against_boolean", {query, documents, metadata}, utf8mb4_bin, true)); + + const auto utf8mb4_general_ci = TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_GENERAL_CI); + ASSERT_COLUMN_EQ( + createColumn({1, 1, 1, 1}), + executeFunction("local_match_against_boolean", {query, documents, metadata}, utf8mb4_general_ci, true)); + + const auto short_query = createConstColumn(4, "+c*"); + const auto short_metadata = createConstColumn(4, make_metadata(2, "c")); + ASSERT_COLUMN_EQ( + createColumn({1, 1, 1, 1}), + executeFunction( + "local_match_against_boolean", + {short_query, documents, short_metadata}, + utf8mb4_general_ci, + true)); +} +CATCH + +TEST_F(TestLocalMatchAgainst, MatchBooleanCollationMatrix) +try +{ + const auto query = createConstColumn(3, "+cafe"); + const auto documents = createColumn({"CAFE", "café", "cafe"}); + const auto prefix_query = createConstColumn(3, "+caf*"); + const auto make_metadata = [](tipb::LocalMatchAgainstBooleanTermType term_type, const String & term) { + tipb::LocalMatchAgainstBooleanQuery boolean_query; + boolean_query.set_parser(tipb::LocalMatchAgainstParser::LocalMatchAgainstParserStandard); + auto * node = boolean_query.add_nodes(); + node->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + node->set_term_type(term_type); + node->set_text(term); + return serializeLocalMatchAgainstBooleanQuery(boolean_query); + }; + const auto word_metadata + = createConstColumn(3, make_metadata(tipb::LocalMatchAgainstBooleanTermWord, "cafe")); + const auto prefix_metadata + = createConstColumn(3, make_metadata(tipb::LocalMatchAgainstBooleanTermPrefix, "caf")); + + const auto utf8mb4_bin = TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_BIN); + ASSERT_COLUMN_EQ( + createColumn({0, 0, 1}), + executeFunction("local_match_against_boolean", {query, documents, word_metadata}, utf8mb4_bin, true)); + ASSERT_COLUMN_EQ( + createColumn({0, 1, 1}), + executeFunction("local_match_against_boolean", {prefix_query, documents, prefix_metadata}, utf8mb4_bin, true)); + + const auto utf8mb4_0900_bin = TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_0900_BIN); + ASSERT_COLUMN_EQ( + createColumn({0, 0, 1}), + executeFunction("local_match_against_boolean", {query, documents, word_metadata}, utf8mb4_0900_bin, true)); + ASSERT_COLUMN_EQ( + createColumn({0, 1, 1}), + executeFunction( + "local_match_against_boolean", + {prefix_query, documents, prefix_metadata}, + utf8mb4_0900_bin, + true)); + + const auto utf8mb4_general_ci = TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_GENERAL_CI); + ASSERT_COLUMN_EQ( + createColumn({1, 1, 1}), + executeFunction("local_match_against_boolean", {query, documents, word_metadata}, utf8mb4_general_ci, true)); + ASSERT_COLUMN_EQ( + createColumn({1, 1, 1}), + executeFunction( + "local_match_against_boolean", + {prefix_query, documents, prefix_metadata}, + utf8mb4_general_ci, + true)); + + const auto utf8mb4_unicode_ci = TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_UNICODE_CI); + ASSERT_COLUMN_EQ( + createColumn({1, 1, 1}), + executeFunction("local_match_against_boolean", {query, documents, word_metadata}, utf8mb4_unicode_ci, true)); + ASSERT_COLUMN_EQ( + createColumn({1, 1, 1}), + executeFunction( + "local_match_against_boolean", + {prefix_query, documents, prefix_metadata}, + utf8mb4_unicode_ci, + true)); + + const auto utf8mb4_0900_ai_ci = TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_0900_AI_CI); + ASSERT_COLUMN_EQ( + createColumn({1, 1, 1}), + executeFunction("local_match_against_boolean", {query, documents, word_metadata}, utf8mb4_0900_ai_ci, true)); + ASSERT_COLUMN_EQ( + createColumn({1, 1, 1}), + executeFunction( + "local_match_against_boolean", + {prefix_query, documents, prefix_metadata}, + utf8mb4_0900_ai_ci, + true)); +} +CATCH + +TEST_F(TestLocalMatchAgainst, MatchBooleanSplitWordModifiers) +try +{ + const auto collator = TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_BIN); + const auto documents = createColumn({"foo only", "bar only", "foo bar", "baz foo", "baz bar", "baz qux"}); + for (const auto occur : + {tipb::LocalMatchAgainstBooleanOccurShould, + tipb::LocalMatchAgainstBooleanOccurMust, + tipb::LocalMatchAgainstBooleanOccurMustNot}) + { + tipb::LocalMatchAgainstBooleanQuery query; + auto * node = query.add_nodes(); + node->set_occur(occur); + node->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + node->set_text("foo.bar"); + if (occur == tipb::LocalMatchAgainstBooleanOccurMustNot) + { + auto * positive = query.add_nodes(); + positive->set_occur(tipb::LocalMatchAgainstBooleanOccurShould); + positive->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + positive->set_text("baz"); + } + const auto expected = occur == tipb::LocalMatchAgainstBooleanOccurShould + ? createColumn({1, 1, 1, 1, 1, 0}) + : occur == tipb::LocalMatchAgainstBooleanOccurMust ? createColumn({0, 0, 1, 0, 0, 0}) + : createColumn({0, 0, 0, 0, 0, 1}); + const auto metadata = createConstColumn(6, serializeLocalMatchAgainstBooleanQuery(query)); + ASSERT_COLUMN_EQ( + expected, + executeFunction( + "local_match_against_boolean", + {createConstColumn(6, "foo.bar"), documents, metadata}, + collator, + true)); + } + // The intersection for a required split word may span MATCH columns. + tipb::LocalMatchAgainstBooleanQuery query; + auto * node = query.add_nodes(); + node->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + node->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + node->set_text("foo.bar"); + ASSERT_COLUMN_EQ( + createColumn({1, 0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(2, "+foo.bar"), + createColumn({"foo", "foo"}), + createColumn({"bar", "qux"}), + createConstColumn(2, serializeLocalMatchAgainstBooleanQuery(query))}, + collator, + true)); +} +CATCH + +TEST_F(TestLocalMatchAgainst, BuiltinStopwordLookupMatchesComparisons) +try +{ + // Independent reference: retain the old compare-based stopword rule, + // including NGRAM's substring and source-code-point-length restriction. + const auto & stopwords = builtin_stopwords; + std::vector candidates; + for (const auto word : stopwords) + candidates.emplace_back(word); + for (const String & word : + {"ABOUT", "The", "THE", "WWW", "WITH", "ábout", "thé", "tĥe", "ｗｉｔｈ", "ｗｗｗ", "aß", "ß", + "xá", "áx", "xＡ", "Ａx", "xáy", "aaa", "AAA", "xy", "xyz", "foo", "数据库"}) + candidates.push_back(word); + for (const auto collation : + {"binary", + "ascii_bin", + "latin1_bin", + "utf8_bin", + "utf8mb4_bin", + "utf8mb4_0900_bin", + "utf8_general_ci", + "utf8mb4_general_ci", + "utf8_unicode_ci", + "utf8mb4_unicode_ci", + "utf8mb4_0900_ai_ci"}) + { + const auto stopword_collator = TiDB::ITiDBCollator::getCollator(collation); + ASSERT_NE(stopword_collator, nullptr); + // Check the key-equality premise directly, including padding and + // combining marks which the STANDARD tokenizer may split away. + String key_buffer; + auto key_copy = [&](const String & text) { + const auto key = stopword_collator->sortKeyFastPath(text.data(), text.size(), key_buffer); + return String(key.data, key.size); + }; + for (const auto word : stopwords) + { + const String source(word); + const auto source_key = key_copy(source); + for (const auto & candidate : candidates) + for (const String & suffix : {"", " ", " ", "\u0301"}) + { + const String text = candidate + suffix; + EXPECT_EQ( + source_key == key_copy(text), + stopword_collator->compare(source.data(), source.size(), text.data(), text.size()) == 0) + << collation << " source=" << source << " candidate=" << text; + } + } + auto equals_stopword = [&](std::string_view token, size_t length, bool restrict_length) { + return std::any_of(stopwords.begin(), stopwords.end(), [&](const auto word) { + return (!restrict_length || length == word.size()) + && stopword_collator->compare(token.data(), token.size(), word.data(), word.size()) == 0; + }); + }; + for (const auto size : {0U, 1U, 2U, 3U}) + for (const auto & candidate : candidates) + { + SCOPED_TRACE(fmt::format("stopword_collation={} size={} candidate={}", collation, size, candidate)); + std::vector boundaries{0}; + for (size_t offset = 0; offset < candidate.size();) + { + offset += UTF8::utf8Decode(candidate.data() + offset, candidate.size() - offset).second; + boundaries.push_back(offset); + } + bool retained = size == 0 && !equals_stopword(candidate, boundaries.size() - 1, false); + if (size != 0) + for (size_t start = 0; start + size < boundaries.size(); ++start) + { + bool filtered = false; + for (size_t begin = start; begin < start + size; ++begin) + for (size_t end = begin + 1; end <= start + size; ++end) + filtered = filtered + || equals_stopword( + std::string_view(candidate).substr( + boundaries[begin], + boundaries[end] - boundaries[begin]), + end - begin, + true); + retained = retained || !filtered; + } + tipb::LocalMatchAgainstBooleanQuery query; + query.set_parser( + size == 0 ? tipb::LocalMatchAgainstParserStandard : tipb::LocalMatchAgainstParserNgram); + query.set_ngram_token_size(size); + query.set_innodb_ft_min_token_size(1); + query.set_innodb_ft_max_token_size(84); + query.set_stopword_collation(collation); + auto * node = query.add_nodes(); + node->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + node->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + node->set_text(candidate); + // Repeated rows exercise key-buffer reuse. Column and server + // collations are deliberately independent. + for (const auto match_collation : {"utf8mb4_bin", "utf8mb4_general_ci"}) + ASSERT_COLUMN_EQ( + createColumn({retained ? 1.0 : 0.0, retained ? 1.0 : 0.0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(2, candidate), + createColumn({candidate, candidate}), + createConstColumn(2, serializeLocalMatchAgainstBooleanQuery(query))}, + TiDB::ITiDBCollator::getCollator(match_collation), + true)); + } + } +} +CATCH + +TEST_F(TestLocalMatchAgainst, NgramSlidingStopwordWindows) +try +{ + InferredDataVector> documents{ + "", + "z", + "zzaz", + "zzazzzzzzzzzzz", + "zthe", + "zzthezz", + "zzthezzzzzzzzzz", + "zaboutzz", + "zaboutzaboutzzzzzz", + "zzazzizzz", + "za zzzzzzzzzzz", + "zzz!zzz", + "zThezz", + "zTHEzz", + "zthézz", + "ztĥezz", + "zｗｉｔｈzz", + "zｗｗｗzz", + "zázz", + "zＡzz", + "zßzz", + "z\u0301azz", + "z🙃a🙃zz", + "z数据azzz", + "ZTHEZZ", + String("za\0zzzzzzzzzz", 13), + String("zthe\xffzzzzzzzzzz"), + String("\xffzzzzzzzzzz"), + "zzazzazzzzzzzzzzzzz", + "zaboutzzzzzzzzzz", + "z", + {}, + "zzzzzzzzzzzz"}; + // Deterministic mixed runs, ending candidates and repeated rows. The + // reference below never uses the optimized lookup or sliding state. + UInt32 seed = 70485; + const std::array pieces{"z", "a", "i", "the", "about", "thé", "🙃", "!"}; + for (size_t row = 0; row < 24; ++row) + { + String text; + for (size_t i = 0; i < 12; ++i) + { + seed = seed * 1664525 + 1013904223; + text += pieces[(seed >> 16) % pieces.size()]; + } + documents.emplace_back(std::move(text)); + } + + for (const auto stopword_collation : + {"binary", + "ascii_bin", + "latin1_bin", + "utf8_bin", + "utf8mb4_bin", + "utf8mb4_0900_bin", + "utf8_general_ci", + "utf8mb4_general_ci", + "utf8_unicode_ci", + "utf8mb4_unicode_ci", + "utf8mb4_0900_ai_ci"}) + for (const auto match_collation : {"utf8mb4_bin", "utf8mb4_general_ci"}) + for (size_t size = 1; size <= 10; ++size) + { + SCOPED_TRACE(fmt::format("stopwords={} match={} size={}", stopword_collation, match_collation, size)); + const auto stopword_collator = TiDB::ITiDBCollator::getCollator(stopword_collation); + const auto match_collator = TiDB::ITiDBCollator::getCollator(match_collation); + auto prefix = match_collator->pattern(); + prefix->compile("z%", '\\'); + InferredDataVector> expected; + for (const auto & document : documents) + { + bool retained = false; + if (document) + { + const String & text = *document; + size_t run_start = 0; + std::vector boundaries{0}; + auto check_run = [&] { + for (size_t start = 0; start + size < boundaries.size() && !retained; ++start) + { + bool filtered = false; + for (size_t begin = start; begin < start + size && !filtered; ++begin) + for (size_t end = begin + 1; end <= start + size && !filtered; ++end) + { + const auto candidate = std::string_view(text).substr( + run_start + boundaries[begin], + boundaries[end] - boundaries[begin]); + filtered = std::any_of( + builtin_stopwords.begin(), + builtin_stopwords.end(), + [&](auto word) { + return word.size() == end - begin + && stopword_collator->compare( + candidate.data(), + candidate.size(), + word.data(), + word.size()) + == 0; + }); + } + const auto gram = std::string_view(text).substr( + run_start + boundaries[start], + boundaries[start + size] - boundaries[start]); + retained = !filtered && prefix->match(gram.data(), gram.size()); + } + }; + for (size_t offset = 0; offset < text.size();) + { + const auto byte = static_cast(text[offset]); + const auto decoded = UTF8::utf8Decode(text.data() + offset, text.size() - offset); + if (decoded.second == 0 || decoded.first == UTF8::UTF8_Error + || decoded.second > text.size() - offset || (byte >= 0x80 && decoded.second == 1)) + break; + if (byte < 0x80 && !LocalMatchAgainst::isTokenChar(byte)) + { + check_run(); + run_start = ++offset; + boundaries.assign(1, 0); + } + else + { + offset += decoded.second; + boundaries.push_back(offset - run_start); + } + } + check_run(); + } + expected.emplace_back(retained ? 1.0 : 0.0); + } + tipb::LocalMatchAgainstBooleanQuery query; + query.set_parser(tipb::LocalMatchAgainstParserNgram); + query.set_ngram_token_size(size); + query.set_stopword_collation(stopword_collation); + auto * node = query.add_nodes(); + node->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + node->set_term_type(tipb::LocalMatchAgainstBooleanTermPrefix); + node->set_text("z"); + ASSERT_COLUMN_EQ( + createColumn>(expected), + executeFunction( + "local_match_against_boolean", + {createConstColumn(documents.size(), "+z*"), + createColumn>(documents), + createConstColumn(documents.size(), serializeLocalMatchAgainstBooleanQuery(query))}, + match_collator, + true)); + } +} +CATCH + +TEST_F(TestLocalMatchAgainst, MatchBooleanASCIIAndMalformedUTF8) +try +{ + const auto collator = TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_BIN); + const auto documents = createColumn( + {"bar", + "foo bar", + "foobar", + "bar!", + String("bar\0foo", 7), + "bar\xff" + "foo", + "foo\xff" + "bar", + "foo中bar", + ""}); + for (const auto size : {0U, 2U, 3U}) + { + tipb::LocalMatchAgainstBooleanQuery query; + query.set_parser(size == 0 ? tipb::LocalMatchAgainstParserStandard : tipb::LocalMatchAgainstParserNgram); + query.set_ngram_token_size(size); + query.set_stopword_mode(tipb::LocalMatchAgainstStopwordModeDisabled); + auto * node = query.add_nodes(); + node->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + node->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + node->set_text("bar"); + ASSERT_COLUMN_EQ( + size == 0 ? createColumn({1, 1, 0, 1, 1, 1, 1, 0, 0}) + : createColumn({1, 1, 1, 1, 1, 1, 0, 1, 0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(9, "+bar"), + documents, + createConstColumn(9, serializeLocalMatchAgainstBooleanQuery(query))}, + collator, + true)); + } +} +CATCH + +TEST_F(TestLocalMatchAgainst, DocumentViewsPreserveFilteredPhrase) +try +{ + String miss; + for (size_t i = 0; i < 4096; ++i) + miss += "foo xyz zoo "; + for (const bool ngram : {false, true}) + for (const bool stopwords : {false, true}) + for (const auto collation : {"utf8mb4_bin", "utf8mb4_general_ci"}) + { + SCOPED_TRACE(fmt::format("ngram={} stopwords={} collation={}", ngram, stopwords, collation)); + tipb::LocalMatchAgainstBooleanQuery query; + query.set_parser(ngram ? tipb::LocalMatchAgainstParserNgram : tipb::LocalMatchAgainstParserStandard); + query.set_ngram_token_size(3); + query.set_stopword_mode( + stopwords ? tipb::LocalMatchAgainstStopwordModeBuiltin + : tipb::LocalMatchAgainstStopwordModeDisabled); + auto * phrase = query.add_nodes(); + phrase->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + phrase->set_term_type(tipb::LocalMatchAgainstBooleanTermPhrase); + phrase->set_text("foo the zoo"); + auto * excluded = query.add_nodes(); + excluded->set_occur(tipb::LocalMatchAgainstBooleanOccurMustNot); + excluded->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + excluded->set_text("blocked"); + ASSERT_COLUMN_EQ( + createColumn>({1, 1, 1, 0, 0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(5, "+\"foo the zoo\" -blocked"), + createColumn>({miss, "foo the zoo", {}, "", miss}), + createColumn>( + {"foo the zoo", miss, "foo the zoo", "foo the zoo blocked", miss}), + createConstColumn(5, serializeLocalMatchAgainstBooleanQuery(query))}, + TiDB::ITiDBCollator::getCollator(collation), + true)); + } +} +CATCH + +TEST_F(TestLocalMatchAgainst, NgramDocumentViewsAndPositions) +try +{ + // Overlapping grams in separate runs still have consecutive positions; + // short runs contribute none. Exercise vector growth, scratch reuse and + // successive large/small/NULL documents without borrowing scratch memory. + for (const auto size : {1U, 2U, 3U, 4U, 5U, 6U, 7U, 8U, 9U, 10U}) + for (const bool chinese : {false, true}) + for (const auto collation : {"", "utf8mb4_bin", "utf8mb4_general_ci"}) + { + SCOPED_TRACE(fmt::format("size={} chinese={} collation={}", size, chinese, collation)); + const String alphabet = chinese ? "甲乙丙丁戊己庚辛壬癸子丑" : "ABCDEFGHIJKL"; + const size_t width = chinese ? 3 : 1; + const String query_text = alphabet.substr(0, (size + 1) * width); + const String first = query_text.substr(0, size * width); + const String last = query_text.substr(width); + const String short_run(size - 1, 'z'); + const String padding(4096, 'z'); + const auto documents = createColumn>( + {padding + " " + query_text, + query_text, + first + "!" + last, + first + "!" + String(size, 'z') + "!" + last, + first + "!" + short_run + "!" + last, + query_text + "\xff" + padding, + first + "\xff" + last, + {}, + "", + query_text + "!" + padding}); + tipb::LocalMatchAgainstBooleanQuery query; + query.set_parser(tipb::LocalMatchAgainstParserNgram); + query.set_ngram_token_size(size); + query.set_stopword_mode(tipb::LocalMatchAgainstStopwordModeDisabled); + auto * node = query.add_nodes(); + node->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + node->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + node->set_text(query_text); + ASSERT_COLUMN_EQ( + createColumn>({1, 1, 1, 0, 1, 1, 0, 0, 0, 1}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(10, query_text), + documents, + createConstColumn(10, serializeLocalMatchAgainstBooleanQuery(query))}, + String(collation).empty() ? nullptr : TiDB::ITiDBCollator::getCollator(collation), + true)); + } +} +CATCH + +TEST_F(TestLocalMatchAgainst, MatchBooleanUnicodeTokenBoundaries) +try +{ + const auto collator = TiDB::ITiDBCollator::getCollator(TiDB::ITiDBCollator::UTF8MB4_BIN); + const auto documents = createColumn( + {"foo🙃bar", + "foo👁bar", + "foo bar", + "foobar", + "foo𞤀bar", + "foo𝟙bar", + "foo\U0002EBF0bar", + "foo\xff" + "bar"}); + for (const auto size : {0U, 2U, 3U}) + { + tipb::LocalMatchAgainstBooleanQuery query; + query.set_parser(size == 0 ? tipb::LocalMatchAgainstParserStandard : tipb::LocalMatchAgainstParserNgram); + query.set_ngram_token_size(size); + query.set_stopword_mode(tipb::LocalMatchAgainstStopwordModeDisabled); + auto * node = query.add_nodes(); + node->set_occur(tipb::LocalMatchAgainstBooleanOccurMust); + node->set_term_type(tipb::LocalMatchAgainstBooleanTermWord); + node->set_text(size == 0 ? "foo" : "foob"); + ASSERT_COLUMN_EQ( + size == 0 ? createColumn({1, 1, 1, 0, 1, 1, 1, 1}) + : createColumn({0, 0, 0, 1, 0, 0, 0, 0}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(8, node->text()), + documents, + createConstColumn(8, serializeLocalMatchAgainstBooleanQuery(query))}, + collator, + true)); + // TiDB's version-2 NGRAM Boolean lexer emits "bar" for this search. + node->set_text(size == 0 ? "𞤀bar" : "bar"); + ASSERT_COLUMN_EQ( + createColumn({1, 1, 1}), + executeFunction( + "local_match_against_boolean", + {createConstColumn(3, "+𞤀bar"), + createColumn({"foo𞤀bar", "𞤀bar", "foo bar"}), + createConstColumn(3, serializeLocalMatchAgainstBooleanQuery(query))}, + collator, + true)); + } +} +CATCH + +TEST(LocalMatchAgainstTokenChars, ProtocolV1Classification) +{ + for (const UInt32 code_point : {0x1F643U, 0x1F441U, 0x301U, 0xFFFDU, 0x2EBF0U, 0x110000U}) + EXPECT_FALSE(LocalMatchAgainst::isTokenChar(code_point)) << code_point; + for (const UInt32 code_point : {0x5FU, 0x4E2DU, 0xE9U, 0x1D7D9U, 0x1E900U}) + EXPECT_TRUE(LocalMatchAgainst::isTokenChar(code_point)) << code_point; + UInt32 previous = 127; + for (const auto & span : LocalMatchAgainst::token_char_ranges) + { + ASSERT_GT(span.first, previous); + ASSERT_LE(span.first, span.last); + for (UInt32 code_point = previous + 1; code_point < span.first; ++code_point) + ASSERT_FALSE(LocalMatchAgainst::isTokenChar(code_point)) << code_point; + for (UInt32 code_point = span.first; code_point <= span.last; ++code_point) + ASSERT_TRUE(LocalMatchAgainst::isTokenChar(code_point)) << code_point; + previous = span.last; + } + for (UInt32 code_point = previous + 1; code_point <= 0x10FFFFU; ++code_point) + ASSERT_FALSE(LocalMatchAgainst::isTokenChar(code_point)) << code_point; + + // FNV-1a over one 0/1 byte per code point, U+0000..U+10FFFF. The paired + // TiDB tests hash Go's Unicode 15.0.0 IsLetter/IsNumber/'_' classification + // identically. This fixed regression checksum must not change in v1, + // even when the header is regenerated or either toolchain is upgraded. + std::uint64_t fingerprint = 14695981039346656037ULL; + for (UInt32 code_point = 0; code_point <= 0x10FFFFU; ++code_point) + { + fingerprint ^= static_cast(LocalMatchAgainst::isTokenChar(code_point)); + fingerprint *= 1099511628211ULL; + } + EXPECT_EQ(0x71f51f3810b3b529ULL, fingerprint); +} + +} // namespace DB::tests diff --git a/dbms/src/Storages/KVStore/MultiRaft/Disagg/RaftLogManager.h b/dbms/src/Storages/KVStore/MultiRaft/Disagg/RaftLogManager.h index 09b6a60e5d7..3b151d8062e 100644 --- a/dbms/src/Storages/KVStore/MultiRaft/Disagg/RaftLogManager.h +++ b/dbms/src/Storages/KVStore/MultiRaft/Disagg/RaftLogManager.h @@ -18,6 +18,8 @@ #include #include +#include +#include #include namespace DB diff --git a/libs/libcommon/include/common/demangle.h b/libs/libcommon/include/common/demangle.h index d47fa290179..e3772b4ce01 100644 --- a/libs/libcommon/include/common/demangle.h +++ b/libs/libcommon/include/common/demangle.h @@ -14,6 +14,7 @@ #pragma once +#include #include #include diff --git a/tests/docker/cluster.yaml b/tests/docker/cluster.yaml index dd2f1dbf33f..0f727a7cf9c 100644 --- a/tests/docker/cluster.yaml +++ b/tests/docker/cluster.yaml @@ -16,7 +16,7 @@ version: '2.3' services: pd0: - image: hub.pingcap.net/qa/pd:${PD_BRANCH:-master} + image: ${PD_IMAGE:?PD_IMAGE must be set} security_opt: - seccomp:unconfined volumes: @@ -35,7 +35,7 @@ services: - --log-file=/log/pd.log restart: on-failure tikv0: - image: hub.pingcap.net/qa/tikv:${TIKV_BRANCH:-master} + image: ${TIKV_IMAGE:?TIKV_IMAGE must be set} security_opt: - seccomp:unconfined volumes: @@ -55,7 +55,7 @@ services: - "pd0" restart: on-failure tidb0: - image: hub.pingcap.net/qa/tidb:${TIDB_BRANCH:-master} + image: ${TIDB_IMAGE:?TIDB_IMAGE must be set} security_opt: - seccomp:unconfined volumes: diff --git a/tests/docker/cluster_disable_new_collation.yaml b/tests/docker/cluster_disable_new_collation.yaml index 8dd13e2c0a8..ef79caf2995 100644 --- a/tests/docker/cluster_disable_new_collation.yaml +++ b/tests/docker/cluster_disable_new_collation.yaml @@ -16,7 +16,7 @@ version: '2.3' services: pd0: - image: hub.pingcap.net/qa/pd:${PD_BRANCH:-master} + image: ${PD_IMAGE:?PD_IMAGE must be set} security_opt: - seccomp:unconfined volumes: @@ -35,7 +35,7 @@ services: - --log-file=/log/pd.log restart: on-failure tikv0: - image: hub.pingcap.net/qa/tikv:${TIKV_BRANCH:-master} + image: ${TIKV_IMAGE:?TIKV_IMAGE must be set} security_opt: - seccomp:unconfined volumes: @@ -55,7 +55,7 @@ services: - "pd0" restart: on-failure tidb0: - image: hub.pingcap.net/qa/tidb:${TIDB_BRANCH:-master} + image: ${TIDB_IMAGE:?TIDB_IMAGE must be set} security_opt: - seccomp:unconfined volumes: diff --git a/tests/docker/cluster_new_collation.yaml b/tests/docker/cluster_new_collation.yaml index 153b40f327b..17a2ebf207d 100644 --- a/tests/docker/cluster_new_collation.yaml +++ b/tests/docker/cluster_new_collation.yaml @@ -16,7 +16,7 @@ version: '2.3' services: pd0: - image: hub.pingcap.net/qa/pd:${PD_BRANCH:-master} + image: ${PD_IMAGE:?PD_IMAGE must be set} security_opt: - seccomp:unconfined volumes: @@ -35,7 +35,7 @@ services: - --log-file=/log/pd.log restart: on-failure tikv0: - image: hub.pingcap.net/qa/tikv:${TIKV_BRANCH:-master} + image: ${TIKV_IMAGE:?TIKV_IMAGE must be set} security_opt: - seccomp:unconfined volumes: @@ -55,7 +55,7 @@ services: - "pd0" restart: on-failure tidb0: - image: hub.pingcap.net/qa/tidb:${TIDB_BRANCH:-master} + image: ${TIDB_IMAGE:?TIDB_IMAGE must be set} security_opt: - seccomp:unconfined volumes: diff --git a/tests/docker/cluster_tidb_fail_point.yaml b/tests/docker/cluster_tidb_fail_point.yaml index faec3c48898..c5817670086 100644 --- a/tests/docker/cluster_tidb_fail_point.yaml +++ b/tests/docker/cluster_tidb_fail_point.yaml @@ -16,7 +16,7 @@ version: '2.3' services: pd0: - image: hub.pingcap.net/qa/pd:${PD_BRANCH:-master} + image: ${PD_IMAGE:?PD_IMAGE must be set} security_opt: - seccomp:unconfined volumes: @@ -35,7 +35,7 @@ services: - --log-file=/log/pd.log restart: on-failure tikv0: - image: hub.pingcap.net/qa/tikv:${TIKV_BRANCH:-master} + image: ${TIKV_IMAGE:?TIKV_IMAGE must be set} security_opt: - seccomp:unconfined volumes: @@ -55,7 +55,7 @@ services: - "pd0" restart: on-failure tidb0: - image: hub.pingcap.net/qa/tidb:${TIDB_BRANCH:-master}-failpoint + image: ${TIDB_IMAGE:?TIDB_IMAGE must be set}-failpoint security_opt: - seccomp:unconfined environment: