From 224f0dbccbb90b915e62afecbcb9be427d04a9d2 Mon Sep 17 00:00:00 2001 From: happenlee Date: Mon, 20 Jul 2026 19:30:02 +0800 Subject: [PATCH 01/20] [feature](runtime filter) Support single-column runtime filter bucket pruning Issue Number: None Related PR: None Problem Summary: Runtime filters could prune table partitions but still created and scheduled scanners for every hash-distributed bucket. Add FE eligibility metadata for direct targets on a single HASH distribution column and use exact IN values in BE to compute Doris CRC bucket indexes. Initial and late filters skip nonmatching tablet scanners. Composite distribution and non-invertible Bloom filters conservatively fall back. Support runtime-filter bucket pruning for exact filters on single-column HASH-distributed OLAP scans. It can be disabled with enable_runtime_filter_bucket_prune. - Test: Unit Test and Regression test - FE unit test: RuntimeFilterBucketPruneClassifierTest (6 tests) - BE unit test: RuntimeFilterBucketPrunerTest (5 tests) - Regression test: query_p0/runtime_filter/rf_bucket_pruning - Build: ./build.sh --fe and ./build.sh --be -j 48 - Static analysis: run-clang-tidy.sh on modified C++ translation units - Behavior changed: Yes. Eligible runtime filters skip nonmatching hash buckets. - Does this need documentation: No --- be/src/exec/operator/olap_scan_operator.cpp | 40 +++- be/src/exec/operator/olap_scan_operator.h | 7 + .../runtime_filter_bucket_pruner.cpp | 164 +++++++++++++++ .../runtime_filter_bucket_pruner.h | 58 ++++++ be/src/exec/scan/olap_scanner.cpp | 9 + be/src/exec/scan/olap_scanner.h | 2 + be/src/exec/scan/scanner.h | 3 + be/src/exec/scan/scanner_scheduler.cpp | 8 +- .../runtime_filter_bucket_pruner_test.cpp | 193 ++++++++++++++++++ .../RuntimeFilterBucketPruneClassifier.java | 123 +++++++++++ .../translator/RuntimeFilterTranslator.java | 10 + .../apache/doris/planner/OlapScanNode.java | 40 +++- .../apache/doris/planner/RuntimeFilter.java | 17 ++ .../org/apache/doris/qe/SessionVariable.java | 16 ++ ...untimeFilterBucketPruneClassifierTest.java | 123 +++++++++++ gensrc/thrift/PaloInternalService.thrift | 1 + gensrc/thrift/PlanNodes.thrift | 10 + .../runtime_filter/rf_bucket_pruning.out | 4 + .../runtime_filter/rf_bucket_pruning.groovy | 126 ++++++++++++ 19 files changed, 945 insertions(+), 9 deletions(-) create mode 100644 be/src/exec/runtime_filter/runtime_filter_bucket_pruner.cpp create mode 100644 be/src/exec/runtime_filter/runtime_filter_bucket_pruner.h create mode 100644 be/test/exec/runtime_filter/runtime_filter_bucket_pruner_test.cpp create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/RuntimeFilterBucketPruneClassifier.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/RuntimeFilterBucketPruneClassifierTest.java create mode 100644 regression-test/data/query_p0/runtime_filter/rf_bucket_pruning.out create mode 100644 regression-test/suites/query_p0/runtime_filter/rf_bucket_pruning.groovy diff --git a/be/src/exec/operator/olap_scan_operator.cpp b/be/src/exec/operator/olap_scan_operator.cpp index 292ca468c39ec3..5cbd1f9116834b 100644 --- a/be/src/exec/operator/olap_scan_operator.cpp +++ b/be/src/exec/operator/olap_scan_operator.cpp @@ -130,6 +130,8 @@ Status OlapScanLocalState::_init_profile() { _scan_rows = ADD_COUNTER(custom_profile(), "ScanRows", TUnit::UNIT); _tablets_pruned_by_rf_counter = ADD_COUNTER(custom_profile(), "TabletsPrunedByRuntimeFilter", TUnit::UNIT); + _buckets_pruned_by_rf_counter = + ADD_COUNTER(custom_profile(), "BucketsPrunedByRuntimeFilter", TUnit::UNIT); // 1. init segment profile _segment_profile.reset(new RuntimeProfile("SegmentIterator")); @@ -655,7 +657,7 @@ Status OlapScanLocalState::_init_scanners(std::list* scanners) { _cond_ranges.emplace_back(new doris::OlapScanRange()); } - // Filter out tablets whose partitions have been pruned by runtime filters. + // Filter out tablets whose partitions or buckets have been pruned by runtime filters. // // TODO(rf-partition-prune): this happens after OlapScanLocalState::init() // has already executed _sync_cloud_tablets() (in cloud mode that performs @@ -670,13 +672,16 @@ Status OlapScanLocalState::_init_scanners(std::list* scanners) { // the tablet, (b) acquire ready-at-start RFs before _sync_cloud_tablets() // and run partition pruning there to filter _scan_ranges by partition_id // so the heavy per-tablet work is skipped for pruned partitions. - if (_rf_partition_pruner.pruned_partition_count() > 0) { + if (_rf_partition_pruner.pruned_partition_count() > 0 || + _rf_bucket_pruner.pruned_tablet_count() > 0) { DCHECK_EQ(_tablets.size(), _scan_ranges.size()); DCHECK_EQ(_tablets.size(), _read_sources.size()); size_t write_idx = 0; for (size_t read_idx = 0; read_idx < _tablets.size(); ++read_idx) { int64_t pid = _tablets[read_idx].tablet->partition_id(); - if (!_rf_partition_pruner.is_partition_pruned(pid)) { + int64_t tablet_id = _tablets[read_idx].tablet->tablet_id(); + if (!_rf_partition_pruner.is_partition_pruned(pid) && + !_rf_bucket_pruner.is_tablet_pruned(tablet_id)) { if (write_idx != read_idx) { _tablets[write_idx] = std::move(_tablets[read_idx]); _scan_ranges[write_idx] = std::move(_scan_ranges[read_idx]); @@ -1114,10 +1119,39 @@ void OlapScanLocalState::set_scan_ranges(RuntimeState* state, for (auto& scan_range : scan_ranges) { DCHECK(scan_range.scan_range.__isset.palo_scan_range); _scan_ranges.emplace_back(new TPaloScanRange(scan_range.scan_range.palo_scan_range)); + const auto& palo_scan_range = scan_range.scan_range.palo_scan_range; + if (palo_scan_range.__isset.bucket_seq || palo_scan_range.__isset.bucket_num) { + DORIS_CHECK(palo_scan_range.__isset.bucket_seq); + DORIS_CHECK(palo_scan_range.__isset.bucket_num); + _rf_bucket_prune_ranges.emplace_back(palo_scan_range.tablet_id, + palo_scan_range.bucket_seq, + palo_scan_range.bucket_num); + } COUNTER_UPDATE(_tablet_counter, 1); } } +Status OlapScanLocalState::_on_runtime_filter_update() { + RETURN_IF_ERROR(Base::_on_runtime_filter_update()); + if (!state()->query_options().enable_runtime_filter_bucket_prune || + _rf_bucket_prune_ranges.empty()) { + return Status::OK(); + } + + int64_t newly_pruned = 0; + RETURN_IF_ERROR(_rf_bucket_pruner.prune_by_runtime_filters( + _rf_bucket_prune_ranges, _conjuncts, _parent->runtime_filter_descs(), + _parent->node_id(), state()->runtime_filter_max_in_num(), &newly_pruned)); + if (newly_pruned > 0) { + COUNTER_SET(_buckets_pruned_by_rf_counter, _rf_bucket_pruner.pruned_tablet_count()); + } + return Status::OK(); +} + +bool OlapScanLocalState::_is_tablet_pruned_by_runtime_filter(int64_t tablet_id) const { + return _rf_bucket_pruner.is_tablet_pruned(tablet_id); +} + static std::string tablets_id_to_string( const std::vector>& scan_ranges) { if (scan_ranges.empty()) { diff --git a/be/src/exec/operator/olap_scan_operator.h b/be/src/exec/operator/olap_scan_operator.h index 4716c78cf8b7c5..57f902e1a17e94 100644 --- a/be/src/exec/operator/olap_scan_operator.h +++ b/be/src/exec/operator/olap_scan_operator.h @@ -27,6 +27,7 @@ #include "common/status.h" #include "exec/operator/operator.h" #include "exec/operator/scan_operator.h" +#include "exec/runtime_filter/runtime_filter_bucket_pruner.h" #include "runtime/runtime_profile.h" #include "storage/index/snii/snii_prx_profile.h" #include "storage/olap_scan_common.h" @@ -78,6 +79,7 @@ class OlapScanLocalState final : public ScanLocalState { const std::vector& scan_ranges) override; Status _init_profile() override; Status _process_conjuncts(RuntimeState* state) override; + Status _on_runtime_filter_update() override; bool _is_key_column(const std::string& col_name) override; bool can_push_down_column_predicate(const SlotDescriptor* slot) override; @@ -135,7 +137,11 @@ class OlapScanLocalState final : public ScanLocalState { Status _build_key_ranges_and_filters(); + bool _is_tablet_pruned_by_runtime_filter(int64_t tablet_id) const; + std::vector> _scan_ranges; + std::vector _rf_bucket_prune_ranges; + RuntimeFilterBucketPruner _rf_bucket_pruner; std::vector _sync_statistics; MonotonicStopWatch _sync_cloud_tablets_watcher; std::shared_ptr _cloud_tablet_dependency; @@ -154,6 +160,7 @@ class OlapScanLocalState final : public ScanLocalState { snii::SniiPhraseRuntimeProfileCounters _snii_phrase_profile_counters; RuntimeProfile::Counter* _tablet_counter = nullptr; + RuntimeProfile::Counter* _buckets_pruned_by_rf_counter = nullptr; RuntimeProfile::Counter* _key_range_counter = nullptr; RuntimeProfile::Counter* _reader_init_timer = nullptr; RuntimeProfile::Counter* _scanner_init_timer = nullptr; diff --git a/be/src/exec/runtime_filter/runtime_filter_bucket_pruner.cpp b/be/src/exec/runtime_filter/runtime_filter_bucket_pruner.cpp new file mode 100644 index 00000000000000..3bce95b1053c7a --- /dev/null +++ b/be/src/exec/runtime_filter/runtime_filter_bucket_pruner.cpp @@ -0,0 +1,164 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "exec/runtime_filter/runtime_filter_bucket_pruner.h" + +#include + +#include +#include +#include + +#include "core/column/column.h" +#include "core/data_type/data_type.h" +#include "core/data_type/primitive_type.h" +#include "core/string_ref.h" +#include "exprs/hybrid_set.h" +#include "exprs/runtime_filter_expr.h" +#include "exprs/vexpr.h" +#include "exprs/vexpr_context.h" +#include "exprs/vslot_ref.h" + +namespace doris { + +static void materialize_hashes(const VExprSPtr& target_expr, HybridSetBase* hybrid_set, + std::vector* hashes) { + DORIS_CHECK(target_expr != nullptr); + DORIS_CHECK(hybrid_set != nullptr); + + const DataTypePtr& data_type = target_expr->data_type(); + MutableColumnPtr column = data_type->create_column(); + PrimitiveType primitive_type = data_type->get_primitive_type(); + auto* iter = hybrid_set->begin(); + while (iter->has_next()) { + const void* value = iter->get_value(); + DORIS_CHECK(value != nullptr); + if (is_string_type(primitive_type)) { + const auto* string_value = reinterpret_cast(value); + column->insert_data(string_value->data, string_value->size); + } else { + column->insert_data(reinterpret_cast(value), 0); + } + iter->next(); + } + if (hybrid_set->contain_null() && data_type->is_nullable()) { + column->insert_default(); + } + + hashes->assign(column->size(), 0); + if (!hashes->empty()) { + column->update_crcs_with_value(hashes->data(), primitive_type, + static_cast(column->size())); + } +} + +Status RuntimeFilterBucketPruner::prune_by_runtime_filters( + const std::vector& ranges, + const VExprContextSPtrs& conjuncts, const std::vector& rf_descs, + int scan_node_id, int max_in_num, int64_t* newly_pruned_count) { + *newly_pruned_count = 0; + if (ranges.empty()) { + return Status::OK(); + } + + phmap::flat_hash_set eligible_filter_ids; + for (const auto& desc : rf_descs) { + if (desc.__isset.bucket_pruning_target_ids && + desc.bucket_pruning_target_ids.contains(scan_node_id)) { + eligible_filter_ids.insert(desc.filter_id); + } + } + if (eligible_filter_ids.empty()) { + return Status::OK(); + } + + phmap::flat_hash_set newly_pruned; + for (const auto& conjunct_ctx : conjuncts) { + VExprSPtr root = conjunct_ctx->root(); + if (!root->is_rf_wrapper()) { + continue; + } + auto* wrapper = assert_cast(root.get()); + if (!eligible_filter_ids.contains(wrapper->filter_id())) { + continue; + } + + VExprSPtr impl = root->get_impl(); + DORIS_CHECK(impl != nullptr); + std::shared_ptr hybrid_set = impl->get_set_func(); + if (hybrid_set == nullptr) { + // IN_OR_BLOOM may become a Bloom filter at runtime. A Bloom filter + // cannot be inverted to a safe finite bucket set. + continue; + } + if (hybrid_set->size() > max_in_num) { + continue; + } + + DORIS_CHECK_EQ(impl->children().size(), 1); + VExprSPtr target_expr = impl->children()[0]; + DORIS_CHECK_EQ(target_expr->node_type(), TExprNodeType::SLOT_REF); + + std::vector hashes; + materialize_hashes(target_expr, hybrid_set.get(), &hashes); + phmap::flat_hash_map> selected_buckets_by_num; + for (const auto& range : ranges) { + if (newly_pruned.contains(range.tablet_id)) { + continue; + } + DORIS_CHECK_GT(range.bucket_num, 0); + DORIS_CHECK_GE(range.bucket_seq, 0); + DORIS_CHECK_LT(range.bucket_seq, range.bucket_num); + + auto [selected_it, inserted] = selected_buckets_by_num.try_emplace(range.bucket_num); + if (inserted) { + auto& selected_buckets = selected_it->second; + selected_buckets.reserve( + std::min(hashes.size(), static_cast(range.bucket_num))); + for (uint32_t hash : hashes) { + selected_buckets.insert( + static_cast(hash % static_cast(range.bucket_num))); + } + } + if (!selected_it->second.contains(range.bucket_seq)) { + newly_pruned.insert(range.tablet_id); + } + } + } + + if (!newly_pruned.empty()) { + std::unique_lock lock(_prune_mutex); + for (int64_t tablet_id : newly_pruned) { + if (_pruned_tablet_ids.insert(tablet_id).second) { + ++*newly_pruned_count; + } + } + } + return Status::OK(); +} + +bool RuntimeFilterBucketPruner::is_tablet_pruned(int64_t tablet_id) const { + std::shared_lock lock(_prune_mutex); + return _pruned_tablet_ids.contains(tablet_id); +} + +int64_t RuntimeFilterBucketPruner::pruned_tablet_count() const { + std::shared_lock lock(_prune_mutex); + return static_cast(_pruned_tablet_ids.size()); +} + +} // namespace doris diff --git a/be/src/exec/runtime_filter/runtime_filter_bucket_pruner.h b/be/src/exec/runtime_filter/runtime_filter_bucket_pruner.h new file mode 100644 index 00000000000000..b70f48480debff --- /dev/null +++ b/be/src/exec/runtime_filter/runtime_filter_bucket_pruner.h @@ -0,0 +1,58 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "common/status.h" +#include "exec/common/hash_table/phmap_fwd_decl.h" +#include "exprs/vexpr_fwd.h" + +namespace doris { + +struct TRuntimeFilterDesc; + +struct RuntimeFilterBucketPruneRange { + int64_t tablet_id = 0; + int32_t bucket_seq = 0; + int32_t bucket_num = 0; +}; + +// Per-scan-instance state for single-column HASH bucket pruning. Runtime filters +// are conjunctive, so each exact IN filter can monotonically add tablet ids to +// the pruned set without retaining or combining its original value set. +// is_tablet_pruned() is safe to call concurrently with the serialized pruning +// updates performed by ScanLocalStateBase. +class RuntimeFilterBucketPruner { +public: + Status prune_by_runtime_filters(const std::vector& ranges, + const VExprContextSPtrs& conjuncts, + const std::vector& rf_descs, + int scan_node_id, int max_in_num, int64_t* newly_pruned_count); + + bool is_tablet_pruned(int64_t tablet_id) const; + int64_t pruned_tablet_count() const; + +private: + phmap::flat_hash_set _pruned_tablet_ids; + mutable std::shared_mutex _prune_mutex; +}; + +} // namespace doris diff --git a/be/src/exec/scan/olap_scanner.cpp b/be/src/exec/scan/olap_scanner.cpp index 7282724cb54928..f2dd3ce06600b7 100644 --- a/be/src/exec/scan/olap_scanner.cpp +++ b/be/src/exec/scan/olap_scanner.cpp @@ -657,6 +657,15 @@ bool OlapScanner::check_partition_pruned() const { return _local_state->is_partition_pruned(_tablet_reader_params.tablet->partition_id()); } +bool OlapScanner::check_bucket_pruned() const { + if (!_local_state) { + return false; + } + auto* olap_local_state = assert_cast(_local_state); + return olap_local_state->_is_tablet_pruned_by_runtime_filter( + _tablet_reader_params.tablet->tablet_id()); +} + doris::TabletStorageType OlapScanner::get_storage_type() { if (config::is_cloud_mode()) { // we don't have cold storage in cloud mode, all storage is treated as local diff --git a/be/src/exec/scan/olap_scanner.h b/be/src/exec/scan/olap_scanner.h index 337c2c2cfea2ff..7973541b6d51fc 100644 --- a/be/src/exec/scan/olap_scanner.h +++ b/be/src/exec/scan/olap_scanner.h @@ -96,6 +96,8 @@ class OlapScanner : public Scanner { bool check_partition_pruned() const override; + bool check_bucket_pruned() const override; + void update_realtime_counters() override; protected: diff --git a/be/src/exec/scan/scanner.h b/be/src/exec/scan/scanner.h index d53fad8c80d264..639164c6e6070d 100644 --- a/be/src/exec/scan/scanner.h +++ b/be/src/exec/scan/scanner.h @@ -215,6 +215,9 @@ class Scanner { // Overridden by OlapScanner to check partition pruning state. virtual bool check_partition_pruned() const { return false; } + // Returns true if this scanner's bucket has been pruned by a runtime filter. + virtual bool check_bucket_pruned() const { return false; } + bool need_to_close() const { return _need_to_close; } void mark_to_need_to_close() { diff --git a/be/src/exec/scan/scanner_scheduler.cpp b/be/src/exec/scan/scanner_scheduler.cpp index 3036cbdf6ee099..9d3cb60473fb2c 100644 --- a/be/src/exec/scan/scanner_scheduler.cpp +++ b/be/src/exec/scan/scanner_scheduler.cpp @@ -183,7 +183,7 @@ void ScannerScheduler::_scanner_scan(std::shared_ptr ctx, // so better to also check low memory and clear free blocks here. if (ctx->low_memory_mode()) { ctx->clear_free_blocks(); } - if (scanner->check_partition_pruned()) { eos = true; } + if (scanner->check_partition_pruned() || scanner->check_bucket_pruned()) { eos = true; } if (!eos && !scanner->has_prepared()) { status = scanner->prepare(); @@ -208,8 +208,10 @@ void ScannerScheduler::_scanner_scan(std::shared_ptr ctx, } } - // After processing late RFs, check if this scanner's partition was pruned. - if (!eos && scanner->check_partition_pruned()) { eos = true; } + // After processing late RFs, check if this scanner's partition or bucket was pruned. + if (!eos && (scanner->check_partition_pruned() || scanner->check_bucket_pruned())) { + eos = true; + } size_t raw_bytes_threshold = config::doris_scanner_row_bytes; if (ctx->low_memory_mode()) { diff --git a/be/test/exec/runtime_filter/runtime_filter_bucket_pruner_test.cpp b/be/test/exec/runtime_filter/runtime_filter_bucket_pruner_test.cpp new file mode 100644 index 00000000000000..e54e185d3b9164 --- /dev/null +++ b/be/test/exec/runtime_filter/runtime_filter_bucket_pruner_test.cpp @@ -0,0 +1,193 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "exec/runtime_filter/runtime_filter_bucket_pruner.h" + +#include + +#include +#include +#include +#include +#include + +#include "core/column/column_vector.h" +#include "core/data_type/data_type_number.h" +#include "exec/runtime_filter/runtime_filter_definitions.h" +#include "exprs/create_predicate_function.h" +#include "exprs/runtime_filter_expr.h" +#include "exprs/vdirect_in_predicate.h" +#include "exprs/vexpr_context.h" +#include "exprs/vslot_ref.h" + +namespace doris { + +class RuntimeFilterBucketPrunerTest : public testing::Test { +protected: + static constexpr int SCAN_NODE_ID = 10; + + VExprContextSPtr make_in_conjunct(int filter_id, const std::vector& values) { + std::shared_ptr set(create_set(TYPE_INT, false)); + for (const int32_t value : values) { + set->insert(&value); + } + + TExprNode node; + node.__set_type(create_type_desc(TYPE_BOOLEAN)); + node.__set_node_type(TExprNodeType::IN_PRED); + node.in_predicate.__set_is_not_in(false); + node.__set_opcode(TExprOpcode::FILTER_IN); + node.__set_is_nullable(false); + auto impl = VDirectInPredicate::create_shared(node, std::move(set), true); + impl->add_child(VSlotRef::create_shared(/*slot_id=*/1, /*column_id=*/0, + /*column_uniq_id=*/1, + std::make_shared(), "dist_col")); + auto wrapper = RuntimeFilterExpr::create_shared(node, impl, 0, false, filter_id); + return std::make_shared(wrapper); + } + + VExprContextSPtr make_non_exact_conjunct(int filter_id) { + TExprNode node; + node.__set_type(create_type_desc(TYPE_BOOLEAN)); + node.__set_node_type(TExprNodeType::BLOOM_PRED); + node.__set_opcode(TExprOpcode::RT_FILTER); + node.__set_is_nullable(false); + auto impl = VDirectInPredicate::create_shared(node, nullptr, true); + impl->add_child(VSlotRef::create_shared(/*slot_id=*/1, /*column_id=*/0, + /*column_uniq_id=*/1, + std::make_shared(), "dist_col")); + auto wrapper = RuntimeFilterExpr::create_shared(node, impl, 0, false, filter_id); + return std::make_shared(wrapper); + } + + TRuntimeFilterDesc bucket_prune_desc(int filter_id) { + TRuntimeFilterDesc desc; + desc.__set_filter_id(filter_id); + desc.__set_bucket_pruning_target_ids({SCAN_NODE_ID}); + return desc; + } + + std::vector four_bucket_ranges() { + std::vector ranges; + for (int32_t bucket_seq = 0; bucket_seq < 4; ++bucket_seq) { + ranges.push_back({100 + bucket_seq, bucket_seq, 4}); + } + return ranges; + } + + int32_t bucket_for_value(int32_t value, int32_t bucket_num) { + auto column = ColumnInt32::create(); + column->insert_value(value); + uint32_t hash = 0; + column->update_crcs_with_value(&hash, TYPE_INT, 1, 0, nullptr); + return static_cast(hash % static_cast(bucket_num)); + } +}; + +TEST_F(RuntimeFilterBucketPrunerTest, ExactInKeepsOnlyMatchingBucket) { + constexpr int filter_id = 7; + constexpr int32_t value = 10; + VExprContextSPtrs conjuncts {make_in_conjunct(filter_id, {value})}; + std::vector rf_descs {bucket_prune_desc(filter_id)}; + + RuntimeFilterBucketPruner pruner; + int64_t newly_pruned = 0; + ASSERT_TRUE(pruner.prune_by_runtime_filters(four_bucket_ranges(), conjuncts, rf_descs, + SCAN_NODE_ID, /*max_in_num=*/1024, &newly_pruned) + .ok()); + + EXPECT_EQ(newly_pruned, 3); + EXPECT_EQ(pruner.pruned_tablet_count(), 3); + int32_t selected_bucket = bucket_for_value(value, 4); + for (int32_t bucket_seq = 0; bucket_seq < 4; ++bucket_seq) { + EXPECT_EQ(pruner.is_tablet_pruned(100 + bucket_seq), bucket_seq != selected_bucket); + } + + ASSERT_TRUE(pruner.prune_by_runtime_filters(four_bucket_ranges(), conjuncts, rf_descs, + SCAN_NODE_ID, /*max_in_num=*/1024, &newly_pruned) + .ok()); + EXPECT_EQ(newly_pruned, 0); +} + +TEST_F(RuntimeFilterBucketPrunerTest, SupportsDifferentBucketCountsAcrossPartitions) { + constexpr int filter_id = 11; + constexpr int32_t value = 10; + VExprContextSPtrs conjuncts {make_in_conjunct(filter_id, {value})}; + std::vector rf_descs {bucket_prune_desc(filter_id)}; + std::vector ranges; + for (int32_t bucket_seq = 0; bucket_seq < 4; ++bucket_seq) { + ranges.push_back({100 + bucket_seq, bucket_seq, 4}); + } + for (int32_t bucket_seq = 0; bucket_seq < 7; ++bucket_seq) { + ranges.push_back({200 + bucket_seq, bucket_seq, 7}); + } + + RuntimeFilterBucketPruner pruner; + int64_t newly_pruned = 0; + ASSERT_TRUE(pruner.prune_by_runtime_filters(ranges, conjuncts, rf_descs, SCAN_NODE_ID, + /*max_in_num=*/1024, &newly_pruned) + .ok()); + + EXPECT_EQ(newly_pruned, 9); + EXPECT_FALSE(pruner.is_tablet_pruned(100 + bucket_for_value(value, 4))); + EXPECT_FALSE(pruner.is_tablet_pruned(200 + bucket_for_value(value, 7))); +} + +TEST_F(RuntimeFilterBucketPrunerTest, EmptyExactInPrunesAllBuckets) { + constexpr int filter_id = 8; + VExprContextSPtrs conjuncts {make_in_conjunct(filter_id, {})}; + std::vector rf_descs {bucket_prune_desc(filter_id)}; + + RuntimeFilterBucketPruner pruner; + int64_t newly_pruned = 0; + ASSERT_TRUE(pruner.prune_by_runtime_filters(four_bucket_ranges(), conjuncts, rf_descs, + SCAN_NODE_ID, /*max_in_num=*/1024, &newly_pruned) + .ok()); + EXPECT_EQ(newly_pruned, 4); + EXPECT_EQ(pruner.pruned_tablet_count(), 4); +} + +TEST_F(RuntimeFilterBucketPrunerTest, NonExactRuntimeRepresentationIsIgnored) { + constexpr int filter_id = 9; + VExprContextSPtrs conjuncts {make_non_exact_conjunct(filter_id)}; + std::vector rf_descs {bucket_prune_desc(filter_id)}; + + RuntimeFilterBucketPruner pruner; + int64_t newly_pruned = 0; + ASSERT_TRUE(pruner.prune_by_runtime_filters(four_bucket_ranges(), conjuncts, rf_descs, + SCAN_NODE_ID, /*max_in_num=*/1024, &newly_pruned) + .ok()); + EXPECT_EQ(newly_pruned, 0); + EXPECT_EQ(pruner.pruned_tablet_count(), 0); +} + +TEST_F(RuntimeFilterBucketPrunerTest, DescriptorMustMarkScanAsEligible) { + constexpr int filter_id = 10; + VExprContextSPtrs conjuncts {make_in_conjunct(filter_id, {10})}; + TRuntimeFilterDesc desc; + desc.__set_filter_id(filter_id); + + RuntimeFilterBucketPruner pruner; + int64_t newly_pruned = 0; + ASSERT_TRUE(pruner.prune_by_runtime_filters(four_bucket_ranges(), conjuncts, {desc}, + SCAN_NODE_ID, /*max_in_num=*/1024, &newly_pruned) + .ok()); + EXPECT_EQ(newly_pruned, 0); + EXPECT_EQ(pruner.pruned_tablet_count(), 0); +} + +} // namespace doris diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/RuntimeFilterBucketPruneClassifier.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/RuntimeFilterBucketPruneClassifier.java new file mode 100644 index 00000000000000..dbe44910ba173f --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/RuntimeFilterBucketPruneClassifier.java @@ -0,0 +1,123 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.glue.translator; + +import org.apache.doris.analysis.Expr; +import org.apache.doris.analysis.SlotRef; +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.DistributionInfo; +import org.apache.doris.catalog.HashDistributionInfo; +import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.Partition; +import org.apache.doris.planner.OlapScanNode; +import org.apache.doris.planner.PlanNode; +import org.apache.doris.thrift.TRuntimeFilterType; + +/** Classifies direct single-column HASH targets for BE-side runtime-filter bucket pruning. */ +final class RuntimeFilterBucketPruneClassifier { + private RuntimeFilterBucketPruneClassifier() { + } + + static Classification classify(TRuntimeFilterType filterType, Expr targetExpr, PlanNode scanNode) { + if (filterType != TRuntimeFilterType.IN && filterType != TRuntimeFilterType.IN_OR_BLOOM) { + return Classification.unsupported("runtime filter is not IN or IN_OR_BLOOM"); + } + if (!(scanNode instanceof OlapScanNode)) { + return Classification.unsupported("target scan is not an OlapScanNode"); + } + if (!(targetExpr instanceof SlotRef)) { + return Classification.unsupported("target expression is not a direct SlotRef"); + } + + Column targetColumn = ((SlotRef) targetExpr).getColumn(); + if (targetColumn == null) { + return Classification.unsupported("target SlotRef has no column"); + } + + OlapScanNode olapScanNode = (OlapScanNode) scanNode; + OlapTable table = olapScanNode.getOlapTable(); + if (table == null || olapScanNode.getSelectedPartitionIds().isEmpty()) { + return Classification.unsupported("target scan has no selected partitions"); + } + + Column distributionColumn = null; + for (Long partitionId : olapScanNode.getSelectedPartitionIds()) { + Partition partition = table.getPartition(partitionId); + if (partition == null) { + return Classification.unsupported("selected partition does not exist"); + } + DistributionInfo distributionInfo = partition.getDistributionInfo(); + if (!(distributionInfo instanceof HashDistributionInfo)) { + return Classification.unsupported("distribution type is not HASH"); + } + HashDistributionInfo hashDistributionInfo = (HashDistributionInfo) distributionInfo; + if (hashDistributionInfo.getDistributionColumns().size() != 1) { + return Classification.unsupported("HASH distribution is not single-column"); + } + Column currentDistributionColumn = hashDistributionInfo.getDistributionColumns().get(0); + if (!sameColumn(targetColumn, currentDistributionColumn)) { + return Classification.unsupported("target SlotRef is not the HASH distribution column"); + } + if (distributionColumn != null && !sameColumn(distributionColumn, currentDistributionColumn)) { + return Classification.unsupported("selected partitions use different distribution columns"); + } + distributionColumn = currentDistributionColumn; + } + return Classification.supported(); + } + + private static boolean sameColumn(Column targetColumn, Column distributionColumn) { + if (targetColumn == distributionColumn || targetColumn.equals(distributionColumn)) { + return true; + } + int targetUniqueId = targetColumn.getUniqueId(); + int distributionUniqueId = distributionColumn.getUniqueId(); + if (targetUniqueId != Column.COLUMN_UNIQUE_ID_INIT_VALUE + && distributionUniqueId != Column.COLUMN_UNIQUE_ID_INIT_VALUE + && targetUniqueId == distributionUniqueId) { + return true; + } + return targetColumn.tryGetBaseColumnName().equalsIgnoreCase(distributionColumn.getName()); + } + + static final class Classification { + private final boolean canPruneBuckets; + private final String unsupportedReason; + + private Classification(boolean canPruneBuckets, String unsupportedReason) { + this.canPruneBuckets = canPruneBuckets; + this.unsupportedReason = unsupportedReason; + } + + static Classification supported() { + return new Classification(true, ""); + } + + static Classification unsupported(String reason) { + return new Classification(false, reason); + } + + boolean canPruneBuckets() { + return canPruneBuckets; + } + + String getUnsupportedReason() { + return unsupportedReason; + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/RuntimeFilterTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/RuntimeFilterTranslator.java index 6558541f0d723e..266dc1310ec309 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/RuntimeFilterTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/RuntimeFilterTranslator.java @@ -249,6 +249,11 @@ private void createLegacyRuntimeFilterFromGroup(List group, RuntimeFilterPartitionPruneClassifier.classify( head.getType(), targetExpr, nereidsTargetExprList.get(i), scanNode); setPartitionPruningMetadata(origFilter, scanNode, classification); + RuntimeFilterBucketPruneClassifier.Classification bucketClassification = + RuntimeFilterBucketPruneClassifier.classify(head.getType(), targetExpr, scanNode); + if (bucketClassification.canPruneBuckets()) { + origFilter.markTargetCanPruneBuckets(scanNode.getId()); + } } origFilter.setBloomFilterSizeCalculatedByNdv(head.isBloomFilterSizeCalculatedByNdv()); setWaitTimeMs(origFilter, head.isNonBlocking(), isLocalTarget); @@ -352,6 +357,11 @@ public void createLegacyRuntimeFilter(RuntimeFilter filter, PlanNode node, PlanT RuntimeFilterPartitionPruneClassifier.classify( filter.getType(), targetExpr, filter.getTargetExpressions().get(i), scanNode); setPartitionPruningMetadata(origFilter, scanNode, classification); + RuntimeFilterBucketPruneClassifier.Classification bucketClassification = + RuntimeFilterBucketPruneClassifier.classify(filter.getType(), targetExpr, scanNode); + if (bucketClassification.canPruneBuckets()) { + origFilter.markTargetCanPruneBuckets(scanNode.getId()); + } } origFilter.setBloomFilterSizeCalculatedByNdv(filter.isBloomFilterSizeCalculatedByNdv()); setWaitTimeMs(origFilter, filter.isNonBlocking(), isLocalTarget); diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java index 45ace61d3569c2..4edc08361f3ade 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java @@ -211,6 +211,7 @@ public class OlapScanNode extends ScanNode { private TableSample tableSample; private Map tabletId2BucketSeq = Maps.newHashMap(); + private Map tabletId2BucketNum = Maps.newHashMap(); // a bucket seq may map to many tablets, and each tablet has a // TScanRangeLocations. public ArrayListMultimap bucketSeq2locations = ArrayListMultimap.create(); @@ -1062,9 +1063,13 @@ private void computeTabletInfo() throws UserException { scanTabletIds.addAll(allTabletIds); } - if (!isPointQuery()) { - for (int i = 0; i < allTabletIds.size(); i++) { - tabletId2BucketSeq.put(allTabletIds.get(i), i); + for (int i = 0; i < allTabletIds.size(); i++) { + tabletId2BucketSeq.put(allTabletIds.get(i), i); + } + if (partition.getDistributionInfo() instanceof HashDistributionInfo) { + int bucketNum = ((HashDistributionInfo) partition.getDistributionInfo()).getBucketNum(); + for (Long tabletId : allTabletIds) { + tabletId2BucketNum.put(tabletId, bucketNum); } } @@ -1095,6 +1100,7 @@ public List lazyEvaluateRangeLocations() throws UserExcepti scanBackendOrderBySelection = false; scanTabletIds.clear(); tabletId2BucketSeq.clear(); + tabletId2BucketNum.clear(); bucketSeq2locations.clear(); bucketSeq2Bytes.clear(); scanReplicaIds.clear(); @@ -1482,6 +1488,11 @@ protected void toThrift(TPlanNode msg) { && hasRfDrivingPartitionPruning()) { setPartitionBoundariesForRuntimeFilter(msg.olap_scan_node); } + if (rfPruneCtx != null + && rfPruneCtx.getSessionVariable().isEnableRuntimeFilterBucketPrune() + && hasRfDrivingBucketPruning()) { + setRuntimeFilterBucketPruneParameters(); + } super.toThrift(msg); } @@ -1532,6 +1543,29 @@ void setPartitionBoundariesForRuntimeFilter(TOlapScanNode olapScanNode) { } } + private boolean hasRfDrivingBucketPruning() { + PlanNodeId myId = this.getId(); + for (RuntimeFilter rf : runtimeFilters) { + if (rf.canPruneBucketsFor(myId)) { + return true; + } + } + return false; + } + + private void setRuntimeFilterBucketPruneParameters() { + for (TScanRangeLocations locations : scanRangeLocations) { + TPaloScanRange scanRange = locations.getScanRange().getPaloScanRange(); + Integer bucketSeq = tabletId2BucketSeq.get(scanRange.getTabletId()); + Integer bucketNum = tabletId2BucketNum.get(scanRange.getTabletId()); + Preconditions.checkState(bucketSeq != null && bucketNum != null && bucketNum > 0, + "missing bucket metadata for runtime-filter bucket pruning, tablet=%s", + scanRange.getTabletId()); + scanRange.setBucketSeq(bucketSeq); + scanRange.setBucketNum(bucketNum); + } + } + private List buildPartitionBoundariesForRuntimeFilter() { PartitionInfo partitionInfo = olapTable.getPartitionInfo(); PartitionType partType = partitionInfo.getType(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/RuntimeFilter.java b/fe/fe-core/src/main/java/org/apache/doris/planner/RuntimeFilter.java index 77bb4ffcdc1ae3..24ea1cf9d11a1c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/RuntimeFilter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/RuntimeFilter.java @@ -146,6 +146,7 @@ public FilterSizeLimits(SessionVariable sessionVariable) { private final Map> targetPartitionMonotonicityByScanId = new HashMap<>(); private final Set partitionPruningTargetScanIds = new HashSet<>(); + private final Set bucketPruningTargetScanIds = new HashSet<>(); /** * Internal representation of a runtime filter target. @@ -391,6 +392,14 @@ public TRuntimeFilterDesc toThrift() { } } + boolean enableRfBucketPrune = rfPruneCtx != null + && rfPruneCtx.getSessionVariable().isEnableRuntimeFilterBucketPrune(); + if (enableRfBucketPrune && !bucketPruningTargetScanIds.isEmpty()) { + tFilter.setBucketPruningTargetIds(bucketPruningTargetScanIds.stream() + .map(PlanNodeId::asInt) + .collect(Collectors.toSet())); + } + return tFilter; } @@ -422,6 +431,14 @@ public boolean canPrunePartitionsFor(PlanNodeId scanNodeId) { return partitionPruningTargetScanIds.contains(scanNodeId); } + public void markTargetCanPruneBuckets(PlanNodeId scanNodeId) { + bucketPruningTargetScanIds.add(scanNodeId); + } + + public boolean canPruneBucketsFor(PlanNodeId scanNodeId) { + return bucketPruningTargetScanIds.contains(scanNodeId); + } + public boolean hasTargets() { return !targets.isEmpty(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index 09990761712672..c5b503e9f144b0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -470,6 +470,9 @@ public String toString() { public static final String ENABLE_RUNTIME_FILTER_PARTITION_PRUNE = "enable_runtime_filter_partition_prune"; + public static final String ENABLE_RUNTIME_FILTER_BUCKET_PRUNE = + "enable_runtime_filter_bucket_prune"; + public static final String ENABLE_PRUNE_NESTED_COLUMN = "enable_prune_nested_column"; static final String SESSION_CONTEXT = "session_context"; @@ -2176,6 +2179,9 @@ public boolean isEnableHboNonStrictMatchingMode() { fuzzy = true) public boolean enableRuntimeFilterPartitionPrune = true; + @VarAttrDef.VarAttr(name = ENABLE_RUNTIME_FILTER_BUCKET_PRUNE, needForward = true, fuzzy = true) + public boolean enableRuntimeFilterBucketPrune = true; + /** * The client can pass some special information by setting this session variable in the format: "k1:v1;k2:v2". * For example, trace_id can be passed to trace the query request sent by the user. @@ -3710,6 +3716,7 @@ public void initFuzzyModeVariables() { this.enableParallelScan = random.nextInt(2) == 0; this.enableRuntimeFilterPrune = (randomInt % 10) == 0; this.enableRuntimeFilterPartitionPrune = (randomInt % 2) == 0; + this.enableRuntimeFilterBucketPrune = (randomInt % 2) == 0; this.runtimeFilterTreePublishMaxSendBytes = Util.getRandomLong(0, 64L * 1024L * 1024L, 128L * 1024L * 1024L, 256L * 1024L * 1024L); @@ -5291,6 +5298,14 @@ public void setEnableRuntimeFilterPartitionPrune(boolean enableRuntimeFilterPart this.enableRuntimeFilterPartitionPrune = enableRuntimeFilterPartitionPrune; } + public boolean isEnableRuntimeFilterBucketPrune() { + return enableRuntimeFilterBucketPrune; + } + + public void setEnableRuntimeFilterBucketPrune(boolean enableRuntimeFilterBucketPrune) { + this.enableRuntimeFilterBucketPrune = enableRuntimeFilterBucketPrune; + } + public void setFragmentTransmissionCompressionCodec(String codec) { this.fragmentTransmissionCompressionCodec = codec; } @@ -5669,6 +5684,7 @@ public TQueryOptions toThrift() { tResult.setIgnoreRuntimeFilterError(ignoreRuntimeFilterError); tResult.setProfileLevel(getProfileLevel()); tResult.setEnableRuntimeFilterPartitionPrune(enableRuntimeFilterPartitionPrune); + tResult.setEnableRuntimeFilterBucketPrune(enableRuntimeFilterBucketPrune); tResult.setMinimumOperatorMemoryRequiredKb(minimumOperatorMemoryRequiredKB); tResult.setExchangeMultiBlocksByteSize(exchangeMultiBlocksByteSize); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/RuntimeFilterBucketPruneClassifierTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/RuntimeFilterBucketPruneClassifierTest.java new file mode 100644 index 00000000000000..f34a5fc4c5cbb4 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/RuntimeFilterBucketPruneClassifierTest.java @@ -0,0 +1,123 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.glue.translator; + +import org.apache.doris.analysis.SlotDescriptor; +import org.apache.doris.analysis.SlotId; +import org.apache.doris.analysis.SlotRef; +import org.apache.doris.analysis.TupleId; +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.HashDistributionInfo; +import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.Partition; +import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.catalog.RandomDistributionInfo; +import org.apache.doris.planner.OlapScanNode; +import org.apache.doris.thrift.TRuntimeFilterType; + +import com.google.common.collect.ImmutableList; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +class RuntimeFilterBucketPruneClassifierTest { + @Test + void testSingleColumnHashInSupported() { + Column distributionColumn = new Column("dist_col", PrimitiveType.INT); + RuntimeFilterBucketPruneClassifier.Classification classification = classify( + TRuntimeFilterType.IN, distributionColumn, + new HashDistributionInfo(8, ImmutableList.of(distributionColumn))); + + Assertions.assertTrue(classification.canPruneBuckets()); + } + + @Test + void testInOrBloomSupportedAtPlanTime() { + Column distributionColumn = new Column("dist_col", PrimitiveType.INT); + RuntimeFilterBucketPruneClassifier.Classification classification = classify( + TRuntimeFilterType.IN_OR_BLOOM, distributionColumn, + new HashDistributionInfo(8, ImmutableList.of(distributionColumn))); + + Assertions.assertTrue(classification.canPruneBuckets()); + } + + @Test + void testBloomRejected() { + Column distributionColumn = new Column("dist_col", PrimitiveType.INT); + RuntimeFilterBucketPruneClassifier.Classification classification = classify( + TRuntimeFilterType.BLOOM, distributionColumn, + new HashDistributionInfo(8, ImmutableList.of(distributionColumn))); + + Assertions.assertFalse(classification.canPruneBuckets()); + Assertions.assertTrue(classification.getUnsupportedReason().contains("IN")); + } + + @Test + void testCompositeHashRejected() { + Column distributionColumn = new Column("dist_col", PrimitiveType.INT); + Column secondDistributionColumn = new Column("dist_col_2", PrimitiveType.INT); + RuntimeFilterBucketPruneClassifier.Classification classification = classify( + TRuntimeFilterType.IN, distributionColumn, + new HashDistributionInfo(8, + ImmutableList.of(distributionColumn, secondDistributionColumn))); + + Assertions.assertFalse(classification.canPruneBuckets()); + Assertions.assertTrue(classification.getUnsupportedReason().contains("single-column")); + } + + @Test + void testNonDistributionTargetRejected() { + Column distributionColumn = new Column("dist_col", PrimitiveType.INT); + Column targetColumn = new Column("value_col", PrimitiveType.INT); + RuntimeFilterBucketPruneClassifier.Classification classification = classify( + TRuntimeFilterType.IN, targetColumn, + new HashDistributionInfo(8, ImmutableList.of(distributionColumn))); + + Assertions.assertFalse(classification.canPruneBuckets()); + Assertions.assertTrue(classification.getUnsupportedReason().contains("distribution column")); + } + + @Test + void testRandomDistributionRejected() { + Column targetColumn = new Column("dist_col", PrimitiveType.INT); + RuntimeFilterBucketPruneClassifier.Classification classification = classify( + TRuntimeFilterType.IN, targetColumn, new RandomDistributionInfo(8)); + + Assertions.assertFalse(classification.canPruneBuckets()); + Assertions.assertTrue(classification.getUnsupportedReason().contains("not HASH")); + } + + private RuntimeFilterBucketPruneClassifier.Classification classify( + TRuntimeFilterType filterType, Column targetColumn, + org.apache.doris.catalog.DistributionInfo distributionInfo) { + SlotDescriptor slotDescriptor = new SlotDescriptor(new SlotId(1), new TupleId(1)); + slotDescriptor.setColumn(targetColumn); + slotDescriptor.setType(targetColumn.getType()); + SlotRef targetSlot = new SlotRef(slotDescriptor); + + OlapTable table = Mockito.mock(OlapTable.class); + Partition partition = Mockito.mock(Partition.class); + OlapScanNode scanNode = Mockito.mock(OlapScanNode.class); + Mockito.when(scanNode.getOlapTable()).thenReturn(table); + Mockito.when(scanNode.getSelectedPartitionIds()).thenReturn(ImmutableList.of(1L)); + Mockito.when(table.getPartition(1L)).thenReturn(partition); + Mockito.when(partition.getDistributionInfo()).thenReturn(distributionInfo); + + return RuntimeFilterBucketPruneClassifier.classify(filterType, targetSlot, scanNode); + } +} diff --git a/gensrc/thrift/PaloInternalService.thrift b/gensrc/thrift/PaloInternalService.thrift index fd16b4b7598861..07944139285228 100644 --- a/gensrc/thrift/PaloInternalService.thrift +++ b/gensrc/thrift/PaloInternalService.thrift @@ -518,6 +518,7 @@ struct TQueryOptions { 230: optional bool supports_external_file_report_ack = false; // Fall back to RE2 when Hyperscan cannot compile a regular expression. 231: optional bool enable_hyperscan_fallback = true; + 232: optional bool enable_runtime_filter_bucket_prune = true; // For cloud, to control if the content would be written into file cache // In write path, to control if the content would be written into file cache. // In read path, read from file cache or remote storage when execute query. diff --git a/gensrc/thrift/PlanNodes.thrift b/gensrc/thrift/PlanNodes.thrift index b2702dd0ed4c83..65c70fd519e84c 100644 --- a/gensrc/thrift/PlanNodes.thrift +++ b/gensrc/thrift/PlanNodes.thrift @@ -102,6 +102,11 @@ struct TPaloScanRange { 10: optional i64 start_tso 11: optional i64 end_tso 12: optional TBinlogScanType binlog_scan_type + // Bucket metadata for BE-side runtime-filter bucket pruning. These fields + // are populated only when the scan has an eligible single-column HASH + // distribution runtime-filter target. + 13: optional i32 bucket_seq + 14: optional i32 bucket_num } enum TFileFormatType { @@ -1659,6 +1664,11 @@ struct TRuntimeFilterDesc { // slice and must be merged before being applied. Computed truthfully by FE after local // exchange planning; replaces inferring this from the target scan's is_serial_operator. 21: optional bool force_local_merge; + + // Scan node ids whose target is a direct SlotRef on the only HASH + // distribution column. BE still verifies that the delivered filter has an + // exact IN set before using it for bucket pruning. + 22: optional set bucket_pruning_target_ids; } diff --git a/regression-test/data/query_p0/runtime_filter/rf_bucket_pruning.out b/regression-test/data/query_p0/runtime_filter/rf_bucket_pruning.out new file mode 100644 index 00000000000000..4dadd5b6a170a8 --- /dev/null +++ b/regression-test/data/query_p0/runtime_filter/rf_bucket_pruning.out @@ -0,0 +1,4 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !bucket_result -- +3 30 + diff --git a/regression-test/suites/query_p0/runtime_filter/rf_bucket_pruning.groovy b/regression-test/suites/query_p0/runtime_filter/rf_bucket_pruning.groovy new file mode 100644 index 00000000000000..7717e3d4c5e7e0 --- /dev/null +++ b/regression-test/suites/query_p0/runtime_filter/rf_bucket_pruning.groovy @@ -0,0 +1,126 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.action.ProfileAction + +suite("rf_bucket_pruning", "nonConcurrent") { + sql "set enable_runtime_filter_prune=false" + sql "set enable_runtime_filter_partition_prune=false" + sql "set enable_runtime_filter_bucket_prune=true" + sql "set runtime_filter_wait_infinitely=true" + sql "set runtime_filter_type='IN'" + sql "set disable_join_reorder=true" + sql "set enable_profile=true" + sql "set profile_level=2" + sql "set parallel_pipeline_task_num=1" + + sql "drop table if exists rf_bucket_prune_fact" + sql """ + CREATE TABLE rf_bucket_prune_fact ( + k INT NOT NULL, + v INT NOT NULL + ) + DISTRIBUTED BY HASH(k) BUCKETS 8 + PROPERTIES("replication_num" = "1") + """ + sql "drop table if exists rf_bucket_prune_composite" + sql """ + CREATE TABLE rf_bucket_prune_composite ( + k INT NOT NULL, + v INT NOT NULL + ) + DISTRIBUTED BY HASH(k, v) BUCKETS 8 + PROPERTIES("replication_num" = "1") + """ + sql "drop table if exists rf_bucket_prune_dim" + sql """ + CREATE TABLE rf_bucket_prune_dim ( + k INT NOT NULL + ) + DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES("replication_num" = "1") + """ + + sql """ + INSERT INTO rf_bucket_prune_fact VALUES + (1, 10), (2, 20), (3, 30), (4, 40), + (5, 50), (6, 60), (7, 70), (8, 80) + """ + sql """ + INSERT INTO rf_bucket_prune_composite VALUES + (1, 10), (2, 20), (3, 30), (4, 40), + (5, 50), (6, 60), (7, 70), (8, 80) + """ + sql "INSERT INTO rf_bucket_prune_dim VALUES (3)" + + order_qt_bucket_result """ + SELECT f.k, f.v + FROM rf_bucket_prune_fact f + JOIN [broadcast] rf_bucket_prune_dim d ON f.k = d.k + ORDER BY f.k, f.v + """ + + def profileAction = new ProfileAction(context) + def getProfileByToken = { String token -> + String profileContent = "" + for (int attempt = 0; attempt < 60; attempt++) { + List profileData = profileAction.getProfileList() + for (final def profileItem in profileData) { + if (profileItem["Sql Statement"].toString().contains(token)) { + def currentProfile = profileAction.getProfile(profileItem["Profile ID"].toString()) + if (currentProfile != "") { + profileContent = currentProfile + } + if (profileItem["Profile Completion State"]?.toString() == "COMPLETE" + && profileContent.contains("BucketsPrunedByRuntimeFilter")) { + return profileContent + } + break + } + } + Thread.sleep(500) + } + return profileContent + } + def extractPrunedBuckets = { String profile -> + def values = (profile =~ /-\s*BucketsPrunedByRuntimeFilter:\s*(\d+)/) + .collect { it[1].toLong() } + return values.isEmpty() ? 0L : values.sum() + } + def runProfileQuery = { String tableName -> + def token = UUID.randomUUID().toString() + sql """ + SELECT "${token}", COUNT(*) + FROM ${tableName} f + JOIN [broadcast] rf_bucket_prune_dim d ON f.k = d.k + """ + def profile = getProfileByToken(token) + assertTrue(profile != "", "Profile not found for ${token}") + assertTrue(profile.contains("BucketsPrunedByRuntimeFilter"), + "Bucket-pruning counter not found for ${token}") + return extractPrunedBuckets(profile) + } + + assertTrue(runProfileQuery("rf_bucket_prune_fact") > 0, + "single-column HASH distribution should be pruned") + assertTrue(runProfileQuery("rf_bucket_prune_composite") == 0, + "multi-column HASH distribution must not be pruned") + + sql "set enable_runtime_filter_bucket_prune=false" + assertTrue(runProfileQuery("rf_bucket_prune_fact") == 0, + "disabled runtime-filter bucket pruning must not prune buckets") +} From 883316bb5963913008945a5206ddee5ab4785fb9 Mon Sep 17 00:00:00 2001 From: happenlee Date: Wed, 12 Aug 2026 10:42:09 +0800 Subject: [PATCH 02/20] [fix](runtime filter) Address bucket pruning review feedback ### What problem does this PR solve? Issue Number: None Related PR: #65837 Problem Summary: Runtime filter bucket pruning compared distribution columns with an asymmetric name fallback and could accept columns with conflicting unique IDs. Compare set unique IDs first and use symmetric base-column names only when IDs are unavailable. Clarify fixed-width and null-aware hashing contracts, and cover both behaviors with focused tests. ### Release note None ### Check List (For Author) - Test: Unit Test - FE RuntimeFilterBucketPruneClassifierTest (8 tests) - BE RuntimeFilterBucketPrunerTest (6 tests) - Behavior changed: Yes (bucket pruning is disabled when target and distribution unique IDs conflict; query results are unchanged) - Does this need documentation: No --- .../runtime_filter_bucket_pruner.cpp | 3 ++ .../runtime_filter_bucket_pruner_test.cpp | 47 +++++++++++++++++++ .../RuntimeFilterBucketPruneClassifier.java | 10 ++-- ...untimeFilterBucketPruneClassifierTest.java | 31 ++++++++++++ 4 files changed, 86 insertions(+), 5 deletions(-) diff --git a/be/src/exec/runtime_filter/runtime_filter_bucket_pruner.cpp b/be/src/exec/runtime_filter/runtime_filter_bucket_pruner.cpp index 3bce95b1053c7a..4ac1140f35251f 100644 --- a/be/src/exec/runtime_filter/runtime_filter_bucket_pruner.cpp +++ b/be/src/exec/runtime_filter/runtime_filter_bucket_pruner.cpp @@ -51,11 +51,14 @@ static void materialize_hashes(const VExprSPtr& target_expr, HybridSetBase* hybr const auto* string_value = reinterpret_cast(value); column->insert_data(string_value->data, string_value->size); } else { + // ColumnVector::insert_data ignores length for fixed-length values. column->insert_data(reinterpret_cast(value), 0); } iter->next(); } if (hybrid_set->contain_null() && data_type->is_nullable()) { + // contain_null() is true only for a null-aware filter. Keep the bucket that owns + // NULL probe rows by hashing NULL with the same nullable CRC semantics as partitioning. column->insert_default(); } diff --git a/be/test/exec/runtime_filter/runtime_filter_bucket_pruner_test.cpp b/be/test/exec/runtime_filter/runtime_filter_bucket_pruner_test.cpp index e54e185d3b9164..a4cc925e381b10 100644 --- a/be/test/exec/runtime_filter/runtime_filter_bucket_pruner_test.cpp +++ b/be/test/exec/runtime_filter/runtime_filter_bucket_pruner_test.cpp @@ -26,6 +26,7 @@ #include #include "core/column/column_vector.h" +#include "core/data_type/data_type_nullable.h" #include "core/data_type/data_type_number.h" #include "exec/runtime_filter/runtime_filter_definitions.h" #include "exprs/create_predicate_function.h" @@ -74,6 +75,24 @@ class RuntimeFilterBucketPrunerTest : public testing::Test { return std::make_shared(wrapper); } + VExprContextSPtr make_null_aware_in_conjunct(int filter_id) { + std::shared_ptr set(create_set(TYPE_INT, true)); + set->insert(static_cast(nullptr)); + + TExprNode node; + node.__set_type(create_type_desc(TYPE_BOOLEAN)); + node.__set_node_type(TExprNodeType::NULL_AWARE_IN_PRED); + node.in_predicate.__set_is_not_in(false); + node.__set_opcode(TExprOpcode::FILTER_IN); + node.__set_is_nullable(false); + auto impl = VDirectInPredicate::create_shared(node, std::move(set), true); + impl->add_child(VSlotRef::create_shared( + /*slot_id=*/1, /*column_id=*/0, /*column_uniq_id=*/1, + std::make_shared(std::make_shared()), "dist_col")); + auto wrapper = RuntimeFilterExpr::create_shared(node, impl, 0, false, filter_id); + return std::make_shared(wrapper); + } + TRuntimeFilterDesc bucket_prune_desc(int filter_id) { TRuntimeFilterDesc desc; desc.__set_filter_id(filter_id); @@ -96,6 +115,15 @@ class RuntimeFilterBucketPrunerTest : public testing::Test { column->update_crcs_with_value(&hash, TYPE_INT, 1, 0, nullptr); return static_cast(hash % static_cast(bucket_num)); } + + int32_t bucket_for_null(int32_t bucket_num) { + auto column = std::make_shared(std::make_shared()) + ->create_column(); + column->insert_default(); + uint32_t hash = 0; + column->update_crcs_with_value(&hash, TYPE_INT, 1, 0, nullptr); + return static_cast(hash % static_cast(bucket_num)); + } }; TEST_F(RuntimeFilterBucketPrunerTest, ExactInKeepsOnlyMatchingBucket) { @@ -161,6 +189,25 @@ TEST_F(RuntimeFilterBucketPrunerTest, EmptyExactInPrunesAllBuckets) { EXPECT_EQ(pruner.pruned_tablet_count(), 4); } +TEST_F(RuntimeFilterBucketPrunerTest, NullAwareInKeepsNullBucket) { + constexpr int filter_id = 12; + VExprContextSPtrs conjuncts {make_null_aware_in_conjunct(filter_id)}; + std::vector rf_descs {bucket_prune_desc(filter_id)}; + + RuntimeFilterBucketPruner pruner; + int64_t newly_pruned = 0; + ASSERT_TRUE(pruner.prune_by_runtime_filters(four_bucket_ranges(), conjuncts, rf_descs, + SCAN_NODE_ID, /*max_in_num=*/1024, &newly_pruned) + .ok()); + + EXPECT_EQ(newly_pruned, 3); + EXPECT_EQ(pruner.pruned_tablet_count(), 3); + int32_t null_bucket = bucket_for_null(4); + for (int32_t bucket_seq = 0; bucket_seq < 4; ++bucket_seq) { + EXPECT_EQ(pruner.is_tablet_pruned(100 + bucket_seq), bucket_seq != null_bucket); + } +} + TEST_F(RuntimeFilterBucketPrunerTest, NonExactRuntimeRepresentationIsIgnored) { constexpr int filter_id = 9; VExprContextSPtrs conjuncts {make_non_exact_conjunct(filter_id)}; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/RuntimeFilterBucketPruneClassifier.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/RuntimeFilterBucketPruneClassifier.java index dbe44910ba173f..58c1e0632626e6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/RuntimeFilterBucketPruneClassifier.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/RuntimeFilterBucketPruneClassifier.java @@ -82,17 +82,17 @@ static Classification classify(TRuntimeFilterType filterType, Expr targetExpr, P } private static boolean sameColumn(Column targetColumn, Column distributionColumn) { - if (targetColumn == distributionColumn || targetColumn.equals(distributionColumn)) { + if (targetColumn == distributionColumn) { return true; } int targetUniqueId = targetColumn.getUniqueId(); int distributionUniqueId = distributionColumn.getUniqueId(); if (targetUniqueId != Column.COLUMN_UNIQUE_ID_INIT_VALUE - && distributionUniqueId != Column.COLUMN_UNIQUE_ID_INIT_VALUE - && targetUniqueId == distributionUniqueId) { - return true; + && distributionUniqueId != Column.COLUMN_UNIQUE_ID_INIT_VALUE) { + return targetUniqueId == distributionUniqueId; } - return targetColumn.tryGetBaseColumnName().equalsIgnoreCase(distributionColumn.getName()); + return targetColumn.tryGetBaseColumnName() + .equalsIgnoreCase(distributionColumn.tryGetBaseColumnName()); } static final class Classification { diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/RuntimeFilterBucketPruneClassifierTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/RuntimeFilterBucketPruneClassifierTest.java index f34a5fc4c5cbb4..2b6bb49a4ebd3f 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/RuntimeFilterBucketPruneClassifierTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/RuntimeFilterBucketPruneClassifierTest.java @@ -92,6 +92,37 @@ void testNonDistributionTargetRejected() { Assertions.assertTrue(classification.getUnsupportedReason().contains("distribution column")); } + @Test + void testBaseColumnNamesComparedSymmetrically() { + Column baseColumn = new Column("base_col", PrimitiveType.INT); + SlotDescriptor baseSlotDescriptor = new SlotDescriptor(new SlotId(2), new TupleId(2)); + baseSlotDescriptor.setColumn(baseColumn); + baseSlotDescriptor.setType(baseColumn.getType()); + + Column distributionColumn = new Column("mv_dist_col", PrimitiveType.INT); + distributionColumn.setDefineExpr(new SlotRef(baseSlotDescriptor)); + RuntimeFilterBucketPruneClassifier.Classification classification = classify( + TRuntimeFilterType.IN, baseColumn, + new HashDistributionInfo(8, ImmutableList.of(distributionColumn))); + + Assertions.assertTrue(classification.canPruneBuckets()); + } + + @Test + void testDifferentUniqueIdsRejectedBeforeNameFallback() { + Column distributionColumn = new Column("dist_col", PrimitiveType.INT); + distributionColumn.setUniqueId(1); + Column targetColumn = new Column("dist_col", PrimitiveType.INT); + targetColumn.setUniqueId(2); + + RuntimeFilterBucketPruneClassifier.Classification classification = classify( + TRuntimeFilterType.IN, targetColumn, + new HashDistributionInfo(8, ImmutableList.of(distributionColumn))); + + Assertions.assertFalse(classification.canPruneBuckets()); + Assertions.assertTrue(classification.getUnsupportedReason().contains("distribution column")); + } + @Test void testRandomDistributionRejected() { Column targetColumn = new Column("dist_col", PrimitiveType.INT); From a09815379769329a274f18be2e46f6b41ec8e35e Mon Sep 17 00:00:00 2001 From: happenlee Date: Wed, 12 Aug 2026 11:32:55 +0800 Subject: [PATCH 03/20] [fix](runtime filter) Reject cross-index column ID collisions ### What problem does this PR solve? Issue Number: None Related PR: #65837 Problem Summary: Runtime filter bucket pruning treated column unique IDs as table-wide identities, but Doris assigns them independently per materialized index. A selected rollup column could therefore share an ID with a different base distribution column and incorrectly enable bucket pruning, causing matching rows to be skipped. Match columns by logical base-column name across index namespaces and require exact type equality. Add focused rollup ID-collision and type-mismatch coverage. ### Release note None ### Check List (For Author) - Test: Unit Test - FE RuntimeFilterBucketPruneClassifierTest (10 tests) - Behavior changed: Yes (unsafe bucket pruning is disabled for cross-index column ID collisions and type mismatches) - Does this need documentation: No --- .../RuntimeFilterBucketPruneClassifier.java | 9 +---- ...untimeFilterBucketPruneClassifierTest.java | 39 ++++++++++++++++++- 2 files changed, 40 insertions(+), 8 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/RuntimeFilterBucketPruneClassifier.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/RuntimeFilterBucketPruneClassifier.java index 58c1e0632626e6..8317bdaaccb27f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/RuntimeFilterBucketPruneClassifier.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/RuntimeFilterBucketPruneClassifier.java @@ -85,14 +85,9 @@ private static boolean sameColumn(Column targetColumn, Column distributionColumn if (targetColumn == distributionColumn) { return true; } - int targetUniqueId = targetColumn.getUniqueId(); - int distributionUniqueId = distributionColumn.getUniqueId(); - if (targetUniqueId != Column.COLUMN_UNIQUE_ID_INIT_VALUE - && distributionUniqueId != Column.COLUMN_UNIQUE_ID_INIT_VALUE) { - return targetUniqueId == distributionUniqueId; - } return targetColumn.tryGetBaseColumnName() - .equalsIgnoreCase(distributionColumn.tryGetBaseColumnName()); + .equalsIgnoreCase(distributionColumn.tryGetBaseColumnName()) + && targetColumn.getType().equals(distributionColumn.getType()); } static final class Classification { diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/RuntimeFilterBucketPruneClassifierTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/RuntimeFilterBucketPruneClassifierTest.java index 2b6bb49a4ebd3f..d6cff3b1a15d87 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/RuntimeFilterBucketPruneClassifierTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/RuntimeFilterBucketPruneClassifierTest.java @@ -109,7 +109,7 @@ void testBaseColumnNamesComparedSymmetrically() { } @Test - void testDifferentUniqueIdsRejectedBeforeNameFallback() { + void testDifferentUniqueIdsAllowedForSameBaseColumn() { Column distributionColumn = new Column("dist_col", PrimitiveType.INT); distributionColumn.setUniqueId(1); Column targetColumn = new Column("dist_col", PrimitiveType.INT); @@ -119,6 +119,43 @@ void testDifferentUniqueIdsRejectedBeforeNameFallback() { TRuntimeFilterType.IN, targetColumn, new HashDistributionInfo(8, ImmutableList.of(distributionColumn))); + Assertions.assertTrue(classification.canPruneBuckets()); + } + + @Test + void testRollupUniqueIdCollisionRejectedForDifferentBaseColumns() { + Column baseDistributionColumn = new Column("k2", PrimitiveType.INT); + baseDistributionColumn.setUniqueId(1); + + Column baseTargetColumn = new Column("k1", PrimitiveType.INT); + SlotDescriptor baseSlotDescriptor = new SlotDescriptor(new SlotId(2), new TupleId(2)); + baseSlotDescriptor.setColumn(baseTargetColumn); + baseSlotDescriptor.setType(baseTargetColumn.getType()); + Column rollupTargetColumn = new Column("mv_k1", PrimitiveType.INT); + rollupTargetColumn.setUniqueId(1); + rollupTargetColumn.setDefineExpr(new SlotRef(baseSlotDescriptor)); + + RuntimeFilterBucketPruneClassifier.Classification classification = classify( + TRuntimeFilterType.IN, rollupTargetColumn, + new HashDistributionInfo(8, ImmutableList.of(baseDistributionColumn))); + + Assertions.assertFalse(classification.canPruneBuckets()); + Assertions.assertTrue(classification.getUnsupportedReason().contains("distribution column")); + } + + @Test + void testRollupWithDifferentTypeRejected() { + Column baseDistributionColumn = new Column("dist_col", PrimitiveType.INT); + SlotDescriptor baseSlotDescriptor = new SlotDescriptor(new SlotId(2), new TupleId(2)); + baseSlotDescriptor.setColumn(baseDistributionColumn); + baseSlotDescriptor.setType(baseDistributionColumn.getType()); + Column rollupTargetColumn = new Column("mv_dist_col", PrimitiveType.BIGINT); + rollupTargetColumn.setDefineExpr(new SlotRef(baseSlotDescriptor)); + + RuntimeFilterBucketPruneClassifier.Classification classification = classify( + TRuntimeFilterType.IN, rollupTargetColumn, + new HashDistributionInfo(8, ImmutableList.of(baseDistributionColumn))); + Assertions.assertFalse(classification.canPruneBuckets()); Assertions.assertTrue(classification.getUnsupportedReason().contains("distribution column")); } From eeb95e2f4846ba8942938c80a7ced22823e91361 Mon Sep 17 00:00:00 2001 From: happenlee Date: Wed, 12 Aug 2026 13:36:45 +0800 Subject: [PATCH 04/20] [fix](runtime filter) Unify scan pruning checks Issue Number: None Related PR: #65837 Problem Summary: Runtime-filter partition and bucket pruning exposed separate scanner APIs, reprocessed all accumulated filters while holding the conjunct lock, retained duplicate all-tablet metadata for ordinary scans, and built that metadata for point queries. With F staggered filters, late-arrival pruning performed F(F+1)/2 filter passes. Unify the scheduler-facing API as is_pruned_by_runtime_filter(), process only the newly appended immutable filter slice outside the conjunct lock so the work is F passes, compact bucket number and sequence into one map, exclude point queries from that map, and bypass the bucket-pruner lock for ineligible scans. None - Test: Unit Test - BE RuntimeFilterBucketPrunerTest and RuntimeFilterPartitionPrunerTest (17 tests) - FE RuntimeFilterBucketPruneClassifierTest (10 tests) - BE and FE Release compilation - BE clang-format and format check - Behavior changed: No - Does this need documentation: No --- be/src/exec/operator/olap_scan_operator.cpp | 17 ++-- be/src/exec/operator/olap_scan_operator.h | 4 +- be/src/exec/operator/scan_operator.cpp | 50 +++++------ be/src/exec/operator/scan_operator.h | 19 ++-- .../runtime_filter_bucket_pruner.h | 3 +- .../runtime_filter_partition_pruner.cpp | 90 ++++++++++--------- .../runtime_filter_partition_pruner.h | 4 +- be/src/exec/scan/olap_scanner.cpp | 14 +-- be/src/exec/scan/olap_scanner.h | 4 +- be/src/exec/scan/scanner.h | 8 +- be/src/exec/scan/scanner_scheduler.cpp | 8 +- .../apache/doris/planner/OlapScanNode.java | 43 +++++---- 12 files changed, 127 insertions(+), 137 deletions(-) diff --git a/be/src/exec/operator/olap_scan_operator.cpp b/be/src/exec/operator/olap_scan_operator.cpp index 5cbd1f9116834b..28e356d2a640f8 100644 --- a/be/src/exec/operator/olap_scan_operator.cpp +++ b/be/src/exec/operator/olap_scan_operator.cpp @@ -680,8 +680,7 @@ Status OlapScanLocalState::_init_scanners(std::list* scanners) { for (size_t read_idx = 0; read_idx < _tablets.size(); ++read_idx) { int64_t pid = _tablets[read_idx].tablet->partition_id(); int64_t tablet_id = _tablets[read_idx].tablet->tablet_id(); - if (!_rf_partition_pruner.is_partition_pruned(pid) && - !_rf_bucket_pruner.is_tablet_pruned(tablet_id)) { + if (!_is_tablet_pruned_by_runtime_filter(pid, tablet_id)) { if (write_idx != read_idx) { _tablets[write_idx] = std::move(_tablets[read_idx]); _scan_ranges[write_idx] = std::move(_scan_ranges[read_idx]); @@ -1131,8 +1130,8 @@ void OlapScanLocalState::set_scan_ranges(RuntimeState* state, } } -Status OlapScanLocalState::_on_runtime_filter_update() { - RETURN_IF_ERROR(Base::_on_runtime_filter_update()); +Status OlapScanLocalState::_on_runtime_filter_update(const VExprContextSPtrs& new_conjuncts) { + RETURN_IF_ERROR(Base::_on_runtime_filter_update(new_conjuncts)); if (!state()->query_options().enable_runtime_filter_bucket_prune || _rf_bucket_prune_ranges.empty()) { return Status::OK(); @@ -1140,7 +1139,7 @@ Status OlapScanLocalState::_on_runtime_filter_update() { int64_t newly_pruned = 0; RETURN_IF_ERROR(_rf_bucket_pruner.prune_by_runtime_filters( - _rf_bucket_prune_ranges, _conjuncts, _parent->runtime_filter_descs(), + _rf_bucket_prune_ranges, new_conjuncts, _parent->runtime_filter_descs(), _parent->node_id(), state()->runtime_filter_max_in_num(), &newly_pruned)); if (newly_pruned > 0) { COUNTER_SET(_buckets_pruned_by_rf_counter, _rf_bucket_pruner.pruned_tablet_count()); @@ -1148,8 +1147,12 @@ Status OlapScanLocalState::_on_runtime_filter_update() { return Status::OK(); } -bool OlapScanLocalState::_is_tablet_pruned_by_runtime_filter(int64_t tablet_id) const { - return _rf_bucket_pruner.is_tablet_pruned(tablet_id); +bool OlapScanLocalState::_is_tablet_pruned_by_runtime_filter(int64_t partition_id, + int64_t tablet_id) const { + if (_rf_partition_pruner.is_partition_pruned(partition_id)) { + return true; + } + return !_rf_bucket_prune_ranges.empty() && _rf_bucket_pruner.is_tablet_pruned(tablet_id); } static std::string tablets_id_to_string( diff --git a/be/src/exec/operator/olap_scan_operator.h b/be/src/exec/operator/olap_scan_operator.h index 57f902e1a17e94..78bc8e256109c2 100644 --- a/be/src/exec/operator/olap_scan_operator.h +++ b/be/src/exec/operator/olap_scan_operator.h @@ -79,7 +79,7 @@ class OlapScanLocalState final : public ScanLocalState { const std::vector& scan_ranges) override; Status _init_profile() override; Status _process_conjuncts(RuntimeState* state) override; - Status _on_runtime_filter_update() override; + Status _on_runtime_filter_update(const VExprContextSPtrs& new_conjuncts) override; bool _is_key_column(const std::string& col_name) override; bool can_push_down_column_predicate(const SlotDescriptor* slot) override; @@ -137,7 +137,7 @@ class OlapScanLocalState final : public ScanLocalState { Status _build_key_ranges_and_filters(); - bool _is_tablet_pruned_by_runtime_filter(int64_t tablet_id) const; + bool _is_tablet_pruned_by_runtime_filter(int64_t partition_id, int64_t tablet_id) const; std::vector> _scan_ranges; std::vector _rf_bucket_prune_ranges; diff --git a/be/src/exec/operator/scan_operator.cpp b/be/src/exec/operator/scan_operator.cpp index a6458289d7ef5f..3d7c9f5e44b640 100644 --- a/be/src/exec/operator/scan_operator.cpp +++ b/be/src/exec/operator/scan_operator.cpp @@ -74,23 +74,24 @@ bool ScanLocalState::should_run_serial() const { Status ScanLocalStateBase::update_late_arrival_runtime_filter(RuntimeState* state, int& arrived_rf_num) { - // Lock needed because _conjuncts can be accessed concurrently by multiple scanner threads - LockGuard lock(_conjuncts_lock); - size_t conjuncts_before = _conjuncts.size(); - RETURN_IF_ERROR(_helper.try_append_late_arrival_runtime_filter( - state, _parent->operator_row_desc_before_projection(), arrived_rf_num, _conjuncts)); - if (state->enable_adjust_conjunct_order_by_cost()) { - std::ranges::stable_sort(_conjuncts, [](const auto& a, const auto& b) { - return a->execute_cost() < b->execute_cost(); - }); - }; - // Only re-run partition pruning when try_append_late_arrival_runtime_filter - // actually appended new conjuncts. Otherwise this hook would re-scan all - // partition boundaries on every scheduler pass while there are still - // unapplied RFs (Scanner::_applied_rf_num is not advanced here), wasting - // CPU re-evaluating the same set of RFs against the same boundaries. - if (_conjuncts.size() > conjuncts_before) { - RETURN_IF_ERROR(_on_runtime_filter_update()); + VExprContextSPtrs new_conjuncts; + { + // Lock needed because _conjuncts can be accessed concurrently by multiple scanner threads. + LockGuard lock(_conjuncts_lock); + size_t conjuncts_before = _conjuncts.size(); + RETURN_IF_ERROR(_helper.try_append_late_arrival_runtime_filter( + state, _parent->operator_row_desc_before_projection(), arrived_rf_num, _conjuncts)); + if (_conjuncts.size() > conjuncts_before) { + new_conjuncts.assign(_conjuncts.begin() + conjuncts_before, _conjuncts.end()); + } + if (state->enable_adjust_conjunct_order_by_cost()) { + std::ranges::stable_sort(_conjuncts, [](const auto& a, const auto& b) { + return a->execute_cost() < b->execute_cost(); + }); + } + } + if (!new_conjuncts.empty()) { + RETURN_IF_ERROR(_on_runtime_filter_update(new_conjuncts)); } return Status::OK(); } @@ -105,19 +106,15 @@ Status ScanLocalStateBase::clone_conjunct_ctxs(VExprContextSPtrs& scanner_conjun return Status::OK(); } -bool ScanLocalStateBase::is_partition_pruned(int64_t partition_id) const { - return _rf_partition_pruner.is_partition_pruned(partition_id); -} - -Status ScanLocalStateBase::_on_runtime_filter_update() { +Status ScanLocalStateBase::_on_runtime_filter_update(const VExprContextSPtrs& new_conjuncts) { const auto* parsed = _parent->parsed_partition_boundaries(); if (parsed != nullptr && !parsed->empty()) { - RETURN_IF_ERROR(_do_partition_pruning_by_rf()); + RETURN_IF_ERROR(_do_partition_pruning_by_rf(new_conjuncts)); } return Status::OK(); } -Status ScanLocalStateBase::_do_partition_pruning_by_rf() { +Status ScanLocalStateBase::_do_partition_pruning_by_rf(const VExprContextSPtrs& conjuncts) { if (!_state->query_options().enable_runtime_filter_partition_prune) { return Status::OK(); } @@ -127,7 +124,7 @@ Status ScanLocalStateBase::_do_partition_pruning_by_rf() { } int64_t newly_pruned = 0; RETURN_IF_ERROR(_rf_partition_pruner.prune_by_runtime_filters( - *parsed, _conjuncts, _parent->runtime_filter_descs(), _parent->node_id(), + *parsed, conjuncts, _parent->runtime_filter_descs(), _parent->node_id(), &newly_pruned)); if (newly_pruned > 0) { COUNTER_SET(_partitions_pruned_by_rf_counter, @@ -236,7 +233,8 @@ Status ScanLocalState::open(RuntimeState* state) { RETURN_IF_ERROR(_helper.acquire_runtime_filter(state, _conjuncts, p.operator_row_desc_before_projection())); if (_conjuncts.size() > conjuncts_before) { - RETURN_IF_ERROR(_on_runtime_filter_update()); + VExprContextSPtrs new_conjuncts(_conjuncts.begin() + conjuncts_before, _conjuncts.end()); + RETURN_IF_ERROR(_on_runtime_filter_update(new_conjuncts)); } // Disable condition cache in topn filter valid. TODO:: Try to support the topn filter in condition cache diff --git a/be/src/exec/operator/scan_operator.h b/be/src/exec/operator/scan_operator.h index 71dd4da466d4c3..d496a6b3176333 100644 --- a/be/src/exec/operator/scan_operator.h +++ b/be/src/exec/operator/scan_operator.h @@ -98,10 +98,6 @@ class ScanLocalStateBase : public PipelineXLocalState<> { [[nodiscard]] virtual int min_scanners_concurrency(RuntimeState* state) const; [[nodiscard]] virtual ScannerScheduler* scan_scheduler(RuntimeState* state) const; - // Thread-safe check whether a partition has been pruned by runtime filter. - // Callable from any scan type's scanner in scheduling threads. - bool is_partition_pruned(int64_t partition_id) const; - [[nodiscard]] std::string get_name() { return _parent->get_name(); } uint64_t get_condition_cache_digest() const { return _condition_cache_digest; } @@ -116,12 +112,12 @@ class ScanLocalStateBase : public PipelineXLocalState<> { virtual Status _init_profile() = 0; - // Hook for subclasses to react after new runtime filters are appended. - // Called inside update_late_arrival_runtime_filter() while _conjuncts_lock is held. - // Default implementation runs partition pruning on the newly appended RFs. - virtual Status _on_runtime_filter_update(); + // Hook for subclasses to process only the runtime-filter conjuncts appended by the + // current update. The shared pointers keep this immutable snapshot alive after + // _conjuncts_lock is released. + virtual Status _on_runtime_filter_update(const VExprContextSPtrs& new_conjuncts); - Status _do_partition_pruning_by_rf(); + Status _do_partition_pruning_by_rf(const VExprContextSPtrs& conjuncts); std::atomic _opened {false}; @@ -296,10 +292,7 @@ class ScanLocalState : public ScanLocalStateBase { friend class Scanner; Status _init_profile() override; - virtual Status _process_conjuncts(RuntimeState* state) { - RETURN_IF_ERROR(_do_partition_pruning_by_rf()); - return _normalize_conjuncts(state); - } + virtual Status _process_conjuncts(RuntimeState* state) { return _normalize_conjuncts(state); } virtual bool _should_push_down_common_expr(const VExprSPtr&) { return false; } virtual bool can_push_down_column_predicate(const SlotDescriptor* slot) { diff --git a/be/src/exec/runtime_filter/runtime_filter_bucket_pruner.h b/be/src/exec/runtime_filter/runtime_filter_bucket_pruner.h index b70f48480debff..5db7f881f643b5 100644 --- a/be/src/exec/runtime_filter/runtime_filter_bucket_pruner.h +++ b/be/src/exec/runtime_filter/runtime_filter_bucket_pruner.h @@ -38,8 +38,7 @@ struct RuntimeFilterBucketPruneRange { // Per-scan-instance state for single-column HASH bucket pruning. Runtime filters // are conjunctive, so each exact IN filter can monotonically add tablet ids to // the pruned set without retaining or combining its original value set. -// is_tablet_pruned() is safe to call concurrently with the serialized pruning -// updates performed by ScanLocalStateBase. +// Both pruning updates and is_tablet_pruned() are safe to call concurrently. class RuntimeFilterBucketPruner { public: Status prune_by_runtime_filters(const std::vector& ranges, diff --git a/be/src/exec/runtime_filter/runtime_filter_partition_pruner.cpp b/be/src/exec/runtime_filter/runtime_filter_partition_pruner.cpp index 6e6d43472c526a..74ced3a3acef8b 100644 --- a/be/src/exec/runtime_filter/runtime_filter_partition_pruner.cpp +++ b/be/src/exec/runtime_filter/runtime_filter_partition_pruner.cpp @@ -854,66 +854,70 @@ Status RuntimeFilterPartitionPruner::prune_by_runtime_filters( } } - // This function is serialized by _conjuncts_lock in the caller, so our reads - // of _pruned_partition_ids never race with our writes below. The only concurrent - // readers are is_partition_pruned() calls (under shared_lock), which are - // properly synchronized by the unique_lock we take when inserting. phmap::flat_hash_set newly_pruned; + { + // _try_prune_by_single_rf() skips IDs already published by previous filters. + // Hold a shared lock while it reads that set; concurrent updates merge their + // local results under the unique lock below. + std::shared_lock lock(_prune_mutex); + for (const auto& conjunct_ctx : conjuncts) { + VExprSPtr root = conjunct_ctx->root(); + if (!root->is_rf_wrapper()) { + continue; + } - for (const auto& conjunct_ctx : conjuncts) { - VExprSPtr root = conjunct_ctx->root(); - if (!root->is_rf_wrapper()) { - continue; - } + VExprSPtr impl = root->get_impl(); + if (!impl) { + continue; + } - VExprSPtr impl = root->get_impl(); - if (!impl) { - continue; - } + if (impl->children().empty()) { + continue; + } - if (impl->children().empty()) { - continue; - } + auto* wrapper_root = assert_cast(root.get()); + int filter_id = wrapper_root->filter_id(); - auto* wrapper_root = assert_cast(root.get()); - int filter_id = wrapper_root->filter_id(); + VExprSPtr target_subtree = impl->children()[0]; - VExprSPtr target_subtree = impl->children()[0]; + auto partition_mono_it = filter_id_to_partition_monotonicity.find(filter_id); + // For LIST partition targets, this metadata is also the per-partition eligibility map; + // BE projects every finite LIST value, so the direction itself is neutral there. + bool has_partition_mono = + partition_mono_it != filter_id_to_partition_monotonicity.end() && + !partition_mono_it->second.empty(); + if (!has_partition_mono) { + continue; + } - auto partition_mono_it = filter_id_to_partition_monotonicity.find(filter_id); - // For LIST partition targets, this metadata is also the per-partition eligibility map; - // BE projects every finite LIST value, so the direction itself is neutral there. - bool has_partition_mono = partition_mono_it != filter_id_to_partition_monotonicity.end() && - !partition_mono_it->second.empty(); - if (!has_partition_mono) { - continue; - } + const VSlotRef* leaf_slot = find_unique_slot_ref(target_subtree.get()); + DORIS_CHECK(leaf_slot != nullptr); - const VSlotRef* leaf_slot = find_unique_slot_ref(target_subtree.get()); - DORIS_CHECK(leaf_slot != nullptr); + SlotId leaf_slot_id = leaf_slot->slot_id(); + DORIS_CHECK(slot_to_boundaries.contains(leaf_slot_id)); - SlotId leaf_slot_id = leaf_slot->slot_id(); - DORIS_CHECK(slot_to_boundaries.contains(leaf_slot_id)); + int leaf_column_id = leaf_slot->column_id(); - int leaf_column_id = leaf_slot->column_id(); + std::shared_ptr> projected; + RETURN_IF_ERROR(parsed.get_or_compute_projected_boundaries( + filter_id, target_subtree, leaf_slot_id, leaf_column_id, + partition_mono_it->second, conjunct_ctx.get(), &projected)); - std::shared_ptr> projected; - RETURN_IF_ERROR(parsed.get_or_compute_projected_boundaries( - filter_id, target_subtree, leaf_slot_id, leaf_column_id, partition_mono_it->second, - conjunct_ctx.get(), &projected)); + if (projected == nullptr || projected->empty()) { + continue; + } - if (projected == nullptr || projected->empty()) { - continue; + _try_prune_by_single_rf(*projected, impl, newly_pruned); } - - _try_prune_by_single_rf(*projected, impl, newly_pruned); } - auto count = static_cast(newly_pruned.size()); - if (count > 0) { + int64_t count = 0; + if (!newly_pruned.empty()) { std::unique_lock lock(_prune_mutex); for (int64_t pid : newly_pruned) { - _pruned_partition_ids.insert(pid); + if (_pruned_partition_ids.insert(pid).second) { + ++count; + } } } *newly_pruned_count = count; diff --git a/be/src/exec/runtime_filter/runtime_filter_partition_pruner.h b/be/src/exec/runtime_filter/runtime_filter_partition_pruner.h index 9e3d0150db1232..6493baec72cb4a 100644 --- a/be/src/exec/runtime_filter/runtime_filter_partition_pruner.h +++ b/be/src/exec/runtime_filter/runtime_filter_partition_pruner.h @@ -133,8 +133,8 @@ class ParsedPartitionBoundaries { // `OperatorXBase::parsed_partition_boundaries()`. The owner (ScanLocalStateBase) // passes the parsed object into `prune_by_runtime_filters` on each call. // -// Thread safety: `is_partition_pruned()` is safe to call concurrently with -// `prune_by_runtime_filters()` via an internal shared_mutex. +// Thread safety: pruning updates and lookups are safe to call concurrently via +// an internal shared_mutex. class RuntimeFilterPartitionPruner { public: RuntimeFilterPartitionPruner() = default; diff --git a/be/src/exec/scan/olap_scanner.cpp b/be/src/exec/scan/olap_scanner.cpp index f2dd3ce06600b7..635495cb43eadf 100644 --- a/be/src/exec/scan/olap_scanner.cpp +++ b/be/src/exec/scan/olap_scanner.cpp @@ -650,19 +650,11 @@ Status OlapScanner::_init_read_schema() { return Status::OK(); } -bool OlapScanner::check_partition_pruned() const { - if (!_local_state) { - return false; - } - return _local_state->is_partition_pruned(_tablet_reader_params.tablet->partition_id()); -} - -bool OlapScanner::check_bucket_pruned() const { - if (!_local_state) { - return false; - } +bool OlapScanner::is_pruned_by_runtime_filter() const { + DCHECK(_local_state != nullptr); auto* olap_local_state = assert_cast(_local_state); return olap_local_state->_is_tablet_pruned_by_runtime_filter( + _tablet_reader_params.tablet->partition_id(), _tablet_reader_params.tablet->tablet_id()); } diff --git a/be/src/exec/scan/olap_scanner.h b/be/src/exec/scan/olap_scanner.h index 7973541b6d51fc..67fd4ef64d8e87 100644 --- a/be/src/exec/scan/olap_scanner.h +++ b/be/src/exec/scan/olap_scanner.h @@ -94,9 +94,7 @@ class OlapScanner : public Scanner { doris::TabletStorageType get_storage_type() override; - bool check_partition_pruned() const override; - - bool check_bucket_pruned() const override; + bool is_pruned_by_runtime_filter() const override; void update_realtime_counters() override; diff --git a/be/src/exec/scan/scanner.h b/be/src/exec/scan/scanner.h index 639164c6e6070d..3ec462dbd3c574 100644 --- a/be/src/exec/scan/scanner.h +++ b/be/src/exec/scan/scanner.h @@ -211,12 +211,8 @@ class Scanner { return doris::TabletStorageType::STORAGE_TYPE_REMOTE; } - // Returns true if this scanner's partition has been pruned by a runtime filter. - // Overridden by OlapScanner to check partition pruning state. - virtual bool check_partition_pruned() const { return false; } - - // Returns true if this scanner's bucket has been pruned by a runtime filter. - virtual bool check_bucket_pruned() const { return false; } + // Returns true if this scanner's scan range has been pruned by a runtime filter. + virtual bool is_pruned_by_runtime_filter() const { return false; } bool need_to_close() const { return _need_to_close; } diff --git a/be/src/exec/scan/scanner_scheduler.cpp b/be/src/exec/scan/scanner_scheduler.cpp index 9d3cb60473fb2c..4a7afda12d8b85 100644 --- a/be/src/exec/scan/scanner_scheduler.cpp +++ b/be/src/exec/scan/scanner_scheduler.cpp @@ -183,7 +183,7 @@ void ScannerScheduler::_scanner_scan(std::shared_ptr ctx, // so better to also check low memory and clear free blocks here. if (ctx->low_memory_mode()) { ctx->clear_free_blocks(); } - if (scanner->check_partition_pruned() || scanner->check_bucket_pruned()) { eos = true; } + if (scanner->is_pruned_by_runtime_filter()) { eos = true; } if (!eos && !scanner->has_prepared()) { status = scanner->prepare(); @@ -208,10 +208,8 @@ void ScannerScheduler::_scanner_scan(std::shared_ptr ctx, } } - // After processing late RFs, check if this scanner's partition or bucket was pruned. - if (!eos && (scanner->check_partition_pruned() || scanner->check_bucket_pruned())) { - eos = true; - } + // After processing late RFs, check whether this scanner's scan range was pruned. + if (!eos && scanner->is_pruned_by_runtime_filter()) { eos = true; } size_t raw_bytes_threshold = config::doris_scanner_row_bytes; if (ctx->low_memory_mode()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java index 4edc08361f3ade..32399e7b80af09 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java @@ -210,8 +210,8 @@ public class OlapScanNode extends ScanNode { private Set nereidsPrunedTabletIds = Sets.newHashSet(); private TableSample tableSample; - private Map tabletId2BucketSeq = Maps.newHashMap(); - private Map tabletId2BucketNum = Maps.newHashMap(); + // Pack bucket number and sequence into one value to avoid retaining two all-tablet maps. + private Map tabletId2BucketInfo = Maps.newHashMap(); // a bucket seq may map to many tablets, and each tablet has a // TScanRangeLocations. public ArrayListMultimap bucketSeq2locations = ArrayListMultimap.create(); @@ -798,7 +798,9 @@ private void addScanRangeLocations(Partition partition, private void addBucketSeqStatsIfNeeded(long tabletId, TScanRangeLocations locations, long oneReplicaBytes) { if (!isPointQuery()) { - Integer bucketSeq = tabletId2BucketSeq.get(tabletId); + long bucketInfo = Preconditions.checkNotNull(tabletId2BucketInfo.get(tabletId), + "missing bucket metadata for tablet %s", tabletId); + int bucketSeq = decodeBucketSeq(bucketInfo); bucketSeq2locations.put(bucketSeq, locations); bucketSeq2Bytes.merge(bucketSeq, oneReplicaBytes, Long::sum); } @@ -1063,13 +1065,10 @@ private void computeTabletInfo() throws UserException { scanTabletIds.addAll(allTabletIds); } - for (int i = 0; i < allTabletIds.size(); i++) { - tabletId2BucketSeq.put(allTabletIds.get(i), i); - } - if (partition.getDistributionInfo() instanceof HashDistributionInfo) { - int bucketNum = ((HashDistributionInfo) partition.getDistributionInfo()).getBucketNum(); - for (Long tabletId : allTabletIds) { - tabletId2BucketNum.put(tabletId, bucketNum); + if (!isPointQuery()) { + int bucketNum = partition.getDistributionInfo().getBucketNum(); + for (int i = 0; i < allTabletIds.size(); i++) { + tabletId2BucketInfo.put(allTabletIds.get(i), encodeBucketInfo(i, bucketNum)); } } @@ -1099,8 +1098,7 @@ public List lazyEvaluateRangeLocations() throws UserExcepti selectionHint = null; scanBackendOrderBySelection = false; scanTabletIds.clear(); - tabletId2BucketSeq.clear(); - tabletId2BucketNum.clear(); + tabletId2BucketInfo.clear(); bucketSeq2locations.clear(); bucketSeq2Bytes.clear(); scanReplicaIds.clear(); @@ -1556,16 +1554,27 @@ private boolean hasRfDrivingBucketPruning() { private void setRuntimeFilterBucketPruneParameters() { for (TScanRangeLocations locations : scanRangeLocations) { TPaloScanRange scanRange = locations.getScanRange().getPaloScanRange(); - Integer bucketSeq = tabletId2BucketSeq.get(scanRange.getTabletId()); - Integer bucketNum = tabletId2BucketNum.get(scanRange.getTabletId()); - Preconditions.checkState(bucketSeq != null && bucketNum != null && bucketNum > 0, + Long bucketInfo = tabletId2BucketInfo.get(scanRange.getTabletId()); + Preconditions.checkState(bucketInfo != null && decodeBucketNum(bucketInfo) > 0, "missing bucket metadata for runtime-filter bucket pruning, tablet=%s", scanRange.getTabletId()); - scanRange.setBucketSeq(bucketSeq); - scanRange.setBucketNum(bucketNum); + scanRange.setBucketSeq(decodeBucketSeq(bucketInfo)); + scanRange.setBucketNum(decodeBucketNum(bucketInfo)); } } + private static long encodeBucketInfo(int bucketSeq, int bucketNum) { + return ((long) bucketNum << Integer.SIZE) | Integer.toUnsignedLong(bucketSeq); + } + + private static int decodeBucketSeq(long bucketInfo) { + return (int) bucketInfo; + } + + private static int decodeBucketNum(long bucketInfo) { + return (int) (bucketInfo >>> Integer.SIZE); + } + private List buildPartitionBoundariesForRuntimeFilter() { PartitionInfo partitionInfo = olapTable.getPartitionInfo(); PartitionType partType = partitionInfo.getType(); From 325d63c12af858ed3338109112018826f829fc66 Mon Sep 17 00:00:00 2001 From: happenlee Date: Wed, 12 Aug 2026 16:45:16 +0800 Subject: [PATCH 05/20] [fix](be) Serialize runtime-filter pruning with scanner cloning Issue Number: None Related PR: #65837 Problem Summary: Late runtime-filter conjuncts became visible to scanner cloning before partition projection completed. VExprContext::clone() opens the shared expression root while projection executes it, so releasing the conjunct lock before pruning allowed concurrent mutation and reads of expression node state. Keep the conjunct lock through pruning while continuing to process only the newly appended filter slice, preserving linear work across staggered arrivals. None - Test: Unit Test - BE ScannerLateArrivalRfTest, RuntimeFilterPartitionPrunerTest, and RuntimeFilterBucketPrunerTest (18 tests) - BE clang-format and format check - BE clang-tidy on modified C++ files - Behavior changed: No - Does this need documentation: No --- be/src/exec/operator/scan_operator.cpp | 34 +++++++++++++------------- be/src/exec/operator/scan_operator.h | 6 ++--- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/be/src/exec/operator/scan_operator.cpp b/be/src/exec/operator/scan_operator.cpp index 3d7c9f5e44b640..3259ec4a19f9fa 100644 --- a/be/src/exec/operator/scan_operator.cpp +++ b/be/src/exec/operator/scan_operator.cpp @@ -74,26 +74,26 @@ bool ScanLocalState::should_run_serial() const { Status ScanLocalStateBase::update_late_arrival_runtime_filter(RuntimeState* state, int& arrived_rf_num) { + // Lock needed because _conjuncts can be accessed concurrently by multiple scanner threads. + LockGuard lock(_conjuncts_lock); + size_t conjuncts_before = _conjuncts.size(); + RETURN_IF_ERROR(_helper.try_append_late_arrival_runtime_filter( + state, _parent->operator_row_desc_before_projection(), arrived_rf_num, _conjuncts)); VExprContextSPtrs new_conjuncts; - { - // Lock needed because _conjuncts can be accessed concurrently by multiple scanner threads. - LockGuard lock(_conjuncts_lock); - size_t conjuncts_before = _conjuncts.size(); - RETURN_IF_ERROR(_helper.try_append_late_arrival_runtime_filter( - state, _parent->operator_row_desc_before_projection(), arrived_rf_num, _conjuncts)); - if (_conjuncts.size() > conjuncts_before) { - new_conjuncts.assign(_conjuncts.begin() + conjuncts_before, _conjuncts.end()); - } - if (state->enable_adjust_conjunct_order_by_cost()) { - std::ranges::stable_sort(_conjuncts, [](const auto& a, const auto& b) { - return a->execute_cost() < b->execute_cost(); - }); - } + if (_conjuncts.size() > conjuncts_before) { + new_conjuncts.assign(_conjuncts.begin() + conjuncts_before, _conjuncts.end()); } - if (!new_conjuncts.empty()) { - RETURN_IF_ERROR(_on_runtime_filter_update(new_conjuncts)); + if (state->enable_adjust_conjunct_order_by_cost()) { + std::ranges::stable_sort(_conjuncts, [](const auto& a, const auto& b) { + return a->execute_cost() < b->execute_cost(); + }); } - return Status::OK(); + if (new_conjuncts.empty()) { + return Status::OK(); + } + // Partition projection executes the shared expression tree. Keep it serialized with + // clone_conjunct_ctxs(), whose VExprContext::clone() opens that same tree. + return _on_runtime_filter_update(new_conjuncts); } Status ScanLocalStateBase::clone_conjunct_ctxs(VExprContextSPtrs& scanner_conjuncts) { diff --git a/be/src/exec/operator/scan_operator.h b/be/src/exec/operator/scan_operator.h index d496a6b3176333..4532e8773a83b4 100644 --- a/be/src/exec/operator/scan_operator.h +++ b/be/src/exec/operator/scan_operator.h @@ -112,9 +112,9 @@ class ScanLocalStateBase : public PipelineXLocalState<> { virtual Status _init_profile() = 0; - // Hook for subclasses to process only the runtime-filter conjuncts appended by the - // current update. The shared pointers keep this immutable snapshot alive after - // _conjuncts_lock is released. + // Hook for subclasses to process only the runtime-filter conjuncts appended by the current + // update. Late-arrival updates call this while holding _conjuncts_lock because pruning may + // execute expression nodes shared with scanner clones. virtual Status _on_runtime_filter_update(const VExprContextSPtrs& new_conjuncts); Status _do_partition_pruning_by_rf(const VExprContextSPtrs& conjuncts); From b790883ba0dedc8844634018c64bd077ad983e56 Mon Sep 17 00:00:00 2001 From: happenlee Date: Wed, 12 Aug 2026 20:17:24 +0800 Subject: [PATCH 06/20] [fix](runtime filter) Reuse bucket-prune hashes across consumers ### What problem does this PR solve? Issue Number: None Related PR: #65837 Problem Summary: Every scan local state materialized and hashed the same finalized exact runtime-filter set independently, point-query scans could be classified for bucket pruning without the metadata required during serialization, and bucket tests derived expectations through the same hashing API as production pruning. Cache immutable bucket hashes once per finalized wrapper and target nullability, reject point-query targets during classification, and validate routing hashes through the write path plus an end-to-end nullable null-aware join. ### Release note None ### Check List (For Author) - Test: Unit Test and Regression test - BE RuntimeFilterBucketPrunerTest (7 tests) - FE RuntimeFilterBucketPruneClassifierTest (11 tests) - Regression test query_p0/runtime_filter/rf_bucket_pruning, generated and verified against $run_path/doris - Release build: ./build.sh --be --fe -j 48 - BE clang-format, format check, and clang-tidy on modified C++ files - Behavior changed: No - Does this need documentation: No --- .../runtime_filter_bucket_pruner.cpp | 50 ++---------- .../runtime_filter_consumer.cpp | 2 +- .../runtime_filter/runtime_filter_wrapper.cpp | 46 +++++++++++ .../runtime_filter/runtime_filter_wrapper.h | 12 +++ be/src/exprs/runtime_filter_expr.cpp | 15 +++- be/src/exprs/runtime_filter_expr.h | 8 +- .../runtime_filter_bucket_pruner_test.cpp | 78 ++++++++++++++----- .../RuntimeFilterBucketPruneClassifier.java | 3 + ...untimeFilterBucketPruneClassifierTest.java | 18 +++++ .../runtime_filter/rf_bucket_pruning.out | 3 + .../runtime_filter/rf_bucket_pruning.groovy | 38 +++++++-- 11 files changed, 201 insertions(+), 72 deletions(-) diff --git a/be/src/exec/runtime_filter/runtime_filter_bucket_pruner.cpp b/be/src/exec/runtime_filter/runtime_filter_bucket_pruner.cpp index 4ac1140f35251f..99d00ce47c56d8 100644 --- a/be/src/exec/runtime_filter/runtime_filter_bucket_pruner.cpp +++ b/be/src/exec/runtime_filter/runtime_filter_bucket_pruner.cpp @@ -23,10 +23,6 @@ #include #include -#include "core/column/column.h" -#include "core/data_type/data_type.h" -#include "core/data_type/primitive_type.h" -#include "core/string_ref.h" #include "exprs/hybrid_set.h" #include "exprs/runtime_filter_expr.h" #include "exprs/vexpr.h" @@ -35,40 +31,6 @@ namespace doris { -static void materialize_hashes(const VExprSPtr& target_expr, HybridSetBase* hybrid_set, - std::vector* hashes) { - DORIS_CHECK(target_expr != nullptr); - DORIS_CHECK(hybrid_set != nullptr); - - const DataTypePtr& data_type = target_expr->data_type(); - MutableColumnPtr column = data_type->create_column(); - PrimitiveType primitive_type = data_type->get_primitive_type(); - auto* iter = hybrid_set->begin(); - while (iter->has_next()) { - const void* value = iter->get_value(); - DORIS_CHECK(value != nullptr); - if (is_string_type(primitive_type)) { - const auto* string_value = reinterpret_cast(value); - column->insert_data(string_value->data, string_value->size); - } else { - // ColumnVector::insert_data ignores length for fixed-length values. - column->insert_data(reinterpret_cast(value), 0); - } - iter->next(); - } - if (hybrid_set->contain_null() && data_type->is_nullable()) { - // contain_null() is true only for a null-aware filter. Keep the bucket that owns - // NULL probe rows by hashing NULL with the same nullable CRC semantics as partitioning. - column->insert_default(); - } - - hashes->assign(column->size(), 0); - if (!hashes->empty()) { - column->update_crcs_with_value(hashes->data(), primitive_type, - static_cast(column->size())); - } -} - Status RuntimeFilterBucketPruner::prune_by_runtime_filters( const std::vector& ranges, const VExprContextSPtrs& conjuncts, const std::vector& rf_descs, @@ -95,8 +57,8 @@ Status RuntimeFilterBucketPruner::prune_by_runtime_filters( if (!root->is_rf_wrapper()) { continue; } - auto* wrapper = assert_cast(root.get()); - if (!eligible_filter_ids.contains(wrapper->filter_id())) { + auto* rf_expr = assert_cast(root.get()); + if (!eligible_filter_ids.contains(rf_expr->filter_id())) { continue; } @@ -116,8 +78,8 @@ Status RuntimeFilterBucketPruner::prune_by_runtime_filters( VExprSPtr target_expr = impl->children()[0]; DORIS_CHECK_EQ(target_expr->node_type(), TExprNodeType::SLOT_REF); - std::vector hashes; - materialize_hashes(target_expr, hybrid_set.get(), &hashes); + std::shared_ptr> hashes = + rf_expr->get_bucket_prune_hashes(target_expr->data_type()); phmap::flat_hash_map> selected_buckets_by_num; for (const auto& range : ranges) { if (newly_pruned.contains(range.tablet_id)) { @@ -131,8 +93,8 @@ Status RuntimeFilterBucketPruner::prune_by_runtime_filters( if (inserted) { auto& selected_buckets = selected_it->second; selected_buckets.reserve( - std::min(hashes.size(), static_cast(range.bucket_num))); - for (uint32_t hash : hashes) { + std::min(hashes->size(), static_cast(range.bucket_num))); + for (uint32_t hash : *hashes) { selected_buckets.insert( static_cast(hash % static_cast(range.bucket_num))); } diff --git a/be/src/exec/runtime_filter/runtime_filter_consumer.cpp b/be/src/exec/runtime_filter/runtime_filter_consumer.cpp index 84fcc621ace697..70b61372c52612 100644 --- a/be/src/exec/runtime_filter/runtime_filter_consumer.cpp +++ b/be/src/exec/runtime_filter/runtime_filter_consumer.cpp @@ -104,7 +104,7 @@ Status RuntimeFilterConsumer::_get_push_exprs(std::vector& in_pred->add_child(probe_ctx->root()); auto wrapper = RuntimeFilterExpr::create_shared( node, in_pred, get_in_list_ignore_thredhold(_wrapper->hybrid_set()->size()), - null_aware, _wrapper->filter_id(), sampling_frequency); + null_aware, _wrapper->filter_id(), sampling_frequency, _wrapper); container.push_back(wrapper); break; } diff --git a/be/src/exec/runtime_filter/runtime_filter_wrapper.cpp b/be/src/exec/runtime_filter/runtime_filter_wrapper.cpp index 109fa6f0171878..bf5ba0fc7a029c 100644 --- a/be/src/exec/runtime_filter/runtime_filter_wrapper.cpp +++ b/be/src/exec/runtime_filter/runtime_filter_wrapper.cpp @@ -18,6 +18,7 @@ #include "exec/runtime_filter/runtime_filter_wrapper.h" #include "core/data_type/define_primitive_type.h" +#include "core/string_ref.h" #include "exec/runtime_filter/runtime_filter_definitions.h" #include "exprs/create_predicate_function.h" #include "exprs/function/cast/cast_to_date_or_datetime_impl.hpp" @@ -615,6 +616,51 @@ bool RuntimeFilterWrapper::contain_null() const { return false; } +std::shared_ptr> +RuntimeFilterWrapper::get_or_compute_bucket_prune_hashes(const DataTypePtr& target_type) const { + DORIS_CHECK(_state.load() == State::READY); + DORIS_CHECK(_hybrid_set != nullptr); + DORIS_CHECK(target_type != nullptr); + PrimitiveType primitive_type = target_type->get_primitive_type(); + DORIS_CHECK_EQ(primitive_type, _column_return_type); + + bool is_nullable = target_type->is_nullable(); + auto& once = is_nullable ? _nullable_bucket_prune_hashes_once + : _non_nullable_bucket_prune_hashes_once; + auto& cached_hashes = + is_nullable ? _nullable_bucket_prune_hashes : _non_nullable_bucket_prune_hashes; + std::call_once(once, [&] { + MutableColumnPtr column = target_type->create_column(); + auto* iter = _hybrid_set->begin(); + while (iter->has_next()) { + const void* value = iter->get_value(); + DORIS_CHECK(value != nullptr); + if (is_string_type(primitive_type)) { + const auto* string_value = reinterpret_cast(value); + column->insert_data(string_value->data, string_value->size); + } else { + // ColumnVector::insert_data ignores length for fixed-length values. + column->insert_data(reinterpret_cast(value), 0); + } + iter->next(); + } + if (_hybrid_set->contain_null() && is_nullable) { + // A null-aware filter can match NULL probe rows, so retain the bucket selected by + // the same nullable CRC semantics used during tablet routing. + column->insert_default(); + } + + auto hashes = std::make_shared>(column->size(), 0); + if (!hashes->empty()) { + column->update_crcs_with_value(hashes->data(), primitive_type, + static_cast(column->size())); + } + cached_hashes = std::move(hashes); + }); + DORIS_CHECK(cached_hashes != nullptr); + return cached_hashes; +} + std::string RuntimeFilterWrapper::debug_string() const { auto type_string = _filter_type == RuntimeFilterType::IN_OR_BLOOM_FILTER ? fmt::format("{}({})", filter_type_to_string(_filter_type), diff --git a/be/src/exec/runtime_filter/runtime_filter_wrapper.h b/be/src/exec/runtime_filter/runtime_filter_wrapper.h index 3d22afe5ff7556..acb65c09e6c66e 100644 --- a/be/src/exec/runtime_filter/runtime_filter_wrapper.h +++ b/be/src/exec/runtime_filter/runtime_filter_wrapper.h @@ -19,8 +19,12 @@ #include +#include +#include + #include "common/status.h" #include "core/column/column.h" +#include "core/data_type/data_type.h" #include "exec/runtime_filter/runtime_filter_definitions.h" #include "exec/runtime_filter/utils.h" #include "exprs/vexpr_fwd.h" @@ -82,6 +86,9 @@ class RuntimeFilterWrapper { bool contain_null() const; + std::shared_ptr> get_or_compute_bucket_prune_hashes( + const DataTypePtr& target_type) const; + bool disable_always_true_logic() const { return _disable_always_true_logic; } std::string debug_string() const; @@ -157,5 +164,10 @@ class RuntimeFilterWrapper { // on state is thread-safe. std::atomic _state; AtomicStatus _reason; + + mutable std::once_flag _non_nullable_bucket_prune_hashes_once; + mutable std::once_flag _nullable_bucket_prune_hashes_once; + mutable std::shared_ptr> _non_nullable_bucket_prune_hashes; + mutable std::shared_ptr> _nullable_bucket_prune_hashes; }; } // namespace doris diff --git a/be/src/exprs/runtime_filter_expr.cpp b/be/src/exprs/runtime_filter_expr.cpp index 1e729ed5e45f88..e491b4329f3d76 100644 --- a/be/src/exprs/runtime_filter_expr.cpp +++ b/be/src/exprs/runtime_filter_expr.cpp @@ -30,6 +30,7 @@ #include "core/data_type/data_type.h" #include "core/types.h" #include "exec/common/util.hpp" +#include "exec/runtime_filter/runtime_filter_wrapper.h" #include "exprs/vslot_ref.h" #include "runtime/runtime_profile.h" #include "storage/index/zone_map/zonemap_eval_context.h" @@ -58,13 +59,15 @@ namespace doris { class VExprContext; RuntimeFilterExpr::RuntimeFilterExpr(const TExprNode& node, VExprSPtr impl, double ignore_thredhold, - bool null_aware, int filter_id, int sampling_frequency) + bool null_aware, int filter_id, int sampling_frequency, + std::shared_ptr runtime_filter_wrapper) : VExpr(node), _impl(std::move(impl)), _ignore_thredhold(ignore_thredhold), _null_aware(null_aware), _filter_id(filter_id), - _sampling_frequency(sampling_frequency) { + _sampling_frequency(sampling_frequency), + _runtime_filter_wrapper(std::move(runtime_filter_wrapper)) { DORIS_CHECK(_impl != nullptr); } @@ -75,13 +78,19 @@ Status RuntimeFilterExpr::clone_node(VExprSPtr* cloned_expr) const { RETURN_IF_ERROR(_impl->deep_clone(&cloned_impl)); auto cloned_runtime_filter = RuntimeFilterExpr::create_shared( clone_texpr_node(), std::move(cloned_impl), _ignore_thredhold, _null_aware, _filter_id, - _sampling_frequency); + _sampling_frequency, _runtime_filter_wrapper); cloned_runtime_filter->attach_profile_counter(_rf_input_rows, _rf_filter_rows, _always_true_filter_rows); *cloned_expr = std::move(cloned_runtime_filter); return Status::OK(); } +std::shared_ptr> RuntimeFilterExpr::get_bucket_prune_hashes( + const DataTypePtr& target_type) const { + DORIS_CHECK(_runtime_filter_wrapper != nullptr); + return _runtime_filter_wrapper->get_or_compute_bucket_prune_hashes(target_type); +} + Status RuntimeFilterExpr::prepare(RuntimeState* state, const RowDescriptor& desc, VExprContext* context) { RETURN_IF_ERROR_OR_PREPARED(_impl->prepare(state, desc, context)); diff --git a/be/src/exprs/runtime_filter_expr.h b/be/src/exprs/runtime_filter_expr.h index 6deb123668932e..32004500f9ca1f 100644 --- a/be/src/exprs/runtime_filter_expr.h +++ b/be/src/exprs/runtime_filter_expr.h @@ -46,6 +46,7 @@ double get_bloom_filter_ignore_thredhold(); namespace doris { class Block; +class RuntimeFilterWrapper; class VExprContext; class RuntimeFilterExpr final : public VExpr { @@ -54,7 +55,8 @@ class RuntimeFilterExpr final : public VExpr { public: RuntimeFilterExpr(const TExprNode& node, VExprSPtr impl, double ignore_thredhold, bool null_aware, int filter_id, - int sampling_frequency = RuntimeFilterSelectivity::DISABLE_SAMPLING); + int sampling_frequency = RuntimeFilterSelectivity::DISABLE_SAMPLING, + std::shared_ptr runtime_filter_wrapper = nullptr); ~RuntimeFilterExpr() override = default; Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector, size_t count, ColumnPtr& result_column) const override; @@ -124,6 +126,9 @@ class RuntimeFilterExpr final : public VExpr { int filter_id() const { return _filter_id; } + std::shared_ptr> get_bucket_prune_hashes( + const DataTypePtr& target_type) const; + std::shared_ptr predicate_filtered_rows_counter() const { return _rf_filter_rows; } @@ -152,6 +157,7 @@ class RuntimeFilterExpr final : public VExpr { bool _null_aware; int _filter_id; int _sampling_frequency; + std::shared_ptr _runtime_filter_wrapper; }; using RuntimeFilterExprPtr = std::shared_ptr; diff --git a/be/test/exec/runtime_filter/runtime_filter_bucket_pruner_test.cpp b/be/test/exec/runtime_filter/runtime_filter_bucket_pruner_test.cpp index a4cc925e381b10..0240c5faf1953b 100644 --- a/be/test/exec/runtime_filter/runtime_filter_bucket_pruner_test.cpp +++ b/be/test/exec/runtime_filter/runtime_filter_bucket_pruner_test.cpp @@ -29,11 +29,14 @@ #include "core/data_type/data_type_nullable.h" #include "core/data_type/data_type_number.h" #include "exec/runtime_filter/runtime_filter_definitions.h" +#include "exec/runtime_filter/runtime_filter_wrapper.h" #include "exprs/create_predicate_function.h" #include "exprs/runtime_filter_expr.h" #include "exprs/vdirect_in_predicate.h" #include "exprs/vexpr_context.h" #include "exprs/vslot_ref.h" +#include "util/hash_util.hpp" +#include "util/raw_value.h" namespace doris { @@ -41,10 +44,30 @@ class RuntimeFilterBucketPrunerTest : public testing::Test { protected: static constexpr int SCAN_NODE_ID = 10; - VExprContextSPtr make_in_conjunct(int filter_id, const std::vector& values) { - std::shared_ptr set(create_set(TYPE_INT, false)); + std::shared_ptr make_in_wrapper(int filter_id, + const std::vector& values, + bool null_aware = false) { + RuntimeFilterParams params {.filter_id = filter_id, + .filter_type = RuntimeFilterType::IN_FILTER, + .column_return_type = TYPE_INT, + .null_aware = null_aware, + .max_in_num = 1024}; + auto wrapper = std::make_shared(¶ms); for (const int32_t value : values) { - set->insert(&value); + wrapper->hybrid_set()->insert(&value); + } + if (null_aware) { + wrapper->hybrid_set()->insert(static_cast(nullptr)); + } + wrapper->set_state(RuntimeFilterWrapper::State::READY); + return wrapper; + } + + VExprContextSPtr make_in_conjunct( + int filter_id, const std::vector& values, + std::shared_ptr runtime_filter_wrapper = nullptr) { + if (runtime_filter_wrapper == nullptr) { + runtime_filter_wrapper = make_in_wrapper(filter_id, values); } TExprNode node; @@ -53,11 +76,14 @@ class RuntimeFilterBucketPrunerTest : public testing::Test { node.in_predicate.__set_is_not_in(false); node.__set_opcode(TExprOpcode::FILTER_IN); node.__set_is_nullable(false); - auto impl = VDirectInPredicate::create_shared(node, std::move(set), true); + auto impl = + VDirectInPredicate::create_shared(node, runtime_filter_wrapper->hybrid_set(), true); impl->add_child(VSlotRef::create_shared(/*slot_id=*/1, /*column_id=*/0, /*column_uniq_id=*/1, std::make_shared(), "dist_col")); - auto wrapper = RuntimeFilterExpr::create_shared(node, impl, 0, false, filter_id); + auto wrapper = RuntimeFilterExpr::create_shared(node, impl, 0, false, filter_id, + RuntimeFilterSelectivity::DISABLE_SAMPLING, + std::move(runtime_filter_wrapper)); return std::make_shared(wrapper); } @@ -76,8 +102,7 @@ class RuntimeFilterBucketPrunerTest : public testing::Test { } VExprContextSPtr make_null_aware_in_conjunct(int filter_id) { - std::shared_ptr set(create_set(TYPE_INT, true)); - set->insert(static_cast(nullptr)); + auto runtime_filter_wrapper = make_in_wrapper(filter_id, {}, true); TExprNode node; node.__set_type(create_type_desc(TYPE_BOOLEAN)); @@ -85,11 +110,14 @@ class RuntimeFilterBucketPrunerTest : public testing::Test { node.in_predicate.__set_is_not_in(false); node.__set_opcode(TExprOpcode::FILTER_IN); node.__set_is_nullable(false); - auto impl = VDirectInPredicate::create_shared(node, std::move(set), true); + auto impl = + VDirectInPredicate::create_shared(node, runtime_filter_wrapper->hybrid_set(), true); impl->add_child(VSlotRef::create_shared( /*slot_id=*/1, /*column_id=*/0, /*column_uniq_id=*/1, std::make_shared(std::make_shared()), "dist_col")); - auto wrapper = RuntimeFilterExpr::create_shared(node, impl, 0, false, filter_id); + auto wrapper = RuntimeFilterExpr::create_shared(node, impl, 0, false, filter_id, + RuntimeFilterSelectivity::DISABLE_SAMPLING, + std::move(runtime_filter_wrapper)); return std::make_shared(wrapper); } @@ -109,23 +137,37 @@ class RuntimeFilterBucketPrunerTest : public testing::Test { } int32_t bucket_for_value(int32_t value, int32_t bucket_num) { - auto column = ColumnInt32::create(); - column->insert_value(value); - uint32_t hash = 0; - column->update_crcs_with_value(&hash, TYPE_INT, 1, 0, nullptr); + uint32_t hash = RawValue::zlib_crc32(&value, sizeof(value), TYPE_INT, 0); return static_cast(hash % static_cast(bucket_num)); } int32_t bucket_for_null(int32_t bucket_num) { - auto column = std::make_shared(std::make_shared()) - ->create_column(); - column->insert_default(); - uint32_t hash = 0; - column->update_crcs_with_value(&hash, TYPE_INT, 1, 0, nullptr); + uint32_t hash = HashUtil::zlib_crc_hash_null(0); return static_cast(hash % static_cast(bucket_num)); } }; +TEST_F(RuntimeFilterBucketPrunerTest, ExactSetHashesSharedAcrossConsumers) { + constexpr int filter_id = 13; + auto runtime_filter_wrapper = make_in_wrapper(filter_id, {1, 2, 3}, true); + auto first = make_in_conjunct(filter_id, {}, runtime_filter_wrapper); + auto second = make_in_conjunct(filter_id, {}, runtime_filter_wrapper); + auto target_type = first->root()->get_impl()->children()[0]->data_type(); + + auto first_hashes = assert_cast(first->root().get()) + ->get_bucket_prune_hashes(target_type); + auto second_hashes = assert_cast(second->root().get()) + ->get_bucket_prune_hashes(target_type); + auto nullable_hashes = assert_cast(first->root().get()) + ->get_bucket_prune_hashes(std::make_shared( + std::make_shared())); + + EXPECT_EQ(first_hashes.get(), second_hashes.get()); + EXPECT_EQ(first_hashes->size(), 3); + EXPECT_NE(first_hashes.get(), nullable_hashes.get()); + EXPECT_EQ(nullable_hashes->size(), 4); +} + TEST_F(RuntimeFilterBucketPrunerTest, ExactInKeepsOnlyMatchingBucket) { constexpr int filter_id = 7; constexpr int32_t value = 10; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/RuntimeFilterBucketPruneClassifier.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/RuntimeFilterBucketPruneClassifier.java index 8317bdaaccb27f..d9e4c2aadc4846 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/RuntimeFilterBucketPruneClassifier.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/RuntimeFilterBucketPruneClassifier.java @@ -50,6 +50,9 @@ static Classification classify(TRuntimeFilterType filterType, Expr targetExpr, P } OlapScanNode olapScanNode = (OlapScanNode) scanNode; + if (olapScanNode.isPointQuery()) { + return Classification.unsupported("target scan is a point query"); + } OlapTable table = olapScanNode.getOlapTable(); if (table == null || olapScanNode.getSelectedPartitionIds().isEmpty()) { return Classification.unsupported("target scan has no selected partitions"); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/RuntimeFilterBucketPruneClassifierTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/RuntimeFilterBucketPruneClassifierTest.java index d6cff3b1a15d87..c2a160717dbb8a 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/RuntimeFilterBucketPruneClassifierTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/RuntimeFilterBucketPruneClassifierTest.java @@ -56,6 +56,17 @@ void testInOrBloomSupportedAtPlanTime() { Assertions.assertTrue(classification.canPruneBuckets()); } + @Test + void testPointQueryRejected() { + Column distributionColumn = new Column("dist_col", PrimitiveType.INT); + RuntimeFilterBucketPruneClassifier.Classification classification = classify( + TRuntimeFilterType.IN, distributionColumn, + new HashDistributionInfo(8, ImmutableList.of(distributionColumn)), true); + + Assertions.assertFalse(classification.canPruneBuckets()); + Assertions.assertTrue(classification.getUnsupportedReason().contains("point query")); + } + @Test void testBloomRejected() { Column distributionColumn = new Column("dist_col", PrimitiveType.INT); @@ -173,6 +184,12 @@ void testRandomDistributionRejected() { private RuntimeFilterBucketPruneClassifier.Classification classify( TRuntimeFilterType filterType, Column targetColumn, org.apache.doris.catalog.DistributionInfo distributionInfo) { + return classify(filterType, targetColumn, distributionInfo, false); + } + + private RuntimeFilterBucketPruneClassifier.Classification classify( + TRuntimeFilterType filterType, Column targetColumn, + org.apache.doris.catalog.DistributionInfo distributionInfo, boolean isPointQuery) { SlotDescriptor slotDescriptor = new SlotDescriptor(new SlotId(1), new TupleId(1)); slotDescriptor.setColumn(targetColumn); slotDescriptor.setType(targetColumn.getType()); @@ -181,6 +198,7 @@ private RuntimeFilterBucketPruneClassifier.Classification classify( OlapTable table = Mockito.mock(OlapTable.class); Partition partition = Mockito.mock(Partition.class); OlapScanNode scanNode = Mockito.mock(OlapScanNode.class); + Mockito.when(scanNode.isPointQuery()).thenReturn(isPointQuery); Mockito.when(scanNode.getOlapTable()).thenReturn(table); Mockito.when(scanNode.getSelectedPartitionIds()).thenReturn(ImmutableList.of(1L)); Mockito.when(table.getPartition(1L)).thenReturn(partition); diff --git a/regression-test/data/query_p0/runtime_filter/rf_bucket_pruning.out b/regression-test/data/query_p0/runtime_filter/rf_bucket_pruning.out index 4dadd5b6a170a8..18b5fae2233162 100644 --- a/regression-test/data/query_p0/runtime_filter/rf_bucket_pruning.out +++ b/regression-test/data/query_p0/runtime_filter/rf_bucket_pruning.out @@ -2,3 +2,6 @@ -- !bucket_result -- 3 30 +-- !nullable_bucket_result -- +\N 90 + diff --git a/regression-test/suites/query_p0/runtime_filter/rf_bucket_pruning.groovy b/regression-test/suites/query_p0/runtime_filter/rf_bucket_pruning.groovy index 7717e3d4c5e7e0..edb18b7480ac8c 100644 --- a/regression-test/suites/query_p0/runtime_filter/rf_bucket_pruning.groovy +++ b/regression-test/suites/query_p0/runtime_filter/rf_bucket_pruning.groovy @@ -54,6 +54,23 @@ suite("rf_bucket_pruning", "nonConcurrent") { DISTRIBUTED BY HASH(k) BUCKETS 1 PROPERTIES("replication_num" = "1") """ + sql "drop table if exists rf_bucket_prune_nullable" + sql """ + CREATE TABLE rf_bucket_prune_nullable ( + k INT NULL, + v INT NOT NULL + ) + DISTRIBUTED BY HASH(k) BUCKETS 8 + PROPERTIES("replication_num" = "1") + """ + sql "drop table if exists rf_bucket_prune_dim_nullable" + sql """ + CREATE TABLE rf_bucket_prune_dim_nullable ( + k INT NULL + ) + DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES("replication_num" = "1") + """ sql """ INSERT INTO rf_bucket_prune_fact VALUES @@ -66,6 +83,8 @@ suite("rf_bucket_pruning", "nonConcurrent") { (5, 50), (6, 60), (7, 70), (8, 80) """ sql "INSERT INTO rf_bucket_prune_dim VALUES (3)" + sql "INSERT INTO rf_bucket_prune_nullable VALUES (NULL, 90), (3, 30)" + sql "INSERT INTO rf_bucket_prune_dim_nullable VALUES (NULL)" order_qt_bucket_result """ SELECT f.k, f.v @@ -74,6 +93,13 @@ suite("rf_bucket_pruning", "nonConcurrent") { ORDER BY f.k, f.v """ + order_qt_nullable_bucket_result """ + SELECT f.k, f.v + FROM rf_bucket_prune_nullable f + JOIN [broadcast] rf_bucket_prune_dim_nullable d ON f.k <=> d.k + ORDER BY f.k, f.v + """ + def profileAction = new ProfileAction(context) def getProfileByToken = { String token -> String profileContent = "" @@ -101,12 +127,12 @@ suite("rf_bucket_pruning", "nonConcurrent") { .collect { it[1].toLong() } return values.isEmpty() ? 0L : values.sum() } - def runProfileQuery = { String tableName -> + def runProfileQuery = { String tableName, String dimensionName, String joinOperator -> def token = UUID.randomUUID().toString() sql """ SELECT "${token}", COUNT(*) FROM ${tableName} f - JOIN [broadcast] rf_bucket_prune_dim d ON f.k = d.k + JOIN [broadcast] ${dimensionName} d ON f.k ${joinOperator} d.k """ def profile = getProfileByToken(token) assertTrue(profile != "", "Profile not found for ${token}") @@ -115,12 +141,14 @@ suite("rf_bucket_pruning", "nonConcurrent") { return extractPrunedBuckets(profile) } - assertTrue(runProfileQuery("rf_bucket_prune_fact") > 0, + assertTrue(runProfileQuery("rf_bucket_prune_fact", "rf_bucket_prune_dim", "=") > 0, "single-column HASH distribution should be pruned") - assertTrue(runProfileQuery("rf_bucket_prune_composite") == 0, + assertTrue(runProfileQuery("rf_bucket_prune_composite", "rf_bucket_prune_dim", "=") == 0, "multi-column HASH distribution must not be pruned") + assertTrue(runProfileQuery("rf_bucket_prune_nullable", "rf_bucket_prune_dim_nullable", "<=>") > 0, + "null-aware IN filter should prune buckets while retaining the NULL bucket") sql "set enable_runtime_filter_bucket_prune=false" - assertTrue(runProfileQuery("rf_bucket_prune_fact") == 0, + assertTrue(runProfileQuery("rf_bucket_prune_fact", "rf_bucket_prune_dim", "=") == 0, "disabled runtime-filter bucket pruning must not prune buckets") } From 171e951c893a733fa9a57d400ed339ceb31e7da8 Mon Sep 17 00:00:00 2001 From: happenlee Date: Wed, 12 Aug 2026 20:57:22 +0800 Subject: [PATCH 07/20] [fix](runtime filter) Share one bucket-prune hash cache ### What problem does this PR solve? Issue Number: None Related PR: #65837 Problem Summary: The finalized runtime-filter wrapper cached separate nullable and non-nullable hash vectors even though normal-value hashes are identical. Store one immutable vector containing all normal-value hashes and append the write-path NULL hash whenever the exact set contains NULL. Nullable targets remain precise; non-nullable targets may conservatively retain one extra bucket but never prune matching data. ### Release note None ### Check List (For Author) - Test: Unit Test - BE RuntimeFilterBucketPrunerTest (8 tests) - Release build: ./build.sh --be -j 48 - BE clang-format, format check, and clang-tidy on modified C++ files - Behavior changed: No - Does this need documentation: No --- .../runtime_filter/runtime_filter_wrapper.cpp | 24 +++++++-------- .../runtime_filter/runtime_filter_wrapper.h | 8 ++--- .../runtime_filter_bucket_pruner_test.cpp | 30 +++++++++++++++++-- 3 files changed, 41 insertions(+), 21 deletions(-) diff --git a/be/src/exec/runtime_filter/runtime_filter_wrapper.cpp b/be/src/exec/runtime_filter/runtime_filter_wrapper.cpp index bf5ba0fc7a029c..da018b5a2f7c78 100644 --- a/be/src/exec/runtime_filter/runtime_filter_wrapper.cpp +++ b/be/src/exec/runtime_filter/runtime_filter_wrapper.cpp @@ -22,6 +22,7 @@ #include "exec/runtime_filter/runtime_filter_definitions.h" #include "exprs/create_predicate_function.h" #include "exprs/function/cast/cast_to_date_or_datetime_impl.hpp" +#include "util/hash_util.hpp" namespace doris { RuntimeFilterWrapper::RuntimeFilterWrapper(const RuntimeFilterParams* params) @@ -624,12 +625,7 @@ RuntimeFilterWrapper::get_or_compute_bucket_prune_hashes(const DataTypePtr& targ PrimitiveType primitive_type = target_type->get_primitive_type(); DORIS_CHECK_EQ(primitive_type, _column_return_type); - bool is_nullable = target_type->is_nullable(); - auto& once = is_nullable ? _nullable_bucket_prune_hashes_once - : _non_nullable_bucket_prune_hashes_once; - auto& cached_hashes = - is_nullable ? _nullable_bucket_prune_hashes : _non_nullable_bucket_prune_hashes; - std::call_once(once, [&] { + std::call_once(_bucket_prune_hashes_once, [&] { MutableColumnPtr column = target_type->create_column(); auto* iter = _hybrid_set->begin(); while (iter->has_next()) { @@ -644,21 +640,21 @@ RuntimeFilterWrapper::get_or_compute_bucket_prune_hashes(const DataTypePtr& targ } iter->next(); } - if (_hybrid_set->contain_null() && is_nullable) { - // A null-aware filter can match NULL probe rows, so retain the bucket selected by - // the same nullable CRC semantics used during tablet routing. - column->insert_default(); - } auto hashes = std::make_shared>(column->size(), 0); if (!hashes->empty()) { column->update_crcs_with_value(hashes->data(), primitive_type, static_cast(column->size())); } - cached_hashes = std::move(hashes); + if (_hybrid_set->contain_null()) { + // Keep one shared vector for nullable and non-nullable targets. A non-nullable + // target may retain this extra bucket, but can never lose matching rows. + hashes->push_back(HashUtil::zlib_crc_hash_null(0)); + } + _bucket_prune_hashes = std::move(hashes); }); - DORIS_CHECK(cached_hashes != nullptr); - return cached_hashes; + DORIS_CHECK(_bucket_prune_hashes != nullptr); + return _bucket_prune_hashes; } std::string RuntimeFilterWrapper::debug_string() const { diff --git a/be/src/exec/runtime_filter/runtime_filter_wrapper.h b/be/src/exec/runtime_filter/runtime_filter_wrapper.h index acb65c09e6c66e..fce58a4d2d9b30 100644 --- a/be/src/exec/runtime_filter/runtime_filter_wrapper.h +++ b/be/src/exec/runtime_filter/runtime_filter_wrapper.h @@ -86,6 +86,8 @@ class RuntimeFilterWrapper { bool contain_null() const; + // The shared vector includes the NULL hash whenever the exact set contains NULL, regardless + // of target nullability. A non-nullable target may therefore retain one conservative bucket. std::shared_ptr> get_or_compute_bucket_prune_hashes( const DataTypePtr& target_type) const; @@ -165,9 +167,7 @@ class RuntimeFilterWrapper { std::atomic _state; AtomicStatus _reason; - mutable std::once_flag _non_nullable_bucket_prune_hashes_once; - mutable std::once_flag _nullable_bucket_prune_hashes_once; - mutable std::shared_ptr> _non_nullable_bucket_prune_hashes; - mutable std::shared_ptr> _nullable_bucket_prune_hashes; + mutable std::once_flag _bucket_prune_hashes_once; + mutable std::shared_ptr> _bucket_prune_hashes; }; } // namespace doris diff --git a/be/test/exec/runtime_filter/runtime_filter_bucket_pruner_test.cpp b/be/test/exec/runtime_filter/runtime_filter_bucket_pruner_test.cpp index 0240c5faf1953b..4fd648c7f55cbe 100644 --- a/be/test/exec/runtime_filter/runtime_filter_bucket_pruner_test.cpp +++ b/be/test/exec/runtime_filter/runtime_filter_bucket_pruner_test.cpp @@ -163,9 +163,9 @@ TEST_F(RuntimeFilterBucketPrunerTest, ExactSetHashesSharedAcrossConsumers) { std::make_shared())); EXPECT_EQ(first_hashes.get(), second_hashes.get()); - EXPECT_EQ(first_hashes->size(), 3); - EXPECT_NE(first_hashes.get(), nullable_hashes.get()); - EXPECT_EQ(nullable_hashes->size(), 4); + EXPECT_EQ(first_hashes.get(), nullable_hashes.get()); + ASSERT_EQ(first_hashes->size(), 4); + EXPECT_EQ(first_hashes->back(), HashUtil::zlib_crc_hash_null(0)); } TEST_F(RuntimeFilterBucketPrunerTest, ExactInKeepsOnlyMatchingBucket) { @@ -193,6 +193,30 @@ TEST_F(RuntimeFilterBucketPrunerTest, ExactInKeepsOnlyMatchingBucket) { EXPECT_EQ(newly_pruned, 0); } +TEST_F(RuntimeFilterBucketPrunerTest, NonNullableTargetConservativelyKeepsNullBucket) { + constexpr int filter_id = 14; + constexpr int32_t value = 1; + auto runtime_filter_wrapper = make_in_wrapper(filter_id, {value}, true); + VExprContextSPtrs conjuncts { + make_in_conjunct(filter_id, {}, std::move(runtime_filter_wrapper))}; + std::vector rf_descs {bucket_prune_desc(filter_id)}; + + std::set selected_buckets {bucket_for_value(value, 4), bucket_for_null(4)}; + ASSERT_EQ(selected_buckets.size(), 2); + + RuntimeFilterBucketPruner pruner; + int64_t newly_pruned = 0; + ASSERT_TRUE(pruner.prune_by_runtime_filters(four_bucket_ranges(), conjuncts, rf_descs, + SCAN_NODE_ID, /*max_in_num=*/1024, &newly_pruned) + .ok()); + + EXPECT_EQ(newly_pruned, 2); + for (int32_t bucket_seq = 0; bucket_seq < 4; ++bucket_seq) { + EXPECT_EQ(pruner.is_tablet_pruned(100 + bucket_seq), + !selected_buckets.contains(bucket_seq)); + } +} + TEST_F(RuntimeFilterBucketPrunerTest, SupportsDifferentBucketCountsAcrossPartitions) { constexpr int filter_id = 11; constexpr int32_t value = 10; From d3b0eb150ff11dca4b15b5c275246f7bdd81ba4f Mon Sep 17 00:00:00 2001 From: happenlee Date: Thu, 13 Aug 2026 16:55:21 +0800 Subject: [PATCH 08/20] [fix](regression) Wait for complete bucket-pruning profile The profile list can report COMPLETE before the detailed profile has incorporated the final BE counters. Reuse ProfileAction.getProfileBySql so the suite waits for a complete detailed profile containing BucketsPrunedByRuntimeFilter. --- .../runtime_filter/rf_bucket_pruning.groovy | 20 +------------------ 1 file changed, 1 insertion(+), 19 deletions(-) diff --git a/regression-test/suites/query_p0/runtime_filter/rf_bucket_pruning.groovy b/regression-test/suites/query_p0/runtime_filter/rf_bucket_pruning.groovy index edb18b7480ac8c..9419c00dffb6c2 100644 --- a/regression-test/suites/query_p0/runtime_filter/rf_bucket_pruning.groovy +++ b/regression-test/suites/query_p0/runtime_filter/rf_bucket_pruning.groovy @@ -102,25 +102,7 @@ suite("rf_bucket_pruning", "nonConcurrent") { def profileAction = new ProfileAction(context) def getProfileByToken = { String token -> - String profileContent = "" - for (int attempt = 0; attempt < 60; attempt++) { - List profileData = profileAction.getProfileList() - for (final def profileItem in profileData) { - if (profileItem["Sql Statement"].toString().contains(token)) { - def currentProfile = profileAction.getProfile(profileItem["Profile ID"].toString()) - if (currentProfile != "") { - profileContent = currentProfile - } - if (profileItem["Profile Completion State"]?.toString() == "COMPLETE" - && profileContent.contains("BucketsPrunedByRuntimeFilter")) { - return profileContent - } - break - } - } - Thread.sleep(500) - } - return profileContent + return profileAction.getProfileBySql(token, ["BucketsPrunedByRuntimeFilter"]) } def extractPrunedBuckets = { String profile -> def values = (profile =~ /-\s*BucketsPrunedByRuntimeFilter:\s*(\d+)/) From a6a75e070d54291950548cdc228b74d4938df942 Mon Sep 17 00:00:00 2001 From: happenlee Date: Thu, 13 Aug 2026 19:53:19 +0800 Subject: [PATCH 09/20] [fix](runtime filter) Address bucket pruning review feedback ### What problem does this PR solve? Issue Number: None Related PR: #65837 Problem Summary: Disabled runtime-filter bucket pruning still classified every selected partition even though the resulting metadata was discarded. Gate bucket classification before catalog access. Add translator-level coverage for grouped target identity, cast suppression, and enabled/disabled Thrift serialization, plus a deterministic BE scheduler test that publishes an exact filter after parallel probe tasks enter prepare and verifies correct rows and bucket pruning. ### Release note None ### Check List (For Author) - Test: Unit Test - BE: ScannerLateArrivalRfTest.bucket_pruning_after_probe_tasks_start - FE: RuntimeFilterTranslatorBucketPruneTest (4 tests) - Static analysis: run-clang-tidy.sh on scanner_late_arrival_rf_test.cpp - Format: build-support/check-format.sh - Behavior changed: No (disabled queries avoid discarded catalog classification work) - Does this need documentation: No --- .../scan/scanner_late_arrival_rf_test.cpp | 198 +++++++++++++++ .../translator/RuntimeFilterTranslator.java | 24 +- ...untimeFilterTranslatorBucketPruneTest.java | 235 ++++++++++++++++++ 3 files changed, 447 insertions(+), 10 deletions(-) create mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/RuntimeFilterTranslatorBucketPruneTest.java diff --git a/be/test/exec/scan/scanner_late_arrival_rf_test.cpp b/be/test/exec/scan/scanner_late_arrival_rf_test.cpp index 8327aaffc26556..5971431d82e524 100644 --- a/be/test/exec/scan/scanner_late_arrival_rf_test.cpp +++ b/be/test/exec/scan/scanner_late_arrival_rf_test.cpp @@ -18,23 +18,31 @@ #include #include +#include +#include #include +#include #include "common/object_pool.h" #include "core/data_type/data_type_factory.hpp" #include "core/data_type/data_type_number.h" #include "exec/operator/mock_scan_operator.h" +#include "exec/operator/olap_scan_operator.h" #include "exec/runtime_filter/runtime_filter_consumer.h" #include "exec/runtime_filter/runtime_filter_consumer_helper.h" #include "exec/runtime_filter/runtime_filter_producer.h" #include "exec/runtime_filter/runtime_filter_test_utils.h" #include "exec/scan/scanner.h" +#include "exec/scan/scanner_context.h" +#include "exec/scan/scanner_scheduler.h" #include "runtime/descriptors.h" #include "runtime/exec_env.h" #include "testutil/column_helper.h" +#include "testutil/desc_tbl_builder.h" #include "testutil/mock/mock_descriptors.h" #include "testutil/mock/mock_runtime_state.h" #include "testutil/mock/mock_slot_ref.h" +#include "util/raw_value.h" namespace doris { @@ -70,6 +78,57 @@ class TestScanner final : public Scanner { std::list _blocks; }; +class LateBucketScanner final : public Scanner { +public: + LateBucketScanner(RuntimeState* state, OlapScanLocalState* local_state, int64_t tablet_id, + bool has_matching_row, RuntimeProfile* profile, std::latch* prepare_started, + std::latch* filter_published) + : Scanner(state, local_state, -1, profile), + _olap_local_state(local_state), + _tablet_id(tablet_id), + _has_matching_row(has_matching_row), + _prepare_started(prepare_started), + _filter_published(filter_published) {} + + bool is_pruned_by_runtime_filter() const override { + return _olap_local_state->_is_tablet_pruned_by_runtime_filter(1, _tablet_id); + } + + int read_calls() const { return _read_calls.load(); } + +protected: + Status _prepare_impl() override { + _prepare_started->count_down(); + _filter_published->wait(); + return Scanner::_prepare_impl(); + } + + Status _get_block_impl(RuntimeState* /*state*/, Block* block, bool* eof) override { + ++_read_calls; + if (_returned) { + *eof = true; + return Status::OK(); + } + _returned = true; + *eof = false; + if (_has_matching_row) { + *block = ColumnHelper::create_block({7}); + } else { + *block = ColumnHelper::create_block(std::vector {}); + } + return Status::OK(); + } + +private: + OlapScanLocalState* _olap_local_state; + int64_t _tablet_id; + bool _has_matching_row; + std::latch* _prepare_started; + std::latch* _filter_published; + std::atomic _read_calls {0}; + bool _returned = false; +}; + class ScannerLateArrivalRfTest : public RuntimeFilterTest { public: void SetUp() override { @@ -140,6 +199,145 @@ TEST_F(ScannerLateArrivalRfTest, applied_rf_num_advances_after_late_arrival) { ASSERT_TRUE(scanner->_conjuncts.empty()); } +TEST_F(ScannerLateArrivalRfTest, bucket_pruning_after_probe_tasks_start) { + constexpr int scan_node_id = 0; + constexpr int bucket_num = 4; + constexpr int filter_value = 7; + + auto desc = TRuntimeFilterDescBuilder().add_planId_to_target_expr(scan_node_id).build(); + desc.__set_bucket_pruning_target_ids({scan_node_id}); + + ObjectPool pool; + DescriptorTblBuilder desc_builder(&pool); + desc_builder.declare_tuple() << TupleDescBuilder::SlotType {std::make_shared(), + "dist_col"}; + DescriptorTbl* desc_tbl = desc_builder.build(); + ASSERT_NE(desc_tbl, nullptr); + + TOlapScanNode olap_scan_node; + olap_scan_node.__set_tuple_id(0); + olap_scan_node.__set_keyType(TKeysType::DUP_KEYS); + olap_scan_node.__set_key_column_name({"dist_col"}); + olap_scan_node.__set_key_column_type({TPrimitiveType::INT}); + + TPlanNode plan_node; + plan_node.__set_node_id(scan_node_id); + plan_node.__set_node_type(TPlanNodeType::OLAP_SCAN_NODE); + plan_node.__set_num_children(0); + plan_node.__set_limit(-1); + plan_node.__set_row_tuples({0}); + plan_node.__set_runtime_filters({desc}); + plan_node.__set_olap_scan_node(olap_scan_node); + + auto op = std::make_shared(&pool, plan_node, 0, *desc_tbl, bucket_num, + TQueryCacheParam {}); + auto* state = _runtime_states[0].get(); + state->set_desc_tbl(desc_tbl); + TQueryOptions query_options = + TQueryOptionsBuilder().set_runtime_filter_max_in_num(1024).build(); + query_options.__set_enable_runtime_filter_bucket_prune(true); + state->set_query_options(query_options); + + auto local_state = OlapScanLocalState::create_shared(state, op.get()); + std::vector> rf_dependencies; + ASSERT_TRUE(local_state->_helper.init(state, true, 0, 0, rf_dependencies, "").ok()); + ASSERT_TRUE( + local_state->_helper + .acquire_runtime_filter(state, local_state->_conjuncts, op->row_descriptor()) + .ok()); + ASSERT_TRUE(local_state->_conjuncts.empty()); + auto task_exec_ctx = std::make_shared(); + state->set_task_execution_context(task_exec_ctx); + for (int bucket_seq = 0; bucket_seq < bucket_num; ++bucket_seq) { + local_state->_rf_bucket_prune_ranges.push_back({100 + bucket_seq, bucket_seq, bucket_num}); + } + RuntimeProfile scan_profile("late bucket scan"); + local_state->_buckets_pruned_by_rf_counter = + ADD_COUNTER(&scan_profile, "BucketsPrunedByRuntimeFilter", TUnit::UNIT); + local_state->_scan_timer = ADD_TIMER(&scan_profile, "ScannerGetBlockTime"); + local_state->_scan_cpu_timer = ADD_TIMER(&scan_profile, "ScannerCpuTime"); + local_state->_filter_timer = ADD_TIMER(&scan_profile, "ScannerFilterTime"); + local_state->_rows_read_counter = ADD_COUNTER(&scan_profile, "RowsRead", TUnit::UNIT); + + uint32_t hash = RawValue::zlib_crc32(&filter_value, sizeof(filter_value), TYPE_INT, 0); + int selected_bucket = static_cast(hash % bucket_num); + std::latch prepare_started(bucket_num); + std::latch filter_published(1); + std::list> scanner_delegates; + std::vector> scanners; + for (int bucket_seq = 0; bucket_seq < bucket_num; ++bucket_seq) { + auto scanner = std::make_shared( + state, local_state.get(), 100 + bucket_seq, bucket_seq == selected_bucket, + &scan_profile, &prepare_started, &filter_published); + ASSERT_TRUE(scanner->init(state, {}).ok()); + scanners.push_back(scanner); + ScannerSPtr scanner_base = scanner; + scanner_delegates.push_back(std::make_shared(scanner_base)); + } + + auto dependency = Dependency::create_shared(0, 0, "late bucket scan dependency"); + std::atomic shared_limit {-1}; + auto scanner_context = ScannerContext::create_shared( + state, local_state.get(), desc_tbl->get_tuple_descriptor(0), nullptr, scanner_delegates, + -1, dependency, &shared_limit, nullptr, nullptr, 0, false, bucket_num); + scanner_context->_newly_create_free_blocks_num = + ADD_COUNTER(&scan_profile, "NewlyCreatedFreeBlocks", TUnit::UNIT); + scanner_context->_scanner_memory_used_counter = + ADD_COUNTER(&scan_profile, "ScannerMemoryUsed", TUnit::BYTES); + scanner_context->_max_bytes_in_queue = 10 * 1024 * 1024; + std::vector> tasks; + for (const auto& scanner_delegate : scanner_delegates) { + auto task = std::make_shared(scanner_delegate); + task->set_state(ScanTask::State::IN_FLIGHT); + tasks.push_back(std::move(task)); + } + scanner_context->_in_flight_tasks_num = bucket_num; + + std::vector probe_threads; + for (const auto& task : tasks) { + probe_threads.emplace_back([scanner_context, task] { + ScannerScheduler::_scanner_scan(scanner_context, task); + }); + } + + // Every task has passed the scheduler's pre-prepare pruning check while the RF is not ready. + prepare_started.wait(); + ASSERT_EQ(local_state->_rf_bucket_pruner.pruned_tablet_count(), 0); + + std::shared_ptr producer; + ASSERT_TRUE(RuntimeFilterProducer::create(_query_ctx.get(), &desc, &producer).ok()); + ASSERT_TRUE(producer->init(1).ok()); + auto filter_column = ColumnInt32::create(); + filter_column->insert_value(filter_value); + ASSERT_TRUE(producer->insert(std::move(filter_column), 0).ok()); + producer->set_wrapper_state_and_ready_to_publish(RuntimeFilterWrapper::State::READY); + local_state->_helper._consumers[0]->signal(producer.get()); + filter_published.count_down(); + + for (auto& thread : probe_threads) { + thread.join(); + } + + for (int bucket_seq = 0; bucket_seq < bucket_num; ++bucket_seq) { + if (bucket_seq == selected_bucket) { + EXPECT_GT(scanners[bucket_seq]->read_calls(), 0); + EXPECT_NE(tasks[bucket_seq]->cached_block, nullptr); + } else { + EXPECT_EQ(scanners[bucket_seq]->read_calls(), 0); + } + } + int64_t result_rows = 0; + for (const auto& task : tasks) { + if (task->cached_block != nullptr) { + result_rows += task->cached_block->rows(); + } + } + ASSERT_EQ(result_rows, 1); + ASSERT_GT(local_state->_buckets_pruned_by_rf_counter->value(), 0); + ASSERT_GT(local_state->_rf_bucket_pruner.pruned_tablet_count(), 0); + ASSERT_LT(local_state->_rf_bucket_pruner.pruned_tablet_count(), bucket_num); +} + TEST(ScannerProjectionTest, merges_padding_block_when_limit_eos_without_extra_flag) { ObjectPool pool; auto data_type = std::make_shared(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/RuntimeFilterTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/RuntimeFilterTranslator.java index 266dc1310ec309..1a10cfdccb8e5b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/RuntimeFilterTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/RuntimeFilterTranslator.java @@ -249,11 +249,7 @@ private void createLegacyRuntimeFilterFromGroup(List group, RuntimeFilterPartitionPruneClassifier.classify( head.getType(), targetExpr, nereidsTargetExprList.get(i), scanNode); setPartitionPruningMetadata(origFilter, scanNode, classification); - RuntimeFilterBucketPruneClassifier.Classification bucketClassification = - RuntimeFilterBucketPruneClassifier.classify(head.getType(), targetExpr, scanNode); - if (bucketClassification.canPruneBuckets()) { - origFilter.markTargetCanPruneBuckets(scanNode.getId()); - } + setBucketPruningMetadata(origFilter, scanNode, head.getType(), targetExpr); } origFilter.setBloomFilterSizeCalculatedByNdv(head.isBloomFilterSizeCalculatedByNdv()); setWaitTimeMs(origFilter, head.isNonBlocking(), isLocalTarget); @@ -357,11 +353,7 @@ public void createLegacyRuntimeFilter(RuntimeFilter filter, PlanNode node, PlanT RuntimeFilterPartitionPruneClassifier.classify( filter.getType(), targetExpr, filter.getTargetExpressions().get(i), scanNode); setPartitionPruningMetadata(origFilter, scanNode, classification); - RuntimeFilterBucketPruneClassifier.Classification bucketClassification = - RuntimeFilterBucketPruneClassifier.classify(filter.getType(), targetExpr, scanNode); - if (bucketClassification.canPruneBuckets()) { - origFilter.markTargetCanPruneBuckets(scanNode.getId()); - } + setBucketPruningMetadata(origFilter, scanNode, filter.getType(), targetExpr); } origFilter.setBloomFilterSizeCalculatedByNdv(filter.isBloomFilterSizeCalculatedByNdv()); setWaitTimeMs(origFilter, filter.isNonBlocking(), isLocalTarget); @@ -403,6 +395,18 @@ private void setPartitionPruningMetadata(org.apache.doris.planner.RuntimeFilter scanNode.getId(), classification.getPartitionMonotonicity()); } + private void setBucketPruningMetadata(org.apache.doris.planner.RuntimeFilter runtimeFilter, + ScanNode scanNode, TRuntimeFilterType filterType, Expr targetExpr) { + if (!context.getSessionVariable().isEnableRuntimeFilterBucketPrune()) { + return; + } + RuntimeFilterBucketPruneClassifier.Classification classification = + RuntimeFilterBucketPruneClassifier.classify(filterType, targetExpr, scanNode); + if (classification.canPruneBuckets()) { + runtimeFilter.markTargetCanPruneBuckets(scanNode.getId()); + } + } + private void setWaitTimeMs(org.apache.doris.planner.RuntimeFilter filter, boolean isNonBlocking, boolean isLocalTarget) { if (isNonBlocking) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/RuntimeFilterTranslatorBucketPruneTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/RuntimeFilterTranslatorBucketPruneTest.java new file mode 100644 index 00000000000000..46f7ec109f49cc --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/RuntimeFilterTranslatorBucketPruneTest.java @@ -0,0 +1,235 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.glue.translator; + +import org.apache.doris.analysis.SlotRef; +import org.apache.doris.analysis.TupleDescriptor; +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.HashDistributionInfo; +import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.Partition; +import org.apache.doris.catalog.PartitionInfo; +import org.apache.doris.catalog.PartitionType; +import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.common.IdGenerator; +import org.apache.doris.nereids.processor.post.RuntimeFilterContext; +import org.apache.doris.nereids.trees.expressions.Add; +import org.apache.doris.nereids.trees.expressions.ExprId; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral; +import org.apache.doris.nereids.trees.plans.physical.AbstractPhysicalPlan; +import org.apache.doris.nereids.trees.plans.physical.PhysicalRelation; +import org.apache.doris.nereids.trees.plans.physical.RuntimeFilter; +import org.apache.doris.nereids.types.DataType; +import org.apache.doris.nereids.types.IntegerType; +import org.apache.doris.planner.OlapScanNode; +import org.apache.doris.planner.PlanFragment; +import org.apache.doris.planner.PlanFragmentId; +import org.apache.doris.planner.PlanNode; +import org.apache.doris.planner.PlanNodeId; +import org.apache.doris.planner.RuntimeFilterId; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.SessionVariable; +import org.apache.doris.thrift.TExprNodeType; +import org.apache.doris.thrift.TMinMaxRuntimeFilterType; +import org.apache.doris.thrift.TRuntimeFilterDesc; +import org.apache.doris.thrift.TRuntimeFilterType; + +import com.google.common.collect.ImmutableList; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import java.util.List; + +class RuntimeFilterTranslatorBucketPruneTest { + private static final int SCAN_NODE_ID = 7; + + private ConnectContext previousContext; + private SessionVariable sessionVariable; + + @BeforeEach + void setUp() { + previousContext = ConnectContext.get(); + sessionVariable = new SessionVariable(); + sessionVariable.setEnableRuntimeFilterPartitionPrune(false); + sessionVariable.setEnableRuntimeFilterBucketPrune(true); + ConnectContext connectContext = new ConnectContext(); + connectContext.setSessionVariable(sessionVariable); + connectContext.setThreadLocalInfo(); + } + + @AfterEach + void tearDown() { + ConnectContext.remove(); + if (previousContext != null) { + previousContext.setThreadLocalInfo(); + } + } + + @Test + void testGroupedSameTargetSerializesOneExpressionAndBucketTarget() { + TranslatorHarness harness = new TranslatorHarness(); + SlotReference target = harness.addTargetSlot("dist_col", harness.distributionColumn, + IntegerType.INSTANCE); + + TRuntimeFilterDesc desc = harness.translate(ImmutableList.of( + harness.newFilter(target, target), harness.newFilter(target, target))); + + Assertions.assertEquals(1, desc.planId_to_target_expr.size()); + Assertions.assertEquals(firstLegacySlotId(harness, target), + desc.planId_to_target_expr.get(SCAN_NODE_ID).nodes.get(0).slot_ref.slot_id); + Assertions.assertTrue(desc.isSetBucketPruningTargetIds()); + Assertions.assertEquals(ImmutableList.of(SCAN_NODE_ID), + desc.bucket_pruning_target_ids.stream().sorted().collect(java.util.stream.Collectors.toList())); + } + + @Test + void testGroupedDifferentTargetsKeepThriftMapButSuppressBucketTarget() { + TranslatorHarness harness = new TranslatorHarness(); + SlotReference distributionTarget = harness.addTargetSlot("dist_col", harness.distributionColumn, + IntegerType.INSTANCE); + Column valueColumn = new Column("value_col", PrimitiveType.INT); + SlotReference valueTarget = harness.addTargetSlot("value_col", valueColumn, IntegerType.INSTANCE); + + TRuntimeFilterDesc desc = harness.translate(ImmutableList.of( + harness.newFilter(distributionTarget, distributionTarget), + harness.newFilter(valueTarget, valueTarget))); + + Assertions.assertEquals(1, desc.planId_to_target_expr.size()); + Assertions.assertEquals(firstLegacySlotId(harness, valueTarget), + desc.planId_to_target_expr.get(SCAN_NODE_ID).nodes.get(0).slot_ref.slot_id); + Assertions.assertFalse(desc.isSetBucketPruningTargetIds()); + } + + @Test + void testCastAfterNonIdentityTargetSuppressesBucketTarget() { + TranslatorHarness harness = new TranslatorHarness(PrimitiveType.BIGINT); + SlotReference target = harness.addTargetSlot("dist_col", harness.distributionColumn, + IntegerType.INSTANCE); + + TRuntimeFilterDesc desc = harness.translate(ImmutableList.of( + harness.newFilter(target, new Add(target, new IntegerLiteral(1))))); + + Assertions.assertEquals(TExprNodeType.CAST_EXPR, + desc.planId_to_target_expr.get(SCAN_NODE_ID).nodes.get(0).node_type); + Assertions.assertFalse(desc.isSetBucketPruningTargetIds()); + } + + @Test + void testDisabledFeatureSkipsClassificationAndSerialization() { + sessionVariable.setEnableRuntimeFilterBucketPrune(false); + TranslatorHarness harness = new TranslatorHarness(); + SlotReference target = harness.addTargetSlot("dist_col", harness.distributionColumn, + IntegerType.INSTANCE); + Mockito.clearInvocations(harness.scanNode); + + TRuntimeFilterDesc desc = harness.translate(ImmutableList.of(harness.newFilter(target, target))); + + Mockito.verify(harness.scanNode, Mockito.never()).isPointQuery(); + Mockito.verify(harness.scanNode, Mockito.never()).getSelectedPartitionIds(); + Assertions.assertEquals(1, desc.planId_to_target_expr.size()); + Assertions.assertFalse(desc.isSetBucketPruningTargetIds()); + } + + private static int firstLegacySlotId(TranslatorHarness harness, SlotReference target) { + return harness.translatorContext.findSlotRef(target.getExprId()).getSlotId().asInt(); + } + + private class TranslatorHarness { + private final Column distributionColumn = new Column("dist_col", PrimitiveType.INT); + private final OlapTable table = Mockito.mock(OlapTable.class); + private final Partition partition = Mockito.mock(Partition.class); + private final OlapScanNode scanNode = Mockito.mock(OlapScanNode.class); + private final PlanNode builderNode = Mockito.mock(PlanNode.class); + private final AbstractPhysicalPlan nereidsBuilder = Mockito.mock(AbstractPhysicalPlan.class); + private final PhysicalRelation targetRelation = Mockito.mock(PhysicalRelation.class); + private final PlanTranslatorContext translatorContext = new PlanTranslatorContext(); + private final RuntimeFilterContext runtimeFilterContext; + private final RuntimeFilterTranslator translator; + private final IdGenerator filterIdGenerator = RuntimeFilterId.createGenerator(); + private final SlotReference source; + private final TupleDescriptor targetTuple = translatorContext.generateTupleDesc(); + private int nextExprId = 1; + + TranslatorHarness() { + this(PrimitiveType.INT); + } + + TranslatorHarness(PrimitiveType sourceType) { + runtimeFilterContext = new RuntimeFilterContext(sessionVariable, filterIdGenerator); + translator = new RuntimeFilterTranslator(runtimeFilterContext); + + PartitionInfo partitionInfo = Mockito.mock(PartitionInfo.class); + Mockito.when(partitionInfo.getType()).thenReturn(PartitionType.UNPARTITIONED); + Mockito.when(table.getPartitionInfo()).thenReturn(partitionInfo); + Mockito.when(table.getPartition(1L)).thenReturn(partition); + Mockito.when(partition.getDistributionInfo()).thenReturn( + new HashDistributionInfo(8, ImmutableList.of(distributionColumn))); + Mockito.when(scanNode.getOlapTable()).thenReturn(table); + Mockito.when(scanNode.getSelectedPartitionIds()).thenReturn(ImmutableList.of(1L)); + Mockito.when(scanNode.getId()).thenReturn(new PlanNodeId(SCAN_NODE_ID)); + + PlanFragment fragment = Mockito.mock(PlanFragment.class); + PlanFragmentId fragmentId = new PlanFragmentId(3); + Mockito.when(fragment.getFragmentId()).thenReturn(fragmentId); + Mockito.doCallRealMethod().when(builderNode).setFragment(fragment); + Mockito.doCallRealMethod().when(scanNode).setFragment(fragment); + builderNode.setFragment(fragment); + scanNode.setFragment(fragment); + Mockito.when(builderNode.getFragment()).thenReturn(fragment); + Mockito.when(scanNode.getFragment()).thenReturn(fragment); + Mockito.when(builderNode.getFragmentId()).thenReturn(fragmentId); + Mockito.when(scanNode.getFragmentId()).thenReturn(fragmentId); + + Column sourceColumn = new Column("src", sourceType); + source = addSlot("src", sourceColumn, DataType.fromCatalogType(sourceColumn.getType()), + translatorContext.generateTupleDesc()); + } + + SlotReference addTargetSlot(String name, Column column, DataType dataType) { + SlotReference target = addSlot(name, column, dataType, targetTuple); + SlotRef targetSlotRef = translatorContext.findSlotRef(target.getExprId()); + runtimeFilterContext.getExprIdToOlapScanNodeSlotRef().put(target.getExprId(), targetSlotRef); + runtimeFilterContext.getScanNodeOfLegacyRuntimeFilterTarget().put(target, scanNode); + return target; + } + + RuntimeFilter newFilter(SlotReference target, Expression targetExpression) { + return new RuntimeFilter(filterIdGenerator.getNextId(), source, target, targetExpression, + TRuntimeFilterType.IN, 0, nereidsBuilder, 10, false, + TMinMaxRuntimeFilterType.MIN_MAX, targetRelation); + } + + TRuntimeFilterDesc translate(List filters) { + translator.createLegacyRuntimeFilters(filters, builderNode, translatorContext); + Assertions.assertEquals(1, runtimeFilterContext.getLegacyFilters().size()); + return runtimeFilterContext.getLegacyFilters().get(0).toThrift(); + } + + private SlotReference addSlot(String name, Column column, DataType dataType, TupleDescriptor tuple) { + SlotReference slot = new SlotReference(new ExprId(nextExprId++), name, dataType, + false, ImmutableList.of("t"), table, column, table, column); + translatorContext.createSlotDesc(tuple, slot); + return slot; + } + } +} From 1f759d91e0650836535f7fb49402db020c53fd77 Mon Sep 17 00:00:00 2001 From: happenlee Date: Thu, 13 Aug 2026 21:47:39 +0800 Subject: [PATCH 10/20] [fix](runtime filter) Isolate merged bucket-pruning state ### What problem does this PR solve? Issue Number: None Related PR: #65837 Problem Summary: A local consumer could materialize bucket hashes on the first producer while the merger still aliased and mutated that producer wrapper. The cached partial hash set could then be published as the merged filter and incorrectly prune rows from later producers. Give the merger independent initialized storage and deep-copy Bloom filters on first merge. Hash exact values directly with the write-routing CRC into allocator-aware storage, avoiding a temporary copy of wide strings. Reuse the owned TPaloScanRange metadata instead of retaining a second per-tablet tuple, attach FE bucket metadata only once across worker serialization, and make the parallel late-arrival test release and join workers unconditionally. ### Release note None ### Check List (For Author) - Test: Unit Test - BE: RuntimeFilterMergerTest, RuntimeFilterBucketPrunerTest, ScannerLateArrivalRfTest (24 tests) - FE: OlapScanNodeTest and RuntimeFilterTranslatorBucketPruneTest (12 tests) - Static analysis: build-support/run-clang-tidy.sh on 13 modified C++ files - Format: build-support/clang-format.sh and build-support/check-format.sh - Behavior changed: Yes (prevents stale partial hashes from pruning matching buckets and removes redundant planning/runtime memory work) - Does this need documentation: No --- be/src/exec/operator/olap_scan_operator.cpp | 24 +++---- be/src/exec/operator/olap_scan_operator.h | 2 +- .../runtime_filter_bucket_pruner.cpp | 72 ++++++++++++------- .../runtime_filter_bucket_pruner.h | 10 +-- .../runtime_filter/runtime_filter_merger.h | 14 +--- .../runtime_filter/runtime_filter_wrapper.cpp | 25 +++---- .../runtime_filter/runtime_filter_wrapper.h | 5 +- be/src/exprs/bloom_filter_func.h | 12 ++++ be/src/exprs/runtime_filter_expr.cpp | 2 +- be/src/exprs/runtime_filter_expr.h | 3 +- .../runtime_filter_bucket_pruner_test.cpp | 45 ++++++++++-- .../runtime_filter_merger_test.cpp | 51 +++++++++++++ .../scan/scanner_late_arrival_rf_test.cpp | 40 ++++++++--- .../apache/doris/planner/OlapScanNode.java | 8 ++- .../doris/planner/OlapScanNodeTest.java | 39 ++++++++++ 15 files changed, 261 insertions(+), 91 deletions(-) diff --git a/be/src/exec/operator/olap_scan_operator.cpp b/be/src/exec/operator/olap_scan_operator.cpp index 28e356d2a640f8..6871fe4e79c6f8 100644 --- a/be/src/exec/operator/olap_scan_operator.cpp +++ b/be/src/exec/operator/olap_scan_operator.cpp @@ -1115,17 +1115,12 @@ void OlapScanLocalState::set_scan_ranges(RuntimeState* state, } } - for (auto& scan_range : scan_ranges) { + for (const auto& scan_range : scan_ranges) { DCHECK(scan_range.scan_range.__isset.palo_scan_range); _scan_ranges.emplace_back(new TPaloScanRange(scan_range.scan_range.palo_scan_range)); const auto& palo_scan_range = scan_range.scan_range.palo_scan_range; - if (palo_scan_range.__isset.bucket_seq || palo_scan_range.__isset.bucket_num) { - DORIS_CHECK(palo_scan_range.__isset.bucket_seq); - DORIS_CHECK(palo_scan_range.__isset.bucket_num); - _rf_bucket_prune_ranges.emplace_back(palo_scan_range.tablet_id, - palo_scan_range.bucket_seq, - palo_scan_range.bucket_num); - } + DCHECK_EQ(palo_scan_range.__isset.bucket_seq, palo_scan_range.__isset.bucket_num); + DCHECK_EQ(palo_scan_range.__isset.bucket_seq, _scan_ranges.front()->__isset.bucket_seq); COUNTER_UPDATE(_tablet_counter, 1); } } @@ -1133,14 +1128,14 @@ void OlapScanLocalState::set_scan_ranges(RuntimeState* state, Status OlapScanLocalState::_on_runtime_filter_update(const VExprContextSPtrs& new_conjuncts) { RETURN_IF_ERROR(Base::_on_runtime_filter_update(new_conjuncts)); if (!state()->query_options().enable_runtime_filter_bucket_prune || - _rf_bucket_prune_ranges.empty()) { + !_has_runtime_filter_bucket_prune_metadata()) { return Status::OK(); } int64_t newly_pruned = 0; RETURN_IF_ERROR(_rf_bucket_pruner.prune_by_runtime_filters( - _rf_bucket_prune_ranges, new_conjuncts, _parent->runtime_filter_descs(), - _parent->node_id(), state()->runtime_filter_max_in_num(), &newly_pruned)); + _scan_ranges, new_conjuncts, _parent->runtime_filter_descs(), _parent->node_id(), + state()->runtime_filter_max_in_num(), &newly_pruned)); if (newly_pruned > 0) { COUNTER_SET(_buckets_pruned_by_rf_counter, _rf_bucket_pruner.pruned_tablet_count()); } @@ -1152,7 +1147,12 @@ bool OlapScanLocalState::_is_tablet_pruned_by_runtime_filter(int64_t partition_i if (_rf_partition_pruner.is_partition_pruned(partition_id)) { return true; } - return !_rf_bucket_prune_ranges.empty() && _rf_bucket_pruner.is_tablet_pruned(tablet_id); + return _has_runtime_filter_bucket_prune_metadata() && + _rf_bucket_pruner.is_tablet_pruned(tablet_id); +} + +bool OlapScanLocalState::_has_runtime_filter_bucket_prune_metadata() const { + return !_scan_ranges.empty() && _scan_ranges.front()->__isset.bucket_seq; } static std::string tablets_id_to_string( diff --git a/be/src/exec/operator/olap_scan_operator.h b/be/src/exec/operator/olap_scan_operator.h index 78bc8e256109c2..a92f8cdf794d4e 100644 --- a/be/src/exec/operator/olap_scan_operator.h +++ b/be/src/exec/operator/olap_scan_operator.h @@ -138,9 +138,9 @@ class OlapScanLocalState final : public ScanLocalState { Status _build_key_ranges_and_filters(); bool _is_tablet_pruned_by_runtime_filter(int64_t partition_id, int64_t tablet_id) const; + bool _has_runtime_filter_bucket_prune_metadata() const; std::vector> _scan_ranges; - std::vector _rf_bucket_prune_ranges; RuntimeFilterBucketPruner _rf_bucket_pruner; std::vector _sync_statistics; MonotonicStopWatch _sync_cloud_tablets_watcher; diff --git a/be/src/exec/runtime_filter/runtime_filter_bucket_pruner.cpp b/be/src/exec/runtime_filter/runtime_filter_bucket_pruner.cpp index 99d00ce47c56d8..f06a1477f5841e 100644 --- a/be/src/exec/runtime_filter/runtime_filter_bucket_pruner.cpp +++ b/be/src/exec/runtime_filter/runtime_filter_bucket_pruner.cpp @@ -30,9 +30,52 @@ #include "exprs/vslot_ref.h" namespace doris { +namespace { + +using SelectedBuckets = phmap::flat_hash_set; +using SelectedBucketsByNum = phmap::flat_hash_map; + +const SelectedBuckets& get_selected_buckets(int32_t bucket_num, const DorisVector& hashes, + SelectedBucketsByNum* selected_buckets_by_num) { + auto [selected_it, inserted] = selected_buckets_by_num->try_emplace(bucket_num); + if (inserted) { + auto& selected_buckets = selected_it->second; + selected_buckets.reserve(std::min(hashes.size(), static_cast(bucket_num))); + for (uint32_t hash : hashes) { + selected_buckets.insert(static_cast(hash % static_cast(bucket_num))); + } + } + return selected_it->second; +} + +void collect_pruned_tablets(const std::vector>& ranges, + const DorisVector& hashes, + phmap::flat_hash_set* newly_pruned) { + SelectedBucketsByNum selected_buckets_by_num; + for (const auto& range_ptr : ranges) { + DCHECK(range_ptr != nullptr); + const auto& range = *range_ptr; + if (newly_pruned->contains(range.tablet_id)) { + continue; + } + DCHECK(range.__isset.bucket_seq); + DCHECK(range.__isset.bucket_num); + DCHECK_GT(range.bucket_num, 0); + DCHECK_GE(range.bucket_seq, 0); + DCHECK_LT(range.bucket_seq, range.bucket_num); + + const auto& selected_buckets = + get_selected_buckets(range.bucket_num, hashes, &selected_buckets_by_num); + if (!selected_buckets.contains(range.bucket_seq)) { + newly_pruned->insert(range.tablet_id); + } + } +} + +} // namespace Status RuntimeFilterBucketPruner::prune_by_runtime_filters( - const std::vector& ranges, + const std::vector>& ranges, const VExprContextSPtrs& conjuncts, const std::vector& rf_descs, int scan_node_id, int max_in_num, int64_t* newly_pruned_count) { *newly_pruned_count = 0; @@ -78,31 +121,8 @@ Status RuntimeFilterBucketPruner::prune_by_runtime_filters( VExprSPtr target_expr = impl->children()[0]; DORIS_CHECK_EQ(target_expr->node_type(), TExprNodeType::SLOT_REF); - std::shared_ptr> hashes = - rf_expr->get_bucket_prune_hashes(target_expr->data_type()); - phmap::flat_hash_map> selected_buckets_by_num; - for (const auto& range : ranges) { - if (newly_pruned.contains(range.tablet_id)) { - continue; - } - DORIS_CHECK_GT(range.bucket_num, 0); - DORIS_CHECK_GE(range.bucket_seq, 0); - DORIS_CHECK_LT(range.bucket_seq, range.bucket_num); - - auto [selected_it, inserted] = selected_buckets_by_num.try_emplace(range.bucket_num); - if (inserted) { - auto& selected_buckets = selected_it->second; - selected_buckets.reserve( - std::min(hashes->size(), static_cast(range.bucket_num))); - for (uint32_t hash : *hashes) { - selected_buckets.insert( - static_cast(hash % static_cast(range.bucket_num))); - } - } - if (!selected_it->second.contains(range.bucket_seq)) { - newly_pruned.insert(range.tablet_id); - } - } + auto hashes = rf_expr->get_bucket_prune_hashes(target_expr->data_type()); + collect_pruned_tablets(ranges, *hashes, &newly_pruned); } if (!newly_pruned.empty()) { diff --git a/be/src/exec/runtime_filter/runtime_filter_bucket_pruner.h b/be/src/exec/runtime_filter/runtime_filter_bucket_pruner.h index 5db7f881f643b5..b04a0fbd0d8d6c 100644 --- a/be/src/exec/runtime_filter/runtime_filter_bucket_pruner.h +++ b/be/src/exec/runtime_filter/runtime_filter_bucket_pruner.h @@ -18,6 +18,7 @@ #pragma once #include +#include #include #include @@ -28,12 +29,7 @@ namespace doris { struct TRuntimeFilterDesc; - -struct RuntimeFilterBucketPruneRange { - int64_t tablet_id = 0; - int32_t bucket_seq = 0; - int32_t bucket_num = 0; -}; +class TPaloScanRange; // Per-scan-instance state for single-column HASH bucket pruning. Runtime filters // are conjunctive, so each exact IN filter can monotonically add tablet ids to @@ -41,7 +37,7 @@ struct RuntimeFilterBucketPruneRange { // Both pruning updates and is_tablet_pruned() are safe to call concurrently. class RuntimeFilterBucketPruner { public: - Status prune_by_runtime_filters(const std::vector& ranges, + Status prune_by_runtime_filters(const std::vector>& ranges, const VExprContextSPtrs& conjuncts, const std::vector& rf_descs, int scan_node_id, int max_in_num, int64_t* newly_pruned_count); diff --git a/be/src/exec/runtime_filter/runtime_filter_merger.h b/be/src/exec/runtime_filter/runtime_filter_merger.h index a7bd3605f7165b..3b78c182a5016e 100644 --- a/be/src/exec/runtime_filter/runtime_filter_merger.h +++ b/be/src/exec/runtime_filter/runtime_filter_merger.h @@ -39,12 +39,7 @@ class RuntimeFilterMerger : public RuntimeFilter { static Status create(const QueryContext* query_ctx, const TRuntimeFilterDesc* desc, std::shared_ptr* res) { *res = std::shared_ptr(new RuntimeFilterMerger(query_ctx, desc)); - VExprContextSPtr build_ctx; - RETURN_IF_ERROR(VExpr::create_expr_tree(desc->src_expr, build_ctx)); - (*res)->_wrapper = std::make_shared( - build_ctx->root()->data_type()->get_primitive_type(), (*res)->_runtime_filter_type, - desc->filter_id, RuntimeFilterWrapper::State::UNINITED); - return Status::OK(); + return (*res)->_init_with_desc(desc, &query_ctx->query_options()); } std::string debug_string() override { @@ -69,12 +64,7 @@ class RuntimeFilterMerger : public RuntimeFilter { if (_received_producer_num == _expected_producer_num) { _rf_state = State::READY; } - if (_wrapper->get_state() == RuntimeFilterWrapper::State::UNINITED) { - _wrapper = other->_wrapper; - return Status::OK(); - } - auto st = _wrapper->merge(other->_wrapper.get()); - return st; + return _wrapper->merge(other->_wrapper.get()); } // Only raise the expected producer count. RuntimeFilterMgr may compute the diff --git a/be/src/exec/runtime_filter/runtime_filter_wrapper.cpp b/be/src/exec/runtime_filter/runtime_filter_wrapper.cpp index da018b5a2f7c78..0be2d57a4f949b 100644 --- a/be/src/exec/runtime_filter/runtime_filter_wrapper.cpp +++ b/be/src/exec/runtime_filter/runtime_filter_wrapper.cpp @@ -23,6 +23,7 @@ #include "exprs/create_predicate_function.h" #include "exprs/function/cast/cast_to_date_or_datetime_impl.hpp" #include "util/hash_util.hpp" +#include "util/raw_value.h" namespace doris { RuntimeFilterWrapper::RuntimeFilterWrapper(const RuntimeFilterParams* params) @@ -166,7 +167,11 @@ Status RuntimeFilterWrapper::merge(const RuntimeFilterWrapper* other) { break; } case RuntimeFilterType::BLOOM_FILTER: { - RETURN_IF_ERROR(_bloom_filter_func->merge(other->_bloom_filter_func.get())); + if (_state == State::UNINITED) { + RETURN_IF_ERROR(_bloom_filter_func->deep_copy(other->_bloom_filter_func.get())); + } else { + RETURN_IF_ERROR(_bloom_filter_func->merge(other->_bloom_filter_func.get())); + } break; } case RuntimeFilterType::IN_OR_BLOOM_FILTER: { @@ -190,7 +195,7 @@ Status RuntimeFilterWrapper::merge(const RuntimeFilterWrapper* other) { } } else { // case1&case2: use input bf directly and insert hybrid set data into bf - _bloom_filter_func = other->_bloom_filter_func; + RETURN_IF_ERROR(_bloom_filter_func->deep_copy(other->_bloom_filter_func.get())); RETURN_IF_ERROR(_change_to_bloom_filter()); } } else { @@ -617,7 +622,7 @@ bool RuntimeFilterWrapper::contain_null() const { return false; } -std::shared_ptr> +std::shared_ptr> RuntimeFilterWrapper::get_or_compute_bucket_prune_hashes(const DataTypePtr& target_type) const { DORIS_CHECK(_state.load() == State::READY); DORIS_CHECK(_hybrid_set != nullptr); @@ -626,26 +631,22 @@ RuntimeFilterWrapper::get_or_compute_bucket_prune_hashes(const DataTypePtr& targ DORIS_CHECK_EQ(primitive_type, _column_return_type); std::call_once(_bucket_prune_hashes_once, [&] { - MutableColumnPtr column = target_type->create_column(); + auto hashes = std::make_shared>(); + hashes->reserve(_hybrid_set->size() + (_hybrid_set->contain_null() ? 1 : 0)); auto* iter = _hybrid_set->begin(); while (iter->has_next()) { const void* value = iter->get_value(); DORIS_CHECK(value != nullptr); if (is_string_type(primitive_type)) { const auto* string_value = reinterpret_cast(value); - column->insert_data(string_value->data, string_value->size); + hashes->push_back(RawValue::zlib_crc32(string_value->data, string_value->size, + primitive_type, 0)); } else { - // ColumnVector::insert_data ignores length for fixed-length values. - column->insert_data(reinterpret_cast(value), 0); + hashes->push_back(RawValue::zlib_crc32(value, 0, primitive_type, 0)); } iter->next(); } - auto hashes = std::make_shared>(column->size(), 0); - if (!hashes->empty()) { - column->update_crcs_with_value(hashes->data(), primitive_type, - static_cast(column->size())); - } if (_hybrid_set->contain_null()) { // Keep one shared vector for nullable and non-nullable targets. A non-nullable // target may retain this extra bucket, but can never lose matching rows. diff --git a/be/src/exec/runtime_filter/runtime_filter_wrapper.h b/be/src/exec/runtime_filter/runtime_filter_wrapper.h index fce58a4d2d9b30..44e6e7905df767 100644 --- a/be/src/exec/runtime_filter/runtime_filter_wrapper.h +++ b/be/src/exec/runtime_filter/runtime_filter_wrapper.h @@ -24,6 +24,7 @@ #include "common/status.h" #include "core/column/column.h" +#include "core/custom_allocator.h" #include "core/data_type/data_type.h" #include "exec/runtime_filter/runtime_filter_definitions.h" #include "exec/runtime_filter/utils.h" @@ -88,7 +89,7 @@ class RuntimeFilterWrapper { // The shared vector includes the NULL hash whenever the exact set contains NULL, regardless // of target nullability. A non-nullable target may therefore retain one conservative bucket. - std::shared_ptr> get_or_compute_bucket_prune_hashes( + std::shared_ptr> get_or_compute_bucket_prune_hashes( const DataTypePtr& target_type) const; bool disable_always_true_logic() const { return _disable_always_true_logic; } @@ -168,6 +169,6 @@ class RuntimeFilterWrapper { AtomicStatus _reason; mutable std::once_flag _bucket_prune_hashes_once; - mutable std::shared_ptr> _bucket_prune_hashes; + mutable std::shared_ptr> _bucket_prune_hashes; }; } // namespace doris diff --git a/be/src/exprs/bloom_filter_func.h b/be/src/exprs/bloom_filter_func.h index 7e5eaeecb7e59f..47d1ec609685ce 100644 --- a/be/src/exprs/bloom_filter_func.h +++ b/be/src/exprs/bloom_filter_func.h @@ -101,6 +101,18 @@ class BloomFilterFuncBase : public FilterBase { return _bloom_filter->merge(other->_bloom_filter.get()); } + Status deep_copy(BloomFilterFuncBase* other) { + DORIS_CHECK(other != nullptr); + DORIS_CHECK(other->_bloom_filter != nullptr); + DORIS_CHECK_GT(other->_bloom_filter_alloced, 0); + + _bloom_filter_alloced = other->_bloom_filter_alloced; + _bloom_filter_length = other->_bloom_filter_length; + _bloom_filter.reset(BloomFilterAdaptor::create(_null_aware)); + RETURN_IF_ERROR(_bloom_filter->init(_bloom_filter_alloced)); + return merge(other); + } + Status assign(butil::IOBufAsZeroCopyInputStream* data, const size_t data_size, bool contain_null) { if (_bloom_filter == nullptr) { diff --git a/be/src/exprs/runtime_filter_expr.cpp b/be/src/exprs/runtime_filter_expr.cpp index e491b4329f3d76..54f2d17ef70c4d 100644 --- a/be/src/exprs/runtime_filter_expr.cpp +++ b/be/src/exprs/runtime_filter_expr.cpp @@ -85,7 +85,7 @@ Status RuntimeFilterExpr::clone_node(VExprSPtr* cloned_expr) const { return Status::OK(); } -std::shared_ptr> RuntimeFilterExpr::get_bucket_prune_hashes( +std::shared_ptr> RuntimeFilterExpr::get_bucket_prune_hashes( const DataTypePtr& target_type) const { DORIS_CHECK(_runtime_filter_wrapper != nullptr); return _runtime_filter_wrapper->get_or_compute_bucket_prune_hashes(target_type); diff --git a/be/src/exprs/runtime_filter_expr.h b/be/src/exprs/runtime_filter_expr.h index 32004500f9ca1f..544e9237fa9e17 100644 --- a/be/src/exprs/runtime_filter_expr.h +++ b/be/src/exprs/runtime_filter_expr.h @@ -28,6 +28,7 @@ #include "common/config.h" #include "common/status.h" +#include "core/custom_allocator.h" #include "exec/runtime_filter/runtime_filter_selectivity.h" #include "exprs/function_context.h" #include "exprs/vexpr.h" @@ -126,7 +127,7 @@ class RuntimeFilterExpr final : public VExpr { int filter_id() const { return _filter_id; } - std::shared_ptr> get_bucket_prune_hashes( + std::shared_ptr> get_bucket_prune_hashes( const DataTypePtr& target_type) const; std::shared_ptr predicate_filtered_rows_counter() const { diff --git a/be/test/exec/runtime_filter/runtime_filter_bucket_pruner_test.cpp b/be/test/exec/runtime_filter/runtime_filter_bucket_pruner_test.cpp index 4fd648c7f55cbe..3b30cb5130a91f 100644 --- a/be/test/exec/runtime_filter/runtime_filter_bucket_pruner_test.cpp +++ b/be/test/exec/runtime_filter/runtime_filter_bucket_pruner_test.cpp @@ -17,6 +17,7 @@ #include "exec/runtime_filter/runtime_filter_bucket_pruner.h" +#include #include #include @@ -28,6 +29,8 @@ #include "core/column/column_vector.h" #include "core/data_type/data_type_nullable.h" #include "core/data_type/data_type_number.h" +#include "core/data_type/data_type_string.h" +#include "core/string_ref.h" #include "exec/runtime_filter/runtime_filter_definitions.h" #include "exec/runtime_filter/runtime_filter_wrapper.h" #include "exprs/create_predicate_function.h" @@ -128,10 +131,19 @@ class RuntimeFilterBucketPrunerTest : public testing::Test { return desc; } - std::vector four_bucket_ranges() { - std::vector ranges; + std::unique_ptr bucket_range(int64_t tablet_id, int32_t bucket_seq, + int32_t bucket_num) { + auto range = std::make_unique(); + range->__set_tablet_id(tablet_id); + range->__set_bucket_seq(bucket_seq); + range->__set_bucket_num(bucket_num); + return range; + } + + std::vector> four_bucket_ranges() { + std::vector> ranges; for (int32_t bucket_seq = 0; bucket_seq < 4; ++bucket_seq) { - ranges.push_back({100 + bucket_seq, bucket_seq, 4}); + ranges.push_back(bucket_range(100 + bucket_seq, bucket_seq, 4)); } return ranges; } @@ -168,6 +180,27 @@ TEST_F(RuntimeFilterBucketPrunerTest, ExactSetHashesSharedAcrossConsumers) { EXPECT_EQ(first_hashes->back(), HashUtil::zlib_crc_hash_null(0)); } +TEST_F(RuntimeFilterBucketPrunerTest, StringHashesMatchWriteRoutingWithoutMaterialization) { + RuntimeFilterParams params {.filter_id = 15, + .filter_type = RuntimeFilterType::IN_FILTER, + .column_return_type = TYPE_STRING, + .null_aware = false, + .max_in_num = 1024}; + auto wrapper = std::make_shared(¶ms); + std::vector values {std::string(64 * 1024, 'x'), "bucket-prune"}; + std::set expected_hashes; + for (const auto& value : values) { + StringRef value_ref(value); + wrapper->hybrid_set()->insert(&value_ref); + expected_hashes.insert(RawValue::zlib_crc32(value.data(), value.size(), TYPE_STRING, 0)); + } + wrapper->set_state(RuntimeFilterWrapper::State::READY); + + auto hashes = wrapper->get_or_compute_bucket_prune_hashes(std::make_shared()); + + EXPECT_EQ(std::set(hashes->begin(), hashes->end()), expected_hashes); +} + TEST_F(RuntimeFilterBucketPrunerTest, ExactInKeepsOnlyMatchingBucket) { constexpr int filter_id = 7; constexpr int32_t value = 10; @@ -222,12 +255,12 @@ TEST_F(RuntimeFilterBucketPrunerTest, SupportsDifferentBucketCountsAcrossPartiti constexpr int32_t value = 10; VExprContextSPtrs conjuncts {make_in_conjunct(filter_id, {value})}; std::vector rf_descs {bucket_prune_desc(filter_id)}; - std::vector ranges; + std::vector> ranges; for (int32_t bucket_seq = 0; bucket_seq < 4; ++bucket_seq) { - ranges.push_back({100 + bucket_seq, bucket_seq, 4}); + ranges.push_back(bucket_range(100 + bucket_seq, bucket_seq, 4)); } for (int32_t bucket_seq = 0; bucket_seq < 7; ++bucket_seq) { - ranges.push_back({200 + bucket_seq, bucket_seq, 7}); + ranges.push_back(bucket_range(200 + bucket_seq, bucket_seq, 7)); } RuntimeFilterBucketPruner pruner; diff --git a/be/test/exec/runtime_filter/runtime_filter_merger_test.cpp b/be/test/exec/runtime_filter/runtime_filter_merger_test.cpp index 3d0bb701c8b1e9..410999eaa0734e 100644 --- a/be/test/exec/runtime_filter/runtime_filter_merger_test.cpp +++ b/be/test/exec/runtime_filter/runtime_filter_merger_test.cpp @@ -20,8 +20,13 @@ #include #include +#include + +#include "core/column/column_vector.h" +#include "core/data_type/data_type_number.h" #include "exec/runtime_filter/runtime_filter_producer.h" #include "exec/runtime_filter/runtime_filter_test_utils.h" +#include "util/raw_value.h" namespace doris { @@ -190,4 +195,50 @@ TEST_F(RuntimeFilterMergerTest, serialize_max_only) { test_serialize(RuntimeFilterWrapper::State::READY, desc); } +TEST_F(RuntimeFilterMergerTest, partial_merge_does_not_alias_producer_hash_cache) { + auto desc = TRuntimeFilterDescBuilder().set_type(TRuntimeFilterType::IN).build(); + std::shared_ptr merger; + FAIL_IF_ERROR_OR_CATCH_EXCEPTION(RuntimeFilterMerger::create(_query_ctx.get(), &desc, &merger)); + merger->increase_expected_producer_num(2); + + std::shared_ptr first_producer; + FAIL_IF_ERROR_OR_CATCH_EXCEPTION( + _runtime_states[0]->register_producer_runtime_filter(desc, &first_producer)); + FAIL_IF_ERROR_OR_CATCH_EXCEPTION(first_producer->init(1)); + auto first_column = ColumnInt32::create(); + first_column->insert_value(1); + FAIL_IF_ERROR_OR_CATCH_EXCEPTION(first_producer->insert(std::move(first_column), 0)); + first_producer->set_wrapper_state_and_ready_to_publish(RuntimeFilterWrapper::State::READY); + + bool ready = false; + FAIL_IF_ERROR_OR_CATCH_EXCEPTION(merger->merge_from(first_producer.get(), &ready)); + ASSERT_FALSE(ready); + auto first_wrapper = first_producer->wrapper(); + auto first_hashes = + first_wrapper->get_or_compute_bucket_prune_hashes(std::make_shared()); + ASSERT_EQ(first_hashes->size(), 1); + ASSERT_NE(merger->_wrapper.get(), first_wrapper.get()); + + std::shared_ptr second_producer; + FAIL_IF_ERROR_OR_CATCH_EXCEPTION( + _runtime_states[1]->register_producer_runtime_filter(desc, &second_producer)); + FAIL_IF_ERROR_OR_CATCH_EXCEPTION(second_producer->init(1)); + auto second_column = ColumnInt32::create(); + second_column->insert_value(2); + FAIL_IF_ERROR_OR_CATCH_EXCEPTION(second_producer->insert(std::move(second_column), 0)); + second_producer->set_wrapper_state_and_ready_to_publish(RuntimeFilterWrapper::State::READY); + FAIL_IF_ERROR_OR_CATCH_EXCEPTION(merger->merge_from(second_producer.get(), &ready)); + ASSERT_TRUE(ready); + + auto merged_hashes = + merger->_wrapper->get_or_compute_bucket_prune_hashes(std::make_shared()); + ASSERT_EQ(first_hashes->size(), 1); + ASSERT_EQ(merged_hashes->size(), 2); + std::set expected_hashes; + for (int32_t value : {1, 2}) { + expected_hashes.insert(RawValue::zlib_crc32(&value, sizeof(value), TYPE_INT, 0)); + } + EXPECT_EQ(std::set(merged_hashes->begin(), merged_hashes->end()), expected_hashes); +} + } // namespace doris diff --git a/be/test/exec/scan/scanner_late_arrival_rf_test.cpp b/be/test/exec/scan/scanner_late_arrival_rf_test.cpp index 5971431d82e524..2040d816bc3d7e 100644 --- a/be/test/exec/scan/scanner_late_arrival_rf_test.cpp +++ b/be/test/exec/scan/scanner_late_arrival_rf_test.cpp @@ -42,6 +42,7 @@ #include "testutil/mock/mock_descriptors.h" #include "testutil/mock/mock_runtime_state.h" #include "testutil/mock/mock_slot_ref.h" +#include "util/defer_op.h" #include "util/raw_value.h" namespace doris { @@ -249,7 +250,11 @@ TEST_F(ScannerLateArrivalRfTest, bucket_pruning_after_probe_tasks_start) { auto task_exec_ctx = std::make_shared(); state->set_task_execution_context(task_exec_ctx); for (int bucket_seq = 0; bucket_seq < bucket_num; ++bucket_seq) { - local_state->_rf_bucket_prune_ranges.push_back({100 + bucket_seq, bucket_seq, bucket_num}); + auto scan_range = std::make_unique(); + scan_range->__set_tablet_id(100 + bucket_seq); + scan_range->__set_bucket_seq(bucket_seq); + scan_range->__set_bucket_num(bucket_num); + local_state->_scan_ranges.push_back(std::move(scan_range)); } RuntimeProfile scan_profile("late bucket scan"); local_state->_buckets_pruned_by_rf_counter = @@ -293,7 +298,26 @@ TEST_F(ScannerLateArrivalRfTest, bucket_pruning_after_probe_tasks_start) { } scanner_context->_in_flight_tasks_num = bucket_num; + std::shared_ptr producer; + ASSERT_TRUE(RuntimeFilterProducer::create(_query_ctx.get(), &desc, &producer).ok()); + ASSERT_TRUE(producer->init(1).ok()); + auto filter_column = ColumnInt32::create(); + filter_column->insert_value(filter_value); + ASSERT_TRUE(producer->insert(std::move(filter_column), 0).ok()); + producer->set_wrapper_state_and_ready_to_publish(RuntimeFilterWrapper::State::READY); + std::vector probe_threads; + bool filter_released = false; + Defer probe_thread_guard {[&] { + if (!filter_released) { + filter_published.count_down(); + } + for (auto& thread : probe_threads) { + if (thread.joinable()) { + thread.join(); + } + } + }}; for (const auto& task : tasks) { probe_threads.emplace_back([scanner_context, task] { ScannerScheduler::_scanner_scan(scanner_context, task); @@ -302,20 +326,16 @@ TEST_F(ScannerLateArrivalRfTest, bucket_pruning_after_probe_tasks_start) { // Every task has passed the scheduler's pre-prepare pruning check while the RF is not ready. prepare_started.wait(); - ASSERT_EQ(local_state->_rf_bucket_pruner.pruned_tablet_count(), 0); + EXPECT_EQ(local_state->_rf_bucket_pruner.pruned_tablet_count(), 0); - std::shared_ptr producer; - ASSERT_TRUE(RuntimeFilterProducer::create(_query_ctx.get(), &desc, &producer).ok()); - ASSERT_TRUE(producer->init(1).ok()); - auto filter_column = ColumnInt32::create(); - filter_column->insert_value(filter_value); - ASSERT_TRUE(producer->insert(std::move(filter_column), 0).ok()); - producer->set_wrapper_state_and_ready_to_publish(RuntimeFilterWrapper::State::READY); local_state->_helper._consumers[0]->signal(producer.get()); filter_published.count_down(); + filter_released = true; for (auto& thread : probe_threads) { - thread.join(); + if (thread.joinable()) { + thread.join(); + } } for (int bucket_seq = 0; bucket_seq < bucket_num; ++bucket_seq) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java index 32399e7b80af09..59c5f10539a40d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java @@ -212,6 +212,7 @@ public class OlapScanNode extends ScanNode { // Pack bucket number and sequence into one value to avoid retaining two all-tablet maps. private Map tabletId2BucketInfo = Maps.newHashMap(); + private boolean runtimeFilterBucketPruneParametersSet = false; // a bucket seq may map to many tablets, and each tablet has a // TScanRangeLocations. public ArrayListMultimap bucketSeq2locations = ArrayListMultimap.create(); @@ -1551,7 +1552,11 @@ private boolean hasRfDrivingBucketPruning() { return false; } - private void setRuntimeFilterBucketPruneParameters() { + @VisibleForTesting + synchronized void setRuntimeFilterBucketPruneParameters() { + if (runtimeFilterBucketPruneParametersSet) { + return; + } for (TScanRangeLocations locations : scanRangeLocations) { TPaloScanRange scanRange = locations.getScanRange().getPaloScanRange(); Long bucketInfo = tabletId2BucketInfo.get(scanRange.getTabletId()); @@ -1561,6 +1566,7 @@ private void setRuntimeFilterBucketPruneParameters() { scanRange.setBucketSeq(decodeBucketSeq(bucketInfo)); scanRange.setBucketNum(decodeBucketNum(bucketInfo)); } + runtimeFilterBucketPruneParametersSet = true; } private static long encodeBucketInfo(int bucketSeq, int bucketNum) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/OlapScanNodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/OlapScanNodeTest.java index 89d734e7d08e21..d5493d8dc7ac84 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/planner/OlapScanNodeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/planner/OlapScanNodeTest.java @@ -38,7 +38,10 @@ import org.apache.doris.common.AnalysisException; import org.apache.doris.datasource.InternalCatalog; import org.apache.doris.thrift.TOlapScanNode; +import org.apache.doris.thrift.TPaloScanRange; import org.apache.doris.thrift.TPartitionBoundary; +import org.apache.doris.thrift.TScanRange; +import org.apache.doris.thrift.TScanRangeLocations; import com.google.common.collect.Lists; import com.google.common.collect.Maps; @@ -277,6 +280,42 @@ public void testRuntimeFilterPartitionBoundariesUsePlanningSnapshot() throws Ana Assert.assertEquals("p_target,p_after", scanNode.getSelectedPartitionNamesForExplain()); } + @Test + public void testRuntimeFilterBucketMetadataAttachedOnceAcrossWorkers() throws Exception { + OlapTable table = Mockito.mock(OlapTable.class); + Mockito.when(table.getName()).thenReturn("rf_bucket_fact"); + Mockito.when(table.getDistributionColumnNames()).thenReturn(Collections.emptySet()); + + TupleDescriptor tupleDescriptor = new TupleDescriptor(new TupleId(1)); + tupleDescriptor.setTable(table); + OlapScanNode scanNode = new OlapScanNode( + new PlanNodeId(1), tupleDescriptor, "rfBucketScanNode", ScanContext.EMPTY); + + TPaloScanRange paloScanRange = new TPaloScanRange(); + paloScanRange.setTabletId(10L); + TScanRange scanRange = new TScanRange(); + scanRange.setPaloScanRange(paloScanRange); + TScanRangeLocations locations = new TScanRangeLocations(); + locations.setScanRange(scanRange); + scanNode.scanRangeLocations.add(locations); + + java.lang.reflect.Field bucketInfoField = + OlapScanNode.class.getDeclaredField("tabletId2BucketInfo"); + bucketInfoField.setAccessible(true); + @SuppressWarnings("unchecked") + Map bucketInfo = (Map) bucketInfoField.get(scanNode); + bucketInfo.put(10L, ((long) 4 << Integer.SIZE) | 2L); + + // Nereids serializes the same plan once per worker. The second call must reuse the + // metadata attached by the first worker instead of walking the global ranges again. + scanNode.setRuntimeFilterBucketPruneParameters(); + bucketInfo.clear(); + scanNode.setRuntimeFilterBucketPruneParameters(); + + Assert.assertEquals(2, paloScanRange.getBucketSeq()); + Assert.assertEquals(4, paloScanRange.getBucketNum()); + } + private Partition mockPartition(String name) { Partition partition = Mockito.mock(Partition.class); Mockito.when(partition.getName()).thenReturn(name); From a6c799db58b3f9bfefd45504514181465726076e Mon Sep 17 00:00:00 2001 From: happenlee Date: Thu, 13 Aug 2026 22:11:42 +0800 Subject: [PATCH 11/20] Revert "[fix](runtime filter) Isolate merged bucket-pruning state" This reverts commit 452b51734266dd229bc65d05aaccc95d3c2e93b6. --- be/src/exec/operator/olap_scan_operator.cpp | 24 +++---- be/src/exec/operator/olap_scan_operator.h | 2 +- .../runtime_filter_bucket_pruner.cpp | 72 +++++++------------ .../runtime_filter_bucket_pruner.h | 10 ++- .../runtime_filter/runtime_filter_merger.h | 14 +++- .../runtime_filter/runtime_filter_wrapper.cpp | 25 ++++--- .../runtime_filter/runtime_filter_wrapper.h | 5 +- be/src/exprs/bloom_filter_func.h | 12 ---- be/src/exprs/runtime_filter_expr.cpp | 2 +- be/src/exprs/runtime_filter_expr.h | 3 +- .../runtime_filter_bucket_pruner_test.cpp | 45 ++---------- .../runtime_filter_merger_test.cpp | 51 ------------- .../scan/scanner_late_arrival_rf_test.cpp | 40 +++-------- .../apache/doris/planner/OlapScanNode.java | 8 +-- .../doris/planner/OlapScanNodeTest.java | 39 ---------- 15 files changed, 91 insertions(+), 261 deletions(-) diff --git a/be/src/exec/operator/olap_scan_operator.cpp b/be/src/exec/operator/olap_scan_operator.cpp index 6871fe4e79c6f8..28e356d2a640f8 100644 --- a/be/src/exec/operator/olap_scan_operator.cpp +++ b/be/src/exec/operator/olap_scan_operator.cpp @@ -1115,12 +1115,17 @@ void OlapScanLocalState::set_scan_ranges(RuntimeState* state, } } - for (const auto& scan_range : scan_ranges) { + for (auto& scan_range : scan_ranges) { DCHECK(scan_range.scan_range.__isset.palo_scan_range); _scan_ranges.emplace_back(new TPaloScanRange(scan_range.scan_range.palo_scan_range)); const auto& palo_scan_range = scan_range.scan_range.palo_scan_range; - DCHECK_EQ(palo_scan_range.__isset.bucket_seq, palo_scan_range.__isset.bucket_num); - DCHECK_EQ(palo_scan_range.__isset.bucket_seq, _scan_ranges.front()->__isset.bucket_seq); + if (palo_scan_range.__isset.bucket_seq || palo_scan_range.__isset.bucket_num) { + DORIS_CHECK(palo_scan_range.__isset.bucket_seq); + DORIS_CHECK(palo_scan_range.__isset.bucket_num); + _rf_bucket_prune_ranges.emplace_back(palo_scan_range.tablet_id, + palo_scan_range.bucket_seq, + palo_scan_range.bucket_num); + } COUNTER_UPDATE(_tablet_counter, 1); } } @@ -1128,14 +1133,14 @@ void OlapScanLocalState::set_scan_ranges(RuntimeState* state, Status OlapScanLocalState::_on_runtime_filter_update(const VExprContextSPtrs& new_conjuncts) { RETURN_IF_ERROR(Base::_on_runtime_filter_update(new_conjuncts)); if (!state()->query_options().enable_runtime_filter_bucket_prune || - !_has_runtime_filter_bucket_prune_metadata()) { + _rf_bucket_prune_ranges.empty()) { return Status::OK(); } int64_t newly_pruned = 0; RETURN_IF_ERROR(_rf_bucket_pruner.prune_by_runtime_filters( - _scan_ranges, new_conjuncts, _parent->runtime_filter_descs(), _parent->node_id(), - state()->runtime_filter_max_in_num(), &newly_pruned)); + _rf_bucket_prune_ranges, new_conjuncts, _parent->runtime_filter_descs(), + _parent->node_id(), state()->runtime_filter_max_in_num(), &newly_pruned)); if (newly_pruned > 0) { COUNTER_SET(_buckets_pruned_by_rf_counter, _rf_bucket_pruner.pruned_tablet_count()); } @@ -1147,12 +1152,7 @@ bool OlapScanLocalState::_is_tablet_pruned_by_runtime_filter(int64_t partition_i if (_rf_partition_pruner.is_partition_pruned(partition_id)) { return true; } - return _has_runtime_filter_bucket_prune_metadata() && - _rf_bucket_pruner.is_tablet_pruned(tablet_id); -} - -bool OlapScanLocalState::_has_runtime_filter_bucket_prune_metadata() const { - return !_scan_ranges.empty() && _scan_ranges.front()->__isset.bucket_seq; + return !_rf_bucket_prune_ranges.empty() && _rf_bucket_pruner.is_tablet_pruned(tablet_id); } static std::string tablets_id_to_string( diff --git a/be/src/exec/operator/olap_scan_operator.h b/be/src/exec/operator/olap_scan_operator.h index a92f8cdf794d4e..78bc8e256109c2 100644 --- a/be/src/exec/operator/olap_scan_operator.h +++ b/be/src/exec/operator/olap_scan_operator.h @@ -138,9 +138,9 @@ class OlapScanLocalState final : public ScanLocalState { Status _build_key_ranges_and_filters(); bool _is_tablet_pruned_by_runtime_filter(int64_t partition_id, int64_t tablet_id) const; - bool _has_runtime_filter_bucket_prune_metadata() const; std::vector> _scan_ranges; + std::vector _rf_bucket_prune_ranges; RuntimeFilterBucketPruner _rf_bucket_pruner; std::vector _sync_statistics; MonotonicStopWatch _sync_cloud_tablets_watcher; diff --git a/be/src/exec/runtime_filter/runtime_filter_bucket_pruner.cpp b/be/src/exec/runtime_filter/runtime_filter_bucket_pruner.cpp index f06a1477f5841e..99d00ce47c56d8 100644 --- a/be/src/exec/runtime_filter/runtime_filter_bucket_pruner.cpp +++ b/be/src/exec/runtime_filter/runtime_filter_bucket_pruner.cpp @@ -30,52 +30,9 @@ #include "exprs/vslot_ref.h" namespace doris { -namespace { - -using SelectedBuckets = phmap::flat_hash_set; -using SelectedBucketsByNum = phmap::flat_hash_map; - -const SelectedBuckets& get_selected_buckets(int32_t bucket_num, const DorisVector& hashes, - SelectedBucketsByNum* selected_buckets_by_num) { - auto [selected_it, inserted] = selected_buckets_by_num->try_emplace(bucket_num); - if (inserted) { - auto& selected_buckets = selected_it->second; - selected_buckets.reserve(std::min(hashes.size(), static_cast(bucket_num))); - for (uint32_t hash : hashes) { - selected_buckets.insert(static_cast(hash % static_cast(bucket_num))); - } - } - return selected_it->second; -} - -void collect_pruned_tablets(const std::vector>& ranges, - const DorisVector& hashes, - phmap::flat_hash_set* newly_pruned) { - SelectedBucketsByNum selected_buckets_by_num; - for (const auto& range_ptr : ranges) { - DCHECK(range_ptr != nullptr); - const auto& range = *range_ptr; - if (newly_pruned->contains(range.tablet_id)) { - continue; - } - DCHECK(range.__isset.bucket_seq); - DCHECK(range.__isset.bucket_num); - DCHECK_GT(range.bucket_num, 0); - DCHECK_GE(range.bucket_seq, 0); - DCHECK_LT(range.bucket_seq, range.bucket_num); - - const auto& selected_buckets = - get_selected_buckets(range.bucket_num, hashes, &selected_buckets_by_num); - if (!selected_buckets.contains(range.bucket_seq)) { - newly_pruned->insert(range.tablet_id); - } - } -} - -} // namespace Status RuntimeFilterBucketPruner::prune_by_runtime_filters( - const std::vector>& ranges, + const std::vector& ranges, const VExprContextSPtrs& conjuncts, const std::vector& rf_descs, int scan_node_id, int max_in_num, int64_t* newly_pruned_count) { *newly_pruned_count = 0; @@ -121,8 +78,31 @@ Status RuntimeFilterBucketPruner::prune_by_runtime_filters( VExprSPtr target_expr = impl->children()[0]; DORIS_CHECK_EQ(target_expr->node_type(), TExprNodeType::SLOT_REF); - auto hashes = rf_expr->get_bucket_prune_hashes(target_expr->data_type()); - collect_pruned_tablets(ranges, *hashes, &newly_pruned); + std::shared_ptr> hashes = + rf_expr->get_bucket_prune_hashes(target_expr->data_type()); + phmap::flat_hash_map> selected_buckets_by_num; + for (const auto& range : ranges) { + if (newly_pruned.contains(range.tablet_id)) { + continue; + } + DORIS_CHECK_GT(range.bucket_num, 0); + DORIS_CHECK_GE(range.bucket_seq, 0); + DORIS_CHECK_LT(range.bucket_seq, range.bucket_num); + + auto [selected_it, inserted] = selected_buckets_by_num.try_emplace(range.bucket_num); + if (inserted) { + auto& selected_buckets = selected_it->second; + selected_buckets.reserve( + std::min(hashes->size(), static_cast(range.bucket_num))); + for (uint32_t hash : *hashes) { + selected_buckets.insert( + static_cast(hash % static_cast(range.bucket_num))); + } + } + if (!selected_it->second.contains(range.bucket_seq)) { + newly_pruned.insert(range.tablet_id); + } + } } if (!newly_pruned.empty()) { diff --git a/be/src/exec/runtime_filter/runtime_filter_bucket_pruner.h b/be/src/exec/runtime_filter/runtime_filter_bucket_pruner.h index b04a0fbd0d8d6c..5db7f881f643b5 100644 --- a/be/src/exec/runtime_filter/runtime_filter_bucket_pruner.h +++ b/be/src/exec/runtime_filter/runtime_filter_bucket_pruner.h @@ -18,7 +18,6 @@ #pragma once #include -#include #include #include @@ -29,7 +28,12 @@ namespace doris { struct TRuntimeFilterDesc; -class TPaloScanRange; + +struct RuntimeFilterBucketPruneRange { + int64_t tablet_id = 0; + int32_t bucket_seq = 0; + int32_t bucket_num = 0; +}; // Per-scan-instance state for single-column HASH bucket pruning. Runtime filters // are conjunctive, so each exact IN filter can monotonically add tablet ids to @@ -37,7 +41,7 @@ class TPaloScanRange; // Both pruning updates and is_tablet_pruned() are safe to call concurrently. class RuntimeFilterBucketPruner { public: - Status prune_by_runtime_filters(const std::vector>& ranges, + Status prune_by_runtime_filters(const std::vector& ranges, const VExprContextSPtrs& conjuncts, const std::vector& rf_descs, int scan_node_id, int max_in_num, int64_t* newly_pruned_count); diff --git a/be/src/exec/runtime_filter/runtime_filter_merger.h b/be/src/exec/runtime_filter/runtime_filter_merger.h index 3b78c182a5016e..a7bd3605f7165b 100644 --- a/be/src/exec/runtime_filter/runtime_filter_merger.h +++ b/be/src/exec/runtime_filter/runtime_filter_merger.h @@ -39,7 +39,12 @@ class RuntimeFilterMerger : public RuntimeFilter { static Status create(const QueryContext* query_ctx, const TRuntimeFilterDesc* desc, std::shared_ptr* res) { *res = std::shared_ptr(new RuntimeFilterMerger(query_ctx, desc)); - return (*res)->_init_with_desc(desc, &query_ctx->query_options()); + VExprContextSPtr build_ctx; + RETURN_IF_ERROR(VExpr::create_expr_tree(desc->src_expr, build_ctx)); + (*res)->_wrapper = std::make_shared( + build_ctx->root()->data_type()->get_primitive_type(), (*res)->_runtime_filter_type, + desc->filter_id, RuntimeFilterWrapper::State::UNINITED); + return Status::OK(); } std::string debug_string() override { @@ -64,7 +69,12 @@ class RuntimeFilterMerger : public RuntimeFilter { if (_received_producer_num == _expected_producer_num) { _rf_state = State::READY; } - return _wrapper->merge(other->_wrapper.get()); + if (_wrapper->get_state() == RuntimeFilterWrapper::State::UNINITED) { + _wrapper = other->_wrapper; + return Status::OK(); + } + auto st = _wrapper->merge(other->_wrapper.get()); + return st; } // Only raise the expected producer count. RuntimeFilterMgr may compute the diff --git a/be/src/exec/runtime_filter/runtime_filter_wrapper.cpp b/be/src/exec/runtime_filter/runtime_filter_wrapper.cpp index 0be2d57a4f949b..da018b5a2f7c78 100644 --- a/be/src/exec/runtime_filter/runtime_filter_wrapper.cpp +++ b/be/src/exec/runtime_filter/runtime_filter_wrapper.cpp @@ -23,7 +23,6 @@ #include "exprs/create_predicate_function.h" #include "exprs/function/cast/cast_to_date_or_datetime_impl.hpp" #include "util/hash_util.hpp" -#include "util/raw_value.h" namespace doris { RuntimeFilterWrapper::RuntimeFilterWrapper(const RuntimeFilterParams* params) @@ -167,11 +166,7 @@ Status RuntimeFilterWrapper::merge(const RuntimeFilterWrapper* other) { break; } case RuntimeFilterType::BLOOM_FILTER: { - if (_state == State::UNINITED) { - RETURN_IF_ERROR(_bloom_filter_func->deep_copy(other->_bloom_filter_func.get())); - } else { - RETURN_IF_ERROR(_bloom_filter_func->merge(other->_bloom_filter_func.get())); - } + RETURN_IF_ERROR(_bloom_filter_func->merge(other->_bloom_filter_func.get())); break; } case RuntimeFilterType::IN_OR_BLOOM_FILTER: { @@ -195,7 +190,7 @@ Status RuntimeFilterWrapper::merge(const RuntimeFilterWrapper* other) { } } else { // case1&case2: use input bf directly and insert hybrid set data into bf - RETURN_IF_ERROR(_bloom_filter_func->deep_copy(other->_bloom_filter_func.get())); + _bloom_filter_func = other->_bloom_filter_func; RETURN_IF_ERROR(_change_to_bloom_filter()); } } else { @@ -622,7 +617,7 @@ bool RuntimeFilterWrapper::contain_null() const { return false; } -std::shared_ptr> +std::shared_ptr> RuntimeFilterWrapper::get_or_compute_bucket_prune_hashes(const DataTypePtr& target_type) const { DORIS_CHECK(_state.load() == State::READY); DORIS_CHECK(_hybrid_set != nullptr); @@ -631,22 +626,26 @@ RuntimeFilterWrapper::get_or_compute_bucket_prune_hashes(const DataTypePtr& targ DORIS_CHECK_EQ(primitive_type, _column_return_type); std::call_once(_bucket_prune_hashes_once, [&] { - auto hashes = std::make_shared>(); - hashes->reserve(_hybrid_set->size() + (_hybrid_set->contain_null() ? 1 : 0)); + MutableColumnPtr column = target_type->create_column(); auto* iter = _hybrid_set->begin(); while (iter->has_next()) { const void* value = iter->get_value(); DORIS_CHECK(value != nullptr); if (is_string_type(primitive_type)) { const auto* string_value = reinterpret_cast(value); - hashes->push_back(RawValue::zlib_crc32(string_value->data, string_value->size, - primitive_type, 0)); + column->insert_data(string_value->data, string_value->size); } else { - hashes->push_back(RawValue::zlib_crc32(value, 0, primitive_type, 0)); + // ColumnVector::insert_data ignores length for fixed-length values. + column->insert_data(reinterpret_cast(value), 0); } iter->next(); } + auto hashes = std::make_shared>(column->size(), 0); + if (!hashes->empty()) { + column->update_crcs_with_value(hashes->data(), primitive_type, + static_cast(column->size())); + } if (_hybrid_set->contain_null()) { // Keep one shared vector for nullable and non-nullable targets. A non-nullable // target may retain this extra bucket, but can never lose matching rows. diff --git a/be/src/exec/runtime_filter/runtime_filter_wrapper.h b/be/src/exec/runtime_filter/runtime_filter_wrapper.h index 44e6e7905df767..fce58a4d2d9b30 100644 --- a/be/src/exec/runtime_filter/runtime_filter_wrapper.h +++ b/be/src/exec/runtime_filter/runtime_filter_wrapper.h @@ -24,7 +24,6 @@ #include "common/status.h" #include "core/column/column.h" -#include "core/custom_allocator.h" #include "core/data_type/data_type.h" #include "exec/runtime_filter/runtime_filter_definitions.h" #include "exec/runtime_filter/utils.h" @@ -89,7 +88,7 @@ class RuntimeFilterWrapper { // The shared vector includes the NULL hash whenever the exact set contains NULL, regardless // of target nullability. A non-nullable target may therefore retain one conservative bucket. - std::shared_ptr> get_or_compute_bucket_prune_hashes( + std::shared_ptr> get_or_compute_bucket_prune_hashes( const DataTypePtr& target_type) const; bool disable_always_true_logic() const { return _disable_always_true_logic; } @@ -169,6 +168,6 @@ class RuntimeFilterWrapper { AtomicStatus _reason; mutable std::once_flag _bucket_prune_hashes_once; - mutable std::shared_ptr> _bucket_prune_hashes; + mutable std::shared_ptr> _bucket_prune_hashes; }; } // namespace doris diff --git a/be/src/exprs/bloom_filter_func.h b/be/src/exprs/bloom_filter_func.h index 47d1ec609685ce..7e5eaeecb7e59f 100644 --- a/be/src/exprs/bloom_filter_func.h +++ b/be/src/exprs/bloom_filter_func.h @@ -101,18 +101,6 @@ class BloomFilterFuncBase : public FilterBase { return _bloom_filter->merge(other->_bloom_filter.get()); } - Status deep_copy(BloomFilterFuncBase* other) { - DORIS_CHECK(other != nullptr); - DORIS_CHECK(other->_bloom_filter != nullptr); - DORIS_CHECK_GT(other->_bloom_filter_alloced, 0); - - _bloom_filter_alloced = other->_bloom_filter_alloced; - _bloom_filter_length = other->_bloom_filter_length; - _bloom_filter.reset(BloomFilterAdaptor::create(_null_aware)); - RETURN_IF_ERROR(_bloom_filter->init(_bloom_filter_alloced)); - return merge(other); - } - Status assign(butil::IOBufAsZeroCopyInputStream* data, const size_t data_size, bool contain_null) { if (_bloom_filter == nullptr) { diff --git a/be/src/exprs/runtime_filter_expr.cpp b/be/src/exprs/runtime_filter_expr.cpp index 54f2d17ef70c4d..e491b4329f3d76 100644 --- a/be/src/exprs/runtime_filter_expr.cpp +++ b/be/src/exprs/runtime_filter_expr.cpp @@ -85,7 +85,7 @@ Status RuntimeFilterExpr::clone_node(VExprSPtr* cloned_expr) const { return Status::OK(); } -std::shared_ptr> RuntimeFilterExpr::get_bucket_prune_hashes( +std::shared_ptr> RuntimeFilterExpr::get_bucket_prune_hashes( const DataTypePtr& target_type) const { DORIS_CHECK(_runtime_filter_wrapper != nullptr); return _runtime_filter_wrapper->get_or_compute_bucket_prune_hashes(target_type); diff --git a/be/src/exprs/runtime_filter_expr.h b/be/src/exprs/runtime_filter_expr.h index 544e9237fa9e17..32004500f9ca1f 100644 --- a/be/src/exprs/runtime_filter_expr.h +++ b/be/src/exprs/runtime_filter_expr.h @@ -28,7 +28,6 @@ #include "common/config.h" #include "common/status.h" -#include "core/custom_allocator.h" #include "exec/runtime_filter/runtime_filter_selectivity.h" #include "exprs/function_context.h" #include "exprs/vexpr.h" @@ -127,7 +126,7 @@ class RuntimeFilterExpr final : public VExpr { int filter_id() const { return _filter_id; } - std::shared_ptr> get_bucket_prune_hashes( + std::shared_ptr> get_bucket_prune_hashes( const DataTypePtr& target_type) const; std::shared_ptr predicate_filtered_rows_counter() const { diff --git a/be/test/exec/runtime_filter/runtime_filter_bucket_pruner_test.cpp b/be/test/exec/runtime_filter/runtime_filter_bucket_pruner_test.cpp index 3b30cb5130a91f..4fd648c7f55cbe 100644 --- a/be/test/exec/runtime_filter/runtime_filter_bucket_pruner_test.cpp +++ b/be/test/exec/runtime_filter/runtime_filter_bucket_pruner_test.cpp @@ -17,7 +17,6 @@ #include "exec/runtime_filter/runtime_filter_bucket_pruner.h" -#include #include #include @@ -29,8 +28,6 @@ #include "core/column/column_vector.h" #include "core/data_type/data_type_nullable.h" #include "core/data_type/data_type_number.h" -#include "core/data_type/data_type_string.h" -#include "core/string_ref.h" #include "exec/runtime_filter/runtime_filter_definitions.h" #include "exec/runtime_filter/runtime_filter_wrapper.h" #include "exprs/create_predicate_function.h" @@ -131,19 +128,10 @@ class RuntimeFilterBucketPrunerTest : public testing::Test { return desc; } - std::unique_ptr bucket_range(int64_t tablet_id, int32_t bucket_seq, - int32_t bucket_num) { - auto range = std::make_unique(); - range->__set_tablet_id(tablet_id); - range->__set_bucket_seq(bucket_seq); - range->__set_bucket_num(bucket_num); - return range; - } - - std::vector> four_bucket_ranges() { - std::vector> ranges; + std::vector four_bucket_ranges() { + std::vector ranges; for (int32_t bucket_seq = 0; bucket_seq < 4; ++bucket_seq) { - ranges.push_back(bucket_range(100 + bucket_seq, bucket_seq, 4)); + ranges.push_back({100 + bucket_seq, bucket_seq, 4}); } return ranges; } @@ -180,27 +168,6 @@ TEST_F(RuntimeFilterBucketPrunerTest, ExactSetHashesSharedAcrossConsumers) { EXPECT_EQ(first_hashes->back(), HashUtil::zlib_crc_hash_null(0)); } -TEST_F(RuntimeFilterBucketPrunerTest, StringHashesMatchWriteRoutingWithoutMaterialization) { - RuntimeFilterParams params {.filter_id = 15, - .filter_type = RuntimeFilterType::IN_FILTER, - .column_return_type = TYPE_STRING, - .null_aware = false, - .max_in_num = 1024}; - auto wrapper = std::make_shared(¶ms); - std::vector values {std::string(64 * 1024, 'x'), "bucket-prune"}; - std::set expected_hashes; - for (const auto& value : values) { - StringRef value_ref(value); - wrapper->hybrid_set()->insert(&value_ref); - expected_hashes.insert(RawValue::zlib_crc32(value.data(), value.size(), TYPE_STRING, 0)); - } - wrapper->set_state(RuntimeFilterWrapper::State::READY); - - auto hashes = wrapper->get_or_compute_bucket_prune_hashes(std::make_shared()); - - EXPECT_EQ(std::set(hashes->begin(), hashes->end()), expected_hashes); -} - TEST_F(RuntimeFilterBucketPrunerTest, ExactInKeepsOnlyMatchingBucket) { constexpr int filter_id = 7; constexpr int32_t value = 10; @@ -255,12 +222,12 @@ TEST_F(RuntimeFilterBucketPrunerTest, SupportsDifferentBucketCountsAcrossPartiti constexpr int32_t value = 10; VExprContextSPtrs conjuncts {make_in_conjunct(filter_id, {value})}; std::vector rf_descs {bucket_prune_desc(filter_id)}; - std::vector> ranges; + std::vector ranges; for (int32_t bucket_seq = 0; bucket_seq < 4; ++bucket_seq) { - ranges.push_back(bucket_range(100 + bucket_seq, bucket_seq, 4)); + ranges.push_back({100 + bucket_seq, bucket_seq, 4}); } for (int32_t bucket_seq = 0; bucket_seq < 7; ++bucket_seq) { - ranges.push_back(bucket_range(200 + bucket_seq, bucket_seq, 7)); + ranges.push_back({200 + bucket_seq, bucket_seq, 7}); } RuntimeFilterBucketPruner pruner; diff --git a/be/test/exec/runtime_filter/runtime_filter_merger_test.cpp b/be/test/exec/runtime_filter/runtime_filter_merger_test.cpp index 410999eaa0734e..3d0bb701c8b1e9 100644 --- a/be/test/exec/runtime_filter/runtime_filter_merger_test.cpp +++ b/be/test/exec/runtime_filter/runtime_filter_merger_test.cpp @@ -20,13 +20,8 @@ #include #include -#include - -#include "core/column/column_vector.h" -#include "core/data_type/data_type_number.h" #include "exec/runtime_filter/runtime_filter_producer.h" #include "exec/runtime_filter/runtime_filter_test_utils.h" -#include "util/raw_value.h" namespace doris { @@ -195,50 +190,4 @@ TEST_F(RuntimeFilterMergerTest, serialize_max_only) { test_serialize(RuntimeFilterWrapper::State::READY, desc); } -TEST_F(RuntimeFilterMergerTest, partial_merge_does_not_alias_producer_hash_cache) { - auto desc = TRuntimeFilterDescBuilder().set_type(TRuntimeFilterType::IN).build(); - std::shared_ptr merger; - FAIL_IF_ERROR_OR_CATCH_EXCEPTION(RuntimeFilterMerger::create(_query_ctx.get(), &desc, &merger)); - merger->increase_expected_producer_num(2); - - std::shared_ptr first_producer; - FAIL_IF_ERROR_OR_CATCH_EXCEPTION( - _runtime_states[0]->register_producer_runtime_filter(desc, &first_producer)); - FAIL_IF_ERROR_OR_CATCH_EXCEPTION(first_producer->init(1)); - auto first_column = ColumnInt32::create(); - first_column->insert_value(1); - FAIL_IF_ERROR_OR_CATCH_EXCEPTION(first_producer->insert(std::move(first_column), 0)); - first_producer->set_wrapper_state_and_ready_to_publish(RuntimeFilterWrapper::State::READY); - - bool ready = false; - FAIL_IF_ERROR_OR_CATCH_EXCEPTION(merger->merge_from(first_producer.get(), &ready)); - ASSERT_FALSE(ready); - auto first_wrapper = first_producer->wrapper(); - auto first_hashes = - first_wrapper->get_or_compute_bucket_prune_hashes(std::make_shared()); - ASSERT_EQ(first_hashes->size(), 1); - ASSERT_NE(merger->_wrapper.get(), first_wrapper.get()); - - std::shared_ptr second_producer; - FAIL_IF_ERROR_OR_CATCH_EXCEPTION( - _runtime_states[1]->register_producer_runtime_filter(desc, &second_producer)); - FAIL_IF_ERROR_OR_CATCH_EXCEPTION(second_producer->init(1)); - auto second_column = ColumnInt32::create(); - second_column->insert_value(2); - FAIL_IF_ERROR_OR_CATCH_EXCEPTION(second_producer->insert(std::move(second_column), 0)); - second_producer->set_wrapper_state_and_ready_to_publish(RuntimeFilterWrapper::State::READY); - FAIL_IF_ERROR_OR_CATCH_EXCEPTION(merger->merge_from(second_producer.get(), &ready)); - ASSERT_TRUE(ready); - - auto merged_hashes = - merger->_wrapper->get_or_compute_bucket_prune_hashes(std::make_shared()); - ASSERT_EQ(first_hashes->size(), 1); - ASSERT_EQ(merged_hashes->size(), 2); - std::set expected_hashes; - for (int32_t value : {1, 2}) { - expected_hashes.insert(RawValue::zlib_crc32(&value, sizeof(value), TYPE_INT, 0)); - } - EXPECT_EQ(std::set(merged_hashes->begin(), merged_hashes->end()), expected_hashes); -} - } // namespace doris diff --git a/be/test/exec/scan/scanner_late_arrival_rf_test.cpp b/be/test/exec/scan/scanner_late_arrival_rf_test.cpp index 2040d816bc3d7e..5971431d82e524 100644 --- a/be/test/exec/scan/scanner_late_arrival_rf_test.cpp +++ b/be/test/exec/scan/scanner_late_arrival_rf_test.cpp @@ -42,7 +42,6 @@ #include "testutil/mock/mock_descriptors.h" #include "testutil/mock/mock_runtime_state.h" #include "testutil/mock/mock_slot_ref.h" -#include "util/defer_op.h" #include "util/raw_value.h" namespace doris { @@ -250,11 +249,7 @@ TEST_F(ScannerLateArrivalRfTest, bucket_pruning_after_probe_tasks_start) { auto task_exec_ctx = std::make_shared(); state->set_task_execution_context(task_exec_ctx); for (int bucket_seq = 0; bucket_seq < bucket_num; ++bucket_seq) { - auto scan_range = std::make_unique(); - scan_range->__set_tablet_id(100 + bucket_seq); - scan_range->__set_bucket_seq(bucket_seq); - scan_range->__set_bucket_num(bucket_num); - local_state->_scan_ranges.push_back(std::move(scan_range)); + local_state->_rf_bucket_prune_ranges.push_back({100 + bucket_seq, bucket_seq, bucket_num}); } RuntimeProfile scan_profile("late bucket scan"); local_state->_buckets_pruned_by_rf_counter = @@ -298,26 +293,7 @@ TEST_F(ScannerLateArrivalRfTest, bucket_pruning_after_probe_tasks_start) { } scanner_context->_in_flight_tasks_num = bucket_num; - std::shared_ptr producer; - ASSERT_TRUE(RuntimeFilterProducer::create(_query_ctx.get(), &desc, &producer).ok()); - ASSERT_TRUE(producer->init(1).ok()); - auto filter_column = ColumnInt32::create(); - filter_column->insert_value(filter_value); - ASSERT_TRUE(producer->insert(std::move(filter_column), 0).ok()); - producer->set_wrapper_state_and_ready_to_publish(RuntimeFilterWrapper::State::READY); - std::vector probe_threads; - bool filter_released = false; - Defer probe_thread_guard {[&] { - if (!filter_released) { - filter_published.count_down(); - } - for (auto& thread : probe_threads) { - if (thread.joinable()) { - thread.join(); - } - } - }}; for (const auto& task : tasks) { probe_threads.emplace_back([scanner_context, task] { ScannerScheduler::_scanner_scan(scanner_context, task); @@ -326,16 +302,20 @@ TEST_F(ScannerLateArrivalRfTest, bucket_pruning_after_probe_tasks_start) { // Every task has passed the scheduler's pre-prepare pruning check while the RF is not ready. prepare_started.wait(); - EXPECT_EQ(local_state->_rf_bucket_pruner.pruned_tablet_count(), 0); + ASSERT_EQ(local_state->_rf_bucket_pruner.pruned_tablet_count(), 0); + std::shared_ptr producer; + ASSERT_TRUE(RuntimeFilterProducer::create(_query_ctx.get(), &desc, &producer).ok()); + ASSERT_TRUE(producer->init(1).ok()); + auto filter_column = ColumnInt32::create(); + filter_column->insert_value(filter_value); + ASSERT_TRUE(producer->insert(std::move(filter_column), 0).ok()); + producer->set_wrapper_state_and_ready_to_publish(RuntimeFilterWrapper::State::READY); local_state->_helper._consumers[0]->signal(producer.get()); filter_published.count_down(); - filter_released = true; for (auto& thread : probe_threads) { - if (thread.joinable()) { - thread.join(); - } + thread.join(); } for (int bucket_seq = 0; bucket_seq < bucket_num; ++bucket_seq) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java index 59c5f10539a40d..32399e7b80af09 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java @@ -212,7 +212,6 @@ public class OlapScanNode extends ScanNode { // Pack bucket number and sequence into one value to avoid retaining two all-tablet maps. private Map tabletId2BucketInfo = Maps.newHashMap(); - private boolean runtimeFilterBucketPruneParametersSet = false; // a bucket seq may map to many tablets, and each tablet has a // TScanRangeLocations. public ArrayListMultimap bucketSeq2locations = ArrayListMultimap.create(); @@ -1552,11 +1551,7 @@ private boolean hasRfDrivingBucketPruning() { return false; } - @VisibleForTesting - synchronized void setRuntimeFilterBucketPruneParameters() { - if (runtimeFilterBucketPruneParametersSet) { - return; - } + private void setRuntimeFilterBucketPruneParameters() { for (TScanRangeLocations locations : scanRangeLocations) { TPaloScanRange scanRange = locations.getScanRange().getPaloScanRange(); Long bucketInfo = tabletId2BucketInfo.get(scanRange.getTabletId()); @@ -1566,7 +1561,6 @@ synchronized void setRuntimeFilterBucketPruneParameters() { scanRange.setBucketSeq(decodeBucketSeq(bucketInfo)); scanRange.setBucketNum(decodeBucketNum(bucketInfo)); } - runtimeFilterBucketPruneParametersSet = true; } private static long encodeBucketInfo(int bucketSeq, int bucketNum) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/OlapScanNodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/OlapScanNodeTest.java index d5493d8dc7ac84..89d734e7d08e21 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/planner/OlapScanNodeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/planner/OlapScanNodeTest.java @@ -38,10 +38,7 @@ import org.apache.doris.common.AnalysisException; import org.apache.doris.datasource.InternalCatalog; import org.apache.doris.thrift.TOlapScanNode; -import org.apache.doris.thrift.TPaloScanRange; import org.apache.doris.thrift.TPartitionBoundary; -import org.apache.doris.thrift.TScanRange; -import org.apache.doris.thrift.TScanRangeLocations; import com.google.common.collect.Lists; import com.google.common.collect.Maps; @@ -280,42 +277,6 @@ public void testRuntimeFilterPartitionBoundariesUsePlanningSnapshot() throws Ana Assert.assertEquals("p_target,p_after", scanNode.getSelectedPartitionNamesForExplain()); } - @Test - public void testRuntimeFilterBucketMetadataAttachedOnceAcrossWorkers() throws Exception { - OlapTable table = Mockito.mock(OlapTable.class); - Mockito.when(table.getName()).thenReturn("rf_bucket_fact"); - Mockito.when(table.getDistributionColumnNames()).thenReturn(Collections.emptySet()); - - TupleDescriptor tupleDescriptor = new TupleDescriptor(new TupleId(1)); - tupleDescriptor.setTable(table); - OlapScanNode scanNode = new OlapScanNode( - new PlanNodeId(1), tupleDescriptor, "rfBucketScanNode", ScanContext.EMPTY); - - TPaloScanRange paloScanRange = new TPaloScanRange(); - paloScanRange.setTabletId(10L); - TScanRange scanRange = new TScanRange(); - scanRange.setPaloScanRange(paloScanRange); - TScanRangeLocations locations = new TScanRangeLocations(); - locations.setScanRange(scanRange); - scanNode.scanRangeLocations.add(locations); - - java.lang.reflect.Field bucketInfoField = - OlapScanNode.class.getDeclaredField("tabletId2BucketInfo"); - bucketInfoField.setAccessible(true); - @SuppressWarnings("unchecked") - Map bucketInfo = (Map) bucketInfoField.get(scanNode); - bucketInfo.put(10L, ((long) 4 << Integer.SIZE) | 2L); - - // Nereids serializes the same plan once per worker. The second call must reuse the - // metadata attached by the first worker instead of walking the global ranges again. - scanNode.setRuntimeFilterBucketPruneParameters(); - bucketInfo.clear(); - scanNode.setRuntimeFilterBucketPruneParameters(); - - Assert.assertEquals(2, paloScanRange.getBucketSeq()); - Assert.assertEquals(4, paloScanRange.getBucketNum()); - } - private Partition mockPartition(String name) { Partition partition = Mockito.mock(Partition.class); Mockito.when(partition.getName()).thenReturn(name); From eb82cfa4b1431a5728d7cdf4e548d81189f47fc3 Mon Sep 17 00:00:00 2001 From: happenlee Date: Thu, 13 Aug 2026 23:05:19 +0800 Subject: [PATCH 12/20] [fix](runtime filter) Address bucket pruning review feedback ### What problem does this PR solve? Issue Number: None Related PR: #65837 Problem Summary: Runtime-filter bucket pruning repeated metadata attachment and retained redundant per-tablet state, while late filters could arrive during scanner preparation and still allow reader initialization. Enforce finalized hash-cache usage, document the intentional temporary column materialization, attach FE bucket metadata once, retain compact selected-bucket state, and recheck late filters before scanner open. Also make the concurrent scanner test cleanup unconditional. ### Release note None ### Check List (For Author) - Test: Unit Test - BE: RuntimeFilterBucketPrunerTest and ScannerLateArrivalRfTest (12 tests) - FE: ThriftPlansBuilderTest (4 tests) - BE clang-format/check-format and clang-tidy - Behavior changed: No - Does this need documentation: No --- be/src/exec/operator/olap_scan_operator.cpp | 43 ++++++----- be/src/exec/operator/olap_scan_operator.h | 5 +- .../runtime_filter_bucket_pruner.cpp | 55 +++++++++----- .../runtime_filter_bucket_pruner.h | 22 +++--- .../runtime_filter/runtime_filter_wrapper.cpp | 7 ++ .../runtime_filter/runtime_filter_wrapper.h | 2 + be/src/exec/scan/olap_scanner.cpp | 5 +- be/src/exec/scan/olap_scanner.h | 4 ++ be/src/exec/scan/scanner_scheduler.cpp | 24 +++++-- .../runtime_filter_bucket_pruner_test.cpp | 72 +++++++++++++++---- .../scan/scanner_late_arrival_rf_test.cpp | 66 ++++++++++++----- .../apache/doris/planner/OlapScanNode.java | 11 ++- .../doris/qe/runtime/ThriftPlansBuilder.java | 16 +++++ .../qe/runtime/ThriftPlansBuilderTest.java | 18 +++++ 14 files changed, 257 insertions(+), 93 deletions(-) diff --git a/be/src/exec/operator/olap_scan_operator.cpp b/be/src/exec/operator/olap_scan_operator.cpp index 28e356d2a640f8..e2057690f9b0f6 100644 --- a/be/src/exec/operator/olap_scan_operator.cpp +++ b/be/src/exec/operator/olap_scan_operator.cpp @@ -679,8 +679,9 @@ Status OlapScanLocalState::_init_scanners(std::list* scanners) { size_t write_idx = 0; for (size_t read_idx = 0; read_idx < _tablets.size(); ++read_idx) { int64_t pid = _tablets[read_idx].tablet->partition_id(); - int64_t tablet_id = _tablets[read_idx].tablet->tablet_id(); - if (!_is_tablet_pruned_by_runtime_filter(pid, tablet_id)) { + const auto& scan_range = *_scan_ranges[read_idx]; + if (!_is_tablet_pruned_by_runtime_filter(pid, scan_range.bucket_seq, + scan_range.bucket_num)) { if (write_idx != read_idx) { _tablets[write_idx] = std::move(_tablets[read_idx]); _scan_ranges[write_idx] = std::move(_scan_ranges[read_idx]); @@ -835,6 +836,8 @@ Status OlapScanLocalState::_init_scanners(std::list* scanners) { p._olap_scan_node.is_preaggregation, read_row_binlog, resolve_binlog_scan_type(palo_scan_range), + palo_scan_range.bucket_seq, + palo_scan_range.bucket_num, palo_scan_range.__isset.start_tso ? std::make_optional(palo_scan_range.start_tso) : std::nullopt, @@ -1115,16 +1118,22 @@ void OlapScanLocalState::set_scan_ranges(RuntimeState* state, } } - for (auto& scan_range : scan_ranges) { - DCHECK(scan_range.scan_range.__isset.palo_scan_range); + bool bucket_prune_metadata_initialized = !_scan_ranges.empty(); + for (const auto& scan_range : scan_ranges) { + DORIS_CHECK(scan_range.scan_range.__isset.palo_scan_range); _scan_ranges.emplace_back(new TPaloScanRange(scan_range.scan_range.palo_scan_range)); - const auto& palo_scan_range = scan_range.scan_range.palo_scan_range; - if (palo_scan_range.__isset.bucket_seq || palo_scan_range.__isset.bucket_num) { - DORIS_CHECK(palo_scan_range.__isset.bucket_seq); - DORIS_CHECK(palo_scan_range.__isset.bucket_num); - _rf_bucket_prune_ranges.emplace_back(palo_scan_range.tablet_id, - palo_scan_range.bucket_seq, - palo_scan_range.bucket_num); + const auto& palo_scan_range = *_scan_ranges.back(); + DORIS_CHECK_EQ(palo_scan_range.__isset.bucket_seq, palo_scan_range.__isset.bucket_num); + if (!bucket_prune_metadata_initialized) { + _has_rf_bucket_prune_metadata = palo_scan_range.__isset.bucket_seq; + bucket_prune_metadata_initialized = true; + } else { + DORIS_CHECK_EQ(palo_scan_range.__isset.bucket_seq, _has_rf_bucket_prune_metadata); + } + if (_has_rf_bucket_prune_metadata) { + DORIS_CHECK_GT(palo_scan_range.bucket_num, 0); + DORIS_CHECK_GE(palo_scan_range.bucket_seq, 0); + DORIS_CHECK_LT(palo_scan_range.bucket_seq, palo_scan_range.bucket_num); } COUNTER_UPDATE(_tablet_counter, 1); } @@ -1133,14 +1142,14 @@ void OlapScanLocalState::set_scan_ranges(RuntimeState* state, Status OlapScanLocalState::_on_runtime_filter_update(const VExprContextSPtrs& new_conjuncts) { RETURN_IF_ERROR(Base::_on_runtime_filter_update(new_conjuncts)); if (!state()->query_options().enable_runtime_filter_bucket_prune || - _rf_bucket_prune_ranges.empty()) { + !_has_rf_bucket_prune_metadata || _scan_ranges.empty()) { return Status::OK(); } int64_t newly_pruned = 0; RETURN_IF_ERROR(_rf_bucket_pruner.prune_by_runtime_filters( - _rf_bucket_prune_ranges, new_conjuncts, _parent->runtime_filter_descs(), - _parent->node_id(), state()->runtime_filter_max_in_num(), &newly_pruned)); + _scan_ranges, new_conjuncts, _parent->runtime_filter_descs(), _parent->node_id(), + state()->runtime_filter_max_in_num(), &newly_pruned)); if (newly_pruned > 0) { COUNTER_SET(_buckets_pruned_by_rf_counter, _rf_bucket_pruner.pruned_tablet_count()); } @@ -1148,11 +1157,13 @@ Status OlapScanLocalState::_on_runtime_filter_update(const VExprContextSPtrs& ne } bool OlapScanLocalState::_is_tablet_pruned_by_runtime_filter(int64_t partition_id, - int64_t tablet_id) const { + int32_t bucket_seq, + int32_t bucket_num) const { if (_rf_partition_pruner.is_partition_pruned(partition_id)) { return true; } - return !_rf_bucket_prune_ranges.empty() && _rf_bucket_pruner.is_tablet_pruned(tablet_id); + return _has_rf_bucket_prune_metadata && + _rf_bucket_pruner.is_bucket_pruned(bucket_seq, bucket_num); } static std::string tablets_id_to_string( diff --git a/be/src/exec/operator/olap_scan_operator.h b/be/src/exec/operator/olap_scan_operator.h index 78bc8e256109c2..88440fe5bd8a36 100644 --- a/be/src/exec/operator/olap_scan_operator.h +++ b/be/src/exec/operator/olap_scan_operator.h @@ -137,10 +137,11 @@ class OlapScanLocalState final : public ScanLocalState { Status _build_key_ranges_and_filters(); - bool _is_tablet_pruned_by_runtime_filter(int64_t partition_id, int64_t tablet_id) const; + bool _is_tablet_pruned_by_runtime_filter(int64_t partition_id, int32_t bucket_seq, + int32_t bucket_num) const; std::vector> _scan_ranges; - std::vector _rf_bucket_prune_ranges; + bool _has_rf_bucket_prune_metadata = false; RuntimeFilterBucketPruner _rf_bucket_pruner; std::vector _sync_statistics; MonotonicStopWatch _sync_cloud_tablets_watcher; diff --git a/be/src/exec/runtime_filter/runtime_filter_bucket_pruner.cpp b/be/src/exec/runtime_filter/runtime_filter_bucket_pruner.cpp index 99d00ce47c56d8..88b9c7533f8d1b 100644 --- a/be/src/exec/runtime_filter/runtime_filter_bucket_pruner.cpp +++ b/be/src/exec/runtime_filter/runtime_filter_bucket_pruner.cpp @@ -32,7 +32,7 @@ namespace doris { Status RuntimeFilterBucketPruner::prune_by_runtime_filters( - const std::vector& ranges, + const std::vector>& ranges, const VExprContextSPtrs& conjuncts, const std::vector& rf_descs, int scan_node_id, int max_in_num, int64_t* newly_pruned_count) { *newly_pruned_count = 0; @@ -51,7 +51,6 @@ Status RuntimeFilterBucketPruner::prune_by_runtime_filters( return Status::OK(); } - phmap::flat_hash_set newly_pruned; for (const auto& conjunct_ctx : conjuncts) { VExprSPtr root = conjunct_ctx->root(); if (!root->is_rf_wrapper()) { @@ -80,16 +79,18 @@ Status RuntimeFilterBucketPruner::prune_by_runtime_filters( std::shared_ptr> hashes = rf_expr->get_bucket_prune_hashes(target_expr->data_type()); - phmap::flat_hash_map> selected_buckets_by_num; - for (const auto& range : ranges) { - if (newly_pruned.contains(range.tablet_id)) { - continue; - } + phmap::flat_hash_map> new_selected_buckets_by_num; + for (const auto& range_ptr : ranges) { + DORIS_CHECK(range_ptr != nullptr); + const auto& range = *range_ptr; + DORIS_CHECK(range.__isset.bucket_seq); + DORIS_CHECK(range.__isset.bucket_num); DORIS_CHECK_GT(range.bucket_num, 0); DORIS_CHECK_GE(range.bucket_seq, 0); DORIS_CHECK_LT(range.bucket_seq, range.bucket_num); - auto [selected_it, inserted] = selected_buckets_by_num.try_emplace(range.bucket_num); + auto [selected_it, inserted] = + new_selected_buckets_by_num.try_emplace(range.bucket_num); if (inserted) { auto& selected_buckets = selected_it->second; selected_buckets.reserve( @@ -99,31 +100,49 @@ Status RuntimeFilterBucketPruner::prune_by_runtime_filters( static_cast(hash % static_cast(range.bucket_num))); } } - if (!selected_it->second.contains(range.bucket_seq)) { - newly_pruned.insert(range.tablet_id); - } } - } - if (!newly_pruned.empty()) { std::unique_lock lock(_prune_mutex); - for (int64_t tablet_id : newly_pruned) { - if (_pruned_tablet_ids.insert(tablet_id).second) { + for (const auto& range_ptr : ranges) { + const auto& range = *range_ptr; + auto current_it = _selected_buckets_by_num.find(range.bucket_num); + bool was_selected = current_it == _selected_buckets_by_num.end() || + current_it->second.contains(range.bucket_seq); + if (was_selected && + !new_selected_buckets_by_num.at(range.bucket_num).contains(range.bucket_seq)) { ++*newly_pruned_count; } } + for (auto& [bucket_num, new_selected_buckets] : new_selected_buckets_by_num) { + auto current_it = _selected_buckets_by_num.find(bucket_num); + if (current_it == _selected_buckets_by_num.end()) { + _selected_buckets_by_num.emplace(bucket_num, std::move(new_selected_buckets)); + } else { + for (auto bucket_it = current_it->second.begin(); + bucket_it != current_it->second.end();) { + if (!new_selected_buckets.contains(*bucket_it)) { + bucket_it = current_it->second.erase(bucket_it); + } else { + ++bucket_it; + } + } + } + } + _pruned_tablet_count += *newly_pruned_count; } return Status::OK(); } -bool RuntimeFilterBucketPruner::is_tablet_pruned(int64_t tablet_id) const { +bool RuntimeFilterBucketPruner::is_bucket_pruned(int32_t bucket_seq, int32_t bucket_num) const { std::shared_lock lock(_prune_mutex); - return _pruned_tablet_ids.contains(tablet_id); + auto selected_it = _selected_buckets_by_num.find(bucket_num); + return selected_it != _selected_buckets_by_num.end() && + !selected_it->second.contains(bucket_seq); } int64_t RuntimeFilterBucketPruner::pruned_tablet_count() const { std::shared_lock lock(_prune_mutex); - return static_cast(_pruned_tablet_ids.size()); + return _pruned_tablet_count; } } // namespace doris diff --git a/be/src/exec/runtime_filter/runtime_filter_bucket_pruner.h b/be/src/exec/runtime_filter/runtime_filter_bucket_pruner.h index 5db7f881f643b5..4eb500152dcb85 100644 --- a/be/src/exec/runtime_filter/runtime_filter_bucket_pruner.h +++ b/be/src/exec/runtime_filter/runtime_filter_bucket_pruner.h @@ -18,6 +18,7 @@ #pragma once #include +#include #include #include @@ -27,30 +28,27 @@ namespace doris { +class TPaloScanRange; struct TRuntimeFilterDesc; -struct RuntimeFilterBucketPruneRange { - int64_t tablet_id = 0; - int32_t bucket_seq = 0; - int32_t bucket_num = 0; -}; - // Per-scan-instance state for single-column HASH bucket pruning. Runtime filters -// are conjunctive, so each exact IN filter can monotonically add tablet ids to -// the pruned set without retaining or combining its original value set. -// Both pruning updates and is_tablet_pruned() are safe to call concurrently. +// are conjunctive, so each exact IN filter can monotonically shrink the selected +// bucket set for each bucket count. The retained state is bounded by bucket +// counts rather than the number of tablets across all partitions. +// Both pruning updates and is_bucket_pruned() are safe to call concurrently. class RuntimeFilterBucketPruner { public: - Status prune_by_runtime_filters(const std::vector& ranges, + Status prune_by_runtime_filters(const std::vector>& ranges, const VExprContextSPtrs& conjuncts, const std::vector& rf_descs, int scan_node_id, int max_in_num, int64_t* newly_pruned_count); - bool is_tablet_pruned(int64_t tablet_id) const; + bool is_bucket_pruned(int32_t bucket_seq, int32_t bucket_num) const; int64_t pruned_tablet_count() const; private: - phmap::flat_hash_set _pruned_tablet_ids; + phmap::flat_hash_map> _selected_buckets_by_num; + int64_t _pruned_tablet_count = 0; mutable std::shared_mutex _prune_mutex; }; diff --git a/be/src/exec/runtime_filter/runtime_filter_wrapper.cpp b/be/src/exec/runtime_filter/runtime_filter_wrapper.cpp index da018b5a2f7c78..89db3299425deb 100644 --- a/be/src/exec/runtime_filter/runtime_filter_wrapper.cpp +++ b/be/src/exec/runtime_filter/runtime_filter_wrapper.cpp @@ -126,6 +126,7 @@ bool RuntimeFilterWrapper::build_bf_by_runtime_size() const { } Status RuntimeFilterWrapper::merge(const RuntimeFilterWrapper* other) { + DORIS_CHECK(!_bucket_prune_hashes_started.load()); if (_state == State::DISABLED) { return Status::OK(); } @@ -626,6 +627,12 @@ RuntimeFilterWrapper::get_or_compute_bucket_prune_hashes(const DataTypePtr& targ DORIS_CHECK_EQ(primitive_type, _column_return_type); std::call_once(_bucket_prune_hashes_once, [&] { + _bucket_prune_hashes_started.store(true); + // Materialize the exact-set values into a column so bucket pruning uses the + // column's type-specific CRC implementation. This intentionally makes a + // temporary copy of variable-length values. runtime_filter_max_in_num bounds + // the number of copied values, but not their total byte size; we accept this + // transient memory cost to avoid maintaining a separate per-type hash path. MutableColumnPtr column = target_type->create_column(); auto* iter = _hybrid_set->begin(); while (iter->has_next()) { diff --git a/be/src/exec/runtime_filter/runtime_filter_wrapper.h b/be/src/exec/runtime_filter/runtime_filter_wrapper.h index fce58a4d2d9b30..7fb770e8855c74 100644 --- a/be/src/exec/runtime_filter/runtime_filter_wrapper.h +++ b/be/src/exec/runtime_filter/runtime_filter_wrapper.h @@ -19,6 +19,7 @@ #include +#include #include #include @@ -168,6 +169,7 @@ class RuntimeFilterWrapper { AtomicStatus _reason; mutable std::once_flag _bucket_prune_hashes_once; + mutable std::atomic_bool _bucket_prune_hashes_started = false; mutable std::shared_ptr> _bucket_prune_hashes; }; } // namespace doris diff --git a/be/src/exec/scan/olap_scanner.cpp b/be/src/exec/scan/olap_scanner.cpp index 635495cb43eadf..57b6ba2c054149 100644 --- a/be/src/exec/scan/olap_scanner.cpp +++ b/be/src/exec/scan/olap_scanner.cpp @@ -105,6 +105,8 @@ OlapScanner::OlapScanner(ScanLocalStateBase* parent, OlapScanner::Params&& param .binlog_scan_type = params.binlog_scan_type}), _start_tso(params.start_tso), _end_tso(params.end_tso), + _bucket_seq(params.bucket_seq), + _bucket_num(params.bucket_num), _initial_file_cache_stats(std::move(params.initial_file_cache_stats)) { _tablet_reader_params.set_read_source(std::move(params.read_source), _state->skip_delete_bitmap()); @@ -654,8 +656,7 @@ bool OlapScanner::is_pruned_by_runtime_filter() const { DCHECK(_local_state != nullptr); auto* olap_local_state = assert_cast(_local_state); return olap_local_state->_is_tablet_pruned_by_runtime_filter( - _tablet_reader_params.tablet->partition_id(), - _tablet_reader_params.tablet->tablet_id()); + _tablet_reader_params.tablet->partition_id(), _bucket_seq, _bucket_num); } doris::TabletStorageType OlapScanner::get_storage_type() { diff --git a/be/src/exec/scan/olap_scanner.h b/be/src/exec/scan/olap_scanner.h index 67fd4ef64d8e87..30d2040b7d3587 100644 --- a/be/src/exec/scan/olap_scanner.h +++ b/be/src/exec/scan/olap_scanner.h @@ -80,6 +80,8 @@ class OlapScanner : public Scanner { bool aggregation; bool read_row_binlog = false; TBinlogScanType::type binlog_scan_type = TBinlogScanType::NONE; + int32_t bucket_seq = 0; + int32_t bucket_num = 0; std::optional start_tso; std::optional end_tso; }; @@ -121,6 +123,8 @@ class OlapScanner : public Scanner { std::unique_ptr _tablet_reader; std::optional _start_tso; std::optional _end_tso; + int32_t _bucket_seq; + int32_t _bucket_num; public: io::FileCacheStatistics _initial_file_cache_stats; diff --git a/be/src/exec/scan/scanner_scheduler.cpp b/be/src/exec/scan/scanner_scheduler.cpp index 4a7afda12d8b85..560d85b6d49e97 100644 --- a/be/src/exec/scan/scanner_scheduler.cpp +++ b/be/src/exec/scan/scanner_scheduler.cpp @@ -176,6 +176,13 @@ void ScannerScheduler::_scanner_scan(std::shared_ptr ctx, Status status = Status::OK(); bool eos = false; + auto append_late_arrival_runtime_filter = [&] { + Status rf_status = scanner->try_append_late_arrival_runtime_filter(); + if (!rf_status.ok()) { + LOG(WARNING) << "Failed to append late arrival runtime filter: " + << rf_status.to_string(); + } + }; ASSIGN_STATUS_IF_CATCH_EXCEPTION( RuntimeState* state = ctx->state(); DCHECK(nullptr != state); @@ -192,6 +199,15 @@ void ScannerScheduler::_scanner_scan(std::shared_ptr ctx, } } + // A filter may become ready while prepare() is doing tablet setup. Apply it before + // open() so a newly pruned OLAP scanner never initializes its reader or eagerly reads. + if (!eos && !scanner->is_open()) { + append_late_arrival_runtime_filter(); + if (scanner->is_pruned_by_runtime_filter()) { + eos = true; + } + } + if (!eos && !scanner->is_open()) { status = scanner->open(state); if (!status.ok()) { @@ -200,13 +216,7 @@ void ScannerScheduler::_scanner_scan(std::shared_ptr ctx, scanner->set_opened(); } - if (!eos) { - Status rf_status = scanner->try_append_late_arrival_runtime_filter(); - if (!rf_status.ok()) { - LOG(WARNING) << "Failed to append late arrival runtime filter: " - << rf_status.to_string(); - } - } + if (!eos) { append_late_arrival_runtime_filter(); } // After processing late RFs, check whether this scanner's scan range was pruned. if (!eos && scanner->is_pruned_by_runtime_filter()) { eos = true; } diff --git a/be/test/exec/runtime_filter/runtime_filter_bucket_pruner_test.cpp b/be/test/exec/runtime_filter/runtime_filter_bucket_pruner_test.cpp index 4fd648c7f55cbe..412fc8ca557354 100644 --- a/be/test/exec/runtime_filter/runtime_filter_bucket_pruner_test.cpp +++ b/be/test/exec/runtime_filter/runtime_filter_bucket_pruner_test.cpp @@ -17,6 +17,7 @@ #include "exec/runtime_filter/runtime_filter_bucket_pruner.h" +#include #include #include @@ -43,6 +44,7 @@ namespace doris { class RuntimeFilterBucketPrunerTest : public testing::Test { protected: static constexpr int SCAN_NODE_ID = 10; + using BucketPruneRanges = std::vector>; std::shared_ptr make_in_wrapper(int filter_id, const std::vector& values, @@ -128,10 +130,19 @@ class RuntimeFilterBucketPrunerTest : public testing::Test { return desc; } - std::vector four_bucket_ranges() { - std::vector ranges; + void add_range(BucketPruneRanges* ranges, int64_t tablet_id, int32_t bucket_seq, + int32_t bucket_num) { + auto range = std::make_unique(); + range->__set_tablet_id(tablet_id); + range->__set_bucket_seq(bucket_seq); + range->__set_bucket_num(bucket_num); + ranges->push_back(std::move(range)); + } + + BucketPruneRanges four_bucket_ranges() { + BucketPruneRanges ranges; for (int32_t bucket_seq = 0; bucket_seq < 4; ++bucket_seq) { - ranges.push_back({100 + bucket_seq, bucket_seq, 4}); + add_range(&ranges, 100 + bucket_seq, bucket_seq, 4); } return ranges; } @@ -168,6 +179,17 @@ TEST_F(RuntimeFilterBucketPrunerTest, ExactSetHashesSharedAcrossConsumers) { EXPECT_EQ(first_hashes->back(), HashUtil::zlib_crc_hash_null(0)); } +TEST_F(RuntimeFilterBucketPrunerTest, RejectsMergeAfterBucketHashesStart) { + constexpr int filter_id = 15; + auto wrapper = make_in_wrapper(filter_id, {1}); + auto other = make_in_wrapper(filter_id, {2}); + + static_cast( + wrapper->get_or_compute_bucket_prune_hashes(std::make_shared())); + + EXPECT_DEATH({ static_cast(wrapper->merge(other.get())); }, "Check failed"); +} + TEST_F(RuntimeFilterBucketPrunerTest, ExactInKeepsOnlyMatchingBucket) { constexpr int filter_id = 7; constexpr int32_t value = 10; @@ -184,7 +206,7 @@ TEST_F(RuntimeFilterBucketPrunerTest, ExactInKeepsOnlyMatchingBucket) { EXPECT_EQ(pruner.pruned_tablet_count(), 3); int32_t selected_bucket = bucket_for_value(value, 4); for (int32_t bucket_seq = 0; bucket_seq < 4; ++bucket_seq) { - EXPECT_EQ(pruner.is_tablet_pruned(100 + bucket_seq), bucket_seq != selected_bucket); + EXPECT_EQ(pruner.is_bucket_pruned(bucket_seq, 4), bucket_seq != selected_bucket); } ASSERT_TRUE(pruner.prune_by_runtime_filters(four_bucket_ranges(), conjuncts, rf_descs, @@ -212,8 +234,7 @@ TEST_F(RuntimeFilterBucketPrunerTest, NonNullableTargetConservativelyKeepsNullBu EXPECT_EQ(newly_pruned, 2); for (int32_t bucket_seq = 0; bucket_seq < 4; ++bucket_seq) { - EXPECT_EQ(pruner.is_tablet_pruned(100 + bucket_seq), - !selected_buckets.contains(bucket_seq)); + EXPECT_EQ(pruner.is_bucket_pruned(bucket_seq, 4), !selected_buckets.contains(bucket_seq)); } } @@ -222,12 +243,12 @@ TEST_F(RuntimeFilterBucketPrunerTest, SupportsDifferentBucketCountsAcrossPartiti constexpr int32_t value = 10; VExprContextSPtrs conjuncts {make_in_conjunct(filter_id, {value})}; std::vector rf_descs {bucket_prune_desc(filter_id)}; - std::vector ranges; + BucketPruneRanges ranges; for (int32_t bucket_seq = 0; bucket_seq < 4; ++bucket_seq) { - ranges.push_back({100 + bucket_seq, bucket_seq, 4}); + add_range(&ranges, 100 + bucket_seq, bucket_seq, 4); } for (int32_t bucket_seq = 0; bucket_seq < 7; ++bucket_seq) { - ranges.push_back({200 + bucket_seq, bucket_seq, 7}); + add_range(&ranges, 200 + bucket_seq, bucket_seq, 7); } RuntimeFilterBucketPruner pruner; @@ -237,8 +258,8 @@ TEST_F(RuntimeFilterBucketPrunerTest, SupportsDifferentBucketCountsAcrossPartiti .ok()); EXPECT_EQ(newly_pruned, 9); - EXPECT_FALSE(pruner.is_tablet_pruned(100 + bucket_for_value(value, 4))); - EXPECT_FALSE(pruner.is_tablet_pruned(200 + bucket_for_value(value, 7))); + EXPECT_FALSE(pruner.is_bucket_pruned(bucket_for_value(value, 4), 4)); + EXPECT_FALSE(pruner.is_bucket_pruned(bucket_for_value(value, 7), 7)); } TEST_F(RuntimeFilterBucketPrunerTest, EmptyExactInPrunesAllBuckets) { @@ -270,8 +291,35 @@ TEST_F(RuntimeFilterBucketPrunerTest, NullAwareInKeepsNullBucket) { EXPECT_EQ(pruner.pruned_tablet_count(), 3); int32_t null_bucket = bucket_for_null(4); for (int32_t bucket_seq = 0; bucket_seq < 4; ++bucket_seq) { - EXPECT_EQ(pruner.is_tablet_pruned(100 + bucket_seq), bucket_seq != null_bucket); + EXPECT_EQ(pruner.is_bucket_pruned(bucket_seq, 4), bucket_seq != null_bucket); + } +} + +TEST_F(RuntimeFilterBucketPrunerTest, HighRangeCountRetainsBucketState) { + constexpr int filter_id = 16; + constexpr int32_t value = 10; + constexpr int32_t partition_count = 128; + constexpr int32_t bucket_num = 256; + BucketPruneRanges ranges; + ranges.reserve(partition_count * bucket_num); + for (int32_t partition = 0; partition < partition_count; ++partition) { + for (int32_t bucket_seq = 0; bucket_seq < bucket_num; ++bucket_seq) { + add_range(&ranges, static_cast(partition) * bucket_num + bucket_seq, + bucket_seq, bucket_num); + } } + VExprContextSPtrs conjuncts {make_in_conjunct(filter_id, {value})}; + std::vector rf_descs {bucket_prune_desc(filter_id)}; + + RuntimeFilterBucketPruner pruner; + int64_t newly_pruned = 0; + ASSERT_TRUE(pruner.prune_by_runtime_filters(ranges, conjuncts, rf_descs, SCAN_NODE_ID, + /*max_in_num=*/1024, &newly_pruned) + .ok()); + + EXPECT_EQ(newly_pruned, static_cast(partition_count) * (bucket_num - 1)); + EXPECT_EQ(pruner.pruned_tablet_count(), newly_pruned); + EXPECT_FALSE(pruner.is_bucket_pruned(bucket_for_value(value, bucket_num), bucket_num)); } TEST_F(RuntimeFilterBucketPrunerTest, NonExactRuntimeRepresentationIsIgnored) { diff --git a/be/test/exec/scan/scanner_late_arrival_rf_test.cpp b/be/test/exec/scan/scanner_late_arrival_rf_test.cpp index 5971431d82e524..cd3bbc1d599d1a 100644 --- a/be/test/exec/scan/scanner_late_arrival_rf_test.cpp +++ b/be/test/exec/scan/scanner_late_arrival_rf_test.cpp @@ -42,6 +42,7 @@ #include "testutil/mock/mock_descriptors.h" #include "testutil/mock/mock_runtime_state.h" #include "testutil/mock/mock_slot_ref.h" +#include "util/defer_op.h" #include "util/raw_value.h" namespace doris { @@ -80,21 +81,23 @@ class TestScanner final : public Scanner { class LateBucketScanner final : public Scanner { public: - LateBucketScanner(RuntimeState* state, OlapScanLocalState* local_state, int64_t tablet_id, - bool has_matching_row, RuntimeProfile* profile, std::latch* prepare_started, - std::latch* filter_published) + LateBucketScanner(RuntimeState* state, OlapScanLocalState* local_state, int32_t bucket_seq, + int32_t bucket_num, bool has_matching_row, RuntimeProfile* profile, + std::latch* prepare_started, std::latch* filter_published) : Scanner(state, local_state, -1, profile), _olap_local_state(local_state), - _tablet_id(tablet_id), + _bucket_seq(bucket_seq), + _bucket_num(bucket_num), _has_matching_row(has_matching_row), _prepare_started(prepare_started), _filter_published(filter_published) {} bool is_pruned_by_runtime_filter() const override { - return _olap_local_state->_is_tablet_pruned_by_runtime_filter(1, _tablet_id); + return _olap_local_state->_is_tablet_pruned_by_runtime_filter(1, _bucket_seq, _bucket_num); } int read_calls() const { return _read_calls.load(); } + int open_calls() const { return _open_calls.load(); } protected: Status _prepare_impl() override { @@ -103,6 +106,11 @@ class LateBucketScanner final : public Scanner { return Scanner::_prepare_impl(); } + Status _open_impl(RuntimeState* state) override { + ++_open_calls; + return Scanner::_open_impl(state); + } + Status _get_block_impl(RuntimeState* /*state*/, Block* block, bool* eof) override { ++_read_calls; if (_returned) { @@ -121,11 +129,13 @@ class LateBucketScanner final : public Scanner { private: OlapScanLocalState* _olap_local_state; - int64_t _tablet_id; + int32_t _bucket_seq; + int32_t _bucket_num; bool _has_matching_row; std::latch* _prepare_started; std::latch* _filter_published; std::atomic _read_calls {0}; + std::atomic _open_calls {0}; bool _returned = false; }; @@ -249,8 +259,13 @@ TEST_F(ScannerLateArrivalRfTest, bucket_pruning_after_probe_tasks_start) { auto task_exec_ctx = std::make_shared(); state->set_task_execution_context(task_exec_ctx); for (int bucket_seq = 0; bucket_seq < bucket_num; ++bucket_seq) { - local_state->_rf_bucket_prune_ranges.push_back({100 + bucket_seq, bucket_seq, bucket_num}); + auto scan_range = std::make_unique(); + scan_range->__set_tablet_id(100 + bucket_seq); + scan_range->__set_bucket_seq(bucket_seq); + scan_range->__set_bucket_num(bucket_num); + local_state->_scan_ranges.push_back(std::move(scan_range)); } + local_state->_has_rf_bucket_prune_metadata = true; RuntimeProfile scan_profile("late bucket scan"); local_state->_buckets_pruned_by_rf_counter = ADD_COUNTER(&scan_profile, "BucketsPrunedByRuntimeFilter", TUnit::UNIT); @@ -267,7 +282,7 @@ TEST_F(ScannerLateArrivalRfTest, bucket_pruning_after_probe_tasks_start) { std::vector> scanners; for (int bucket_seq = 0; bucket_seq < bucket_num; ++bucket_seq) { auto scanner = std::make_shared( - state, local_state.get(), 100 + bucket_seq, bucket_seq == selected_bucket, + state, local_state.get(), bucket_seq, bucket_num, bucket_seq == selected_bucket, &scan_profile, &prepare_started, &filter_published); ASSERT_TRUE(scanner->init(state, {}).ok()); scanners.push_back(scanner); @@ -293,7 +308,26 @@ TEST_F(ScannerLateArrivalRfTest, bucket_pruning_after_probe_tasks_start) { } scanner_context->_in_flight_tasks_num = bucket_num; + std::shared_ptr producer; + ASSERT_TRUE(RuntimeFilterProducer::create(_query_ctx.get(), &desc, &producer).ok()); + ASSERT_TRUE(producer->init(1).ok()); + auto filter_column = ColumnInt32::create(); + filter_column->insert_value(filter_value); + ASSERT_TRUE(producer->insert(std::move(filter_column), 0).ok()); + producer->set_wrapper_state_and_ready_to_publish(RuntimeFilterWrapper::State::READY); + std::vector probe_threads; + bool filter_released = false; + Defer probe_thread_guard {[&] { + if (!filter_released) { + filter_published.count_down(); + } + for (auto& thread : probe_threads) { + if (thread.joinable()) { + thread.join(); + } + } + }}; for (const auto& task : tasks) { probe_threads.emplace_back([scanner_context, task] { ScannerScheduler::_scanner_scan(scanner_context, task); @@ -302,27 +336,25 @@ TEST_F(ScannerLateArrivalRfTest, bucket_pruning_after_probe_tasks_start) { // Every task has passed the scheduler's pre-prepare pruning check while the RF is not ready. prepare_started.wait(); - ASSERT_EQ(local_state->_rf_bucket_pruner.pruned_tablet_count(), 0); + EXPECT_EQ(local_state->_rf_bucket_pruner.pruned_tablet_count(), 0); - std::shared_ptr producer; - ASSERT_TRUE(RuntimeFilterProducer::create(_query_ctx.get(), &desc, &producer).ok()); - ASSERT_TRUE(producer->init(1).ok()); - auto filter_column = ColumnInt32::create(); - filter_column->insert_value(filter_value); - ASSERT_TRUE(producer->insert(std::move(filter_column), 0).ok()); - producer->set_wrapper_state_and_ready_to_publish(RuntimeFilterWrapper::State::READY); local_state->_helper._consumers[0]->signal(producer.get()); filter_published.count_down(); + filter_released = true; for (auto& thread : probe_threads) { - thread.join(); + if (thread.joinable()) { + thread.join(); + } } for (int bucket_seq = 0; bucket_seq < bucket_num; ++bucket_seq) { if (bucket_seq == selected_bucket) { + EXPECT_GT(scanners[bucket_seq]->open_calls(), 0); EXPECT_GT(scanners[bucket_seq]->read_calls(), 0); EXPECT_NE(tasks[bucket_seq]->cached_block, nullptr); } else { + EXPECT_EQ(scanners[bucket_seq]->open_calls(), 0); EXPECT_EQ(scanners[bucket_seq]->read_calls(), 0); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java index 32399e7b80af09..b682b329fff60f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java @@ -1486,12 +1486,6 @@ protected void toThrift(TPlanNode msg) { && hasRfDrivingPartitionPruning()) { setPartitionBoundariesForRuntimeFilter(msg.olap_scan_node); } - if (rfPruneCtx != null - && rfPruneCtx.getSessionVariable().isEnableRuntimeFilterBucketPrune() - && hasRfDrivingBucketPruning()) { - setRuntimeFilterBucketPruneParameters(); - } - super.toThrift(msg); } @@ -1551,7 +1545,10 @@ private boolean hasRfDrivingBucketPruning() { return false; } - private void setRuntimeFilterBucketPruneParameters() { + public void setRuntimeFilterBucketPruneParameters() { + if (!hasRfDrivingBucketPruning()) { + return; + } for (TScanRangeLocations locations : scanRangeLocations) { TPaloScanRange scanRange = locations.getScanRange().getPaloScanRange(); Long bucketInfo = tabletId2BucketInfo.get(scanRange.getTabletId()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/ThriftPlansBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/ThriftPlansBuilder.java index 1a239e3122a365..b459233ed95d21 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/ThriftPlansBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/ThriftPlansBuilder.java @@ -41,6 +41,7 @@ import org.apache.doris.planner.DataStreamSink; import org.apache.doris.planner.ExchangeNode; import org.apache.doris.planner.MultiCastDataSink; +import org.apache.doris.planner.OlapScanNode; import org.apache.doris.planner.OlapTableSink; import org.apache.doris.planner.PlanFragment; import org.apache.doris.planner.PlanFragmentId; @@ -117,6 +118,8 @@ public static Map plansToThr // we should set runtime predicate first, then we can use heap sort and to thrift setRuntimePredicateIfNeed(coordinatorContext.scanNodes); + setRuntimeFilterBucketPruneParametersIfNeeded( + coordinatorContext.scanNodes, coordinatorContext.connectContext); int broadcastRuntimeFilterProducerNum = coordinatorContext.connectContext == null ? 0 @@ -205,6 +208,19 @@ static void setRuntimePredicateIfNeed(Collection scanNodes) { } } + static void setRuntimeFilterBucketPruneParametersIfNeeded( + Collection scanNodes, ConnectContext connectContext) { + if (connectContext == null + || !connectContext.getSessionVariable().isEnableRuntimeFilterBucketPrune()) { + return; + } + for (ScanNode scanNode : scanNodes) { + if (scanNode instanceof OlapScanNode) { + ((OlapScanNode) scanNode).setRuntimeFilterBucketPruneParameters(); + } + } + } + private static Supplier> topNFilterToThrift(List topnFilters) { return Suppliers.memoize(() -> { if (CollectionUtils.isEmpty(topnFilters)) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/runtime/ThriftPlansBuilderTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/runtime/ThriftPlansBuilderTest.java index 62a83172f94194..1c3f63ab061b48 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/qe/runtime/ThriftPlansBuilderTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/runtime/ThriftPlansBuilderTest.java @@ -24,10 +24,13 @@ import org.apache.doris.nereids.trees.plans.distribute.worker.job.DefaultScanSource; import org.apache.doris.nereids.trees.plans.distribute.worker.job.LocalShuffleAssignedJob; import org.apache.doris.nereids.trees.plans.distribute.worker.job.UnassignedJob; +import org.apache.doris.planner.OlapScanNode; import org.apache.doris.planner.PlanNodeId; import org.apache.doris.planner.RecursiveCteScanNode; import org.apache.doris.planner.ScanNode; import org.apache.doris.planner.SortNode; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.SessionVariable; import org.apache.doris.thrift.TRecCTETarget; import org.apache.doris.thrift.TUniqueId; @@ -51,6 +54,21 @@ public void testSetRuntimePredicateForNonOlapScanNode() { Mockito.verify(sortNode).setHasRuntimePredicate(); } + @Test + public void testSetRuntimeFilterBucketPruneParametersOnceBeforeWorkerSerialization() { + OlapScanNode olapScanNode = Mockito.mock(OlapScanNode.class); + ScanNode otherScanNode = Mockito.mock(ScanNode.class); + ConnectContext connectContext = Mockito.mock(ConnectContext.class); + SessionVariable sessionVariable = Mockito.mock(SessionVariable.class); + Mockito.when(connectContext.getSessionVariable()).thenReturn(sessionVariable); + Mockito.when(sessionVariable.isEnableRuntimeFilterBucketPrune()).thenReturn(true); + + ThriftPlansBuilder.setRuntimeFilterBucketPruneParametersIfNeeded( + Arrays.asList(olapScanNode, otherScanNode), connectContext); + + Mockito.verify(olapScanNode).setRuntimeFilterBucketPruneParameters(); + } + @Test public void testBuildRecCTETargetsKeepsAllInstancesOnSameBackend() { DistributedPlanWorker worker = Mockito.mock(DistributedPlanWorker.class); From 276da584ad766a822b818df05b20f2775b74af4f Mon Sep 17 00:00:00 2001 From: happenlee Date: Fri, 14 Aug 2026 00:15:24 +0800 Subject: [PATCH 13/20] [fix](be) Fix runtime filter bucket pruning edge cases ### What problem does this PR solve? Issue Number: None Related PR: #65837 Problem Summary: Parallel scanners dropped bucket identity, multiple ready filters overcounted pruned tablets, and scanners pruned after prepare retained reader resources until query teardown. Preserve bucket metadata in the parallel scanner factory, count each newly pruned tablet once, and release prepared-but-unopened scanner resources immediately. ### Release note Fix runtime-filter bucket pruning for parallel scanners and release resources for scanners pruned before open. ### Check List (For Author) - Test: Unit Test - BE unit tests: RuntimeFilterBucketPrunerTest.* and ScannerLateArrivalRfTest.* (15 tests) - Format: build-support/check-format.sh - Static analysis: build-support/run-clang-tidy.sh on modified C++ files - Behavior changed: Yes. Late runtime filters can prune parallel scanners and prepared scanners release resources before open. - Does this need documentation: No --- be/src/exec/operator/olap_scan_operator.cpp | 6 +- .../runtime_filter_bucket_pruner.cpp | 6 +- be/src/exec/scan/olap_scanner.cpp | 17 ++ be/src/exec/scan/olap_scanner.h | 2 + be/src/exec/scan/parallel_scanner_builder.cpp | 4 + be/src/exec/scan/parallel_scanner_builder.h | 14 +- be/src/exec/scan/scanner.h | 8 + be/src/exec/scan/scanner_scheduler.cpp | 1 + .../runtime_filter_bucket_pruner_test.cpp | 23 +++ .../scan/scanner_late_arrival_rf_test.cpp | 177 ++++++++++++++++++ 10 files changed, 252 insertions(+), 6 deletions(-) diff --git a/be/src/exec/operator/olap_scan_operator.cpp b/be/src/exec/operator/olap_scan_operator.cpp index e2057690f9b0f6..b1fb21f764aaf1 100644 --- a/be/src/exec/operator/olap_scan_operator.cpp +++ b/be/src/exec/operator/olap_scan_operator.cpp @@ -745,9 +745,9 @@ Status OlapScanLocalState::_init_scanners(std::list* scanners) { key_ranges.emplace_back(range.get()); } - ParallelScannerBuilder scanner_builder(this, _tablets, _read_sources, _scanner_profile, - key_ranges, state(), p._limit, true, - p._olap_scan_node.is_preaggregation); + ParallelScannerBuilder scanner_builder(this, _tablets, _read_sources, _scan_ranges, + _scanner_profile, key_ranges, state(), p._limit, + true, p._olap_scan_node.is_preaggregation); int max_scanners_count = state()->parallel_scan_max_scanners_count(); diff --git a/be/src/exec/runtime_filter/runtime_filter_bucket_pruner.cpp b/be/src/exec/runtime_filter/runtime_filter_bucket_pruner.cpp index 88b9c7533f8d1b..f22c094baae9e7 100644 --- a/be/src/exec/runtime_filter/runtime_filter_bucket_pruner.cpp +++ b/be/src/exec/runtime_filter/runtime_filter_bucket_pruner.cpp @@ -102,6 +102,7 @@ Status RuntimeFilterBucketPruner::prune_by_runtime_filters( } } + int64_t current_filter_pruned_count = 0; std::unique_lock lock(_prune_mutex); for (const auto& range_ptr : ranges) { const auto& range = *range_ptr; @@ -110,7 +111,7 @@ Status RuntimeFilterBucketPruner::prune_by_runtime_filters( current_it->second.contains(range.bucket_seq); if (was_selected && !new_selected_buckets_by_num.at(range.bucket_num).contains(range.bucket_seq)) { - ++*newly_pruned_count; + ++current_filter_pruned_count; } } for (auto& [bucket_num, new_selected_buckets] : new_selected_buckets_by_num) { @@ -128,7 +129,8 @@ Status RuntimeFilterBucketPruner::prune_by_runtime_filters( } } } - _pruned_tablet_count += *newly_pruned_count; + *newly_pruned_count += current_filter_pruned_count; + _pruned_tablet_count += current_filter_pruned_count; } return Status::OK(); } diff --git a/be/src/exec/scan/olap_scanner.cpp b/be/src/exec/scan/olap_scanner.cpp index 57b6ba2c054149..4be77e71ed62b4 100644 --- a/be/src/exec/scan/olap_scanner.cpp +++ b/be/src/exec/scan/olap_scanner.cpp @@ -659,6 +659,23 @@ bool OlapScanner::is_pruned_by_runtime_filter() const { _tablet_reader_params.tablet->partition_id(), _bucket_seq, _bucket_num); } +void OlapScanner::release_prepared_resources() { + DORIS_CHECK(_has_prepared); + DORIS_CHECK(!_is_open); + + _tablet_reader.reset(); + auto tablet = std::move(_tablet_reader_params.tablet); + _tablet_reader_params = TabletReader::ReaderParams {}; + _tablet_reader_params.tablet = std::move(tablet); + _common_expr_ctxs_push_down.clear(); + _slot_id_to_virtual_column_expr.clear(); + _virtual_column_exprs.clear(); + _score_runtime.reset(); + _ann_topn_runtime.reset(); + + Scanner::release_prepared_resources(); +} + doris::TabletStorageType OlapScanner::get_storage_type() { if (config::is_cloud_mode()) { // we don't have cold storage in cloud mode, all storage is treated as local diff --git a/be/src/exec/scan/olap_scanner.h b/be/src/exec/scan/olap_scanner.h index 30d2040b7d3587..0bdeb634bdbf36 100644 --- a/be/src/exec/scan/olap_scanner.h +++ b/be/src/exec/scan/olap_scanner.h @@ -98,6 +98,8 @@ class OlapScanner : public Scanner { bool is_pruned_by_runtime_filter() const override; + void release_prepared_resources() override; + void update_realtime_counters() override; protected: diff --git a/be/src/exec/scan/parallel_scanner_builder.cpp b/be/src/exec/scan/parallel_scanner_builder.cpp index 9ae9a8bb40342d..54f4722a216d6d 100644 --- a/be/src/exec/scan/parallel_scanner_builder.cpp +++ b/be/src/exec/scan/parallel_scanner_builder.cpp @@ -296,6 +296,8 @@ Status ParallelScannerBuilder::_load() { std::shared_ptr ParallelScannerBuilder::_build_scanner( BaseTabletSPtr tablet, int64_t version, const std::vector& key_ranges, TabletReadSource&& read_source, io::FileCacheStatistics&& initial_file_cache_stats) { + auto bucket_identity = _bucket_identities.find(tablet->tablet_id()); + DORIS_CHECK(bucket_identity != _bucket_identities.end()); OlapScanner::Params params { .state = _state, .profile = _scanner_profile.get(), @@ -308,6 +310,8 @@ std::shared_ptr ParallelScannerBuilder::_build_scanner( .aggregation = _is_preaggregation, .read_row_binlog = false, .binlog_scan_type = TBinlogScanType::NONE, + .bucket_seq = bucket_identity->second.first, + .bucket_num = bucket_identity->second.second, .start_tso = std::nullopt, .end_tso = std::nullopt, }; diff --git a/be/src/exec/scan/parallel_scanner_builder.h b/be/src/exec/scan/parallel_scanner_builder.h index 82b63b07824c3a..59b73bc7e72a7d 100644 --- a/be/src/exec/scan/parallel_scanner_builder.h +++ b/be/src/exec/scan/parallel_scanner_builder.h @@ -43,6 +43,7 @@ class ParallelScannerBuilder { ParallelScannerBuilder(OlapScanLocalState* parent, const std::vector& tablets, std::vector& read_sources, + const std::vector>& scan_ranges, const std::shared_ptr& profile, const std::vector& key_ranges, RuntimeState* state, int64_t limit, bool is_dup_mow_key, bool is_preaggregation) @@ -54,7 +55,17 @@ class ParallelScannerBuilder { _is_preaggregation(is_preaggregation), _tablets(tablets.cbegin(), tablets.cend()), _key_ranges(key_ranges.cbegin(), key_ranges.cend()), - _read_sources(read_sources) {} + _read_sources(read_sources) { + DORIS_CHECK_EQ(_tablets.size(), scan_ranges.size()); + for (size_t i = 0; i < _tablets.size(); ++i) { + DORIS_CHECK(scan_ranges[i] != nullptr); + DORIS_CHECK_EQ(_tablets[i].tablet->tablet_id(), scan_ranges[i]->tablet_id); + auto [_, inserted] = _bucket_identities.try_emplace(scan_ranges[i]->tablet_id, + scan_ranges[i]->bucket_seq, + scan_ranges[i]->bucket_num); + DORIS_CHECK(inserted); + } + } Status build_scanners(std::list& scanners); @@ -111,6 +122,7 @@ class ParallelScannerBuilder { bool _is_preaggregation; std::vector _tablets; std::vector _key_ranges; + std::unordered_map> _bucket_identities; std::unordered_map _all_read_sources; std::vector& _read_sources; }; diff --git a/be/src/exec/scan/scanner.h b/be/src/exec/scan/scanner.h index 3ec462dbd3c574..c63d94122e0f86 100644 --- a/be/src/exec/scan/scanner.h +++ b/be/src/exec/scan/scanner.h @@ -214,6 +214,14 @@ class Scanner { // Returns true if this scanner's scan range has been pruned by a runtime filter. virtual bool is_pruned_by_runtime_filter() const { return false; } + // Releases resources acquired by prepare() when runtime-filter pruning makes open() + // unnecessary. The scanner will not be scheduled again after this call. + virtual void release_prepared_resources() { + DORIS_CHECK(_has_prepared); + DORIS_CHECK(!_is_open); + _has_prepared = false; + } + bool need_to_close() const { return _need_to_close; } void mark_to_need_to_close() { diff --git a/be/src/exec/scan/scanner_scheduler.cpp b/be/src/exec/scan/scanner_scheduler.cpp index 560d85b6d49e97..bde97e955e2438 100644 --- a/be/src/exec/scan/scanner_scheduler.cpp +++ b/be/src/exec/scan/scanner_scheduler.cpp @@ -204,6 +204,7 @@ void ScannerScheduler::_scanner_scan(std::shared_ptr ctx, if (!eos && !scanner->is_open()) { append_late_arrival_runtime_filter(); if (scanner->is_pruned_by_runtime_filter()) { + scanner->release_prepared_resources(); eos = true; } } diff --git a/be/test/exec/runtime_filter/runtime_filter_bucket_pruner_test.cpp b/be/test/exec/runtime_filter/runtime_filter_bucket_pruner_test.cpp index 412fc8ca557354..59737c0c3d6de1 100644 --- a/be/test/exec/runtime_filter/runtime_filter_bucket_pruner_test.cpp +++ b/be/test/exec/runtime_filter/runtime_filter_bucket_pruner_test.cpp @@ -215,6 +215,29 @@ TEST_F(RuntimeFilterBucketPrunerTest, ExactInKeepsOnlyMatchingBucket) { EXPECT_EQ(newly_pruned, 0); } +TEST_F(RuntimeFilterBucketPrunerTest, MultipleFiltersCountEachPrunedBucketOnce) { + constexpr int first_filter_id = 17; + constexpr int second_filter_id = 18; + constexpr int32_t first_value = 10; + int32_t second_value = first_value + 1; + while (bucket_for_value(second_value, 4) == bucket_for_value(first_value, 4)) { + ++second_value; + } + VExprContextSPtrs conjuncts {make_in_conjunct(first_filter_id, {first_value}), + make_in_conjunct(second_filter_id, {second_value})}; + std::vector rf_descs {bucket_prune_desc(first_filter_id), + bucket_prune_desc(second_filter_id)}; + + RuntimeFilterBucketPruner pruner; + int64_t newly_pruned = 0; + ASSERT_TRUE(pruner.prune_by_runtime_filters(four_bucket_ranges(), conjuncts, rf_descs, + SCAN_NODE_ID, /*max_in_num=*/1024, &newly_pruned) + .ok()); + + EXPECT_EQ(newly_pruned, 4); + EXPECT_EQ(pruner.pruned_tablet_count(), 4); +} + TEST_F(RuntimeFilterBucketPrunerTest, NonNullableTargetConservativelyKeepsNullBucket) { constexpr int filter_id = 14; constexpr int32_t value = 1; diff --git a/be/test/exec/scan/scanner_late_arrival_rf_test.cpp b/be/test/exec/scan/scanner_late_arrival_rf_test.cpp index cd3bbc1d599d1a..c420ccec3c3853 100644 --- a/be/test/exec/scan/scanner_late_arrival_rf_test.cpp +++ b/be/test/exec/scan/scanner_late_arrival_rf_test.cpp @@ -32,11 +32,15 @@ #include "exec/runtime_filter/runtime_filter_consumer_helper.h" #include "exec/runtime_filter/runtime_filter_producer.h" #include "exec/runtime_filter/runtime_filter_test_utils.h" +#include "exec/scan/parallel_scanner_builder.h" #include "exec/scan/scanner.h" #include "exec/scan/scanner_context.h" #include "exec/scan/scanner_scheduler.h" #include "runtime/descriptors.h" #include "runtime/exec_env.h" +#include "storage/iterator/block_reader.h" +#include "storage/rowset/rowset_writer.h" +#include "storage/tablet/tablet_meta.h" #include "testutil/column_helper.h" #include "testutil/desc_tbl_builder.h" #include "testutil/mock/mock_descriptors.h" @@ -79,6 +83,60 @@ class TestScanner final : public Scanner { std::list _blocks; }; +class FakeTablet final : public BaseTablet { +public: + FakeTablet(int64_t partition_id, int64_t tablet_id) + : BaseTablet(create_meta(partition_id, tablet_id)) {} + + std::string tablet_path() const override { return ""; } + + bool exceed_version_limit(int32_t /*limit*/) override { return false; } + + Result> create_rowset_writer(RowsetWriterContext& /*context*/, + bool /*vertical*/) override { + return ResultError(Status::NotSupported("fake tablet")); + } + + Result> create_transient_rowset_writer( + const Rowset& /*rowset*/, std::shared_ptr /*partial_update_info*/, + int64_t /*txn_expiration*/ = 0) override { + return ResultError(Status::NotSupported("fake tablet")); + } + + Status capture_rs_readers(const Version& /*spec_version*/, + std::vector* /*rs_splits*/, + const CaptureRowsetOps& /*opts*/) override { + return Status::NotSupported("fake tablet"); + } + + Status save_delete_bitmap(const TabletTxnInfo* /*txn_info*/, int64_t /*txn_id*/, + DeleteBitmapPtr /*delete_bitmap*/, RowsetWriter* /*rowset_writer*/, + const RowsetIdUnorderedSet& /*cur_rowset_ids*/, + int64_t /*lock_id*/ = -1, + int64_t /*next_visible_version*/ = -1) override { + return Status::NotSupported("fake tablet"); + } + + CalcDeleteBitmapExecutor* calc_delete_bitmap_executor() override { return nullptr; } + + void clear_cache() override {} + + Versions calc_missed_versions(int64_t /*spec_version*/, + Versions /*existing_versions*/) const override { + return {}; + } + + size_t tablet_footprint() override { return 0; } + +private: + static TabletMetaSharedPtr create_meta(int64_t partition_id, int64_t tablet_id) { + auto meta = std::make_shared(std::make_shared()); + meta->_partition_id = partition_id; + meta->_tablet_id = tablet_id; + return meta; + } +}; + class LateBucketScanner final : public Scanner { public: LateBucketScanner(RuntimeState* state, OlapScanLocalState* local_state, int32_t bucket_seq, @@ -98,6 +156,12 @@ class LateBucketScanner final : public Scanner { int read_calls() const { return _read_calls.load(); } int open_calls() const { return _open_calls.load(); } + int release_calls() const { return _release_calls.load(); } + + void release_prepared_resources() override { + ++_release_calls; + Scanner::release_prepared_resources(); + } protected: Status _prepare_impl() override { @@ -136,6 +200,7 @@ class LateBucketScanner final : public Scanner { std::latch* _filter_published; std::atomic _read_calls {0}; std::atomic _open_calls {0}; + std::atomic _release_calls {0}; bool _returned = false; }; @@ -352,10 +417,12 @@ TEST_F(ScannerLateArrivalRfTest, bucket_pruning_after_probe_tasks_start) { if (bucket_seq == selected_bucket) { EXPECT_GT(scanners[bucket_seq]->open_calls(), 0); EXPECT_GT(scanners[bucket_seq]->read_calls(), 0); + EXPECT_EQ(scanners[bucket_seq]->release_calls(), 0); EXPECT_NE(tasks[bucket_seq]->cached_block, nullptr); } else { EXPECT_EQ(scanners[bucket_seq]->open_calls(), 0); EXPECT_EQ(scanners[bucket_seq]->read_calls(), 0); + EXPECT_EQ(scanners[bucket_seq]->release_calls(), 1); } } int64_t result_rows = 0; @@ -370,6 +437,116 @@ TEST_F(ScannerLateArrivalRfTest, bucket_pruning_after_probe_tasks_start) { ASSERT_LT(local_state->_rf_bucket_pruner.pruned_tablet_count(), bucket_num); } +TEST_F(ScannerLateArrivalRfTest, parallel_scanner_factory_preserves_bucket_identity) { + constexpr int scan_node_id = 0; + constexpr int32_t bucket_seq = 3; + constexpr int32_t bucket_num = 8; + constexpr int64_t partition_id = 10; + constexpr int64_t tablet_id = 20; + + ObjectPool pool; + DescriptorTblBuilder desc_builder(&pool); + desc_builder.declare_tuple() << TupleDescBuilder::SlotType {std::make_shared(), + "dist_col"}; + DescriptorTbl* desc_tbl = desc_builder.build(); + ASSERT_NE(desc_tbl, nullptr); + + TOlapScanNode olap_scan_node; + olap_scan_node.__set_tuple_id(0); + olap_scan_node.__set_keyType(TKeysType::DUP_KEYS); + TPlanNode plan_node; + plan_node.__set_node_id(scan_node_id); + plan_node.__set_node_type(TPlanNodeType::OLAP_SCAN_NODE); + plan_node.__set_num_children(0); + plan_node.__set_limit(-1); + plan_node.__set_row_tuples({0}); + plan_node.__set_olap_scan_node(olap_scan_node); + + auto op = std::make_shared(&pool, plan_node, 0, *desc_tbl, bucket_num, + TQueryCacheParam {}); + auto* state = _runtime_states[0].get(); + state->set_desc_tbl(desc_tbl); + auto local_state = OlapScanLocalState::create_shared(state, op.get()); + local_state->_has_rf_bucket_prune_metadata = true; + local_state->_rf_bucket_pruner._selected_buckets_by_num[bucket_num] = {bucket_seq - 1}; + + auto tablet = std::make_shared(partition_id, tablet_id); + std::vector tablets {{tablet, 1}}; + std::vector read_sources(1); + std::vector> scan_ranges; + auto scan_range = std::make_unique(); + scan_range->__set_tablet_id(tablet_id); + scan_range->__set_bucket_seq(bucket_seq); + scan_range->__set_bucket_num(bucket_num); + scan_ranges.push_back(std::move(scan_range)); + auto profile = std::make_shared("parallel scanner bucket identity"); + ParallelScannerBuilder builder(local_state.get(), tablets, read_sources, scan_ranges, profile, + {}, state, -1, true, true); + + auto scanner = + builder._build_scanner(tablet, 1, {}, TabletReadSource {}, io::FileCacheStatistics {}); + EXPECT_EQ(scanner->_bucket_seq, bucket_seq); + EXPECT_EQ(scanner->_bucket_num, bucket_num); + EXPECT_TRUE(scanner->is_pruned_by_runtime_filter()); +} + +TEST_F(ScannerLateArrivalRfTest, olap_scanner_releases_prepared_resources_before_open) { + constexpr int scan_node_id = 0; + constexpr int64_t partition_id = 10; + constexpr int64_t tablet_id = 20; + + ObjectPool pool; + DescriptorTblBuilder desc_builder(&pool); + desc_builder.declare_tuple() << TupleDescBuilder::SlotType {std::make_shared(), + "dist_col"}; + DescriptorTbl* desc_tbl = desc_builder.build(); + ASSERT_NE(desc_tbl, nullptr); + + TOlapScanNode olap_scan_node; + olap_scan_node.__set_tuple_id(0); + olap_scan_node.__set_keyType(TKeysType::DUP_KEYS); + TPlanNode plan_node; + plan_node.__set_node_id(scan_node_id); + plan_node.__set_node_type(TPlanNodeType::OLAP_SCAN_NODE); + plan_node.__set_num_children(0); + plan_node.__set_limit(-1); + plan_node.__set_row_tuples({0}); + plan_node.__set_olap_scan_node(olap_scan_node); + + auto op = std::make_shared(&pool, plan_node, 0, *desc_tbl, 1, + TQueryCacheParam {}); + auto* state = _runtime_states[0].get(); + state->set_desc_tbl(desc_tbl); + auto local_state = OlapScanLocalState::create_shared(state, op.get()); + auto tablet = std::make_shared(partition_id, tablet_id); + RuntimeProfile profile("prepared scanner cleanup"); + OlapScanner::Params params; + params.state = state; + params.profile = &profile; + params.tablet = tablet; + params.version = 1; + params.limit = -1; + params.aggregation = true; + auto scanner = OlapScanner::create_shared(local_state.get(), std::move(params)); + ASSERT_TRUE(scanner->Scanner::_prepare_impl().ok()); + scanner->_tablet_reader = std::make_unique(); + scanner->_tablet_reader_params.rs_splits.emplace_back(); + + ASSERT_TRUE(scanner->has_prepared()); + ASSERT_FALSE(scanner->is_open()); + ASSERT_NE(scanner->_tablet_reader, nullptr); + ASSERT_FALSE(scanner->_tablet_reader_params.rs_splits.empty()); + + scanner->release_prepared_resources(); + + EXPECT_FALSE(scanner->has_prepared()); + EXPECT_FALSE(scanner->is_open()); + EXPECT_EQ(scanner->_tablet_reader, nullptr); + EXPECT_TRUE(scanner->_tablet_reader_params.rs_splits.empty()); + EXPECT_EQ(scanner->_tablet_reader_params.tablet, tablet); + scanner->update_realtime_counters(); +} + TEST(ScannerProjectionTest, merges_padding_block_when_limit_eos_without_extra_flag) { ObjectPool pool; auto data_type = std::make_shared(); From a6b2e8c5f50029e2a7b9e101baf3c2963039ed6b Mon Sep 17 00:00:00 2001 From: happenlee Date: Fri, 14 Aug 2026 02:10:15 +0800 Subject: [PATCH 14/20] [fix](runtime filter) Address bucket pruning review feedback ### What problem does this PR solve? Issue Number: None Related PR: #65837 Problem Summary: Bucket pruning could leave constructor-owned OLAP reader inputs attached when bounded concurrency delays a scanner until after a late filter arrives. The FE classifier could also accept a computed MV column when its alias and type collided with the base distribution column, and the parallel scanner builder duplicated bucket identity in a node-based map for every range. Release unopened scanner resources before or after prepare, require direct base SlotRef identity for MV columns, and read bucket identity from aligned owned scan ranges without a duplicate map. ### Release note None ### Check List (For Author) - Test: Unit Test - FE: RuntimeFilterBucketPruneClassifierTest (12 tests) - BE: RuntimeFilterBucketPrunerTest and ScannerLateArrivalRfTest (17 tests) - Formatting: build-support/check-format.sh - Static analysis: build-support/run-clang-tidy.sh on modified C++ files - Behavior changed: Yes. Unsafe computed MV bucket pruning is disabled and unopened pruned scanners release resources promptly. - Does this need documentation: No --- be/src/exec/scan/olap_scanner.cpp | 7 +- be/src/exec/scan/olap_scanner.h | 2 +- be/src/exec/scan/parallel_scanner_builder.cpp | 23 +- be/src/exec/scan/parallel_scanner_builder.h | 8 +- be/src/exec/scan/scanner.h | 7 +- be/src/exec/scan/scanner_scheduler.cpp | 9 +- .../scan/scanner_late_arrival_rf_test.cpp | 218 +++++++++++++++--- .../RuntimeFilterBucketPruneClassifier.java | 24 +- ...untimeFilterBucketPruneClassifierTest.java | 19 ++ 9 files changed, 260 insertions(+), 57 deletions(-) diff --git a/be/src/exec/scan/olap_scanner.cpp b/be/src/exec/scan/olap_scanner.cpp index 4be77e71ed62b4..1359f1880d0aa2 100644 --- a/be/src/exec/scan/olap_scanner.cpp +++ b/be/src/exec/scan/olap_scanner.cpp @@ -659,21 +659,18 @@ bool OlapScanner::is_pruned_by_runtime_filter() const { _tablet_reader_params.tablet->partition_id(), _bucket_seq, _bucket_num); } -void OlapScanner::release_prepared_resources() { - DORIS_CHECK(_has_prepared); +void OlapScanner::release_unopened_resources() { DORIS_CHECK(!_is_open); _tablet_reader.reset(); - auto tablet = std::move(_tablet_reader_params.tablet); _tablet_reader_params = TabletReader::ReaderParams {}; - _tablet_reader_params.tablet = std::move(tablet); _common_expr_ctxs_push_down.clear(); _slot_id_to_virtual_column_expr.clear(); _virtual_column_exprs.clear(); _score_runtime.reset(); _ann_topn_runtime.reset(); - Scanner::release_prepared_resources(); + Scanner::release_unopened_resources(); } doris::TabletStorageType OlapScanner::get_storage_type() { diff --git a/be/src/exec/scan/olap_scanner.h b/be/src/exec/scan/olap_scanner.h index 0bdeb634bdbf36..9eb5f867be7a46 100644 --- a/be/src/exec/scan/olap_scanner.h +++ b/be/src/exec/scan/olap_scanner.h @@ -98,7 +98,7 @@ class OlapScanner : public Scanner { bool is_pruned_by_runtime_filter() const override; - void release_prepared_resources() override; + void release_unopened_resources() override; void update_realtime_counters() override; diff --git a/be/src/exec/scan/parallel_scanner_builder.cpp b/be/src/exec/scan/parallel_scanner_builder.cpp index 54f4722a216d6d..288c51e1802a49 100644 --- a/be/src/exec/scan/parallel_scanner_builder.cpp +++ b/be/src/exec/scan/parallel_scanner_builder.cpp @@ -80,7 +80,9 @@ Status ParallelScannerBuilder::build_scanners(std::list& scanners) Status ParallelScannerBuilder::_build_scanners_by_rowid(std::list& scanners) { DCHECK_GE(_rows_per_scanner, _min_rows_per_scanner); - for (auto&& [tablet, version] : _tablets) { + for (size_t tablet_idx = 0; tablet_idx < _tablets.size(); ++tablet_idx) { + auto&& [tablet, version] = _tablets[tablet_idx]; + const auto& scan_range = *_scan_ranges[tablet_idx]; DCHECK(_all_read_sources.contains(tablet->tablet_id())); auto& entire_read_source = _all_read_sources[tablet->tablet_id()]; @@ -142,7 +144,7 @@ Status ParallelScannerBuilder::_build_scanners_by_rowid(std::list& partitial_read_source.rs_splits.emplace_back(std::move(split)); scanners.emplace_back(_build_scanner( - tablet, version, _key_ranges, + tablet, version, _key_ranges, scan_range, {.rs_splits = std::move(partitial_read_source.rs_splits), .delete_predicates = entire_read_source.delete_predicates, .delete_bitmap = entire_read_source.delete_bitmap}, @@ -189,7 +191,7 @@ Status ParallelScannerBuilder::_build_scanners_by_rowid(std::list& } #endif scanners.emplace_back( - _build_scanner(tablet, version, _key_ranges, + _build_scanner(tablet, version, _key_ranges, scan_range, {.rs_splits = std::move(partitial_read_source.rs_splits), .delete_predicates = entire_read_source.delete_predicates, .delete_bitmap = entire_read_source.delete_bitmap}, @@ -208,7 +210,9 @@ Status ParallelScannerBuilder::_build_scanners_by_rowid(std::list& Status ParallelScannerBuilder::_build_scanners_by_per_segment(std::list& scanners) { DCHECK_GE(_rows_per_scanner, _min_rows_per_scanner); - for (auto&& [tablet, version] : _tablets) { + for (size_t tablet_idx = 0; tablet_idx < _tablets.size(); ++tablet_idx) { + auto&& [tablet, version] = _tablets[tablet_idx]; + const auto& scan_range = *_scan_ranges[tablet_idx]; DCHECK(_all_read_sources.contains(tablet->tablet_id())); auto& entire_read_source = _all_read_sources[tablet->tablet_id()]; @@ -240,7 +244,7 @@ Status ParallelScannerBuilder::_build_scanners_by_per_segment(std::list ParallelScannerBuilder::_build_scanner( BaseTabletSPtr tablet, int64_t version, const std::vector& key_ranges, - TabletReadSource&& read_source, io::FileCacheStatistics&& initial_file_cache_stats) { - auto bucket_identity = _bucket_identities.find(tablet->tablet_id()); - DORIS_CHECK(bucket_identity != _bucket_identities.end()); + const TPaloScanRange& scan_range, TabletReadSource&& read_source, + io::FileCacheStatistics&& initial_file_cache_stats) { OlapScanner::Params params { .state = _state, .profile = _scanner_profile.get(), @@ -310,8 +313,8 @@ std::shared_ptr ParallelScannerBuilder::_build_scanner( .aggregation = _is_preaggregation, .read_row_binlog = false, .binlog_scan_type = TBinlogScanType::NONE, - .bucket_seq = bucket_identity->second.first, - .bucket_num = bucket_identity->second.second, + .bucket_seq = scan_range.bucket_seq, + .bucket_num = scan_range.bucket_num, .start_tso = std::nullopt, .end_tso = std::nullopt, }; diff --git a/be/src/exec/scan/parallel_scanner_builder.h b/be/src/exec/scan/parallel_scanner_builder.h index 59b73bc7e72a7d..60ee95fb267e82 100644 --- a/be/src/exec/scan/parallel_scanner_builder.h +++ b/be/src/exec/scan/parallel_scanner_builder.h @@ -55,15 +55,12 @@ class ParallelScannerBuilder { _is_preaggregation(is_preaggregation), _tablets(tablets.cbegin(), tablets.cend()), _key_ranges(key_ranges.cbegin(), key_ranges.cend()), + _scan_ranges(scan_ranges), _read_sources(read_sources) { DORIS_CHECK_EQ(_tablets.size(), scan_ranges.size()); for (size_t i = 0; i < _tablets.size(); ++i) { DORIS_CHECK(scan_ranges[i] != nullptr); DORIS_CHECK_EQ(_tablets[i].tablet->tablet_id(), scan_ranges[i]->tablet_id); - auto [_, inserted] = _bucket_identities.try_emplace(scan_ranges[i]->tablet_id, - scan_ranges[i]->bucket_seq, - scan_ranges[i]->bucket_num); - DORIS_CHECK(inserted); } } @@ -87,6 +84,7 @@ class ParallelScannerBuilder { std::shared_ptr _build_scanner(BaseTabletSPtr tablet, int64_t version, const std::vector& key_ranges, + const TPaloScanRange& scan_range, TabletReadSource&& read_source, io::FileCacheStatistics&& initial_file_cache_stats); @@ -122,7 +120,7 @@ class ParallelScannerBuilder { bool _is_preaggregation; std::vector _tablets; std::vector _key_ranges; - std::unordered_map> _bucket_identities; + const std::vector>& _scan_ranges; std::unordered_map _all_read_sources; std::vector& _read_sources; }; diff --git a/be/src/exec/scan/scanner.h b/be/src/exec/scan/scanner.h index c63d94122e0f86..927472b2fa207c 100644 --- a/be/src/exec/scan/scanner.h +++ b/be/src/exec/scan/scanner.h @@ -214,10 +214,9 @@ class Scanner { // Returns true if this scanner's scan range has been pruned by a runtime filter. virtual bool is_pruned_by_runtime_filter() const { return false; } - // Releases resources acquired by prepare() when runtime-filter pruning makes open() - // unnecessary. The scanner will not be scheduled again after this call. - virtual void release_prepared_resources() { - DORIS_CHECK(_has_prepared); + // Releases resources owned by a scanner that runtime-filter pruning makes unnecessary before + // open(). The scanner will not be scheduled again after this call. + virtual void release_unopened_resources() { DORIS_CHECK(!_is_open); _has_prepared = false; } diff --git a/be/src/exec/scan/scanner_scheduler.cpp b/be/src/exec/scan/scanner_scheduler.cpp index bde97e955e2438..bc53d83e850d58 100644 --- a/be/src/exec/scan/scanner_scheduler.cpp +++ b/be/src/exec/scan/scanner_scheduler.cpp @@ -190,7 +190,12 @@ void ScannerScheduler::_scanner_scan(std::shared_ptr ctx, // so better to also check low memory and clear free blocks here. if (ctx->low_memory_mode()) { ctx->clear_free_blocks(); } - if (scanner->is_pruned_by_runtime_filter()) { eos = true; } + if (scanner->is_pruned_by_runtime_filter()) { + if (!scanner->is_open()) { + scanner->release_unopened_resources(); + } + eos = true; + } if (!eos && !scanner->has_prepared()) { status = scanner->prepare(); @@ -204,7 +209,7 @@ void ScannerScheduler::_scanner_scan(std::shared_ptr ctx, if (!eos && !scanner->is_open()) { append_late_arrival_runtime_filter(); if (scanner->is_pruned_by_runtime_filter()) { - scanner->release_prepared_resources(); + scanner->release_unopened_resources(); eos = true; } } diff --git a/be/test/exec/scan/scanner_late_arrival_rf_test.cpp b/be/test/exec/scan/scanner_late_arrival_rf_test.cpp index c420ccec3c3853..f6c99795d758f9 100644 --- a/be/test/exec/scan/scanner_late_arrival_rf_test.cpp +++ b/be/test/exec/scan/scanner_late_arrival_rf_test.cpp @@ -39,6 +39,7 @@ #include "runtime/descriptors.h" #include "runtime/exec_env.h" #include "storage/iterator/block_reader.h" +#include "storage/rowset/rowset_meta.h" #include "storage/rowset/rowset_writer.h" #include "storage/tablet/tablet_meta.h" #include "testutil/column_helper.h" @@ -156,15 +157,17 @@ class LateBucketScanner final : public Scanner { int read_calls() const { return _read_calls.load(); } int open_calls() const { return _open_calls.load(); } + int prepare_calls() const { return _prepare_calls.load(); } int release_calls() const { return _release_calls.load(); } - void release_prepared_resources() override { + void release_unopened_resources() override { ++_release_calls; - Scanner::release_prepared_resources(); + Scanner::release_unopened_resources(); } protected: Status _prepare_impl() override { + ++_prepare_calls; _prepare_started->count_down(); _filter_published->wait(); return Scanner::_prepare_impl(); @@ -200,6 +203,7 @@ class LateBucketScanner final : public Scanner { std::latch* _filter_published; std::atomic _read_calls {0}; std::atomic _open_calls {0}; + std::atomic _prepare_calls {0}; std::atomic _release_calls {0}; bool _returned = false; }; @@ -437,6 +441,105 @@ TEST_F(ScannerLateArrivalRfTest, bucket_pruning_after_probe_tasks_start) { ASSERT_LT(local_state->_rf_bucket_pruner.pruned_tablet_count(), bucket_num); } +TEST_F(ScannerLateArrivalRfTest, bounded_concurrency_prunes_scanner_before_first_schedule) { + constexpr int scan_node_id = 0; + constexpr int32_t bucket_num = 2; + constexpr int32_t active_bucket = 0; + constexpr int32_t pending_bucket = 1; + + ObjectPool pool; + DescriptorTblBuilder desc_builder(&pool); + desc_builder.declare_tuple() << TupleDescBuilder::SlotType {std::make_shared(), + "dist_col"}; + DescriptorTbl* desc_tbl = desc_builder.build(); + ASSERT_NE(desc_tbl, nullptr); + + TOlapScanNode olap_scan_node; + olap_scan_node.__set_tuple_id(0); + olap_scan_node.__set_keyType(TKeysType::DUP_KEYS); + TPlanNode plan_node; + plan_node.__set_node_id(scan_node_id); + plan_node.__set_node_type(TPlanNodeType::OLAP_SCAN_NODE); + plan_node.__set_num_children(0); + plan_node.__set_limit(-1); + plan_node.__set_row_tuples({0}); + plan_node.__set_olap_scan_node(olap_scan_node); + + auto op = std::make_shared(&pool, plan_node, 0, *desc_tbl, bucket_num, + TQueryCacheParam {}); + auto* state = _runtime_states[0].get(); + state->set_desc_tbl(desc_tbl); + auto task_exec_ctx = std::make_shared(); + state->set_task_execution_context(task_exec_ctx); + auto local_state = OlapScanLocalState::create_shared(state, op.get()); + local_state->_has_rf_bucket_prune_metadata = true; + + RuntimeProfile scan_profile("bounded late bucket scan"); + local_state->_scan_timer = ADD_TIMER(&scan_profile, "ScannerGetBlockTime"); + local_state->_scan_cpu_timer = ADD_TIMER(&scan_profile, "ScannerCpuTime"); + local_state->_filter_timer = ADD_TIMER(&scan_profile, "ScannerFilterTime"); + local_state->_rows_read_counter = ADD_COUNTER(&scan_profile, "RowsRead", TUnit::UNIT); + + std::latch prepare_started(2); + std::latch filter_published(0); + auto active_scanner = std::make_shared( + state, local_state.get(), active_bucket, bucket_num, true, &scan_profile, + &prepare_started, &filter_published); + auto pending_scanner = std::make_shared( + state, local_state.get(), pending_bucket, bucket_num, false, &scan_profile, + &prepare_started, &filter_published); + ASSERT_TRUE(active_scanner->init(state, {}).ok()); + ASSERT_TRUE(pending_scanner->init(state, {}).ok()); + + ScannerSPtr pending_scanner_base = pending_scanner; + ScannerSPtr active_scanner_base = active_scanner; + std::list> scanner_delegates; + scanner_delegates.push_back(std::make_shared(pending_scanner_base)); + scanner_delegates.push_back(std::make_shared(active_scanner_base)); + + auto dependency = Dependency::create_shared(0, 0, "bounded late bucket scan dependency"); + std::atomic shared_limit {-1}; + auto scanner_context = ScannerContext::create_shared( + state, local_state.get(), desc_tbl->get_tuple_descriptor(0), nullptr, scanner_delegates, + -1, dependency, &shared_limit, nullptr, nullptr, 0, false, 1); + scanner_context->_newly_create_free_blocks_num = + ADD_COUNTER(&scan_profile, "NewlyCreatedFreeBlocks", TUnit::UNIT); + scanner_context->_scanner_memory_used_counter = + ADD_COUNTER(&scan_profile, "ScannerMemoryUsed", TUnit::BYTES); + scanner_context->_max_bytes_in_queue = 10 * 1024 * 1024; + + auto active_task = scanner_context->_pull_next_scan_task(nullptr, 0); + ASSERT_NE(active_task, nullptr); + ASSERT_EQ(active_task->scanner.lock()->_scanner, active_scanner); + EXPECT_EQ(scanner_context->_pull_next_scan_task(nullptr, 1), nullptr); + + active_task->set_state(ScanTask::State::IN_FLIGHT); + scanner_context->_in_flight_tasks_num = 1; + ScannerScheduler::_scanner_scan(scanner_context, active_task); + ASSERT_EQ(active_scanner->prepare_calls(), 1); + ASSERT_EQ(active_scanner->open_calls(), 1); + ASSERT_EQ(scanner_context->_in_flight_tasks_num, 0); + ASSERT_EQ(scanner_context->_completed_tasks.size(), 1); + scanner_context->_completed_tasks.clear(); + + // Model the active scanner applying a newly published filter before the pending scanner gets + // its sole concurrency slot, so the shared pruner already excludes the pending bucket. + local_state->_rf_bucket_pruner._selected_buckets_by_num[bucket_num] = {active_bucket}; + + auto pending_task = scanner_context->_pull_next_scan_task(nullptr, 0); + ASSERT_NE(pending_task, nullptr); + ASSERT_EQ(pending_task->scanner.lock()->_scanner, pending_scanner); + pending_task->set_state(ScanTask::State::IN_FLIGHT); + scanner_context->_in_flight_tasks_num = 1; + ScannerScheduler::_scanner_scan(scanner_context, pending_task); + + EXPECT_EQ(pending_scanner->prepare_calls(), 0); + EXPECT_EQ(pending_scanner->open_calls(), 0); + EXPECT_EQ(pending_scanner->read_calls(), 0); + EXPECT_EQ(pending_scanner->release_calls(), 1); + EXPECT_TRUE(pending_task->is_eos()); +} + TEST_F(ScannerLateArrivalRfTest, parallel_scanner_factory_preserves_bucket_identity) { constexpr int scan_node_id = 0; constexpr int32_t bucket_seq = 3; @@ -483,14 +586,42 @@ TEST_F(ScannerLateArrivalRfTest, parallel_scanner_factory_preserves_bucket_ident ParallelScannerBuilder builder(local_state.get(), tablets, read_sources, scan_ranges, profile, {}, state, -1, true, true); - auto scanner = - builder._build_scanner(tablet, 1, {}, TabletReadSource {}, io::FileCacheStatistics {}); + auto scanner = builder._build_scanner(tablet, 1, {}, *scan_ranges.front(), TabletReadSource {}, + io::FileCacheStatistics {}); EXPECT_EQ(scanner->_bucket_seq, bucket_seq); EXPECT_EQ(scanner->_bucket_num, bucket_num); EXPECT_TRUE(scanner->is_pruned_by_runtime_filter()); } -TEST_F(ScannerLateArrivalRfTest, olap_scanner_releases_prepared_resources_before_open) { +TEST_F(ScannerLateArrivalRfTest, high_cardinality_ineligible_parallel_builder_reuses_scan_ranges) { + constexpr size_t scan_range_count = 20'000; + constexpr int64_t first_tablet_id = 100; + + std::vector tablets; + std::vector read_sources(scan_range_count); + std::vector> scan_ranges; + tablets.reserve(scan_range_count); + scan_ranges.reserve(scan_range_count); + for (size_t i = 0; i < scan_range_count; ++i) { + int64_t tablet_id = first_tablet_id + static_cast(i); + tablets.push_back({std::make_shared(1, tablet_id), 1}); + auto scan_range = std::make_unique(); + scan_range->__set_tablet_id(tablet_id); + scan_ranges.push_back(std::move(scan_range)); + } + std::vector key_ranges; + std::shared_ptr profile; + + ParallelScannerBuilder builder(nullptr, tablets, read_sources, scan_ranges, profile, key_ranges, + nullptr, -1, true, true); + + EXPECT_EQ(&builder._scan_ranges, &scan_ranges); + EXPECT_EQ(builder._scan_ranges.size(), scan_range_count); + EXPECT_FALSE(builder._scan_ranges.front()->__isset.bucket_seq); + EXPECT_FALSE(builder._scan_ranges.front()->__isset.bucket_num); +} + +TEST_F(ScannerLateArrivalRfTest, olap_scanner_releases_resources_before_open) { constexpr int scan_node_id = 0; constexpr int64_t partition_id = 10; constexpr int64_t tablet_id = 20; @@ -518,33 +649,68 @@ TEST_F(ScannerLateArrivalRfTest, olap_scanner_releases_prepared_resources_before auto* state = _runtime_states[0].get(); state->set_desc_tbl(desc_tbl); auto local_state = OlapScanLocalState::create_shared(state, op.get()); - auto tablet = std::make_shared(partition_id, tablet_id); - RuntimeProfile profile("prepared scanner cleanup"); + auto unprepared_tablet = std::make_shared(partition_id, tablet_id); + auto delete_predicate = std::make_shared(); + std::weak_ptr unprepared_tablet_ref = unprepared_tablet; + std::weak_ptr delete_predicate_ref = delete_predicate; + RuntimeProfile profile("unopened scanner cleanup"); OlapScanner::Params params; params.state = state; params.profile = &profile; - params.tablet = tablet; + params.tablet = unprepared_tablet; params.version = 1; + params.read_source.rs_splits.emplace_back(); + params.read_source.delete_predicates.push_back(delete_predicate); params.limit = -1; params.aggregation = true; - auto scanner = OlapScanner::create_shared(local_state.get(), std::move(params)); - ASSERT_TRUE(scanner->Scanner::_prepare_impl().ok()); - scanner->_tablet_reader = std::make_unique(); - scanner->_tablet_reader_params.rs_splits.emplace_back(); - - ASSERT_TRUE(scanner->has_prepared()); - ASSERT_FALSE(scanner->is_open()); - ASSERT_NE(scanner->_tablet_reader, nullptr); - ASSERT_FALSE(scanner->_tablet_reader_params.rs_splits.empty()); - - scanner->release_prepared_resources(); - - EXPECT_FALSE(scanner->has_prepared()); - EXPECT_FALSE(scanner->is_open()); - EXPECT_EQ(scanner->_tablet_reader, nullptr); - EXPECT_TRUE(scanner->_tablet_reader_params.rs_splits.empty()); - EXPECT_EQ(scanner->_tablet_reader_params.tablet, tablet); - scanner->update_realtime_counters(); + auto unprepared_scanner = OlapScanner::create_shared(local_state.get(), std::move(params)); + unprepared_tablet.reset(); + delete_predicate.reset(); + + ASSERT_FALSE(unprepared_scanner->has_prepared()); + ASSERT_FALSE(unprepared_scanner->is_open()); + ASSERT_FALSE(unprepared_tablet_ref.expired()); + ASSERT_FALSE(delete_predicate_ref.expired()); + ASSERT_FALSE(unprepared_scanner->_tablet_reader_params.rs_splits.empty()); + ASSERT_FALSE(unprepared_scanner->_tablet_reader_params.delete_predicates.empty()); + + unprepared_scanner->release_unopened_resources(); + + EXPECT_FALSE(unprepared_scanner->has_prepared()); + EXPECT_FALSE(unprepared_scanner->is_open()); + EXPECT_TRUE(unprepared_tablet_ref.expired()); + EXPECT_TRUE(delete_predicate_ref.expired()); + EXPECT_EQ(unprepared_scanner->_tablet_reader_params.tablet, nullptr); + EXPECT_TRUE(unprepared_scanner->_tablet_reader_params.rs_splits.empty()); + EXPECT_TRUE(unprepared_scanner->_tablet_reader_params.delete_predicates.empty()); + + auto prepared_tablet = std::make_shared(partition_id, tablet_id + 1); + OlapScanner::Params prepared_params; + prepared_params.state = state; + prepared_params.profile = &profile; + prepared_params.tablet = prepared_tablet; + prepared_params.version = 1; + prepared_params.limit = -1; + prepared_params.aggregation = true; + auto prepared_scanner = + OlapScanner::create_shared(local_state.get(), std::move(prepared_params)); + ASSERT_TRUE(prepared_scanner->Scanner::_prepare_impl().ok()); + prepared_scanner->_tablet_reader = std::make_unique(); + prepared_scanner->_tablet_reader_params.rs_splits.emplace_back(); + + ASSERT_TRUE(prepared_scanner->has_prepared()); + ASSERT_FALSE(prepared_scanner->is_open()); + ASSERT_NE(prepared_scanner->_tablet_reader, nullptr); + ASSERT_FALSE(prepared_scanner->_tablet_reader_params.rs_splits.empty()); + + prepared_scanner->release_unopened_resources(); + + EXPECT_FALSE(prepared_scanner->has_prepared()); + EXPECT_FALSE(prepared_scanner->is_open()); + EXPECT_EQ(prepared_scanner->_tablet_reader, nullptr); + EXPECT_TRUE(prepared_scanner->_tablet_reader_params.rs_splits.empty()); + EXPECT_EQ(prepared_scanner->_tablet_reader_params.tablet, nullptr); + prepared_scanner->update_realtime_counters(); } TEST(ScannerProjectionTest, merges_padding_block_when_limit_eos_without_extra_flag) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/RuntimeFilterBucketPruneClassifier.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/RuntimeFilterBucketPruneClassifier.java index d9e4c2aadc4846..0763b971af41d0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/RuntimeFilterBucketPruneClassifier.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/RuntimeFilterBucketPruneClassifier.java @@ -85,14 +85,30 @@ static Classification classify(TRuntimeFilterType filterType, Expr targetExpr, P } private static boolean sameColumn(Column targetColumn, Column distributionColumn) { - if (targetColumn == distributionColumn) { - return true; + Column targetBaseColumn = directBaseColumn(targetColumn); + Column distributionBaseColumn = directBaseColumn(distributionColumn); + if (targetBaseColumn == null || distributionBaseColumn == null) { + return false; } - return targetColumn.tryGetBaseColumnName() - .equalsIgnoreCase(distributionColumn.tryGetBaseColumnName()) + return targetBaseColumn.getName().equalsIgnoreCase(distributionBaseColumn.getName()) && targetColumn.getType().equals(distributionColumn.getType()); } + private static Column directBaseColumn(Column column) { + Expr defineExpr = column.getDefineExpr(); + if (defineExpr == null) { + return column; + } + if (!(defineExpr instanceof SlotRef)) { + return null; + } + Column baseColumn = ((SlotRef) defineExpr).getColumn(); + if (baseColumn == null || baseColumn.isMaterializedViewColumn()) { + return null; + } + return baseColumn; + } + static final class Classification { private final boolean canPruneBuckets; private final String unsupportedReason; diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/RuntimeFilterBucketPruneClassifierTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/RuntimeFilterBucketPruneClassifierTest.java index c2a160717dbb8a..dec5a0878bc3c6 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/RuntimeFilterBucketPruneClassifierTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/RuntimeFilterBucketPruneClassifierTest.java @@ -17,6 +17,7 @@ package org.apache.doris.nereids.glue.translator; +import org.apache.doris.analysis.FunctionCallExpr; import org.apache.doris.analysis.SlotDescriptor; import org.apache.doris.analysis.SlotId; import org.apache.doris.analysis.SlotRef; @@ -119,6 +120,24 @@ void testBaseColumnNamesComparedSymmetrically() { Assertions.assertTrue(classification.canPruneBuckets()); } + @Test + void testComputedMvAliasCollisionRejected() { + Column baseDistributionColumn = new Column("dist_col", PrimitiveType.INT); + SlotDescriptor baseSlotDescriptor = new SlotDescriptor(new SlotId(2), new TupleId(2)); + baseSlotDescriptor.setColumn(baseDistributionColumn); + baseSlotDescriptor.setType(baseDistributionColumn.getType()); + + Column computedMvColumn = new Column("dist_col", PrimitiveType.INT); + computedMvColumn.setDefineExpr(new FunctionCallExpr("abs", + ImmutableList.of(new SlotRef(baseSlotDescriptor)), true)); + RuntimeFilterBucketPruneClassifier.Classification classification = classify( + TRuntimeFilterType.IN, computedMvColumn, + new HashDistributionInfo(8, ImmutableList.of(baseDistributionColumn))); + + Assertions.assertFalse(classification.canPruneBuckets()); + Assertions.assertTrue(classification.getUnsupportedReason().contains("distribution column")); + } + @Test void testDifferentUniqueIdsAllowedForSameBaseColumn() { Column distributionColumn = new Column("dist_col", PrimitiveType.INT); From 1975a15fa6efd7d294d8574dd5efaa335589f9c0 Mon Sep 17 00:00:00 2001 From: happenlee Date: Fri, 14 Aug 2026 11:34:38 +0800 Subject: [PATCH 15/20] [refactor](fe) Generate runtime filter prune metadata in Nereids ### What problem does this PR solve? Issue Number: None Related PR: #65837 Problem Summary: Runtime-filter partition and bucket pruning eligibility was generated during legacy translation and re-gated in multiple Thrift paths, while bucket range metadata was attached from a coordinator-wide pre-pass. This split ownership across planning stages, repeated session checks, and made missing optional bucket metadata abort planning. Generate both pruning decisions once when Nereids creates the filter at its final target scan, carry immutable target metadata through translation, attach bucket fields idempotently from OlapScanNode.toThrift(), and conservatively disable bucket pruning for the scan when range metadata is incomplete. ### Release note None ### Check List (For Author) - Test: Unit Test - FE: RuntimeFilterPruneClassifierTest, RuntimeFilterTranslatorBucketPruneTest, PhysicalPlanTranslatorTest#testRfPartitionPruneSnapshotSurvivesEnablementChange, OlapScanNodeTest, ThriftPlansBuilderTest - FE build: ./build.sh --fe -j 48 - Behavior changed: Yes. Pruning eligibility and session gating are generated once in Nereids; incomplete optional bucket metadata disables only this optimization instead of failing planning. - Does this need documentation: No --- .../RuntimeFilterBucketPruneClassifier.java | 137 ------ ...RuntimeFilterPartitionPruneClassifier.java | 322 ------------- .../translator/RuntimeFilterTranslator.java | 47 +- .../processor/post/RuntimeFilterContext.java | 8 + .../post/RuntimeFilterPruneClassifier.java | 442 ++++++++++++++++++ .../post/RuntimeFilterPushDownVisitor.java | 1 + .../trees/plans/physical/RuntimeFilter.java | 26 ++ .../apache/doris/planner/OlapScanNode.java | 39 +- .../apache/doris/planner/RuntimeFilter.java | 57 +-- .../doris/qe/runtime/ThriftPlansBuilder.java | 16 - .../PhysicalPlanTranslatorTest.java | 4 +- ...untimeFilterBucketPruneClassifierTest.java | 228 --------- ...imeFilterPartitionPruneClassifierTest.java | 163 ------- ...untimeFilterTranslatorBucketPruneTest.java | 16 +- .../RuntimeFilterPruneClassifierTest.java | 377 +++++++++++++++ .../doris/planner/OlapScanNodeTest.java | 79 ++++ .../qe/runtime/ThriftPlansBuilderTest.java | 18 - 17 files changed, 1006 insertions(+), 974 deletions(-) delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/RuntimeFilterBucketPruneClassifier.java delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/RuntimeFilterPartitionPruneClassifier.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterPruneClassifier.java delete mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/RuntimeFilterBucketPruneClassifierTest.java delete mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/RuntimeFilterPartitionPruneClassifierTest.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/processor/post/RuntimeFilterPruneClassifierTest.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/RuntimeFilterBucketPruneClassifier.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/RuntimeFilterBucketPruneClassifier.java deleted file mode 100644 index 0763b971af41d0..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/RuntimeFilterBucketPruneClassifier.java +++ /dev/null @@ -1,137 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.glue.translator; - -import org.apache.doris.analysis.Expr; -import org.apache.doris.analysis.SlotRef; -import org.apache.doris.catalog.Column; -import org.apache.doris.catalog.DistributionInfo; -import org.apache.doris.catalog.HashDistributionInfo; -import org.apache.doris.catalog.OlapTable; -import org.apache.doris.catalog.Partition; -import org.apache.doris.planner.OlapScanNode; -import org.apache.doris.planner.PlanNode; -import org.apache.doris.thrift.TRuntimeFilterType; - -/** Classifies direct single-column HASH targets for BE-side runtime-filter bucket pruning. */ -final class RuntimeFilterBucketPruneClassifier { - private RuntimeFilterBucketPruneClassifier() { - } - - static Classification classify(TRuntimeFilterType filterType, Expr targetExpr, PlanNode scanNode) { - if (filterType != TRuntimeFilterType.IN && filterType != TRuntimeFilterType.IN_OR_BLOOM) { - return Classification.unsupported("runtime filter is not IN or IN_OR_BLOOM"); - } - if (!(scanNode instanceof OlapScanNode)) { - return Classification.unsupported("target scan is not an OlapScanNode"); - } - if (!(targetExpr instanceof SlotRef)) { - return Classification.unsupported("target expression is not a direct SlotRef"); - } - - Column targetColumn = ((SlotRef) targetExpr).getColumn(); - if (targetColumn == null) { - return Classification.unsupported("target SlotRef has no column"); - } - - OlapScanNode olapScanNode = (OlapScanNode) scanNode; - if (olapScanNode.isPointQuery()) { - return Classification.unsupported("target scan is a point query"); - } - OlapTable table = olapScanNode.getOlapTable(); - if (table == null || olapScanNode.getSelectedPartitionIds().isEmpty()) { - return Classification.unsupported("target scan has no selected partitions"); - } - - Column distributionColumn = null; - for (Long partitionId : olapScanNode.getSelectedPartitionIds()) { - Partition partition = table.getPartition(partitionId); - if (partition == null) { - return Classification.unsupported("selected partition does not exist"); - } - DistributionInfo distributionInfo = partition.getDistributionInfo(); - if (!(distributionInfo instanceof HashDistributionInfo)) { - return Classification.unsupported("distribution type is not HASH"); - } - HashDistributionInfo hashDistributionInfo = (HashDistributionInfo) distributionInfo; - if (hashDistributionInfo.getDistributionColumns().size() != 1) { - return Classification.unsupported("HASH distribution is not single-column"); - } - Column currentDistributionColumn = hashDistributionInfo.getDistributionColumns().get(0); - if (!sameColumn(targetColumn, currentDistributionColumn)) { - return Classification.unsupported("target SlotRef is not the HASH distribution column"); - } - if (distributionColumn != null && !sameColumn(distributionColumn, currentDistributionColumn)) { - return Classification.unsupported("selected partitions use different distribution columns"); - } - distributionColumn = currentDistributionColumn; - } - return Classification.supported(); - } - - private static boolean sameColumn(Column targetColumn, Column distributionColumn) { - Column targetBaseColumn = directBaseColumn(targetColumn); - Column distributionBaseColumn = directBaseColumn(distributionColumn); - if (targetBaseColumn == null || distributionBaseColumn == null) { - return false; - } - return targetBaseColumn.getName().equalsIgnoreCase(distributionBaseColumn.getName()) - && targetColumn.getType().equals(distributionColumn.getType()); - } - - private static Column directBaseColumn(Column column) { - Expr defineExpr = column.getDefineExpr(); - if (defineExpr == null) { - return column; - } - if (!(defineExpr instanceof SlotRef)) { - return null; - } - Column baseColumn = ((SlotRef) defineExpr).getColumn(); - if (baseColumn == null || baseColumn.isMaterializedViewColumn()) { - return null; - } - return baseColumn; - } - - static final class Classification { - private final boolean canPruneBuckets; - private final String unsupportedReason; - - private Classification(boolean canPruneBuckets, String unsupportedReason) { - this.canPruneBuckets = canPruneBuckets; - this.unsupportedReason = unsupportedReason; - } - - static Classification supported() { - return new Classification(true, ""); - } - - static Classification unsupported(String reason) { - return new Classification(false, reason); - } - - boolean canPruneBuckets() { - return canPruneBuckets; - } - - String getUnsupportedReason() { - return unsupportedReason; - } - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/RuntimeFilterPartitionPruneClassifier.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/RuntimeFilterPartitionPruneClassifier.java deleted file mode 100644 index 00c61916329e68..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/RuntimeFilterPartitionPruneClassifier.java +++ /dev/null @@ -1,322 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.glue.translator; - -import org.apache.doris.analysis.Expr; -import org.apache.doris.analysis.FunctionCallExpr; -import org.apache.doris.analysis.LiteralExpr; -import org.apache.doris.analysis.SlotRef; -import org.apache.doris.catalog.Column; -import org.apache.doris.catalog.ListPartitionItem; -import org.apache.doris.catalog.OlapTable; -import org.apache.doris.catalog.PartitionInfo; -import org.apache.doris.catalog.PartitionItem; -import org.apache.doris.catalog.PartitionKey; -import org.apache.doris.catalog.PartitionType; -import org.apache.doris.catalog.RangePartitionItem; -import org.apache.doris.nereids.exceptions.AnalysisException; -import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.expressions.Slot; -import org.apache.doris.nereids.trees.expressions.functions.Monotonic; -import org.apache.doris.nereids.trees.expressions.functions.NoneMovableFunction; -import org.apache.doris.nereids.trees.expressions.literal.Literal; -import org.apache.doris.planner.OlapScanNode; -import org.apache.doris.planner.PlanNode; -import org.apache.doris.thrift.TRuntimeFilterType; -import org.apache.doris.thrift.TTargetExprMonotonicity; - -import com.google.common.collect.Range; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** - * Classifies whether one runtime-filter target can safely drive BE-side partition pruning. - * - *

This is the single FE gate for RF partition pruning. It intentionally reasons about the - * final legacy target expression that will be sent to BE, so late-added casts and partition - * expression domains cannot bypass the safety checks. - */ -final class RuntimeFilterPartitionPruneClassifier { - private RuntimeFilterPartitionPruneClassifier() { - } - - static Classification classify(TRuntimeFilterType filterType, Expr targetExpr, - Expression nereidsTargetExpr, PlanNode scanNode) { - if (!(scanNode instanceof OlapScanNode)) { - return Classification.unsupported("target scan is not an OlapScanNode"); - } - - OlapScanNode olapScanNode = (OlapScanNode) scanNode; - OlapTable table = olapScanNode.getOlapTable(); - if (table == null) { - return Classification.unsupported("target scan has no OlapTable"); - } - - PartitionInfo partitionInfo = table.getPartitionInfo(); - PartitionType partType = partitionInfo.getType(); - if (partType != PartitionType.RANGE && partType != PartitionType.LIST) { - return Classification.unsupported("partition type is not RANGE or LIST"); - } - if (filterType == TRuntimeFilterType.BLOOM && partType == PartitionType.RANGE) { - return Classification.unsupported("BLOOM runtime filter does not support RANGE partition pruning"); - } - if (hasUnsupportedAutomaticPartitionExpression(partitionInfo)) { - return Classification.unsupported("automatic partition expression boundary is not modeled"); - } - if (nereidsTargetExpr.containsType(NoneMovableFunction.class)) { - return Classification.unsupported("target expression contains non-movable function"); - } - - if (targetExpr instanceof SlotRef) { - SlotRef slotRef = (SlotRef) targetExpr; - if (!isPartitionColumnSlot(slotRef, partitionInfo.getPartitionColumns())) { - return Classification.unsupported("target SlotRef is not a partition column"); - } - if (!hasSerializedBoundary(slotRef, partitionInfo, partType)) { - return Classification.unsupported("target SlotRef has no serialized partition boundary"); - } - Map partitionMonotonicity = - allSelectedPartitionsIncreasing(olapScanNode, partitionInfo); - if (partitionMonotonicity.isEmpty()) { - return Classification.unsupported("target SlotRef has no prunable selected partitions"); - } - return Classification.supportedPartitions(slotRef, partitionMonotonicity); - } - - SlotRef leafSlot = findUniqueSlotRef(targetExpr); - if (leafSlot == null || !isPartitionColumnSlot(leafSlot, partitionInfo.getPartitionColumns())) { - return Classification.unsupported("target expression is not rooted on one partition column"); - } - if (!hasSerializedBoundary(leafSlot, partitionInfo, partType)) { - return Classification.unsupported("target expression has no serialized partition boundary"); - } - if (partType == PartitionType.LIST) { - if (nereidsTargetExpr.containsNondeterministic()) { - return Classification.unsupported("target expression contains non-deterministic function"); - } - Map partitionMonotonicity = - allSelectedPartitionsIncreasing(olapScanNode, partitionInfo); - if (partitionMonotonicity.isEmpty()) { - return Classification.unsupported("target expression has no prunable selected partitions"); - } - return Classification.supportedPartitions(leafSlot, partitionMonotonicity); - } - - Map partitionMonotonicity = - classifyLocalMonotonicity(nereidsTargetExpr, olapScanNode, partitionInfo, leafSlot); - if (partitionMonotonicity.isEmpty()) { - return Classification.unsupported("target expression is not monotonic on selected partitions"); - } - return Classification.supportedPartitions(leafSlot, partitionMonotonicity); - } - - private static boolean hasUnsupportedAutomaticPartitionExpression(PartitionInfo partitionInfo) { - if (!partitionInfo.enableAutomaticPartition()) { - return false; - } - for (Expr partitionExpr : partitionInfo.getPartitionExprs()) { - if (containsFunctionCall(partitionExpr)) { - return true; - } - } - return false; - } - - private static boolean containsFunctionCall(Expr expr) { - if (expr instanceof FunctionCallExpr) { - return true; - } - for (Expr child : expr.getChildren()) { - if (containsFunctionCall(child)) { - return true; - } - } - return false; - } - - private static boolean hasSerializedBoundary(SlotRef slotRef, PartitionInfo partitionInfo, PartitionType partType) { - if (partType != PartitionType.RANGE) { - return true; - } - List partitionColumns = partitionInfo.getPartitionColumns(); - return !partitionColumns.isEmpty() && sameColumn(slotRef.getColumn(), partitionColumns.get(0)); - } - - private static boolean isPartitionColumnSlot(SlotRef slotRef, List partitionColumns) { - Column targetColumn = slotRef.getColumn(); - if (targetColumn == null) { - return false; - } - for (Column partitionColumn : partitionColumns) { - if (sameColumn(targetColumn, partitionColumn)) { - return true; - } - } - return false; - } - - private static boolean sameColumn(Column targetColumn, Column partitionColumn) { - if (targetColumn == partitionColumn) { - return true; - } - int targetUniqueId = targetColumn.getUniqueId(); - int partitionUniqueId = partitionColumn.getUniqueId(); - if (targetUniqueId != Column.COLUMN_UNIQUE_ID_INIT_VALUE - && partitionUniqueId != Column.COLUMN_UNIQUE_ID_INIT_VALUE - && targetUniqueId == partitionUniqueId) { - return true; - } - return targetColumn.equals(partitionColumn); - } - - private static SlotRef findUniqueSlotRef(Expr expr) { - if (expr instanceof SlotRef) { - return (SlotRef) expr; - } - SlotRef result = null; - for (Expr child : expr.getChildren()) { - SlotRef childSlot = findUniqueSlotRef(child); - if (childSlot == null) { - continue; - } - if (result != null && result.getSlotId().asInt() != childSlot.getSlotId().asInt()) { - return null; - } - result = childSlot; - } - return result; - } - - private static Map classifyLocalMonotonicity( - Expression nereidsTargetExpr, OlapScanNode scanNode, PartitionInfo partitionInfo, SlotRef leafSlot) { - Map result = new HashMap<>(); - if (!(nereidsTargetExpr instanceof Monotonic)) { - return result; - } - - Monotonic monotonic = (Monotonic) nereidsTargetExpr; - int childIndex = monotonic.getMonotonicFunctionChildIndex(); - if (childIndex < 0 || childIndex >= nereidsTargetExpr.arity() - || !(nereidsTargetExpr.child(childIndex) instanceof Slot) - || !hasInputSlotOnlyInMonotonicChild(nereidsTargetExpr, childIndex)) { - return result; - } - - Column partitionColumn = leafSlot.getColumn(); - for (Long partitionId : scanNode.getSelectedPartitionIds()) { - PartitionItem item = partitionInfo.getItem(partitionId); - if (!(item instanceof RangePartitionItem)) { - continue; - } - Range range = ((RangePartitionItem) item).getItems(); - Literal lower = null; - Literal upper = null; - if (range.hasLowerBound() && !range.lowerEndpoint().isMinValue()) { - lower = toNereidsLiteral(range.lowerEndpoint().getKeys().get(0), partitionColumn); - if (lower == null) { - continue; - } - } - if (range.hasUpperBound() && !range.upperEndpoint().isMaxValue()) { - upper = toNereidsLiteral(range.upperEndpoint().getKeys().get(0), partitionColumn); - if (upper == null) { - continue; - } - } - if (monotonic.isMonotonic(lower, upper)) { - result.put(partitionId, monotonic.isPositive() - ? TTargetExprMonotonicity.MONOTONIC_INCREASING - : TTargetExprMonotonicity.MONOTONIC_DECREASING); - } - } - return result; - } - - static boolean hasInputSlotOnlyInMonotonicChild(Expression expression, int monotonicChildIndex) { - for (int i = 0; i < expression.arity(); i++) { - if (i != monotonicChildIndex && !expression.child(i).getInputSlots().isEmpty()) { - return false; - } - } - return true; - } - - private static Map allSelectedPartitionsIncreasing( - OlapScanNode scanNode, PartitionInfo partitionInfo) { - Map result = new HashMap<>(); - for (Long partitionId : scanNode.getSelectedPartitionIds()) { - PartitionItem item = partitionInfo.getItem(partitionId); - if (item == null || (item instanceof ListPartitionItem - && ((ListPartitionItem) item).isDefaultPartition())) { - continue; - } - result.put(partitionId, TTargetExprMonotonicity.MONOTONIC_INCREASING); - } - return result; - } - - private static Literal toNereidsLiteral(LiteralExpr literalExpr, Column column) { - try { - return Literal.fromLegacyLiteral(literalExpr, column.getType()); - } catch (AnalysisException e) { - return null; - } - } - - static final class Classification { - private final boolean canPrunePartitions; - private final SlotRef partitionSlot; - private final Map partitionMonotonicity; - private final String unsupportedReason; - - private Classification(boolean canPrunePartitions, SlotRef partitionSlot, - Map partitionMonotonicity, String unsupportedReason) { - this.canPrunePartitions = canPrunePartitions; - this.partitionSlot = partitionSlot; - this.partitionMonotonicity = partitionMonotonicity; - this.unsupportedReason = unsupportedReason; - } - - static Classification supportedPartitions(SlotRef partitionSlot, - Map partitionMonotonicity) { - return new Classification(true, partitionSlot, partitionMonotonicity, ""); - } - - static Classification unsupported(String reason) { - return new Classification(false, null, new HashMap<>(), reason); - } - - boolean canPrunePartitions() { - return canPrunePartitions; - } - - SlotRef getPartitionSlot() { - return partitionSlot; - } - - Map getPartitionMonotonicity() { - return partitionMonotonicity; - } - - String getUnsupportedReason() { - return unsupportedReason; - } - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/RuntimeFilterTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/RuntimeFilterTranslator.java index 1a10cfdccb8e5b..8912502d1f43ed 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/RuntimeFilterTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/RuntimeFilterTranslator.java @@ -170,8 +170,6 @@ private void createLegacyRuntimeFilterFromGroup(List group, List targetExprList = new ArrayList<>(); List>> targetTupleIdMapList = new ArrayList<>(); List scanNodeList = new ArrayList<>(); - List nereidsTargetExprList = new ArrayList<>(); - List targetAdjustedAfterNonIdentityList = new ArrayList<>(); Map scanNodeTargetExprSql = new LinkedHashMap<>(); Map scanNodeHasDifferentTargets = new LinkedHashMap<>(); boolean hasInvalidTarget = false; @@ -200,16 +198,12 @@ private void createLegacyRuntimeFilterFromGroup(List group, new RuntimeFilterExpressionTranslator(targetSlotRef); targetExpr = curTargetExpression.accept(translator, ctx); } - boolean adjustedAfterNonIdentity = !src.getType().equals(targetExpr.getType()) - && !isIdentityTarget; targetExpr = castTargetToSourceTypeIfNeeded(src, targetExpr); TupleId targetTupleId = targetSlotRef.getDesc().getParentId(); SlotId targetSlotId = targetSlotRef.getSlotId(); scanNodeList.add(scanNode); targetExprList.add(targetExpr); - nereidsTargetExprList.add(curTargetExpression); targetTupleIdMapList.add(ImmutableMap.of(targetTupleId, ImmutableList.of(targetSlotId))); - targetAdjustedAfterNonIdentityList.add(adjustedAfterNonIdentity); int scanNodeId = scanNode.getId().asInt(); String targetExprSql = targetExpr.accept(ExprToSqlVisitor.INSTANCE, ToSqlParams.WITH_TABLE); String existingTargetExprSql = scanNodeTargetExprSql.putIfAbsent(scanNodeId, targetExprSql); @@ -241,15 +235,10 @@ private void createLegacyRuntimeFilterFromGroup(List group, // TRuntimeFilterDesc keys target expressions and pruning metadata by scan node ID. // If one grouped RF has different targets on the same scan, BE cannot match // metadata back to a specific target expression, so skip pruning metadata. - if (scanNodeHasDifferentTargets.getOrDefault(scanNode.getId().asInt(), false) - || targetAdjustedAfterNonIdentityList.get(i)) { + if (scanNodeHasDifferentTargets.getOrDefault(scanNode.getId().asInt(), false)) { continue; } - RuntimeFilterPartitionPruneClassifier.Classification classification = - RuntimeFilterPartitionPruneClassifier.classify( - head.getType(), targetExpr, nereidsTargetExprList.get(i), scanNode); - setPartitionPruningMetadata(origFilter, scanNode, classification); - setBucketPruningMetadata(origFilter, scanNode, head.getType(), targetExpr); + setPruningMetadata(origFilter, scanNode, group.get(i)); } origFilter.setBloomFilterSizeCalculatedByNdv(head.isBloomFilterSizeCalculatedByNdv()); setWaitTimeMs(origFilter, head.isNonBlocking(), isLocalTarget); @@ -288,7 +277,6 @@ public void createLegacyRuntimeFilter(RuntimeFilter filter, PlanNode node, PlanT List targetExprList = new ArrayList<>(); List>> targetTupleIdMapList = new ArrayList<>(); List scanNodeList = new ArrayList<>(); - List targetAdjustedAfterNonIdentityList = new ArrayList<>(); boolean hasInvalidTarget = false; Slot curTargetSlot = filter.getTargetSlot(); Expression curTargetExpression = filter.getTargetExpression(); @@ -314,15 +302,12 @@ public void createLegacyRuntimeFilter(RuntimeFilter filter, PlanNode node, PlanT } // adjust data type - boolean adjustedAfterNonIdentity = !src.getType().equals(targetExpr.getType()) - && !isIdentityTarget; targetExpr = castTargetToSourceTypeIfNeeded(src, targetExpr); TupleId targetTupleId = targetSlotRef.getDesc().getParentId(); SlotId targetSlotId = targetSlotRef.getSlotId(); scanNodeList.add(scanNode); targetExprList.add(targetExpr); targetTupleIdMapList.add(ImmutableMap.of(targetTupleId, ImmutableList.of(targetSlotId))); - targetAdjustedAfterNonIdentityList.add(adjustedAfterNonIdentity); } if (!hasInvalidTarget) { org.apache.doris.planner.RuntimeFilter origFilter @@ -346,14 +331,7 @@ public void createLegacyRuntimeFilter(RuntimeFilter filter, PlanNode node, PlanT Expr targetExpr = targetExprList.get(i); origFilter.addTarget(new RuntimeFilterTarget( scanNode, targetExpr, true, isLocalTarget)); - if (targetAdjustedAfterNonIdentityList.get(i)) { - continue; - } - RuntimeFilterPartitionPruneClassifier.Classification classification = - RuntimeFilterPartitionPruneClassifier.classify( - filter.getType(), targetExpr, filter.getTargetExpressions().get(i), scanNode); - setPartitionPruningMetadata(origFilter, scanNode, classification); - setBucketPruningMetadata(origFilter, scanNode, filter.getType(), targetExpr); + setPruningMetadata(origFilter, scanNode, filter); } origFilter.setBloomFilterSizeCalculatedByNdv(filter.isBloomFilterSizeCalculatedByNdv()); setWaitTimeMs(origFilter, filter.isNonBlocking(), isLocalTarget); @@ -383,26 +361,17 @@ private org.apache.doris.planner.RuntimeFilter finalize(org.apache.doris.planner return origFilter; } - private void setPartitionPruningMetadata(org.apache.doris.planner.RuntimeFilter runtimeFilter, - ScanNode scanNode, RuntimeFilterPartitionPruneClassifier.Classification classification) { - if (classification.canPrunePartitions()) { + private void setPruningMetadata(org.apache.doris.planner.RuntimeFilter runtimeFilter, + ScanNode scanNode, RuntimeFilter nereidsFilter) { + if (nereidsFilter.canPrunePartitions()) { Preconditions.checkState(scanNode instanceof OlapScanNode, "partition-pruning runtime filter target must be an OlapScanNode"); runtimeFilter.markTargetCanPrunePartitions(scanNode.getId()); ((OlapScanNode) scanNode).snapshotPartitionBoundariesForRuntimeFilter(); } runtimeFilter.setTargetPartitionMonotonicity( - scanNode.getId(), classification.getPartitionMonotonicity()); - } - - private void setBucketPruningMetadata(org.apache.doris.planner.RuntimeFilter runtimeFilter, - ScanNode scanNode, TRuntimeFilterType filterType, Expr targetExpr) { - if (!context.getSessionVariable().isEnableRuntimeFilterBucketPrune()) { - return; - } - RuntimeFilterBucketPruneClassifier.Classification classification = - RuntimeFilterBucketPruneClassifier.classify(filterType, targetExpr, scanNode); - if (classification.canPruneBuckets()) { + scanNode.getId(), nereidsFilter.getPartitionMonotonicity()); + if (nereidsFilter.canPruneBuckets()) { runtimeFilter.markTargetCanPruneBuckets(scanNode.getId()); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterContext.java index 0336dd66431bb9..64a321ff03848f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterContext.java @@ -90,6 +90,14 @@ public FilterSizeLimits getLimits() { return limits; } + /** Generate pruning metadata once, while the runtime filter still owns its Nereids target scan. */ + public void generateRuntimeFilterPruneMetadata(RuntimeFilter filter) { + RuntimeFilterPruneClassifier.Classification classification = + RuntimeFilterPruneClassifier.classify(filter, sessionVariable); + filter.setPruningMetadata( + classification.canPruneBuckets(), classification.getPartitionMonotonicity()); + } + public void setTargetExprIdToFilter(ExprId id, RuntimeFilter filter) { Preconditions.checkArgument(filter.getTargetSlot().getExprId() == id); this.targetExprIdToFilter.computeIfAbsent(id, k -> Lists.newArrayList()).add(filter); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterPruneClassifier.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterPruneClassifier.java new file mode 100644 index 00000000000000..02d5445478d517 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterPruneClassifier.java @@ -0,0 +1,442 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.processor.post; + +import org.apache.doris.analysis.Expr; +import org.apache.doris.analysis.FunctionCallExpr; +import org.apache.doris.analysis.LiteralExpr; +import org.apache.doris.analysis.SlotRef; +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.DistributionInfo; +import org.apache.doris.catalog.HashDistributionInfo; +import org.apache.doris.catalog.ListPartitionItem; +import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.Partition; +import org.apache.doris.catalog.PartitionInfo; +import org.apache.doris.catalog.PartitionItem; +import org.apache.doris.catalog.PartitionKey; +import org.apache.doris.catalog.PartitionType; +import org.apache.doris.catalog.RangePartitionItem; +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.functions.Monotonic; +import org.apache.doris.nereids.trees.expressions.functions.NoneMovableFunction; +import org.apache.doris.nereids.trees.expressions.literal.Literal; +import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapScan; +import org.apache.doris.nereids.trees.plans.physical.RuntimeFilter; +import org.apache.doris.qe.SessionVariable; +import org.apache.doris.thrift.TRuntimeFilterType; +import org.apache.doris.thrift.TTargetExprMonotonicity; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Range; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** Generates target-scoped partition and bucket pruning metadata with a Nereids runtime filter. */ +final class RuntimeFilterPruneClassifier { + private RuntimeFilterPruneClassifier() { + } + + static Classification classify(RuntimeFilter filter, SessionVariable sessionVariable) { + BucketClassification bucketClassification = sessionVariable.isEnableRuntimeFilterBucketPrune() + ? classifyBucketPruning(filter) + : BucketClassification.unsupported("runtime-filter bucket pruning is disabled"); + PartitionClassification partitionClassification = sessionVariable.isEnableRuntimeFilterPartitionPrune() + ? classifyPartitionPruning(filter) + : PartitionClassification.unsupported("runtime-filter partition pruning is disabled"); + return new Classification(bucketClassification, partitionClassification); + } + + private static BucketClassification classifyBucketPruning(RuntimeFilter filter) { + if (filter.getType() != TRuntimeFilterType.IN + && filter.getType() != TRuntimeFilterType.IN_OR_BLOOM) { + return BucketClassification.unsupported("runtime filter is not IN or IN_OR_BLOOM"); + } + if (!(filter.getTargetScan() instanceof PhysicalOlapScan)) { + return BucketClassification.unsupported("target scan is not a PhysicalOlapScan"); + } + if (!isFinalTargetDirectSlot(filter)) { + return BucketClassification.unsupported("target expression is not a direct slot"); + } + + Column targetColumn = targetColumn(filter.getTargetSlot()); + if (targetColumn == null) { + return BucketClassification.unsupported("target slot has no column"); + } + + PhysicalOlapScan scan = (PhysicalOlapScan) filter.getTargetScan(); + OlapTable table = scan.getTable(); + if (table == null || scan.getSelectedPartitionIds().isEmpty()) { + return BucketClassification.unsupported("target scan has no selected partitions"); + } + + Column distributionColumn = null; + for (Long partitionId : scan.getSelectedPartitionIds()) { + Partition partition = table.getPartition(partitionId); + if (partition == null) { + return BucketClassification.unsupported("selected partition does not exist"); + } + DistributionInfo distributionInfo = partition.getDistributionInfo(); + if (!(distributionInfo instanceof HashDistributionInfo)) { + return BucketClassification.unsupported("distribution type is not HASH"); + } + HashDistributionInfo hashDistributionInfo = (HashDistributionInfo) distributionInfo; + if (hashDistributionInfo.getDistributionColumns().size() != 1) { + return BucketClassification.unsupported("HASH distribution is not single-column"); + } + Column currentDistributionColumn = hashDistributionInfo.getDistributionColumns().get(0); + if (!sameBucketColumn(targetColumn, currentDistributionColumn)) { + return BucketClassification.unsupported("target slot is not the HASH distribution column"); + } + if (distributionColumn != null + && !sameBucketColumn(distributionColumn, currentDistributionColumn)) { + return BucketClassification.unsupported( + "selected partitions use different distribution columns"); + } + distributionColumn = currentDistributionColumn; + } + return BucketClassification.supported(); + } + + private static PartitionClassification classifyPartitionPruning(RuntimeFilter filter) { + if (!(filter.getTargetScan() instanceof PhysicalOlapScan)) { + return PartitionClassification.unsupported("target scan is not a PhysicalOlapScan"); + } + if (requiresTargetTypeAdjustment(filter) + && !filter.getTargetSlot().equals(filter.getTargetExpression())) { + return PartitionClassification.unsupported( + "non-identity target expression requires a type adjustment"); + } + + PhysicalOlapScan scan = (PhysicalOlapScan) filter.getTargetScan(); + OlapTable table = scan.getTable(); + if (table == null) { + return PartitionClassification.unsupported("target scan has no OlapTable"); + } + + PartitionInfo partitionInfo = table.getPartitionInfo(); + PartitionType partitionType = partitionInfo.getType(); + if (partitionType != PartitionType.RANGE && partitionType != PartitionType.LIST) { + return PartitionClassification.unsupported("partition type is not RANGE or LIST"); + } + if (filter.getType() == TRuntimeFilterType.BLOOM && partitionType == PartitionType.RANGE) { + return PartitionClassification.unsupported( + "BLOOM runtime filter does not support RANGE partition pruning"); + } + if (hasUnsupportedAutomaticPartitionExpression(partitionInfo)) { + return PartitionClassification.unsupported( + "automatic partition expression boundary is not modeled"); + } + + Expression targetExpression = filter.getTargetExpression(); + if (targetExpression.containsType(NoneMovableFunction.class)) { + return PartitionClassification.unsupported( + "target expression contains non-movable function"); + } + Column targetColumn = targetColumn(filter.getTargetSlot()); + if (!isPartitionColumn(targetColumn, partitionInfo.getPartitionColumns())) { + return PartitionClassification.unsupported( + "target expression is not rooted on one partition column"); + } + if (!hasSerializedBoundary(targetColumn, partitionInfo, partitionType)) { + return PartitionClassification.unsupported( + "target expression has no serialized partition boundary"); + } + + if (isFinalTargetDirectSlot(filter)) { + return supportedIncreasingPartitions(scan, partitionInfo, + "target slot has no prunable selected partitions"); + } + if (partitionType == PartitionType.LIST) { + if (targetExpression.containsNondeterministic()) { + return PartitionClassification.unsupported( + "target expression contains non-deterministic function"); + } + return supportedIncreasingPartitions(scan, partitionInfo, + "target expression has no prunable selected partitions"); + } + + Map partitionMonotonicity = + classifyLocalMonotonicity(targetExpression, scan, partitionInfo, targetColumn); + if (partitionMonotonicity.isEmpty()) { + return PartitionClassification.unsupported( + "target expression is not monotonic on selected partitions"); + } + return PartitionClassification.supported(partitionMonotonicity); + } + + private static PartitionClassification supportedIncreasingPartitions( + PhysicalOlapScan scan, PartitionInfo partitionInfo, String emptyReason) { + Map partitionMonotonicity = + allSelectedPartitionsIncreasing(scan, partitionInfo); + return partitionMonotonicity.isEmpty() + ? PartitionClassification.unsupported(emptyReason) + : PartitionClassification.supported(partitionMonotonicity); + } + + private static boolean isFinalTargetDirectSlot(RuntimeFilter filter) { + return filter.getTargetSlot().equals(filter.getTargetExpression()) + && !requiresTargetTypeAdjustment(filter); + } + + private static boolean requiresTargetTypeAdjustment(RuntimeFilter filter) { + return !filter.getSrcExpr().getDataType().toCatalogDataType().equals( + filter.getTargetExpression().getDataType().toCatalogDataType()); + } + + private static Column targetColumn(Slot slot) { + if (!(slot instanceof SlotReference)) { + return null; + } + return ((SlotReference) slot).getOriginalColumn().orElse(null); + } + + private static boolean sameBucketColumn(Column targetColumn, Column distributionColumn) { + Column targetBaseColumn = directBaseColumn(targetColumn); + Column distributionBaseColumn = directBaseColumn(distributionColumn); + if (targetBaseColumn == null || distributionBaseColumn == null) { + return false; + } + return targetBaseColumn.getName().equalsIgnoreCase(distributionBaseColumn.getName()) + && targetColumn.getType().equals(distributionColumn.getType()); + } + + private static Column directBaseColumn(Column column) { + Expr defineExpr = column.getDefineExpr(); + if (defineExpr == null) { + return column; + } + if (!(defineExpr instanceof SlotRef)) { + return null; + } + Column baseColumn = ((SlotRef) defineExpr).getColumn(); + if (baseColumn == null || baseColumn.isMaterializedViewColumn()) { + return null; + } + return baseColumn; + } + + private static boolean hasUnsupportedAutomaticPartitionExpression(PartitionInfo partitionInfo) { + if (!partitionInfo.enableAutomaticPartition()) { + return false; + } + for (Expr partitionExpr : partitionInfo.getPartitionExprs()) { + if (containsFunctionCall(partitionExpr)) { + return true; + } + } + return false; + } + + private static boolean containsFunctionCall(Expr expression) { + if (expression instanceof FunctionCallExpr) { + return true; + } + for (Expr child : expression.getChildren()) { + if (containsFunctionCall(child)) { + return true; + } + } + return false; + } + + private static boolean hasSerializedBoundary( + Column targetColumn, PartitionInfo partitionInfo, PartitionType partitionType) { + if (partitionType != PartitionType.RANGE) { + return true; + } + List partitionColumns = partitionInfo.getPartitionColumns(); + return !partitionColumns.isEmpty() && samePartitionColumn(targetColumn, partitionColumns.get(0)); + } + + private static boolean isPartitionColumn(Column targetColumn, List partitionColumns) { + if (targetColumn == null) { + return false; + } + for (Column partitionColumn : partitionColumns) { + if (samePartitionColumn(targetColumn, partitionColumn)) { + return true; + } + } + return false; + } + + private static boolean samePartitionColumn(Column targetColumn, Column partitionColumn) { + if (targetColumn == partitionColumn) { + return true; + } + int targetUniqueId = targetColumn.getUniqueId(); + int partitionUniqueId = partitionColumn.getUniqueId(); + if (targetUniqueId != Column.COLUMN_UNIQUE_ID_INIT_VALUE + && partitionUniqueId != Column.COLUMN_UNIQUE_ID_INIT_VALUE + && targetUniqueId == partitionUniqueId) { + return true; + } + return targetColumn.equals(partitionColumn); + } + + private static Map classifyLocalMonotonicity( + Expression targetExpression, PhysicalOlapScan scan, + PartitionInfo partitionInfo, Column partitionColumn) { + Map result = new HashMap<>(); + if (!(targetExpression instanceof Monotonic)) { + return result; + } + + Monotonic monotonic = (Monotonic) targetExpression; + int childIndex = monotonic.getMonotonicFunctionChildIndex(); + if (childIndex < 0 || childIndex >= targetExpression.arity() + || !(targetExpression.child(childIndex) instanceof Slot) + || !hasInputSlotOnlyInMonotonicChild(targetExpression, childIndex)) { + return result; + } + + for (Long partitionId : scan.getSelectedPartitionIds()) { + PartitionItem item = partitionInfo.getItem(partitionId); + if (!(item instanceof RangePartitionItem)) { + continue; + } + Range range = ((RangePartitionItem) item).getItems(); + Literal lower = null; + Literal upper = null; + if (range.hasLowerBound() && !range.lowerEndpoint().isMinValue()) { + lower = toNereidsLiteral(range.lowerEndpoint().getKeys().get(0), partitionColumn); + if (lower == null) { + continue; + } + } + if (range.hasUpperBound() && !range.upperEndpoint().isMaxValue()) { + upper = toNereidsLiteral(range.upperEndpoint().getKeys().get(0), partitionColumn); + if (upper == null) { + continue; + } + } + if (monotonic.isMonotonic(lower, upper)) { + result.put(partitionId, monotonic.isPositive() + ? TTargetExprMonotonicity.MONOTONIC_INCREASING + : TTargetExprMonotonicity.MONOTONIC_DECREASING); + } + } + return result; + } + + static boolean hasInputSlotOnlyInMonotonicChild(Expression expression, int monotonicChildIndex) { + for (int i = 0; i < expression.arity(); i++) { + if (i != monotonicChildIndex && !expression.child(i).getInputSlots().isEmpty()) { + return false; + } + } + return true; + } + + private static Map allSelectedPartitionsIncreasing( + PhysicalOlapScan scan, PartitionInfo partitionInfo) { + Map result = new HashMap<>(); + for (Long partitionId : scan.getSelectedPartitionIds()) { + PartitionItem item = partitionInfo.getItem(partitionId); + if (item == null || (item instanceof ListPartitionItem + && ((ListPartitionItem) item).isDefaultPartition())) { + continue; + } + result.put(partitionId, TTargetExprMonotonicity.MONOTONIC_INCREASING); + } + return result; + } + + private static Literal toNereidsLiteral(LiteralExpr literalExpr, Column column) { + try { + return Literal.fromLegacyLiteral(literalExpr, column.getType()); + } catch (AnalysisException e) { + return null; + } + } + + static final class Classification { + private final BucketClassification bucketClassification; + private final PartitionClassification partitionClassification; + + private Classification(BucketClassification bucketClassification, + PartitionClassification partitionClassification) { + this.bucketClassification = bucketClassification; + this.partitionClassification = partitionClassification; + } + + boolean canPruneBuckets() { + return bucketClassification.canPruneBuckets; + } + + boolean canPrunePartitions() { + return !partitionClassification.partitionMonotonicity.isEmpty(); + } + + Map getPartitionMonotonicity() { + return partitionClassification.partitionMonotonicity; + } + + String getBucketUnsupportedReason() { + return bucketClassification.unsupportedReason; + } + + String getPartitionUnsupportedReason() { + return partitionClassification.unsupportedReason; + } + } + + private static final class BucketClassification { + private final boolean canPruneBuckets; + private final String unsupportedReason; + + private BucketClassification(boolean canPruneBuckets, String unsupportedReason) { + this.canPruneBuckets = canPruneBuckets; + this.unsupportedReason = unsupportedReason; + } + + private static BucketClassification supported() { + return new BucketClassification(true, ""); + } + + private static BucketClassification unsupported(String reason) { + return new BucketClassification(false, reason); + } + } + + private static final class PartitionClassification { + private final Map partitionMonotonicity; + private final String unsupportedReason; + + private PartitionClassification( + Map partitionMonotonicity, String unsupportedReason) { + this.partitionMonotonicity = partitionMonotonicity; + this.unsupportedReason = unsupportedReason; + } + + private static PartitionClassification supported( + Map partitionMonotonicity) { + return new PartitionClassification(ImmutableMap.copyOf(partitionMonotonicity), ""); + } + + private static PartitionClassification unsupported(String reason) { + return new PartitionClassification(ImmutableMap.of(), reason); + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterPushDownVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterPushDownVisitor.java index b10b478aba64be..eb5441e2b474a7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterPushDownVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterPushDownVisitor.java @@ -197,6 +197,7 @@ public Boolean visitPhysicalRelation(PhysicalRelation scan, PushDownContext ctx) ctx.srcExpr, scanSlot, ctx.probeExpr, type, ctx.exprOrder, ctx.builderNode, ctx.buildSideNdv, !ctx.hasUnknownColStats, ctx.singleSideMinMax, scan); + ctx.rfContext.generateRuntimeFilterPruneMetadata(filter); scan.addAppliedRuntimeFilter(filter); ctx.rfContext.addJoinToTargetMap(ctx.builderNode, scanSlot.getExprId()); ctx.rfContext.setTargetExprIdToFilter(scanSlot.getExprId(), filter); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/RuntimeFilter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/RuntimeFilter.java index b0745ef2648310..9a83db27ba1113 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/RuntimeFilter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/RuntimeFilter.java @@ -23,11 +23,14 @@ import org.apache.doris.qe.ConnectContext; import org.apache.doris.thrift.TMinMaxRuntimeFilterType; import org.apache.doris.thrift.TRuntimeFilterType; +import org.apache.doris.thrift.TTargetExprMonotonicity; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import java.util.List; +import java.util.Map; /** * runtime filter @@ -53,6 +56,11 @@ public class RuntimeFilter { private boolean nonBlocking; + // Generated once with the runtime filter at its final target scan. Translation only + // maps this target-scoped metadata to the legacy scan node id. + private boolean canPruneBuckets; + private Map partitionMonotonicity = ImmutableMap.of(); + /** * constructor */ @@ -195,4 +203,22 @@ public void setNonBlocking(boolean nonBlocking) { public boolean isBloomFilterSizeCalculatedByNdv() { return bloomFilterSizeCalculatedByNdv; } + + public void setPruningMetadata(boolean canPruneBuckets, + Map partitionMonotonicity) { + this.canPruneBuckets = canPruneBuckets; + this.partitionMonotonicity = ImmutableMap.copyOf(partitionMonotonicity); + } + + public boolean canPruneBuckets() { + return canPruneBuckets; + } + + public boolean canPrunePartitions() { + return !partitionMonotonicity.isEmpty(); + } + + public Map getPartitionMonotonicity() { + return partitionMonotonicity; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java index b682b329fff60f..08f7c8efc92b0a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java @@ -65,6 +65,7 @@ import org.apache.doris.common.FeConstants; import org.apache.doris.common.Pair; import org.apache.doris.common.UserException; +import org.apache.doris.common.util.DebugPointUtil; import org.apache.doris.common.util.DebugUtil; import org.apache.doris.common.util.TimeUtils; import org.apache.doris.nereids.exceptions.ParseException; @@ -128,6 +129,9 @@ // Full scan of an Olap table. public class OlapScanNode extends ScanNode { private static final Logger LOG = LogManager.getLogger(OlapScanNode.class); + @VisibleForTesting + static final String MISSING_RF_BUCKET_METADATA_DEBUG_POINT = + "OlapScanNode.setRuntimeFilterBucketPruneParameters.missingBucketMetadata"; // average compression ratio in doris storage engine private static final int COMPRESSION_RATIO = 5; @@ -212,6 +216,7 @@ public class OlapScanNode extends ScanNode { // Pack bucket number and sequence into one value to avoid retaining two all-tablet maps. private Map tabletId2BucketInfo = Maps.newHashMap(); + private boolean runtimeFilterBucketPruneParametersSet = false; // a bucket seq may map to many tablets, and each tablet has a // TScanRangeLocations. public ArrayListMultimap bucketSeq2locations = ArrayListMultimap.create(); @@ -1479,13 +1484,12 @@ protected void toThrift(TPlanNode msg) { // filter whose target expression can drive partition pruning according // to the FE-side classifier, so we don't bloat thrift for tables with // many partitions but no usable RF target. - // Gated by session variable `enable_runtime_filter_partition_prune`. - ConnectContext rfPruneCtx = ConnectContext.get(); - if (rfPruneCtx != null - && rfPruneCtx.getSessionVariable().isEnableRuntimeFilterPartitionPrune() - && hasRfDrivingPartitionPruning()) { + if (hasRfDrivingPartitionPruning()) { setPartitionBoundariesForRuntimeFilter(msg.olap_scan_node); } + if (hasRfDrivingBucketPruning()) { + setRuntimeFilterBucketPruneParameters(); + } super.toThrift(msg); } @@ -1545,19 +1549,34 @@ private boolean hasRfDrivingBucketPruning() { return false; } - public void setRuntimeFilterBucketPruneParameters() { - if (!hasRfDrivingBucketPruning()) { + @VisibleForTesting + synchronized void setRuntimeFilterBucketPruneParameters() { + if (runtimeFilterBucketPruneParametersSet) { return; } + long debugMissingTabletId = DebugPointUtil.getDebugParamOrDefault( + MISSING_RF_BUCKET_METADATA_DEBUG_POINT, -1L); for (TScanRangeLocations locations : scanRangeLocations) { TPaloScanRange scanRange = locations.getScanRange().getPaloScanRange(); Long bucketInfo = tabletId2BucketInfo.get(scanRange.getTabletId()); - Preconditions.checkState(bucketInfo != null && decodeBucketNum(bucketInfo) > 0, - "missing bucket metadata for runtime-filter bucket pruning, tablet=%s", - scanRange.getTabletId()); + if (scanRange.getTabletId() == debugMissingTabletId) { + bucketInfo = null; + } + if (bucketInfo == null || decodeBucketNum(bucketInfo) <= 0) { + LOG.warn("missing bucket metadata for runtime-filter bucket pruning, " + + "scanNode={}, tablet={}; disable pruning for this scan node", + getId(), scanRange.getTabletId()); + runtimeFilterBucketPruneParametersSet = true; + return; + } + } + for (TScanRangeLocations locations : scanRangeLocations) { + TPaloScanRange scanRange = locations.getScanRange().getPaloScanRange(); + long bucketInfo = tabletId2BucketInfo.get(scanRange.getTabletId()); scanRange.setBucketSeq(decodeBucketSeq(bucketInfo)); scanRange.setBucketNum(decodeBucketNum(bucketInfo)); } + runtimeFilterBucketPruneParametersSet = true; } private static long encodeBucketInfo(int bucketSeq, int bucketNum) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/RuntimeFilter.java b/fe/fe-core/src/main/java/org/apache/doris/planner/RuntimeFilter.java index 24ea1cf9d11a1c..d98224d141acbf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/RuntimeFilter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/RuntimeFilter.java @@ -140,9 +140,8 @@ public FilterSizeLimits(SessionVariable sessionVariable) { private int waitTimeMs = -1; // Per-target monotonicity for BE-side runtime-filter partition pruning, - // keyed by the target scan node's plan id. Filled by the Nereids - // RuntimeFilterPartitionPruneClassifier at translation time from the final - // legacy target expression that will be sent to BE. + // keyed by the target scan node's plan id. Generated with the Nereids + // runtime filter and mapped to the legacy target scan id during translation. private final Map> targetPartitionMonotonicityByScanId = new HashMap<>(); private final Set partitionPruningTargetScanIds = new HashSet<>(); @@ -359,42 +358,34 @@ public TRuntimeFilterDesc toThrift() { } // Per-target, per-partition monotonicity for BE-side partition pruning. Populated - // upstream by RuntimeFilterPartitionPruneClassifier; direct partition + // upstream by RuntimeFilterPruneClassifier; direct partition // column targets are monotonic increasing so BE can use one unified // partition-pruning path. - // Gated by session variable `enable_runtime_filter_partition_prune`. - ConnectContext rfPruneCtx = ConnectContext.get(); - boolean enableRfPartitionPrune = rfPruneCtx != null - && rfPruneCtx.getSessionVariable().isEnableRuntimeFilterPartitionPrune(); - if (enableRfPartitionPrune) { - if (!targetPartitionMonotonicityByScanId.isEmpty()) { - Map> partitionMonoMap = new HashMap<>(); - for (Map.Entry> e - : targetPartitionMonotonicityByScanId.entrySet()) { - List partitionMonoList = new ArrayList<>(); - for (Map.Entry partitionEntry : e.getValue().entrySet()) { - Preconditions.checkArgument( - partitionEntry.getValue() != TTargetExprMonotonicity.NON_MONOTONIC, - "partition pruning monotonicity must not be NON_MONOTONIC"); - TPartitionTargetExprMonotonicity partitionMono = - new TPartitionTargetExprMonotonicity(); - partitionMono.setPartitionId(partitionEntry.getKey()); - partitionMono.setMonotonicity(partitionEntry.getValue()); - partitionMonoList.add(partitionMono); - } - if (!partitionMonoList.isEmpty()) { - partitionMonoMap.put(e.getKey().asInt(), partitionMonoList); - } + if (!targetPartitionMonotonicityByScanId.isEmpty()) { + Map> partitionMonoMap = new HashMap<>(); + for (Map.Entry> e + : targetPartitionMonotonicityByScanId.entrySet()) { + List partitionMonoList = new ArrayList<>(); + for (Map.Entry partitionEntry : e.getValue().entrySet()) { + Preconditions.checkArgument( + partitionEntry.getValue() != TTargetExprMonotonicity.NON_MONOTONIC, + "partition pruning monotonicity must not be NON_MONOTONIC"); + TPartitionTargetExprMonotonicity partitionMono = + new TPartitionTargetExprMonotonicity(); + partitionMono.setPartitionId(partitionEntry.getKey()); + partitionMono.setMonotonicity(partitionEntry.getValue()); + partitionMonoList.add(partitionMono); } - if (!partitionMonoMap.isEmpty()) { - tFilter.setPlanIdToPartitionTargetMonotonicity(partitionMonoMap); + if (!partitionMonoList.isEmpty()) { + partitionMonoMap.put(e.getKey().asInt(), partitionMonoList); } } + if (!partitionMonoMap.isEmpty()) { + tFilter.setPlanIdToPartitionTargetMonotonicity(partitionMonoMap); + } } - boolean enableRfBucketPrune = rfPruneCtx != null - && rfPruneCtx.getSessionVariable().isEnableRuntimeFilterBucketPrune(); - if (enableRfBucketPrune && !bucketPruningTargetScanIds.isEmpty()) { + if (!bucketPruningTargetScanIds.isEmpty()) { tFilter.setBucketPruningTargetIds(bucketPruningTargetScanIds.stream() .map(PlanNodeId::asInt) .collect(Collectors.toSet())); @@ -425,7 +416,7 @@ public void setTargetPartitionMonotonicity(PlanNodeId scanNodeId, * scan node. Used by OlapScanNode.toThrift to decide whether it is worth * serializing partition_boundaries to BE. The single source of truth for * that decision lives in - * RuntimeFilterPartitionPruneClassifier. + * RuntimeFilterPruneClassifier. */ public boolean canPrunePartitionsFor(PlanNodeId scanNodeId) { return partitionPruningTargetScanIds.contains(scanNodeId); diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/ThriftPlansBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/ThriftPlansBuilder.java index b459233ed95d21..1a239e3122a365 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/ThriftPlansBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/runtime/ThriftPlansBuilder.java @@ -41,7 +41,6 @@ import org.apache.doris.planner.DataStreamSink; import org.apache.doris.planner.ExchangeNode; import org.apache.doris.planner.MultiCastDataSink; -import org.apache.doris.planner.OlapScanNode; import org.apache.doris.planner.OlapTableSink; import org.apache.doris.planner.PlanFragment; import org.apache.doris.planner.PlanFragmentId; @@ -118,8 +117,6 @@ public static Map plansToThr // we should set runtime predicate first, then we can use heap sort and to thrift setRuntimePredicateIfNeed(coordinatorContext.scanNodes); - setRuntimeFilterBucketPruneParametersIfNeeded( - coordinatorContext.scanNodes, coordinatorContext.connectContext); int broadcastRuntimeFilterProducerNum = coordinatorContext.connectContext == null ? 0 @@ -208,19 +205,6 @@ static void setRuntimePredicateIfNeed(Collection scanNodes) { } } - static void setRuntimeFilterBucketPruneParametersIfNeeded( - Collection scanNodes, ConnectContext connectContext) { - if (connectContext == null - || !connectContext.getSessionVariable().isEnableRuntimeFilterBucketPrune()) { - return; - } - for (ScanNode scanNode : scanNodes) { - if (scanNode instanceof OlapScanNode) { - ((OlapScanNode) scanNode).setRuntimeFilterBucketPruneParameters(); - } - } - } - private static Supplier> topNFilterToThrift(List topnFilters) { return Suppliers.memoize(() -> { if (CollectionUtils.isEmpty(topnFilters)) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslatorTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslatorTest.java index eb885887ded140..68d2e3457f5254 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslatorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslatorTest.java @@ -404,7 +404,7 @@ public void testRfPartitionPruneSnapshotSurvivesEnablementChange() throws Except boolean oldDisableJoinReorder = connectContext.getSessionVariable().isDisableJoinReorder(); try { connectContext.getSessionVariable().setRuntimeFilterType(TRuntimeFilterType.MIN_MAX.getValue()); - connectContext.getSessionVariable().setEnableRuntimeFilterPartitionPrune(false); + connectContext.getSessionVariable().setEnableRuntimeFilterPartitionPrune(true); connectContext.getSessionVariable().setEnableRuntimeFilterPrune(false); connectContext.getSessionVariable().setDisableJoinReorder(true); @@ -423,7 +423,7 @@ public void testRfPartitionPruneSnapshotSurvivesEnablementChange() throws Except .orElseThrow(); Assertions.assertEquals(2, partitionedScan.getSelectedPartitionIds().size()); - connectContext.getSessionVariable().setEnableRuntimeFilterPartitionPrune(true); + connectContext.getSessionVariable().setEnableRuntimeFilterPartitionPrune(false); TPlanNode thriftScanNode = partitionedScan.treeToThrift().getNodes().get(0); Assertions.assertTrue(thriftScanNode.olap_scan_node.isSetPartitionBoundaries()); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/RuntimeFilterBucketPruneClassifierTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/RuntimeFilterBucketPruneClassifierTest.java deleted file mode 100644 index dec5a0878bc3c6..00000000000000 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/RuntimeFilterBucketPruneClassifierTest.java +++ /dev/null @@ -1,228 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.glue.translator; - -import org.apache.doris.analysis.FunctionCallExpr; -import org.apache.doris.analysis.SlotDescriptor; -import org.apache.doris.analysis.SlotId; -import org.apache.doris.analysis.SlotRef; -import org.apache.doris.analysis.TupleId; -import org.apache.doris.catalog.Column; -import org.apache.doris.catalog.HashDistributionInfo; -import org.apache.doris.catalog.OlapTable; -import org.apache.doris.catalog.Partition; -import org.apache.doris.catalog.PrimitiveType; -import org.apache.doris.catalog.RandomDistributionInfo; -import org.apache.doris.planner.OlapScanNode; -import org.apache.doris.thrift.TRuntimeFilterType; - -import com.google.common.collect.ImmutableList; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; -import org.mockito.Mockito; - -class RuntimeFilterBucketPruneClassifierTest { - @Test - void testSingleColumnHashInSupported() { - Column distributionColumn = new Column("dist_col", PrimitiveType.INT); - RuntimeFilterBucketPruneClassifier.Classification classification = classify( - TRuntimeFilterType.IN, distributionColumn, - new HashDistributionInfo(8, ImmutableList.of(distributionColumn))); - - Assertions.assertTrue(classification.canPruneBuckets()); - } - - @Test - void testInOrBloomSupportedAtPlanTime() { - Column distributionColumn = new Column("dist_col", PrimitiveType.INT); - RuntimeFilterBucketPruneClassifier.Classification classification = classify( - TRuntimeFilterType.IN_OR_BLOOM, distributionColumn, - new HashDistributionInfo(8, ImmutableList.of(distributionColumn))); - - Assertions.assertTrue(classification.canPruneBuckets()); - } - - @Test - void testPointQueryRejected() { - Column distributionColumn = new Column("dist_col", PrimitiveType.INT); - RuntimeFilterBucketPruneClassifier.Classification classification = classify( - TRuntimeFilterType.IN, distributionColumn, - new HashDistributionInfo(8, ImmutableList.of(distributionColumn)), true); - - Assertions.assertFalse(classification.canPruneBuckets()); - Assertions.assertTrue(classification.getUnsupportedReason().contains("point query")); - } - - @Test - void testBloomRejected() { - Column distributionColumn = new Column("dist_col", PrimitiveType.INT); - RuntimeFilterBucketPruneClassifier.Classification classification = classify( - TRuntimeFilterType.BLOOM, distributionColumn, - new HashDistributionInfo(8, ImmutableList.of(distributionColumn))); - - Assertions.assertFalse(classification.canPruneBuckets()); - Assertions.assertTrue(classification.getUnsupportedReason().contains("IN")); - } - - @Test - void testCompositeHashRejected() { - Column distributionColumn = new Column("dist_col", PrimitiveType.INT); - Column secondDistributionColumn = new Column("dist_col_2", PrimitiveType.INT); - RuntimeFilterBucketPruneClassifier.Classification classification = classify( - TRuntimeFilterType.IN, distributionColumn, - new HashDistributionInfo(8, - ImmutableList.of(distributionColumn, secondDistributionColumn))); - - Assertions.assertFalse(classification.canPruneBuckets()); - Assertions.assertTrue(classification.getUnsupportedReason().contains("single-column")); - } - - @Test - void testNonDistributionTargetRejected() { - Column distributionColumn = new Column("dist_col", PrimitiveType.INT); - Column targetColumn = new Column("value_col", PrimitiveType.INT); - RuntimeFilterBucketPruneClassifier.Classification classification = classify( - TRuntimeFilterType.IN, targetColumn, - new HashDistributionInfo(8, ImmutableList.of(distributionColumn))); - - Assertions.assertFalse(classification.canPruneBuckets()); - Assertions.assertTrue(classification.getUnsupportedReason().contains("distribution column")); - } - - @Test - void testBaseColumnNamesComparedSymmetrically() { - Column baseColumn = new Column("base_col", PrimitiveType.INT); - SlotDescriptor baseSlotDescriptor = new SlotDescriptor(new SlotId(2), new TupleId(2)); - baseSlotDescriptor.setColumn(baseColumn); - baseSlotDescriptor.setType(baseColumn.getType()); - - Column distributionColumn = new Column("mv_dist_col", PrimitiveType.INT); - distributionColumn.setDefineExpr(new SlotRef(baseSlotDescriptor)); - RuntimeFilterBucketPruneClassifier.Classification classification = classify( - TRuntimeFilterType.IN, baseColumn, - new HashDistributionInfo(8, ImmutableList.of(distributionColumn))); - - Assertions.assertTrue(classification.canPruneBuckets()); - } - - @Test - void testComputedMvAliasCollisionRejected() { - Column baseDistributionColumn = new Column("dist_col", PrimitiveType.INT); - SlotDescriptor baseSlotDescriptor = new SlotDescriptor(new SlotId(2), new TupleId(2)); - baseSlotDescriptor.setColumn(baseDistributionColumn); - baseSlotDescriptor.setType(baseDistributionColumn.getType()); - - Column computedMvColumn = new Column("dist_col", PrimitiveType.INT); - computedMvColumn.setDefineExpr(new FunctionCallExpr("abs", - ImmutableList.of(new SlotRef(baseSlotDescriptor)), true)); - RuntimeFilterBucketPruneClassifier.Classification classification = classify( - TRuntimeFilterType.IN, computedMvColumn, - new HashDistributionInfo(8, ImmutableList.of(baseDistributionColumn))); - - Assertions.assertFalse(classification.canPruneBuckets()); - Assertions.assertTrue(classification.getUnsupportedReason().contains("distribution column")); - } - - @Test - void testDifferentUniqueIdsAllowedForSameBaseColumn() { - Column distributionColumn = new Column("dist_col", PrimitiveType.INT); - distributionColumn.setUniqueId(1); - Column targetColumn = new Column("dist_col", PrimitiveType.INT); - targetColumn.setUniqueId(2); - - RuntimeFilterBucketPruneClassifier.Classification classification = classify( - TRuntimeFilterType.IN, targetColumn, - new HashDistributionInfo(8, ImmutableList.of(distributionColumn))); - - Assertions.assertTrue(classification.canPruneBuckets()); - } - - @Test - void testRollupUniqueIdCollisionRejectedForDifferentBaseColumns() { - Column baseDistributionColumn = new Column("k2", PrimitiveType.INT); - baseDistributionColumn.setUniqueId(1); - - Column baseTargetColumn = new Column("k1", PrimitiveType.INT); - SlotDescriptor baseSlotDescriptor = new SlotDescriptor(new SlotId(2), new TupleId(2)); - baseSlotDescriptor.setColumn(baseTargetColumn); - baseSlotDescriptor.setType(baseTargetColumn.getType()); - Column rollupTargetColumn = new Column("mv_k1", PrimitiveType.INT); - rollupTargetColumn.setUniqueId(1); - rollupTargetColumn.setDefineExpr(new SlotRef(baseSlotDescriptor)); - - RuntimeFilterBucketPruneClassifier.Classification classification = classify( - TRuntimeFilterType.IN, rollupTargetColumn, - new HashDistributionInfo(8, ImmutableList.of(baseDistributionColumn))); - - Assertions.assertFalse(classification.canPruneBuckets()); - Assertions.assertTrue(classification.getUnsupportedReason().contains("distribution column")); - } - - @Test - void testRollupWithDifferentTypeRejected() { - Column baseDistributionColumn = new Column("dist_col", PrimitiveType.INT); - SlotDescriptor baseSlotDescriptor = new SlotDescriptor(new SlotId(2), new TupleId(2)); - baseSlotDescriptor.setColumn(baseDistributionColumn); - baseSlotDescriptor.setType(baseDistributionColumn.getType()); - Column rollupTargetColumn = new Column("mv_dist_col", PrimitiveType.BIGINT); - rollupTargetColumn.setDefineExpr(new SlotRef(baseSlotDescriptor)); - - RuntimeFilterBucketPruneClassifier.Classification classification = classify( - TRuntimeFilterType.IN, rollupTargetColumn, - new HashDistributionInfo(8, ImmutableList.of(baseDistributionColumn))); - - Assertions.assertFalse(classification.canPruneBuckets()); - Assertions.assertTrue(classification.getUnsupportedReason().contains("distribution column")); - } - - @Test - void testRandomDistributionRejected() { - Column targetColumn = new Column("dist_col", PrimitiveType.INT); - RuntimeFilterBucketPruneClassifier.Classification classification = classify( - TRuntimeFilterType.IN, targetColumn, new RandomDistributionInfo(8)); - - Assertions.assertFalse(classification.canPruneBuckets()); - Assertions.assertTrue(classification.getUnsupportedReason().contains("not HASH")); - } - - private RuntimeFilterBucketPruneClassifier.Classification classify( - TRuntimeFilterType filterType, Column targetColumn, - org.apache.doris.catalog.DistributionInfo distributionInfo) { - return classify(filterType, targetColumn, distributionInfo, false); - } - - private RuntimeFilterBucketPruneClassifier.Classification classify( - TRuntimeFilterType filterType, Column targetColumn, - org.apache.doris.catalog.DistributionInfo distributionInfo, boolean isPointQuery) { - SlotDescriptor slotDescriptor = new SlotDescriptor(new SlotId(1), new TupleId(1)); - slotDescriptor.setColumn(targetColumn); - slotDescriptor.setType(targetColumn.getType()); - SlotRef targetSlot = new SlotRef(slotDescriptor); - - OlapTable table = Mockito.mock(OlapTable.class); - Partition partition = Mockito.mock(Partition.class); - OlapScanNode scanNode = Mockito.mock(OlapScanNode.class); - Mockito.when(scanNode.isPointQuery()).thenReturn(isPointQuery); - Mockito.when(scanNode.getOlapTable()).thenReturn(table); - Mockito.when(scanNode.getSelectedPartitionIds()).thenReturn(ImmutableList.of(1L)); - Mockito.when(table.getPartition(1L)).thenReturn(partition); - Mockito.when(partition.getDistributionInfo()).thenReturn(distributionInfo); - - return RuntimeFilterBucketPruneClassifier.classify(filterType, targetSlot, scanNode); - } -} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/RuntimeFilterPartitionPruneClassifierTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/RuntimeFilterPartitionPruneClassifierTest.java deleted file mode 100644 index 58e32440364c49..00000000000000 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/RuntimeFilterPartitionPruneClassifierTest.java +++ /dev/null @@ -1,163 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.glue.translator; - -import org.apache.doris.analysis.BinaryPredicate; -import org.apache.doris.analysis.Expr; -import org.apache.doris.analysis.FunctionCallExpr; -import org.apache.doris.analysis.IntLiteral; -import org.apache.doris.analysis.SlotDescriptor; -import org.apache.doris.analysis.SlotId; -import org.apache.doris.analysis.SlotRef; -import org.apache.doris.analysis.StringLiteral; -import org.apache.doris.analysis.TupleId; -import org.apache.doris.catalog.Column; -import org.apache.doris.catalog.ListPartitionItem; -import org.apache.doris.catalog.OlapTable; -import org.apache.doris.catalog.PartitionInfo; -import org.apache.doris.catalog.PartitionItem; -import org.apache.doris.catalog.PartitionType; -import org.apache.doris.catalog.PrimitiveType; -import org.apache.doris.catalog.RangePartitionItem; -import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.expressions.GreaterThan; -import org.apache.doris.nereids.trees.expressions.SlotReference; -import org.apache.doris.nereids.trees.expressions.functions.Monotonic; -import org.apache.doris.nereids.trees.expressions.functions.scalar.AssertTrue; -import org.apache.doris.nereids.trees.expressions.functions.scalar.DateTrunc; -import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral; -import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral; -import org.apache.doris.nereids.types.DateTimeV2Type; -import org.apache.doris.nereids.types.IntegerType; -import org.apache.doris.planner.OlapScanNode; -import org.apache.doris.thrift.TRuntimeFilterType; -import org.apache.doris.thrift.TTargetExprMonotonicity; - -import com.google.common.collect.ImmutableList; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; -import org.mockito.Mockito; - -import java.util.Map; -import java.util.function.Function; - -class RuntimeFilterPartitionPruneClassifierTest { - @Test - void testRejectInputSlotOutsideMonotonicChild() { - SlotReference slot = new SlotReference("dt", DateTimeV2Type.SYSTEM_DEFAULT); - DateTrunc dateTrunc = new DateTrunc(slot, slot); - Monotonic monotonic = dateTrunc; - - Assertions.assertEquals(1, dateTrunc.getInputSlots().size()); - Assertions.assertFalse(RuntimeFilterPartitionPruneClassifier.hasInputSlotOnlyInMonotonicChild( - dateTrunc, monotonic.getMonotonicFunctionChildIndex())); - } - - @Test - void testAcceptInputSlotOnlyInMonotonicChild() { - SlotReference slot = new SlotReference("dt", DateTimeV2Type.SYSTEM_DEFAULT); - DateTrunc dateTrunc = new DateTrunc(slot, new VarcharLiteral("day")); - Monotonic monotonic = dateTrunc; - - Assertions.assertTrue(RuntimeFilterPartitionPruneClassifier.hasInputSlotOnlyInMonotonicChild( - dateTrunc, monotonic.getMonotonicFunctionChildIndex())); - } - - @Test - void testBloomRangePartitionUnsupported() { - RuntimeFilterPartitionPruneClassifier.Classification classification = classify( - TRuntimeFilterType.BLOOM, PartitionType.RANGE, RangePartitionItem.DUMMY_ITEM); - - Assertions.assertFalse(classification.canPrunePartitions()); - Assertions.assertTrue(classification.getUnsupportedReason().contains("BLOOM")); - Assertions.assertTrue(classification.getPartitionMonotonicity().isEmpty()); - } - - @Test - void testBloomListPartitionSupported() { - RuntimeFilterPartitionPruneClassifier.Classification classification = classify( - TRuntimeFilterType.BLOOM, PartitionType.LIST, ListPartitionItem.DUMMY_ITEM); - - assertSupportedIncreasingPartitions(classification); - } - - @Test - void testInOrBloomRangePartitionStillSupported() { - RuntimeFilterPartitionPruneClassifier.Classification classification = classify( - TRuntimeFilterType.IN_OR_BLOOM, PartitionType.RANGE, RangePartitionItem.DUMMY_ITEM); - - assertSupportedIncreasingPartitions(classification); - } - - @Test - void testRejectNoneMovableListTargetExpression() { - RuntimeFilterPartitionPruneClassifier.Classification classification = classify( - TRuntimeFilterType.IN, PartitionType.LIST, ListPartitionItem.DUMMY_ITEM, - targetSlot -> new FunctionCallExpr("assert_true", ImmutableList.of( - new BinaryPredicate(BinaryPredicate.Operator.NE, targetSlot, new IntLiteral(0)), - new StringLiteral("rfpp_expr_in_only_error")), false), - targetSlot -> new AssertTrue( - new GreaterThan(targetSlot, new IntegerLiteral(0)), - new VarcharLiteral("rfpp_expr_in_only_error"))); - - Assertions.assertFalse(classification.canPrunePartitions()); - Assertions.assertTrue(classification.getUnsupportedReason().contains("non-movable")); - Assertions.assertTrue(classification.getPartitionMonotonicity().isEmpty()); - } - - private RuntimeFilterPartitionPruneClassifier.Classification classify( - TRuntimeFilterType filterType, PartitionType partitionType, PartitionItem partitionItem) { - return classify(filterType, partitionType, partitionItem, targetSlot -> targetSlot, - targetSlot -> targetSlot); - } - - private RuntimeFilterPartitionPruneClassifier.Classification classify( - TRuntimeFilterType filterType, PartitionType partitionType, PartitionItem partitionItem, - Function legacyTargetFactory, - Function nereidsTargetFactory) { - Column partitionColumn = new Column("part_col", PrimitiveType.INT); - SlotDescriptor slotDescriptor = new SlotDescriptor(new SlotId(1), new TupleId(1)); - slotDescriptor.setColumn(partitionColumn); - slotDescriptor.setType(partitionColumn.getType()); - SlotRef targetSlot = new SlotRef(slotDescriptor); - SlotReference nereidsTarget = new SlotReference("part_col", IntegerType.INSTANCE); - - OlapTable table = Mockito.mock(OlapTable.class); - PartitionInfo partitionInfo = Mockito.mock(PartitionInfo.class); - OlapScanNode scanNode = Mockito.mock(OlapScanNode.class); - Mockito.when(scanNode.getOlapTable()).thenReturn(table); - Mockito.when(scanNode.getSelectedPartitionIds()).thenReturn(ImmutableList.of(1L, 2L)); - Mockito.when(table.getPartitionInfo()).thenReturn(partitionInfo); - Mockito.when(partitionInfo.getType()).thenReturn(partitionType); - Mockito.when(partitionInfo.getPartitionColumns()).thenReturn(ImmutableList.of(partitionColumn)); - Mockito.when(partitionInfo.getItem(1L)).thenReturn(partitionItem); - Mockito.when(partitionInfo.getItem(2L)).thenReturn(partitionItem); - - return RuntimeFilterPartitionPruneClassifier.classify(filterType, - legacyTargetFactory.apply(targetSlot), nereidsTargetFactory.apply(nereidsTarget), scanNode); - } - - private void assertSupportedIncreasingPartitions( - RuntimeFilterPartitionPruneClassifier.Classification classification) { - Assertions.assertTrue(classification.canPrunePartitions()); - Map monotonicity = classification.getPartitionMonotonicity(); - Assertions.assertEquals(2, monotonicity.size()); - Assertions.assertEquals(TTargetExprMonotonicity.MONOTONIC_INCREASING, monotonicity.get(1L)); - Assertions.assertEquals(TTargetExprMonotonicity.MONOTONIC_INCREASING, monotonicity.get(2L)); - } -} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/RuntimeFilterTranslatorBucketPruneTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/RuntimeFilterTranslatorBucketPruneTest.java index 46f7ec109f49cc..43cd6d7bae709d 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/RuntimeFilterTranslatorBucketPruneTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/RuntimeFilterTranslatorBucketPruneTest.java @@ -34,7 +34,7 @@ import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral; import org.apache.doris.nereids.trees.plans.physical.AbstractPhysicalPlan; -import org.apache.doris.nereids.trees.plans.physical.PhysicalRelation; +import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapScan; import org.apache.doris.nereids.trees.plans.physical.RuntimeFilter; import org.apache.doris.nereids.types.DataType; import org.apache.doris.nereids.types.IntegerType; @@ -140,12 +140,12 @@ void testDisabledFeatureSkipsClassificationAndSerialization() { TranslatorHarness harness = new TranslatorHarness(); SlotReference target = harness.addTargetSlot("dist_col", harness.distributionColumn, IntegerType.INSTANCE); - Mockito.clearInvocations(harness.scanNode); + Mockito.clearInvocations(harness.targetRelation); TRuntimeFilterDesc desc = harness.translate(ImmutableList.of(harness.newFilter(target, target))); - Mockito.verify(harness.scanNode, Mockito.never()).isPointQuery(); - Mockito.verify(harness.scanNode, Mockito.never()).getSelectedPartitionIds(); + Mockito.verify(harness.targetRelation, Mockito.never()).getTable(); + Mockito.verify(harness.targetRelation, Mockito.never()).getSelectedPartitionIds(); Assertions.assertEquals(1, desc.planId_to_target_expr.size()); Assertions.assertFalse(desc.isSetBucketPruningTargetIds()); } @@ -161,7 +161,7 @@ private class TranslatorHarness { private final OlapScanNode scanNode = Mockito.mock(OlapScanNode.class); private final PlanNode builderNode = Mockito.mock(PlanNode.class); private final AbstractPhysicalPlan nereidsBuilder = Mockito.mock(AbstractPhysicalPlan.class); - private final PhysicalRelation targetRelation = Mockito.mock(PhysicalRelation.class); + private final PhysicalOlapScan targetRelation = Mockito.mock(PhysicalOlapScan.class); private final PlanTranslatorContext translatorContext = new PlanTranslatorContext(); private final RuntimeFilterContext runtimeFilterContext; private final RuntimeFilterTranslator translator; @@ -187,6 +187,8 @@ private class TranslatorHarness { Mockito.when(scanNode.getOlapTable()).thenReturn(table); Mockito.when(scanNode.getSelectedPartitionIds()).thenReturn(ImmutableList.of(1L)); Mockito.when(scanNode.getId()).thenReturn(new PlanNodeId(SCAN_NODE_ID)); + Mockito.when(targetRelation.getTable()).thenReturn(table); + Mockito.when(targetRelation.getSelectedPartitionIds()).thenReturn(ImmutableList.of(1L)); PlanFragment fragment = Mockito.mock(PlanFragment.class); PlanFragmentId fragmentId = new PlanFragmentId(3); @@ -214,9 +216,11 @@ SlotReference addTargetSlot(String name, Column column, DataType dataType) { } RuntimeFilter newFilter(SlotReference target, Expression targetExpression) { - return new RuntimeFilter(filterIdGenerator.getNextId(), source, target, targetExpression, + RuntimeFilter filter = new RuntimeFilter(filterIdGenerator.getNextId(), source, target, targetExpression, TRuntimeFilterType.IN, 0, nereidsBuilder, 10, false, TMinMaxRuntimeFilterType.MIN_MAX, targetRelation); + runtimeFilterContext.generateRuntimeFilterPruneMetadata(filter); + return filter; } TRuntimeFilterDesc translate(List filters) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/processor/post/RuntimeFilterPruneClassifierTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/processor/post/RuntimeFilterPruneClassifierTest.java new file mode 100644 index 00000000000000..8bf16e7214f898 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/processor/post/RuntimeFilterPruneClassifierTest.java @@ -0,0 +1,377 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.processor.post; + +import org.apache.doris.analysis.FunctionCallExpr; +import org.apache.doris.analysis.SlotDescriptor; +import org.apache.doris.analysis.SlotId; +import org.apache.doris.analysis.SlotRef; +import org.apache.doris.analysis.TupleId; +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.DistributionInfo; +import org.apache.doris.catalog.HashDistributionInfo; +import org.apache.doris.catalog.ListPartitionItem; +import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.Partition; +import org.apache.doris.catalog.PartitionInfo; +import org.apache.doris.catalog.PartitionItem; +import org.apache.doris.catalog.PartitionType; +import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.catalog.RandomDistributionInfo; +import org.apache.doris.catalog.RangePartitionItem; +import org.apache.doris.nereids.trees.expressions.Add; +import org.apache.doris.nereids.trees.expressions.ExprId; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.GreaterThan; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.functions.Monotonic; +import org.apache.doris.nereids.trees.expressions.functions.scalar.AssertTrue; +import org.apache.doris.nereids.trees.expressions.functions.scalar.DateTrunc; +import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral; +import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral; +import org.apache.doris.nereids.trees.plans.physical.AbstractPhysicalPlan; +import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapScan; +import org.apache.doris.nereids.trees.plans.physical.RuntimeFilter; +import org.apache.doris.nereids.types.DataType; +import org.apache.doris.nereids.types.DateTimeV2Type; +import org.apache.doris.planner.RuntimeFilterId; +import org.apache.doris.qe.SessionVariable; +import org.apache.doris.thrift.TMinMaxRuntimeFilterType; +import org.apache.doris.thrift.TRuntimeFilterType; +import org.apache.doris.thrift.TTargetExprMonotonicity; + +import com.google.common.collect.ImmutableList; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import java.util.Map; +import java.util.function.Function; + +class RuntimeFilterPruneClassifierTest { + @Test + void testSingleColumnHashInSupported() { + Column distributionColumn = new Column("dist_col", PrimitiveType.INT); + RuntimeFilterPruneClassifier.Classification classification = classifyBucket( + TRuntimeFilterType.IN, distributionColumn, + new HashDistributionInfo(8, ImmutableList.of(distributionColumn))); + + Assertions.assertTrue(classification.canPruneBuckets()); + } + + @Test + void testInOrBloomSupportedAtPlanTime() { + Column distributionColumn = new Column("dist_col", PrimitiveType.INT); + RuntimeFilterPruneClassifier.Classification classification = classifyBucket( + TRuntimeFilterType.IN_OR_BLOOM, distributionColumn, + new HashDistributionInfo(8, ImmutableList.of(distributionColumn))); + + Assertions.assertTrue(classification.canPruneBuckets()); + } + + @Test + void testBloomAndNonHashDistributionsRejected() { + Column distributionColumn = new Column("dist_col", PrimitiveType.INT); + RuntimeFilterPruneClassifier.Classification bloomClassification = classifyBucket( + TRuntimeFilterType.BLOOM, distributionColumn, + new HashDistributionInfo(8, ImmutableList.of(distributionColumn))); + RuntimeFilterPruneClassifier.Classification randomClassification = classifyBucket( + TRuntimeFilterType.IN, distributionColumn, new RandomDistributionInfo(8)); + + Assertions.assertFalse(bloomClassification.canPruneBuckets()); + Assertions.assertTrue(bloomClassification.getBucketUnsupportedReason().contains("IN")); + Assertions.assertFalse(randomClassification.canPruneBuckets()); + Assertions.assertTrue(randomClassification.getBucketUnsupportedReason().contains("not HASH")); + } + + @Test + void testCompositeHashAndNonDistributionTargetsRejected() { + Column distributionColumn = new Column("dist_col", PrimitiveType.INT); + Column valueColumn = new Column("value_col", PrimitiveType.INT); + RuntimeFilterPruneClassifier.Classification compositeClassification = classifyBucket( + TRuntimeFilterType.IN, distributionColumn, + new HashDistributionInfo(8, ImmutableList.of(distributionColumn, valueColumn))); + RuntimeFilterPruneClassifier.Classification nonDistributionClassification = classifyBucket( + TRuntimeFilterType.IN, valueColumn, + new HashDistributionInfo(8, ImmutableList.of(distributionColumn))); + + Assertions.assertFalse(compositeClassification.canPruneBuckets()); + Assertions.assertTrue(compositeClassification.getBucketUnsupportedReason().contains("single-column")); + Assertions.assertFalse(nonDistributionClassification.canPruneBuckets()); + Assertions.assertTrue(nonDistributionClassification.getBucketUnsupportedReason() + .contains("distribution column")); + } + + @Test + void testDirectMvAliasSupportedButComputedAliasRejected() { + Column baseColumn = new Column("base_col", PrimitiveType.INT); + SlotDescriptor baseSlotDescriptor = new SlotDescriptor(new SlotId(2), new TupleId(2)); + baseSlotDescriptor.setColumn(baseColumn); + baseSlotDescriptor.setType(baseColumn.getType()); + + Column directMvColumn = new Column("mv_base_col", PrimitiveType.INT); + directMvColumn.setDefineExpr(new SlotRef(baseSlotDescriptor)); + RuntimeFilterPruneClassifier.Classification directClassification = classifyBucket( + TRuntimeFilterType.IN, directMvColumn, + new HashDistributionInfo(8, ImmutableList.of(baseColumn))); + + Column computedMvColumn = new Column("base_col", PrimitiveType.INT); + computedMvColumn.setDefineExpr(new FunctionCallExpr("abs", + ImmutableList.of(new SlotRef(baseSlotDescriptor)), true)); + RuntimeFilterPruneClassifier.Classification computedClassification = classifyBucket( + TRuntimeFilterType.IN, computedMvColumn, + new HashDistributionInfo(8, ImmutableList.of(baseColumn))); + + Assertions.assertTrue(directClassification.canPruneBuckets()); + Assertions.assertFalse(computedClassification.canPruneBuckets()); + } + + @Test + void testBaseColumnNamesComparedSymmetrically() { + Column baseColumn = new Column("base_col", PrimitiveType.INT); + SlotDescriptor baseSlotDescriptor = new SlotDescriptor(new SlotId(2), new TupleId(2)); + baseSlotDescriptor.setColumn(baseColumn); + baseSlotDescriptor.setType(baseColumn.getType()); + + Column distributionColumn = new Column("mv_dist_col", PrimitiveType.INT); + distributionColumn.setDefineExpr(new SlotRef(baseSlotDescriptor)); + RuntimeFilterPruneClassifier.Classification classification = classifyBucket( + TRuntimeFilterType.IN, baseColumn, + new HashDistributionInfo(8, ImmutableList.of(distributionColumn))); + + Assertions.assertTrue(classification.canPruneBuckets()); + } + + @Test + void testDifferentUniqueIdsAllowedForSameBaseColumn() { + Column distributionColumn = new Column("dist_col", PrimitiveType.INT); + distributionColumn.setUniqueId(1); + Column targetColumn = new Column("dist_col", PrimitiveType.INT); + targetColumn.setUniqueId(2); + + RuntimeFilterPruneClassifier.Classification classification = classifyBucket( + TRuntimeFilterType.IN, targetColumn, + new HashDistributionInfo(8, ImmutableList.of(distributionColumn))); + + Assertions.assertTrue(classification.canPruneBuckets()); + } + + @Test + void testRollupUniqueIdCollisionRejectedForDifferentBaseColumns() { + Column baseDistributionColumn = new Column("k2", PrimitiveType.INT); + baseDistributionColumn.setUniqueId(1); + + Column baseTargetColumn = new Column("k1", PrimitiveType.INT); + SlotDescriptor baseSlotDescriptor = new SlotDescriptor(new SlotId(2), new TupleId(2)); + baseSlotDescriptor.setColumn(baseTargetColumn); + baseSlotDescriptor.setType(baseTargetColumn.getType()); + Column rollupTargetColumn = new Column("mv_k1", PrimitiveType.INT); + rollupTargetColumn.setUniqueId(1); + rollupTargetColumn.setDefineExpr(new SlotRef(baseSlotDescriptor)); + + RuntimeFilterPruneClassifier.Classification classification = classifyBucket( + TRuntimeFilterType.IN, rollupTargetColumn, + new HashDistributionInfo(8, ImmutableList.of(baseDistributionColumn))); + + Assertions.assertFalse(classification.canPruneBuckets()); + Assertions.assertTrue(classification.getBucketUnsupportedReason().contains("distribution column")); + } + + @Test + void testRollupWithDifferentTypeRejected() { + Column baseDistributionColumn = new Column("dist_col", PrimitiveType.INT); + SlotDescriptor baseSlotDescriptor = new SlotDescriptor(new SlotId(2), new TupleId(2)); + baseSlotDescriptor.setColumn(baseDistributionColumn); + baseSlotDescriptor.setType(baseDistributionColumn.getType()); + Column rollupTargetColumn = new Column("mv_dist_col", PrimitiveType.BIGINT); + rollupTargetColumn.setDefineExpr(new SlotRef(baseSlotDescriptor)); + + RuntimeFilterPruneClassifier.Classification classification = classifyBucket( + TRuntimeFilterType.IN, rollupTargetColumn, + new HashDistributionInfo(8, ImmutableList.of(baseDistributionColumn))); + + Assertions.assertFalse(classification.canPruneBuckets()); + Assertions.assertTrue(classification.getBucketUnsupportedReason().contains("distribution column")); + } + + @Test + void testFeatureGatesAvoidCatalogClassification() { + SessionVariable sessionVariable = new SessionVariable(); + sessionVariable.setEnableRuntimeFilterBucketPrune(false); + sessionVariable.setEnableRuntimeFilterPartitionPrune(false); + Column targetColumn = new Column("dist_col", PrimitiveType.INT); + OlapTable table = Mockito.mock(OlapTable.class); + PhysicalOlapScan scan = Mockito.mock(PhysicalOlapScan.class); + SlotReference target = slot(targetColumn, table, 1); + RuntimeFilter filter = newFilter( + TRuntimeFilterType.IN, target, target, target.getDataType(), scan); + + RuntimeFilterPruneClassifier.Classification classification = + RuntimeFilterPruneClassifier.classify(filter, sessionVariable); + + Assertions.assertFalse(classification.canPruneBuckets()); + Assertions.assertFalse(classification.canPrunePartitions()); + Mockito.verifyNoInteractions(scan); + } + + @Test + void testDirectRangeAndListPartitionTargetsSupported() { + RuntimeFilterPruneClassifier.Classification rangeClassification = classifyPartition( + TRuntimeFilterType.IN_OR_BLOOM, PartitionType.RANGE, RangePartitionItem.DUMMY_ITEM, + target -> target); + RuntimeFilterPruneClassifier.Classification listClassification = classifyPartition( + TRuntimeFilterType.BLOOM, PartitionType.LIST, ListPartitionItem.DUMMY_ITEM, + target -> target); + + assertSupportedIncreasingPartitions(rangeClassification); + assertSupportedIncreasingPartitions(listClassification); + } + + @Test + void testBloomRangePartitionRejected() { + RuntimeFilterPruneClassifier.Classification classification = classifyPartition( + TRuntimeFilterType.BLOOM, PartitionType.RANGE, RangePartitionItem.DUMMY_ITEM, + target -> target); + + Assertions.assertFalse(classification.canPrunePartitions()); + Assertions.assertTrue(classification.getPartitionUnsupportedReason().contains("BLOOM")); + } + + @Test + void testNoneMovableListExpressionRejected() { + RuntimeFilterPruneClassifier.Classification classification = classifyPartition( + TRuntimeFilterType.IN, PartitionType.LIST, ListPartitionItem.DUMMY_ITEM, + target -> new AssertTrue( + new GreaterThan(target, new IntegerLiteral(0)), + new VarcharLiteral("rfpp_expr_in_only_error"))); + + Assertions.assertFalse(classification.canPrunePartitions()); + Assertions.assertTrue(classification.getPartitionUnsupportedReason().contains("non-movable")); + } + + @Test + void testTypeAdjustedNonIdentityTargetRejectedBeforeTranslation() { + Column partitionColumn = new Column("part_col", PrimitiveType.INT); + OlapTable table = Mockito.mock(OlapTable.class); + PartitionInfo partitionInfo = partitionInfo( + partitionColumn, PartitionType.LIST, ListPartitionItem.DUMMY_ITEM); + Mockito.when(table.getPartitionInfo()).thenReturn(partitionInfo); + PhysicalOlapScan scan = scan(table, ImmutableList.of(1L, 2L)); + SlotReference target = slot(partitionColumn, table, 1); + RuntimeFilter filter = newFilter(TRuntimeFilterType.IN, target, + new Add(target, new IntegerLiteral(1)), DateTimeV2Type.SYSTEM_DEFAULT, scan); + SessionVariable sessionVariable = partitionOnlySession(); + + RuntimeFilterPruneClassifier.Classification classification = + RuntimeFilterPruneClassifier.classify(filter, sessionVariable); + + Assertions.assertFalse(classification.canPrunePartitions()); + Assertions.assertTrue(classification.getPartitionUnsupportedReason().contains("type adjustment")); + } + + @Test + void testMonotonicChildMustOwnAllInputSlots() { + SlotReference slot = new SlotReference("dt", DateTimeV2Type.SYSTEM_DEFAULT); + DateTrunc rejected = new DateTrunc(slot, slot); + DateTrunc accepted = new DateTrunc(slot, new VarcharLiteral("day")); + + Assertions.assertFalse(RuntimeFilterPruneClassifier.hasInputSlotOnlyInMonotonicChild( + rejected, ((Monotonic) rejected).getMonotonicFunctionChildIndex())); + Assertions.assertTrue(RuntimeFilterPruneClassifier.hasInputSlotOnlyInMonotonicChild( + accepted, ((Monotonic) accepted).getMonotonicFunctionChildIndex())); + } + + private RuntimeFilterPruneClassifier.Classification classifyBucket( + TRuntimeFilterType filterType, Column targetColumn, DistributionInfo distributionInfo) { + OlapTable table = Mockito.mock(OlapTable.class); + Partition partition = Mockito.mock(Partition.class); + Mockito.when(table.getPartition(1L)).thenReturn(partition); + Mockito.when(partition.getDistributionInfo()).thenReturn(distributionInfo); + PhysicalOlapScan scan = scan(table, ImmutableList.of(1L)); + SlotReference target = slot(targetColumn, table, 1); + RuntimeFilter filter = newFilter( + filterType, target, target, target.getDataType(), scan); + SessionVariable sessionVariable = new SessionVariable(); + sessionVariable.setEnableRuntimeFilterBucketPrune(true); + sessionVariable.setEnableRuntimeFilterPartitionPrune(false); + return RuntimeFilterPruneClassifier.classify(filter, sessionVariable); + } + + private RuntimeFilterPruneClassifier.Classification classifyPartition( + TRuntimeFilterType filterType, PartitionType partitionType, PartitionItem partitionItem, + Function targetFactory) { + Column partitionColumn = new Column("part_col", PrimitiveType.INT); + OlapTable table = Mockito.mock(OlapTable.class); + PartitionInfo partitionInfo = partitionInfo(partitionColumn, partitionType, partitionItem); + Mockito.when(table.getPartitionInfo()).thenReturn(partitionInfo); + PhysicalOlapScan scan = scan(table, ImmutableList.of(1L, 2L)); + SlotReference target = slot(partitionColumn, table, 1); + Expression targetExpression = targetFactory.apply(target); + RuntimeFilter filter = newFilter(filterType, target, + targetExpression, targetExpression.getDataType(), scan); + return RuntimeFilterPruneClassifier.classify(filter, partitionOnlySession()); + } + + private PartitionInfo partitionInfo( + Column partitionColumn, PartitionType partitionType, PartitionItem partitionItem) { + PartitionInfo partitionInfo = Mockito.mock(PartitionInfo.class); + Mockito.when(partitionInfo.getType()).thenReturn(partitionType); + Mockito.when(partitionInfo.getPartitionColumns()).thenReturn(ImmutableList.of(partitionColumn)); + Mockito.when(partitionInfo.getItem(1L)).thenReturn(partitionItem); + Mockito.when(partitionInfo.getItem(2L)).thenReturn(partitionItem); + return partitionInfo; + } + + private PhysicalOlapScan scan(OlapTable table, ImmutableList selectedPartitionIds) { + PhysicalOlapScan scan = Mockito.mock(PhysicalOlapScan.class); + Mockito.when(scan.getTable()).thenReturn(table); + Mockito.when(scan.getSelectedPartitionIds()).thenReturn(selectedPartitionIds); + return scan; + } + + private SlotReference slot(Column column, OlapTable table, int exprId) { + return new SlotReference(new ExprId(exprId), column.getName(), + DataType.fromCatalogType(column.getType()), column.isAllowNull(), ImmutableList.of("t"), + table, column, table, column); + } + + private RuntimeFilter newFilter(TRuntimeFilterType filterType, SlotReference target, + Expression targetExpression, DataType sourceType, PhysicalOlapScan scan) { + SlotReference source = new SlotReference("src", sourceType); + AbstractPhysicalPlan builder = Mockito.mock(AbstractPhysicalPlan.class); + return new RuntimeFilter(RuntimeFilterId.createGenerator().getNextId(), + source, target, targetExpression, filterType, 0, builder, 10, false, + TMinMaxRuntimeFilterType.MIN_MAX, scan); + } + + private SessionVariable partitionOnlySession() { + SessionVariable sessionVariable = new SessionVariable(); + sessionVariable.setEnableRuntimeFilterBucketPrune(false); + sessionVariable.setEnableRuntimeFilterPartitionPrune(true); + return sessionVariable; + } + + private void assertSupportedIncreasingPartitions( + RuntimeFilterPruneClassifier.Classification classification) { + Assertions.assertTrue(classification.canPrunePartitions()); + Map monotonicity = classification.getPartitionMonotonicity(); + Assertions.assertEquals(2, monotonicity.size()); + Assertions.assertEquals(TTargetExprMonotonicity.MONOTONIC_INCREASING, monotonicity.get(1L)); + Assertions.assertEquals(TTargetExprMonotonicity.MONOTONIC_INCREASING, monotonicity.get(2L)); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/OlapScanNodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/OlapScanNodeTest.java index 89d734e7d08e21..c18b821ec116e4 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/planner/OlapScanNodeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/planner/OlapScanNodeTest.java @@ -36,9 +36,14 @@ import org.apache.doris.catalog.RangePartitionItem; import org.apache.doris.catalog.info.TableNameInfo; import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.Config; +import org.apache.doris.common.util.DebugPointUtil; import org.apache.doris.datasource.InternalCatalog; import org.apache.doris.thrift.TOlapScanNode; +import org.apache.doris.thrift.TPaloScanRange; import org.apache.doris.thrift.TPartitionBoundary; +import org.apache.doris.thrift.TScanRange; +import org.apache.doris.thrift.TScanRangeLocations; import com.google.common.collect.Lists; import com.google.common.collect.Maps; @@ -277,6 +282,80 @@ public void testRuntimeFilterPartitionBoundariesUsePlanningSnapshot() throws Ana Assert.assertEquals("p_target,p_after", scanNode.getSelectedPartitionNamesForExplain()); } + @Test + public void testRuntimeFilterBucketMetadataAttachedOnceAcrossWorkers() throws Exception { + OlapScanNode scanNode = newBucketPruneScanNode(10L); + TPaloScanRange paloScanRange = scanNode.scanRangeLocations.get(0) + .getScanRange().getPaloScanRange(); + Map bucketInfo = getBucketInfo(scanNode); + bucketInfo.put(10L, ((long) 4 << Integer.SIZE) | 2L); + + scanNode.setRuntimeFilterBucketPruneParameters(); + bucketInfo.clear(); + scanNode.setRuntimeFilterBucketPruneParameters(); + + Assert.assertEquals(2, paloScanRange.getBucketSeq()); + Assert.assertEquals(4, paloScanRange.getBucketNum()); + } + + @Test + public void testMissingRuntimeFilterBucketMetadataDisablesScanPruning() throws Exception { + OlapScanNode scanNode = newBucketPruneScanNode(10L, 11L); + TPaloScanRange firstScanRange = scanNode.scanRangeLocations.get(0) + .getScanRange().getPaloScanRange(); + TPaloScanRange secondScanRange = scanNode.scanRangeLocations.get(1) + .getScanRange().getPaloScanRange(); + Map bucketInfo = getBucketInfo(scanNode); + bucketInfo.put(10L, ((long) 4 << Integer.SIZE) | 2L); + bucketInfo.put(11L, ((long) 4 << Integer.SIZE) | 3L); + + boolean previousEnableDebugPoints = Config.enable_debug_points; + try { + Config.enable_debug_points = true; + DebugPointUtil.addDebugPointWithValue( + OlapScanNode.MISSING_RF_BUCKET_METADATA_DEBUG_POINT, 11L); + + scanNode.setRuntimeFilterBucketPruneParameters(); + + Assert.assertFalse(firstScanRange.isSetBucketSeq()); + Assert.assertFalse(firstScanRange.isSetBucketNum()); + Assert.assertFalse(secondScanRange.isSetBucketSeq()); + Assert.assertFalse(secondScanRange.isSetBucketNum()); + } finally { + DebugPointUtil.removeDebugPoint(OlapScanNode.MISSING_RF_BUCKET_METADATA_DEBUG_POINT); + Config.enable_debug_points = previousEnableDebugPoints; + } + } + + private OlapScanNode newBucketPruneScanNode(long... tabletIds) { + OlapTable table = Mockito.mock(OlapTable.class); + Mockito.when(table.getName()).thenReturn("rf_bucket_fact"); + Mockito.when(table.getDistributionColumnNames()).thenReturn(Collections.emptySet()); + + TupleDescriptor tupleDescriptor = new TupleDescriptor(new TupleId(1)); + tupleDescriptor.setTable(table); + OlapScanNode scanNode = new OlapScanNode( + new PlanNodeId(1), tupleDescriptor, "rfBucketScanNode", ScanContext.EMPTY); + for (long tabletId : tabletIds) { + TPaloScanRange paloScanRange = new TPaloScanRange(); + paloScanRange.setTabletId(tabletId); + TScanRange scanRange = new TScanRange(); + scanRange.setPaloScanRange(paloScanRange); + TScanRangeLocations locations = new TScanRangeLocations(); + locations.setScanRange(scanRange); + scanNode.scanRangeLocations.add(locations); + } + return scanNode; + } + + @SuppressWarnings("unchecked") + private Map getBucketInfo(OlapScanNode scanNode) throws Exception { + java.lang.reflect.Field bucketInfoField = + OlapScanNode.class.getDeclaredField("tabletId2BucketInfo"); + bucketInfoField.setAccessible(true); + return (Map) bucketInfoField.get(scanNode); + } + private Partition mockPartition(String name) { Partition partition = Mockito.mock(Partition.class); Mockito.when(partition.getName()).thenReturn(name); diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/runtime/ThriftPlansBuilderTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/runtime/ThriftPlansBuilderTest.java index 1c3f63ab061b48..62a83172f94194 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/qe/runtime/ThriftPlansBuilderTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/runtime/ThriftPlansBuilderTest.java @@ -24,13 +24,10 @@ import org.apache.doris.nereids.trees.plans.distribute.worker.job.DefaultScanSource; import org.apache.doris.nereids.trees.plans.distribute.worker.job.LocalShuffleAssignedJob; import org.apache.doris.nereids.trees.plans.distribute.worker.job.UnassignedJob; -import org.apache.doris.planner.OlapScanNode; import org.apache.doris.planner.PlanNodeId; import org.apache.doris.planner.RecursiveCteScanNode; import org.apache.doris.planner.ScanNode; import org.apache.doris.planner.SortNode; -import org.apache.doris.qe.ConnectContext; -import org.apache.doris.qe.SessionVariable; import org.apache.doris.thrift.TRecCTETarget; import org.apache.doris.thrift.TUniqueId; @@ -54,21 +51,6 @@ public void testSetRuntimePredicateForNonOlapScanNode() { Mockito.verify(sortNode).setHasRuntimePredicate(); } - @Test - public void testSetRuntimeFilterBucketPruneParametersOnceBeforeWorkerSerialization() { - OlapScanNode olapScanNode = Mockito.mock(OlapScanNode.class); - ScanNode otherScanNode = Mockito.mock(ScanNode.class); - ConnectContext connectContext = Mockito.mock(ConnectContext.class); - SessionVariable sessionVariable = Mockito.mock(SessionVariable.class); - Mockito.when(connectContext.getSessionVariable()).thenReturn(sessionVariable); - Mockito.when(sessionVariable.isEnableRuntimeFilterBucketPrune()).thenReturn(true); - - ThriftPlansBuilder.setRuntimeFilterBucketPruneParametersIfNeeded( - Arrays.asList(olapScanNode, otherScanNode), connectContext); - - Mockito.verify(olapScanNode).setRuntimeFilterBucketPruneParameters(); - } - @Test public void testBuildRecCTETargetsKeepsAllInstancesOnSameBackend() { DistributedPlanWorker worker = Mockito.mock(DistributedPlanWorker.class); From 1a3d7c0905bba427428072d778c2b230c928ae40 Mon Sep 17 00:00:00 2001 From: happenlee Date: Mon, 17 Aug 2026 16:15:17 +0800 Subject: [PATCH 16/20] [fix](fe) Reject unsafe rollup bucket pruning ### What problem does this PR solve? Issue Number: None Related PR: #65837 Problem Summary: Runtime-filter bucket pruning treated a target column without a define expression as a base column. When a non-base index is selected and the target column provenance cannot be proven, a computed materialized-view column could therefore be classified as the HASH distribution column and cause incorrect tablet pruning. Reject only non-base-index targets without a direct base-column definition, while preserving base-index targets and direct SlotRef rollup targets. ### Release note Prevent runtime-filter bucket pruning when a selected non-base index target has unknown base-column provenance. ### Check List (For Author) - Test: Unit Test - RuntimeFilterPruneClassifierTest - Behavior changed: Yes. Bucket pruning now falls back when a non-base-index target has no direct base-column definition. - Does this need documentation: No --- .../post/RuntimeFilterPruneClassifier.java | 5 ++++ .../RuntimeFilterPruneClassifierTest.java | 23 +++++++++++++++++-- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterPruneClassifier.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterPruneClassifier.java index 02d5445478d517..63f1ae612cb1ae 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterPruneClassifier.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterPruneClassifier.java @@ -89,6 +89,11 @@ private static BucketClassification classifyBucketPruning(RuntimeFilter filter) if (table == null || scan.getSelectedPartitionIds().isEmpty()) { return BucketClassification.unsupported("target scan has no selected partitions"); } + if (scan.getSelectedIndexId() != table.getBaseIndexId() + && targetColumn.getDefineExpr() == null) { + return BucketClassification.unsupported( + "non-base-index target has no direct base-column definition"); + } Column distributionColumn = null; for (Long partitionId : scan.getSelectedPartitionIds()) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/processor/post/RuntimeFilterPruneClassifierTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/processor/post/RuntimeFilterPruneClassifierTest.java index 8bf16e7214f898..f94a2adf7451a3 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/processor/post/RuntimeFilterPruneClassifierTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/processor/post/RuntimeFilterPruneClassifierTest.java @@ -128,19 +128,30 @@ void testDirectMvAliasSupportedButComputedAliasRejected() { directMvColumn.setDefineExpr(new SlotRef(baseSlotDescriptor)); RuntimeFilterPruneClassifier.Classification directClassification = classifyBucket( TRuntimeFilterType.IN, directMvColumn, - new HashDistributionInfo(8, ImmutableList.of(baseColumn))); + new HashDistributionInfo(8, ImmutableList.of(baseColumn)), 1L, 2L); Column computedMvColumn = new Column("base_col", PrimitiveType.INT); computedMvColumn.setDefineExpr(new FunctionCallExpr("abs", ImmutableList.of(new SlotRef(baseSlotDescriptor)), true)); RuntimeFilterPruneClassifier.Classification computedClassification = classifyBucket( TRuntimeFilterType.IN, computedMvColumn, - new HashDistributionInfo(8, ImmutableList.of(baseColumn))); + new HashDistributionInfo(8, ImmutableList.of(baseColumn)), 1L, 2L); Assertions.assertTrue(directClassification.canPruneBuckets()); Assertions.assertFalse(computedClassification.canPruneBuckets()); } + @Test + void testNonBaseIndexWithoutDirectDefinitionRejected() { + Column distributionColumn = new Column("dist_col", PrimitiveType.INT); + RuntimeFilterPruneClassifier.Classification classification = classifyBucket( + TRuntimeFilterType.IN, distributionColumn, + new HashDistributionInfo(8, ImmutableList.of(distributionColumn)), 1L, 2L); + + Assertions.assertFalse(classification.canPruneBuckets()); + Assertions.assertTrue(classification.getBucketUnsupportedReason().contains("no direct base-column")); + } + @Test void testBaseColumnNamesComparedSymmetrically() { Column baseColumn = new Column("base_col", PrimitiveType.INT); @@ -298,11 +309,19 @@ void testMonotonicChildMustOwnAllInputSlots() { private RuntimeFilterPruneClassifier.Classification classifyBucket( TRuntimeFilterType filterType, Column targetColumn, DistributionInfo distributionInfo) { + return classifyBucket(filterType, targetColumn, distributionInfo, 1L, 1L); + } + + private RuntimeFilterPruneClassifier.Classification classifyBucket( + TRuntimeFilterType filterType, Column targetColumn, DistributionInfo distributionInfo, + long baseIndexId, long selectedIndexId) { OlapTable table = Mockito.mock(OlapTable.class); Partition partition = Mockito.mock(Partition.class); + Mockito.when(table.getBaseIndexId()).thenReturn(baseIndexId); Mockito.when(table.getPartition(1L)).thenReturn(partition); Mockito.when(partition.getDistributionInfo()).thenReturn(distributionInfo); PhysicalOlapScan scan = scan(table, ImmutableList.of(1L)); + Mockito.when(scan.getSelectedIndexId()).thenReturn(selectedIndexId); SlotReference target = slot(targetColumn, table, 1); RuntimeFilter filter = newFilter( filterType, target, target, target.getDataType(), scan); From dca09b0952024754b61d58608c4e35b1c0f77b28 Mon Sep 17 00:00:00 2001 From: happenlee Date: Mon, 17 Aug 2026 16:38:25 +0800 Subject: [PATCH 17/20] [fix](fe) Validate partition pruning column provenance ### What problem does this PR solve? Issue Number: None Related PR: #65837 Problem Summary: Runtime-filter partition pruning treated matching Column unique IDs as proof that a target was a partition column. Column unique IDs are allocated independently for base and rollup indexes, so unrelated rollup and partition columns can collide and produce invalid pruning metadata. Resolve a selected-index target to a proven direct base column, compare its name and type with the serialized partition column, and conservatively reject missing, computed, or renamed rollup definitions. ### Release note Prevent runtime-filter partition pruning from using unrelated rollup columns with colliding column IDs. ### Check List (For Author) - Test: Unit Test - RuntimeFilterPruneClassifierTest - Behavior changed: Yes. Partition pruning now requires a target with proven base-column provenance and a directly serializable partition-column identity. - Does this need documentation: No --- .../post/RuntimeFilterPruneClassifier.java | 58 +++++++----- .../RuntimeFilterPruneClassifierTest.java | 93 +++++++++++++++++++ 2 files changed, 127 insertions(+), 24 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterPruneClassifier.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterPruneClassifier.java index 63f1ae612cb1ae..6aed407d96bdd0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterPruneClassifier.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterPruneClassifier.java @@ -89,10 +89,10 @@ private static BucketClassification classifyBucketPruning(RuntimeFilter filter) if (table == null || scan.getSelectedPartitionIds().isEmpty()) { return BucketClassification.unsupported("target scan has no selected partitions"); } - if (scan.getSelectedIndexId() != table.getBaseIndexId() - && targetColumn.getDefineExpr() == null) { + Column targetBaseColumn = directTargetBaseColumn(scan, targetColumn); + if (targetBaseColumn == null) { return BucketClassification.unsupported( - "non-base-index target has no direct base-column definition"); + "target has no direct base-column definition"); } Column distributionColumn = null; @@ -159,11 +159,16 @@ private static PartitionClassification classifyPartitionPruning(RuntimeFilter fi "target expression contains non-movable function"); } Column targetColumn = targetColumn(filter.getTargetSlot()); - if (!isPartitionColumn(targetColumn, partitionInfo.getPartitionColumns())) { + Column targetBaseColumn = directTargetBaseColumn(scan, targetColumn); + if (targetBaseColumn == null) { + return PartitionClassification.unsupported( + "target column has no direct base-column definition"); + } + if (!isPartitionColumn(targetColumn, targetBaseColumn, partitionInfo.getPartitionColumns())) { return PartitionClassification.unsupported( "target expression is not rooted on one partition column"); } - if (!hasSerializedBoundary(targetColumn, partitionInfo, partitionType)) { + if (!hasSerializedBoundary(targetColumn, targetBaseColumn, partitionInfo, partitionType)) { return PartitionClassification.unsupported( "target expression has no serialized partition boundary"); } @@ -216,6 +221,17 @@ private static Column targetColumn(Slot slot) { return ((SlotReference) slot).getOriginalColumn().orElse(null); } + private static Column directTargetBaseColumn(PhysicalOlapScan scan, Column targetColumn) { + if (targetColumn == null) { + return null; + } + if (scan.getSelectedIndexId() != scan.getTable().getBaseIndexId() + && targetColumn.getDefineExpr() == null) { + return null; + } + return directBaseColumn(targetColumn); + } + private static boolean sameBucketColumn(Column targetColumn, Column distributionColumn) { Column targetBaseColumn = directBaseColumn(targetColumn); Column distributionBaseColumn = directBaseColumn(distributionColumn); @@ -266,38 +282,32 @@ private static boolean containsFunctionCall(Expr expression) { } private static boolean hasSerializedBoundary( - Column targetColumn, PartitionInfo partitionInfo, PartitionType partitionType) { + Column targetColumn, Column targetBaseColumn, + PartitionInfo partitionInfo, PartitionType partitionType) { if (partitionType != PartitionType.RANGE) { return true; } List partitionColumns = partitionInfo.getPartitionColumns(); - return !partitionColumns.isEmpty() && samePartitionColumn(targetColumn, partitionColumns.get(0)); + return !partitionColumns.isEmpty() + && samePartitionColumn(targetColumn, targetBaseColumn, partitionColumns.get(0)); } - private static boolean isPartitionColumn(Column targetColumn, List partitionColumns) { - if (targetColumn == null) { - return false; - } + private static boolean isPartitionColumn( + Column targetColumn, Column targetBaseColumn, List partitionColumns) { for (Column partitionColumn : partitionColumns) { - if (samePartitionColumn(targetColumn, partitionColumn)) { + if (samePartitionColumn(targetColumn, targetBaseColumn, partitionColumn)) { return true; } } return false; } - private static boolean samePartitionColumn(Column targetColumn, Column partitionColumn) { - if (targetColumn == partitionColumn) { - return true; - } - int targetUniqueId = targetColumn.getUniqueId(); - int partitionUniqueId = partitionColumn.getUniqueId(); - if (targetUniqueId != Column.COLUMN_UNIQUE_ID_INIT_VALUE - && partitionUniqueId != Column.COLUMN_UNIQUE_ID_INIT_VALUE - && targetUniqueId == partitionUniqueId) { - return true; - } - return targetColumn.equals(partitionColumn); + private static boolean samePartitionColumn( + Column targetColumn, Column targetBaseColumn, Column partitionColumn) { + return targetColumn.getName().equalsIgnoreCase(partitionColumn.getName()) + && targetColumn.getType().equals(partitionColumn.getType()) + && targetBaseColumn.getName().equalsIgnoreCase(partitionColumn.getName()) + && targetBaseColumn.getType().equals(partitionColumn.getType()); } private static Map classifyLocalMonotonicity( diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/processor/post/RuntimeFilterPruneClassifierTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/processor/post/RuntimeFilterPruneClassifierTest.java index f94a2adf7451a3..02157543a6ccb7 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/processor/post/RuntimeFilterPruneClassifierTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/processor/post/RuntimeFilterPruneClassifierTest.java @@ -253,6 +253,71 @@ void testDirectRangeAndListPartitionTargetsSupported() { assertSupportedIncreasingPartitions(listClassification); } + @Test + void testPartitionUniqueIdCollisionRejected() { + Column partitionColumn = new Column("part_col", PrimitiveType.INT); + partitionColumn.setUniqueId(0); + Column baseTargetColumn = new Column("value_col", PrimitiveType.INT); + baseTargetColumn.setUniqueId(1); + Column rollupTargetColumn = directRollupColumn("part_col", baseTargetColumn); + rollupTargetColumn.setUniqueId(0); + + RuntimeFilterPruneClassifier.Classification classification = classifyDirectPartition( + rollupTargetColumn, partitionColumn, 1L, 2L); + + Assertions.assertFalse(classification.canPrunePartitions()); + } + + @Test + void testDirectRollupPartitionColumnSupported() { + Column partitionColumn = new Column("part_col", PrimitiveType.INT); + Column rollupTargetColumn = directRollupColumn("part_col", partitionColumn); + + RuntimeFilterPruneClassifier.Classification classification = classifyDirectPartition( + rollupTargetColumn, partitionColumn, 1L, 2L); + + assertSupportedIncreasingPartitions(classification); + } + + @Test + void testRenamedRollupPartitionColumnRejected() { + Column partitionColumn = new Column("part_col", PrimitiveType.INT); + Column rollupTargetColumn = directRollupColumn("mv_part_col", partitionColumn); + + RuntimeFilterPruneClassifier.Classification classification = classifyDirectPartition( + rollupTargetColumn, partitionColumn, 1L, 2L); + + Assertions.assertFalse(classification.canPrunePartitions()); + } + + @Test + void testNonBasePartitionTargetWithoutDefinitionRejected() { + Column partitionColumn = new Column("part_col", PrimitiveType.INT); + Column rollupTargetColumn = new Column("part_col", PrimitiveType.INT); + + RuntimeFilterPruneClassifier.Classification classification = classifyDirectPartition( + rollupTargetColumn, partitionColumn, 1L, 2L); + + Assertions.assertFalse(classification.canPrunePartitions()); + Assertions.assertTrue(classification.getPartitionUnsupportedReason() + .contains("no direct base-column")); + } + + @Test + void testComputedRollupPartitionTargetRejected() { + Column partitionColumn = new Column("part_col", PrimitiveType.INT); + Column rollupTargetColumn = new Column("part_col", PrimitiveType.INT); + rollupTargetColumn.setDefineExpr(new FunctionCallExpr("abs", + ImmutableList.of(directSlotRef(partitionColumn)), true)); + + RuntimeFilterPruneClassifier.Classification classification = classifyDirectPartition( + rollupTargetColumn, partitionColumn, 1L, 2L); + + Assertions.assertFalse(classification.canPrunePartitions()); + Assertions.assertTrue(classification.getPartitionUnsupportedReason() + .contains("no direct base-column")); + } + @Test void testBloomRangePartitionRejected() { RuntimeFilterPruneClassifier.Classification classification = classifyPartition( @@ -346,6 +411,34 @@ private RuntimeFilterPruneClassifier.Classification classifyPartition( return RuntimeFilterPruneClassifier.classify(filter, partitionOnlySession()); } + private RuntimeFilterPruneClassifier.Classification classifyDirectPartition( + Column targetColumn, Column partitionColumn, long baseIndexId, long selectedIndexId) { + OlapTable table = Mockito.mock(OlapTable.class); + PartitionInfo partitionInfo = partitionInfo( + partitionColumn, PartitionType.RANGE, RangePartitionItem.DUMMY_ITEM); + Mockito.when(table.getBaseIndexId()).thenReturn(baseIndexId); + Mockito.when(table.getPartitionInfo()).thenReturn(partitionInfo); + PhysicalOlapScan scan = scan(table, ImmutableList.of(1L, 2L)); + Mockito.when(scan.getSelectedIndexId()).thenReturn(selectedIndexId); + SlotReference target = slot(targetColumn, table, 1); + RuntimeFilter filter = newFilter( + TRuntimeFilterType.IN, target, target, target.getDataType(), scan); + return RuntimeFilterPruneClassifier.classify(filter, partitionOnlySession()); + } + + private Column directRollupColumn(String name, Column baseColumn) { + Column rollupColumn = new Column(name, baseColumn.getType()); + rollupColumn.setDefineExpr(directSlotRef(baseColumn)); + return rollupColumn; + } + + private SlotRef directSlotRef(Column column) { + SlotDescriptor slotDescriptor = new SlotDescriptor(new SlotId(2), new TupleId(2)); + slotDescriptor.setColumn(column); + slotDescriptor.setType(column.getType()); + return new SlotRef(slotDescriptor); + } + private PartitionInfo partitionInfo( Column partitionColumn, PartitionType partitionType, PartitionItem partitionItem) { PartitionInfo partitionInfo = Mockito.mock(PartitionInfo.class); From 5968e7231729fa450296fb012e5efac1d119e6e4 Mon Sep 17 00:00:00 2001 From: happenlee Date: Mon, 24 Aug 2026 11:32:07 +0800 Subject: [PATCH 18/20] [fix](regression) Remove trailing blank line --- .../data/query_p0/runtime_filter/rf_bucket_pruning.out | 1 - 1 file changed, 1 deletion(-) diff --git a/regression-test/data/query_p0/runtime_filter/rf_bucket_pruning.out b/regression-test/data/query_p0/runtime_filter/rf_bucket_pruning.out index 18b5fae2233162..9020c828089377 100644 --- a/regression-test/data/query_p0/runtime_filter/rf_bucket_pruning.out +++ b/regression-test/data/query_p0/runtime_filter/rf_bucket_pruning.out @@ -4,4 +4,3 @@ -- !nullable_bucket_result -- \N 90 - From 8c648b1a55a2534cb63c79a728fe76e477d49309 Mon Sep 17 00:00:00 2001 From: happenlee Date: Mon, 24 Aug 2026 12:16:55 +0800 Subject: [PATCH 19/20] [fix](be) Remove stale scanner expression cleanup ### What problem does this PR solve? Issue Number: None Related PR: None Problem Summary: OlapScanner no longer owns _slot_id_to_virtual_column_expr after the scan-state refactor, but release_unopened_resources still referenced the removed member. This caused BE compilation to fail. Remove the stale cleanup call while retaining cleanup of scanner-owned virtual column expressions. ### Release note None ### Check List (For Author) - Test: Manual static check - clang-format-16 --dry-run --Werror and git diff --check - Behavior changed: No - Does this need documentation: No --- be/src/exec/scan/olap_scanner.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/be/src/exec/scan/olap_scanner.cpp b/be/src/exec/scan/olap_scanner.cpp index 1359f1880d0aa2..ed049b8c7efb0a 100644 --- a/be/src/exec/scan/olap_scanner.cpp +++ b/be/src/exec/scan/olap_scanner.cpp @@ -665,7 +665,6 @@ void OlapScanner::release_unopened_resources() { _tablet_reader.reset(); _tablet_reader_params = TabletReader::ReaderParams {}; _common_expr_ctxs_push_down.clear(); - _slot_id_to_virtual_column_expr.clear(); _virtual_column_exprs.clear(); _score_runtime.reset(); _ann_topn_runtime.reset(); From ffbded491d751b6bc7b39fba1551376601abb35f Mon Sep 17 00:00:00 2001 From: happenlee Date: Mon, 24 Aug 2026 14:01:22 +0800 Subject: [PATCH 20/20] [fix](test) Align bucket pruning tests with scan APIs ### What problem does this PR solve? Issue Number: None Related PR: #65837 Problem Summary: Recent scan projection and bucket metadata refactors changed the ScannerContext constructor, row descriptor accessor, and OlapScanNode bucket metadata field. The bucket pruning tests still used the old APIs after rebasing, which caused BE test compilation failures and an FE reflection failure. Align the tests with the current production APIs and packed bucket metadata representation. ### Release note None ### Check List (For Author) - Test: Unit Test - BE ASAN UT build compiled scanner_late_arrival_rf_test.cpp successfully; final local link was blocked by an unrelated duplicate getrandom symbol in the local toolchain compatibility library - FE UT checkstyle and test-source compilation passed; local execution was blocked by the JDK Byte Buddy self-attach restriction - Behavior changed: No - Does this need documentation: No --- be/test/exec/scan/scanner_late_arrival_rf_test.cpp | 12 ++++++------ .../OlapScanNodeBackendSelectionConfigTest.java | 3 ++- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/be/test/exec/scan/scanner_late_arrival_rf_test.cpp b/be/test/exec/scan/scanner_late_arrival_rf_test.cpp index f6c99795d758f9..8428a3d629063d 100644 --- a/be/test/exec/scan/scanner_late_arrival_rf_test.cpp +++ b/be/test/exec/scan/scanner_late_arrival_rf_test.cpp @@ -320,10 +320,10 @@ TEST_F(ScannerLateArrivalRfTest, bucket_pruning_after_probe_tasks_start) { auto local_state = OlapScanLocalState::create_shared(state, op.get()); std::vector> rf_dependencies; ASSERT_TRUE(local_state->_helper.init(state, true, 0, 0, rf_dependencies, "").ok()); - ASSERT_TRUE( - local_state->_helper - .acquire_runtime_filter(state, local_state->_conjuncts, op->row_descriptor()) - .ok()); + ASSERT_TRUE(local_state->_helper + .acquire_runtime_filter(state, local_state->_conjuncts, + op->operator_row_desc_before_projection()) + .ok()); ASSERT_TRUE(local_state->_conjuncts.empty()); auto task_exec_ctx = std::make_shared(); state->set_task_execution_context(task_exec_ctx); @@ -362,7 +362,7 @@ TEST_F(ScannerLateArrivalRfTest, bucket_pruning_after_probe_tasks_start) { auto dependency = Dependency::create_shared(0, 0, "late bucket scan dependency"); std::atomic shared_limit {-1}; auto scanner_context = ScannerContext::create_shared( - state, local_state.get(), desc_tbl->get_tuple_descriptor(0), nullptr, scanner_delegates, + state, local_state.get(), desc_tbl->get_tuple_descriptor(0), false, scanner_delegates, -1, dependency, &shared_limit, nullptr, nullptr, 0, false, bucket_num); scanner_context->_newly_create_free_blocks_num = ADD_COUNTER(&scan_profile, "NewlyCreatedFreeBlocks", TUnit::UNIT); @@ -500,7 +500,7 @@ TEST_F(ScannerLateArrivalRfTest, bounded_concurrency_prunes_scanner_before_first auto dependency = Dependency::create_shared(0, 0, "bounded late bucket scan dependency"); std::atomic shared_limit {-1}; auto scanner_context = ScannerContext::create_shared( - state, local_state.get(), desc_tbl->get_tuple_descriptor(0), nullptr, scanner_delegates, + state, local_state.get(), desc_tbl->get_tuple_descriptor(0), false, scanner_delegates, -1, dependency, &shared_limit, nullptr, nullptr, 0, false, 1); scanner_context->_newly_create_free_blocks_num = ADD_COUNTER(&scan_profile, "NewlyCreatedFreeBlocks", TUnit::UNIT); diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/OlapScanNodeBackendSelectionConfigTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/OlapScanNodeBackendSelectionConfigTest.java index df0d2a0de2cc2a..b6db673c0d8260 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/planner/OlapScanNodeBackendSelectionConfigTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/planner/OlapScanNodeBackendSelectionConfigTest.java @@ -366,7 +366,8 @@ private List createScanRanges(Tablet tablet, Mockito.when(partition.getId()).thenReturn(10L); Mockito.when(partition.getVisibleVersion()).thenReturn(10L); - Deencapsulation.setField(scanNode, "tabletId2BucketSeq", ImmutableMap.of(20L, 0)); + Deencapsulation.setField(scanNode, "tabletId2BucketInfo", + ImmutableMap.of(20L, 1L << Integer.SIZE)); Deencapsulation.invoke(scanNode, "addScanRangeLocations", partition, ImmutableList.of(tablet), ImmutableMap.of()); return scanNode.getScanRangeLocations(0);