diff --git a/be/src/exec/operator/olap_scan_operator.cpp b/be/src/exec/operator/olap_scan_operator.cpp index 292ca468c39ec3..b1fb21f764aaf1 100644 --- a/be/src/exec/operator/olap_scan_operator.cpp +++ b/be/src/exec/operator/olap_scan_operator.cpp @@ -130,6 +130,8 @@ Status OlapScanLocalState::_init_profile() { _scan_rows = ADD_COUNTER(custom_profile(), "ScanRows", TUnit::UNIT); _tablets_pruned_by_rf_counter = ADD_COUNTER(custom_profile(), "TabletsPrunedByRuntimeFilter", TUnit::UNIT); + _buckets_pruned_by_rf_counter = + ADD_COUNTER(custom_profile(), "BucketsPrunedByRuntimeFilter", TUnit::UNIT); // 1. init segment profile _segment_profile.reset(new RuntimeProfile("SegmentIterator")); @@ -655,7 +657,7 @@ Status OlapScanLocalState::_init_scanners(std::list* scanners) { _cond_ranges.emplace_back(new doris::OlapScanRange()); } - // Filter out tablets whose partitions have been pruned by runtime filters. + // Filter out tablets whose partitions or buckets have been pruned by runtime filters. // // TODO(rf-partition-prune): this happens after OlapScanLocalState::init() // has already executed _sync_cloud_tablets() (in cloud mode that performs @@ -670,13 +672,16 @@ Status OlapScanLocalState::_init_scanners(std::list* scanners) { // the tablet, (b) acquire ready-at-start RFs before _sync_cloud_tablets() // and run partition pruning there to filter _scan_ranges by partition_id // so the heavy per-tablet work is skipped for pruned partitions. - if (_rf_partition_pruner.pruned_partition_count() > 0) { + if (_rf_partition_pruner.pruned_partition_count() > 0 || + _rf_bucket_pruner.pruned_tablet_count() > 0) { DCHECK_EQ(_tablets.size(), _scan_ranges.size()); DCHECK_EQ(_tablets.size(), _read_sources.size()); size_t write_idx = 0; for (size_t read_idx = 0; read_idx < _tablets.size(); ++read_idx) { int64_t pid = _tablets[read_idx].tablet->partition_id(); - if (!_rf_partition_pruner.is_partition_pruned(pid)) { + const auto& scan_range = *_scan_ranges[read_idx]; + if (!_is_tablet_pruned_by_runtime_filter(pid, scan_range.bucket_seq, + scan_range.bucket_num)) { if (write_idx != read_idx) { _tablets[write_idx] = std::move(_tablets[read_idx]); _scan_ranges[write_idx] = std::move(_scan_ranges[read_idx]); @@ -740,9 +745,9 @@ Status OlapScanLocalState::_init_scanners(std::list* scanners) { key_ranges.emplace_back(range.get()); } - ParallelScannerBuilder scanner_builder(this, _tablets, _read_sources, _scanner_profile, - key_ranges, state(), p._limit, true, - p._olap_scan_node.is_preaggregation); + ParallelScannerBuilder scanner_builder(this, _tablets, _read_sources, _scan_ranges, + _scanner_profile, key_ranges, state(), p._limit, + true, p._olap_scan_node.is_preaggregation); int max_scanners_count = state()->parallel_scan_max_scanners_count(); @@ -831,6 +836,8 @@ Status OlapScanLocalState::_init_scanners(std::list* scanners) { p._olap_scan_node.is_preaggregation, read_row_binlog, resolve_binlog_scan_type(palo_scan_range), + palo_scan_range.bucket_seq, + palo_scan_range.bucket_num, palo_scan_range.__isset.start_tso ? std::make_optional(palo_scan_range.start_tso) : std::nullopt, @@ -1111,13 +1118,54 @@ void OlapScanLocalState::set_scan_ranges(RuntimeState* state, } } - for (auto& scan_range : scan_ranges) { - DCHECK(scan_range.scan_range.__isset.palo_scan_range); + bool bucket_prune_metadata_initialized = !_scan_ranges.empty(); + for (const auto& scan_range : scan_ranges) { + DORIS_CHECK(scan_range.scan_range.__isset.palo_scan_range); _scan_ranges.emplace_back(new TPaloScanRange(scan_range.scan_range.palo_scan_range)); + const auto& palo_scan_range = *_scan_ranges.back(); + DORIS_CHECK_EQ(palo_scan_range.__isset.bucket_seq, palo_scan_range.__isset.bucket_num); + if (!bucket_prune_metadata_initialized) { + _has_rf_bucket_prune_metadata = palo_scan_range.__isset.bucket_seq; + bucket_prune_metadata_initialized = true; + } else { + DORIS_CHECK_EQ(palo_scan_range.__isset.bucket_seq, _has_rf_bucket_prune_metadata); + } + if (_has_rf_bucket_prune_metadata) { + DORIS_CHECK_GT(palo_scan_range.bucket_num, 0); + DORIS_CHECK_GE(palo_scan_range.bucket_seq, 0); + DORIS_CHECK_LT(palo_scan_range.bucket_seq, palo_scan_range.bucket_num); + } COUNTER_UPDATE(_tablet_counter, 1); } } +Status OlapScanLocalState::_on_runtime_filter_update(const VExprContextSPtrs& new_conjuncts) { + RETURN_IF_ERROR(Base::_on_runtime_filter_update(new_conjuncts)); + if (!state()->query_options().enable_runtime_filter_bucket_prune || + !_has_rf_bucket_prune_metadata || _scan_ranges.empty()) { + return Status::OK(); + } + + int64_t newly_pruned = 0; + RETURN_IF_ERROR(_rf_bucket_pruner.prune_by_runtime_filters( + _scan_ranges, new_conjuncts, _parent->runtime_filter_descs(), _parent->node_id(), + state()->runtime_filter_max_in_num(), &newly_pruned)); + if (newly_pruned > 0) { + COUNTER_SET(_buckets_pruned_by_rf_counter, _rf_bucket_pruner.pruned_tablet_count()); + } + return Status::OK(); +} + +bool OlapScanLocalState::_is_tablet_pruned_by_runtime_filter(int64_t partition_id, + int32_t bucket_seq, + int32_t bucket_num) const { + if (_rf_partition_pruner.is_partition_pruned(partition_id)) { + return true; + } + return _has_rf_bucket_prune_metadata && + _rf_bucket_pruner.is_bucket_pruned(bucket_seq, bucket_num); +} + static std::string tablets_id_to_string( const std::vector>& scan_ranges) { if (scan_ranges.empty()) { diff --git a/be/src/exec/operator/olap_scan_operator.h b/be/src/exec/operator/olap_scan_operator.h index 4716c78cf8b7c5..88440fe5bd8a36 100644 --- a/be/src/exec/operator/olap_scan_operator.h +++ b/be/src/exec/operator/olap_scan_operator.h @@ -27,6 +27,7 @@ #include "common/status.h" #include "exec/operator/operator.h" #include "exec/operator/scan_operator.h" +#include "exec/runtime_filter/runtime_filter_bucket_pruner.h" #include "runtime/runtime_profile.h" #include "storage/index/snii/snii_prx_profile.h" #include "storage/olap_scan_common.h" @@ -78,6 +79,7 @@ class OlapScanLocalState final : public ScanLocalState { const std::vector& scan_ranges) override; Status _init_profile() override; Status _process_conjuncts(RuntimeState* state) override; + Status _on_runtime_filter_update(const VExprContextSPtrs& new_conjuncts) override; bool _is_key_column(const std::string& col_name) override; bool can_push_down_column_predicate(const SlotDescriptor* slot) override; @@ -135,7 +137,12 @@ class OlapScanLocalState final : public ScanLocalState { Status _build_key_ranges_and_filters(); + bool _is_tablet_pruned_by_runtime_filter(int64_t partition_id, int32_t bucket_seq, + int32_t bucket_num) const; + std::vector> _scan_ranges; + bool _has_rf_bucket_prune_metadata = false; + RuntimeFilterBucketPruner _rf_bucket_pruner; std::vector _sync_statistics; MonotonicStopWatch _sync_cloud_tablets_watcher; std::shared_ptr _cloud_tablet_dependency; @@ -154,6 +161,7 @@ class OlapScanLocalState final : public ScanLocalState { snii::SniiPhraseRuntimeProfileCounters _snii_phrase_profile_counters; RuntimeProfile::Counter* _tablet_counter = nullptr; + RuntimeProfile::Counter* _buckets_pruned_by_rf_counter = nullptr; RuntimeProfile::Counter* _key_range_counter = nullptr; RuntimeProfile::Counter* _reader_init_timer = nullptr; RuntimeProfile::Counter* _scanner_init_timer = nullptr; diff --git a/be/src/exec/operator/scan_operator.cpp b/be/src/exec/operator/scan_operator.cpp index a6458289d7ef5f..3259ec4a19f9fa 100644 --- a/be/src/exec/operator/scan_operator.cpp +++ b/be/src/exec/operator/scan_operator.cpp @@ -74,25 +74,26 @@ bool ScanLocalState::should_run_serial() const { Status ScanLocalStateBase::update_late_arrival_runtime_filter(RuntimeState* state, int& arrived_rf_num) { - // Lock needed because _conjuncts can be accessed concurrently by multiple scanner threads + // Lock needed because _conjuncts can be accessed concurrently by multiple scanner threads. LockGuard lock(_conjuncts_lock); size_t conjuncts_before = _conjuncts.size(); RETURN_IF_ERROR(_helper.try_append_late_arrival_runtime_filter( state, _parent->operator_row_desc_before_projection(), arrived_rf_num, _conjuncts)); + VExprContextSPtrs new_conjuncts; + if (_conjuncts.size() > conjuncts_before) { + new_conjuncts.assign(_conjuncts.begin() + conjuncts_before, _conjuncts.end()); + } if (state->enable_adjust_conjunct_order_by_cost()) { std::ranges::stable_sort(_conjuncts, [](const auto& a, const auto& b) { return a->execute_cost() < b->execute_cost(); }); - }; - // Only re-run partition pruning when try_append_late_arrival_runtime_filter - // actually appended new conjuncts. Otherwise this hook would re-scan all - // partition boundaries on every scheduler pass while there are still - // unapplied RFs (Scanner::_applied_rf_num is not advanced here), wasting - // CPU re-evaluating the same set of RFs against the same boundaries. - if (_conjuncts.size() > conjuncts_before) { - RETURN_IF_ERROR(_on_runtime_filter_update()); } - return Status::OK(); + if (new_conjuncts.empty()) { + return Status::OK(); + } + // Partition projection executes the shared expression tree. Keep it serialized with + // clone_conjunct_ctxs(), whose VExprContext::clone() opens that same tree. + return _on_runtime_filter_update(new_conjuncts); } Status ScanLocalStateBase::clone_conjunct_ctxs(VExprContextSPtrs& scanner_conjuncts) { @@ -105,19 +106,15 @@ Status ScanLocalStateBase::clone_conjunct_ctxs(VExprContextSPtrs& scanner_conjun return Status::OK(); } -bool ScanLocalStateBase::is_partition_pruned(int64_t partition_id) const { - return _rf_partition_pruner.is_partition_pruned(partition_id); -} - -Status ScanLocalStateBase::_on_runtime_filter_update() { +Status ScanLocalStateBase::_on_runtime_filter_update(const VExprContextSPtrs& new_conjuncts) { const auto* parsed = _parent->parsed_partition_boundaries(); if (parsed != nullptr && !parsed->empty()) { - RETURN_IF_ERROR(_do_partition_pruning_by_rf()); + RETURN_IF_ERROR(_do_partition_pruning_by_rf(new_conjuncts)); } return Status::OK(); } -Status ScanLocalStateBase::_do_partition_pruning_by_rf() { +Status ScanLocalStateBase::_do_partition_pruning_by_rf(const VExprContextSPtrs& conjuncts) { if (!_state->query_options().enable_runtime_filter_partition_prune) { return Status::OK(); } @@ -127,7 +124,7 @@ Status ScanLocalStateBase::_do_partition_pruning_by_rf() { } int64_t newly_pruned = 0; RETURN_IF_ERROR(_rf_partition_pruner.prune_by_runtime_filters( - *parsed, _conjuncts, _parent->runtime_filter_descs(), _parent->node_id(), + *parsed, conjuncts, _parent->runtime_filter_descs(), _parent->node_id(), &newly_pruned)); if (newly_pruned > 0) { COUNTER_SET(_partitions_pruned_by_rf_counter, @@ -236,7 +233,8 @@ Status ScanLocalState::open(RuntimeState* state) { RETURN_IF_ERROR(_helper.acquire_runtime_filter(state, _conjuncts, p.operator_row_desc_before_projection())); if (_conjuncts.size() > conjuncts_before) { - RETURN_IF_ERROR(_on_runtime_filter_update()); + VExprContextSPtrs new_conjuncts(_conjuncts.begin() + conjuncts_before, _conjuncts.end()); + RETURN_IF_ERROR(_on_runtime_filter_update(new_conjuncts)); } // Disable condition cache in topn filter valid. TODO:: Try to support the topn filter in condition cache diff --git a/be/src/exec/operator/scan_operator.h b/be/src/exec/operator/scan_operator.h index 71dd4da466d4c3..4532e8773a83b4 100644 --- a/be/src/exec/operator/scan_operator.h +++ b/be/src/exec/operator/scan_operator.h @@ -98,10 +98,6 @@ class ScanLocalStateBase : public PipelineXLocalState<> { [[nodiscard]] virtual int min_scanners_concurrency(RuntimeState* state) const; [[nodiscard]] virtual ScannerScheduler* scan_scheduler(RuntimeState* state) const; - // Thread-safe check whether a partition has been pruned by runtime filter. - // Callable from any scan type's scanner in scheduling threads. - bool is_partition_pruned(int64_t partition_id) const; - [[nodiscard]] std::string get_name() { return _parent->get_name(); } uint64_t get_condition_cache_digest() const { return _condition_cache_digest; } @@ -116,12 +112,12 @@ class ScanLocalStateBase : public PipelineXLocalState<> { virtual Status _init_profile() = 0; - // Hook for subclasses to react after new runtime filters are appended. - // Called inside update_late_arrival_runtime_filter() while _conjuncts_lock is held. - // Default implementation runs partition pruning on the newly appended RFs. - virtual Status _on_runtime_filter_update(); + // Hook for subclasses to process only the runtime-filter conjuncts appended by the current + // update. Late-arrival updates call this while holding _conjuncts_lock because pruning may + // execute expression nodes shared with scanner clones. + virtual Status _on_runtime_filter_update(const VExprContextSPtrs& new_conjuncts); - Status _do_partition_pruning_by_rf(); + Status _do_partition_pruning_by_rf(const VExprContextSPtrs& conjuncts); std::atomic _opened {false}; @@ -296,10 +292,7 @@ class ScanLocalState : public ScanLocalStateBase { friend class Scanner; Status _init_profile() override; - virtual Status _process_conjuncts(RuntimeState* state) { - RETURN_IF_ERROR(_do_partition_pruning_by_rf()); - return _normalize_conjuncts(state); - } + virtual Status _process_conjuncts(RuntimeState* state) { return _normalize_conjuncts(state); } virtual bool _should_push_down_common_expr(const VExprSPtr&) { return false; } virtual bool can_push_down_column_predicate(const SlotDescriptor* slot) { diff --git a/be/src/exec/runtime_filter/runtime_filter_bucket_pruner.cpp b/be/src/exec/runtime_filter/runtime_filter_bucket_pruner.cpp new file mode 100644 index 00000000000000..f22c094baae9e7 --- /dev/null +++ b/be/src/exec/runtime_filter/runtime_filter_bucket_pruner.cpp @@ -0,0 +1,150 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "exec/runtime_filter/runtime_filter_bucket_pruner.h" + +#include + +#include +#include +#include + +#include "exprs/hybrid_set.h" +#include "exprs/runtime_filter_expr.h" +#include "exprs/vexpr.h" +#include "exprs/vexpr_context.h" +#include "exprs/vslot_ref.h" + +namespace doris { + +Status RuntimeFilterBucketPruner::prune_by_runtime_filters( + const std::vector>& ranges, + const VExprContextSPtrs& conjuncts, const std::vector& rf_descs, + int scan_node_id, int max_in_num, int64_t* newly_pruned_count) { + *newly_pruned_count = 0; + if (ranges.empty()) { + return Status::OK(); + } + + phmap::flat_hash_set eligible_filter_ids; + for (const auto& desc : rf_descs) { + if (desc.__isset.bucket_pruning_target_ids && + desc.bucket_pruning_target_ids.contains(scan_node_id)) { + eligible_filter_ids.insert(desc.filter_id); + } + } + if (eligible_filter_ids.empty()) { + return Status::OK(); + } + + for (const auto& conjunct_ctx : conjuncts) { + VExprSPtr root = conjunct_ctx->root(); + if (!root->is_rf_wrapper()) { + continue; + } + auto* rf_expr = assert_cast(root.get()); + if (!eligible_filter_ids.contains(rf_expr->filter_id())) { + continue; + } + + VExprSPtr impl = root->get_impl(); + DORIS_CHECK(impl != nullptr); + std::shared_ptr hybrid_set = impl->get_set_func(); + if (hybrid_set == nullptr) { + // IN_OR_BLOOM may become a Bloom filter at runtime. A Bloom filter + // cannot be inverted to a safe finite bucket set. + continue; + } + if (hybrid_set->size() > max_in_num) { + continue; + } + + DORIS_CHECK_EQ(impl->children().size(), 1); + VExprSPtr target_expr = impl->children()[0]; + DORIS_CHECK_EQ(target_expr->node_type(), TExprNodeType::SLOT_REF); + + std::shared_ptr> hashes = + rf_expr->get_bucket_prune_hashes(target_expr->data_type()); + phmap::flat_hash_map> new_selected_buckets_by_num; + for (const auto& range_ptr : ranges) { + DORIS_CHECK(range_ptr != nullptr); + const auto& range = *range_ptr; + DORIS_CHECK(range.__isset.bucket_seq); + DORIS_CHECK(range.__isset.bucket_num); + DORIS_CHECK_GT(range.bucket_num, 0); + DORIS_CHECK_GE(range.bucket_seq, 0); + DORIS_CHECK_LT(range.bucket_seq, range.bucket_num); + + auto [selected_it, inserted] = + new_selected_buckets_by_num.try_emplace(range.bucket_num); + if (inserted) { + auto& selected_buckets = selected_it->second; + selected_buckets.reserve( + std::min(hashes->size(), static_cast(range.bucket_num))); + for (uint32_t hash : *hashes) { + selected_buckets.insert( + static_cast(hash % static_cast(range.bucket_num))); + } + } + } + + int64_t current_filter_pruned_count = 0; + std::unique_lock lock(_prune_mutex); + for (const auto& range_ptr : ranges) { + const auto& range = *range_ptr; + auto current_it = _selected_buckets_by_num.find(range.bucket_num); + bool was_selected = current_it == _selected_buckets_by_num.end() || + current_it->second.contains(range.bucket_seq); + if (was_selected && + !new_selected_buckets_by_num.at(range.bucket_num).contains(range.bucket_seq)) { + ++current_filter_pruned_count; + } + } + for (auto& [bucket_num, new_selected_buckets] : new_selected_buckets_by_num) { + auto current_it = _selected_buckets_by_num.find(bucket_num); + if (current_it == _selected_buckets_by_num.end()) { + _selected_buckets_by_num.emplace(bucket_num, std::move(new_selected_buckets)); + } else { + for (auto bucket_it = current_it->second.begin(); + bucket_it != current_it->second.end();) { + if (!new_selected_buckets.contains(*bucket_it)) { + bucket_it = current_it->second.erase(bucket_it); + } else { + ++bucket_it; + } + } + } + } + *newly_pruned_count += current_filter_pruned_count; + _pruned_tablet_count += current_filter_pruned_count; + } + return Status::OK(); +} + +bool RuntimeFilterBucketPruner::is_bucket_pruned(int32_t bucket_seq, int32_t bucket_num) const { + std::shared_lock lock(_prune_mutex); + auto selected_it = _selected_buckets_by_num.find(bucket_num); + return selected_it != _selected_buckets_by_num.end() && + !selected_it->second.contains(bucket_seq); +} + +int64_t RuntimeFilterBucketPruner::pruned_tablet_count() const { + std::shared_lock lock(_prune_mutex); + return _pruned_tablet_count; +} + +} // namespace doris diff --git a/be/src/exec/runtime_filter/runtime_filter_bucket_pruner.h b/be/src/exec/runtime_filter/runtime_filter_bucket_pruner.h new file mode 100644 index 00000000000000..4eb500152dcb85 --- /dev/null +++ b/be/src/exec/runtime_filter/runtime_filter_bucket_pruner.h @@ -0,0 +1,55 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include + +#include "common/status.h" +#include "exec/common/hash_table/phmap_fwd_decl.h" +#include "exprs/vexpr_fwd.h" + +namespace doris { + +class TPaloScanRange; +struct TRuntimeFilterDesc; + +// Per-scan-instance state for single-column HASH bucket pruning. Runtime filters +// are conjunctive, so each exact IN filter can monotonically shrink the selected +// bucket set for each bucket count. The retained state is bounded by bucket +// counts rather than the number of tablets across all partitions. +// Both pruning updates and is_bucket_pruned() are safe to call concurrently. +class RuntimeFilterBucketPruner { +public: + Status prune_by_runtime_filters(const std::vector>& ranges, + const VExprContextSPtrs& conjuncts, + const std::vector& rf_descs, + int scan_node_id, int max_in_num, int64_t* newly_pruned_count); + + bool is_bucket_pruned(int32_t bucket_seq, int32_t bucket_num) const; + int64_t pruned_tablet_count() const; + +private: + phmap::flat_hash_map> _selected_buckets_by_num; + int64_t _pruned_tablet_count = 0; + mutable std::shared_mutex _prune_mutex; +}; + +} // namespace doris diff --git a/be/src/exec/runtime_filter/runtime_filter_consumer.cpp b/be/src/exec/runtime_filter/runtime_filter_consumer.cpp index 84fcc621ace697..70b61372c52612 100644 --- a/be/src/exec/runtime_filter/runtime_filter_consumer.cpp +++ b/be/src/exec/runtime_filter/runtime_filter_consumer.cpp @@ -104,7 +104,7 @@ Status RuntimeFilterConsumer::_get_push_exprs(std::vector& in_pred->add_child(probe_ctx->root()); auto wrapper = RuntimeFilterExpr::create_shared( node, in_pred, get_in_list_ignore_thredhold(_wrapper->hybrid_set()->size()), - null_aware, _wrapper->filter_id(), sampling_frequency); + null_aware, _wrapper->filter_id(), sampling_frequency, _wrapper); container.push_back(wrapper); break; } diff --git a/be/src/exec/runtime_filter/runtime_filter_partition_pruner.cpp b/be/src/exec/runtime_filter/runtime_filter_partition_pruner.cpp index 6e6d43472c526a..74ced3a3acef8b 100644 --- a/be/src/exec/runtime_filter/runtime_filter_partition_pruner.cpp +++ b/be/src/exec/runtime_filter/runtime_filter_partition_pruner.cpp @@ -854,66 +854,70 @@ Status RuntimeFilterPartitionPruner::prune_by_runtime_filters( } } - // This function is serialized by _conjuncts_lock in the caller, so our reads - // of _pruned_partition_ids never race with our writes below. The only concurrent - // readers are is_partition_pruned() calls (under shared_lock), which are - // properly synchronized by the unique_lock we take when inserting. phmap::flat_hash_set newly_pruned; + { + // _try_prune_by_single_rf() skips IDs already published by previous filters. + // Hold a shared lock while it reads that set; concurrent updates merge their + // local results under the unique lock below. + std::shared_lock lock(_prune_mutex); + for (const auto& conjunct_ctx : conjuncts) { + VExprSPtr root = conjunct_ctx->root(); + if (!root->is_rf_wrapper()) { + continue; + } - for (const auto& conjunct_ctx : conjuncts) { - VExprSPtr root = conjunct_ctx->root(); - if (!root->is_rf_wrapper()) { - continue; - } + VExprSPtr impl = root->get_impl(); + if (!impl) { + continue; + } - VExprSPtr impl = root->get_impl(); - if (!impl) { - continue; - } + if (impl->children().empty()) { + continue; + } - if (impl->children().empty()) { - continue; - } + auto* wrapper_root = assert_cast(root.get()); + int filter_id = wrapper_root->filter_id(); - auto* wrapper_root = assert_cast(root.get()); - int filter_id = wrapper_root->filter_id(); + VExprSPtr target_subtree = impl->children()[0]; - VExprSPtr target_subtree = impl->children()[0]; + auto partition_mono_it = filter_id_to_partition_monotonicity.find(filter_id); + // For LIST partition targets, this metadata is also the per-partition eligibility map; + // BE projects every finite LIST value, so the direction itself is neutral there. + bool has_partition_mono = + partition_mono_it != filter_id_to_partition_monotonicity.end() && + !partition_mono_it->second.empty(); + if (!has_partition_mono) { + continue; + } - auto partition_mono_it = filter_id_to_partition_monotonicity.find(filter_id); - // For LIST partition targets, this metadata is also the per-partition eligibility map; - // BE projects every finite LIST value, so the direction itself is neutral there. - bool has_partition_mono = partition_mono_it != filter_id_to_partition_monotonicity.end() && - !partition_mono_it->second.empty(); - if (!has_partition_mono) { - continue; - } + const VSlotRef* leaf_slot = find_unique_slot_ref(target_subtree.get()); + DORIS_CHECK(leaf_slot != nullptr); - const VSlotRef* leaf_slot = find_unique_slot_ref(target_subtree.get()); - DORIS_CHECK(leaf_slot != nullptr); + SlotId leaf_slot_id = leaf_slot->slot_id(); + DORIS_CHECK(slot_to_boundaries.contains(leaf_slot_id)); - SlotId leaf_slot_id = leaf_slot->slot_id(); - DORIS_CHECK(slot_to_boundaries.contains(leaf_slot_id)); + int leaf_column_id = leaf_slot->column_id(); - int leaf_column_id = leaf_slot->column_id(); + std::shared_ptr> projected; + RETURN_IF_ERROR(parsed.get_or_compute_projected_boundaries( + filter_id, target_subtree, leaf_slot_id, leaf_column_id, + partition_mono_it->second, conjunct_ctx.get(), &projected)); - std::shared_ptr> projected; - RETURN_IF_ERROR(parsed.get_or_compute_projected_boundaries( - filter_id, target_subtree, leaf_slot_id, leaf_column_id, partition_mono_it->second, - conjunct_ctx.get(), &projected)); + if (projected == nullptr || projected->empty()) { + continue; + } - if (projected == nullptr || projected->empty()) { - continue; + _try_prune_by_single_rf(*projected, impl, newly_pruned); } - - _try_prune_by_single_rf(*projected, impl, newly_pruned); } - auto count = static_cast(newly_pruned.size()); - if (count > 0) { + int64_t count = 0; + if (!newly_pruned.empty()) { std::unique_lock lock(_prune_mutex); for (int64_t pid : newly_pruned) { - _pruned_partition_ids.insert(pid); + if (_pruned_partition_ids.insert(pid).second) { + ++count; + } } } *newly_pruned_count = count; diff --git a/be/src/exec/runtime_filter/runtime_filter_partition_pruner.h b/be/src/exec/runtime_filter/runtime_filter_partition_pruner.h index 9e3d0150db1232..6493baec72cb4a 100644 --- a/be/src/exec/runtime_filter/runtime_filter_partition_pruner.h +++ b/be/src/exec/runtime_filter/runtime_filter_partition_pruner.h @@ -133,8 +133,8 @@ class ParsedPartitionBoundaries { // `OperatorXBase::parsed_partition_boundaries()`. The owner (ScanLocalStateBase) // passes the parsed object into `prune_by_runtime_filters` on each call. // -// Thread safety: `is_partition_pruned()` is safe to call concurrently with -// `prune_by_runtime_filters()` via an internal shared_mutex. +// Thread safety: pruning updates and lookups are safe to call concurrently via +// an internal shared_mutex. class RuntimeFilterPartitionPruner { public: RuntimeFilterPartitionPruner() = default; diff --git a/be/src/exec/runtime_filter/runtime_filter_wrapper.cpp b/be/src/exec/runtime_filter/runtime_filter_wrapper.cpp index 109fa6f0171878..89db3299425deb 100644 --- a/be/src/exec/runtime_filter/runtime_filter_wrapper.cpp +++ b/be/src/exec/runtime_filter/runtime_filter_wrapper.cpp @@ -18,9 +18,11 @@ #include "exec/runtime_filter/runtime_filter_wrapper.h" #include "core/data_type/define_primitive_type.h" +#include "core/string_ref.h" #include "exec/runtime_filter/runtime_filter_definitions.h" #include "exprs/create_predicate_function.h" #include "exprs/function/cast/cast_to_date_or_datetime_impl.hpp" +#include "util/hash_util.hpp" namespace doris { RuntimeFilterWrapper::RuntimeFilterWrapper(const RuntimeFilterParams* params) @@ -124,6 +126,7 @@ bool RuntimeFilterWrapper::build_bf_by_runtime_size() const { } Status RuntimeFilterWrapper::merge(const RuntimeFilterWrapper* other) { + DORIS_CHECK(!_bucket_prune_hashes_started.load()); if (_state == State::DISABLED) { return Status::OK(); } @@ -615,6 +618,52 @@ bool RuntimeFilterWrapper::contain_null() const { return false; } +std::shared_ptr> +RuntimeFilterWrapper::get_or_compute_bucket_prune_hashes(const DataTypePtr& target_type) const { + DORIS_CHECK(_state.load() == State::READY); + DORIS_CHECK(_hybrid_set != nullptr); + DORIS_CHECK(target_type != nullptr); + PrimitiveType primitive_type = target_type->get_primitive_type(); + DORIS_CHECK_EQ(primitive_type, _column_return_type); + + std::call_once(_bucket_prune_hashes_once, [&] { + _bucket_prune_hashes_started.store(true); + // Materialize the exact-set values into a column so bucket pruning uses the + // column's type-specific CRC implementation. This intentionally makes a + // temporary copy of variable-length values. runtime_filter_max_in_num bounds + // the number of copied values, but not their total byte size; we accept this + // transient memory cost to avoid maintaining a separate per-type hash path. + MutableColumnPtr column = target_type->create_column(); + auto* iter = _hybrid_set->begin(); + while (iter->has_next()) { + const void* value = iter->get_value(); + DORIS_CHECK(value != nullptr); + if (is_string_type(primitive_type)) { + const auto* string_value = reinterpret_cast(value); + column->insert_data(string_value->data, string_value->size); + } else { + // ColumnVector::insert_data ignores length for fixed-length values. + column->insert_data(reinterpret_cast(value), 0); + } + iter->next(); + } + + auto hashes = std::make_shared>(column->size(), 0); + if (!hashes->empty()) { + column->update_crcs_with_value(hashes->data(), primitive_type, + static_cast(column->size())); + } + if (_hybrid_set->contain_null()) { + // Keep one shared vector for nullable and non-nullable targets. A non-nullable + // target may retain this extra bucket, but can never lose matching rows. + hashes->push_back(HashUtil::zlib_crc_hash_null(0)); + } + _bucket_prune_hashes = std::move(hashes); + }); + DORIS_CHECK(_bucket_prune_hashes != nullptr); + return _bucket_prune_hashes; +} + std::string RuntimeFilterWrapper::debug_string() const { auto type_string = _filter_type == RuntimeFilterType::IN_OR_BLOOM_FILTER ? fmt::format("{}({})", filter_type_to_string(_filter_type), diff --git a/be/src/exec/runtime_filter/runtime_filter_wrapper.h b/be/src/exec/runtime_filter/runtime_filter_wrapper.h index 3d22afe5ff7556..7fb770e8855c74 100644 --- a/be/src/exec/runtime_filter/runtime_filter_wrapper.h +++ b/be/src/exec/runtime_filter/runtime_filter_wrapper.h @@ -19,8 +19,13 @@ #include +#include +#include +#include + #include "common/status.h" #include "core/column/column.h" +#include "core/data_type/data_type.h" #include "exec/runtime_filter/runtime_filter_definitions.h" #include "exec/runtime_filter/utils.h" #include "exprs/vexpr_fwd.h" @@ -82,6 +87,11 @@ class RuntimeFilterWrapper { bool contain_null() const; + // The shared vector includes the NULL hash whenever the exact set contains NULL, regardless + // of target nullability. A non-nullable target may therefore retain one conservative bucket. + std::shared_ptr> get_or_compute_bucket_prune_hashes( + const DataTypePtr& target_type) const; + bool disable_always_true_logic() const { return _disable_always_true_logic; } std::string debug_string() const; @@ -157,5 +167,9 @@ class RuntimeFilterWrapper { // on state is thread-safe. std::atomic _state; AtomicStatus _reason; + + mutable std::once_flag _bucket_prune_hashes_once; + mutable std::atomic_bool _bucket_prune_hashes_started = false; + mutable std::shared_ptr> _bucket_prune_hashes; }; } // namespace doris diff --git a/be/src/exec/scan/olap_scanner.cpp b/be/src/exec/scan/olap_scanner.cpp index 7282724cb54928..ed049b8c7efb0a 100644 --- a/be/src/exec/scan/olap_scanner.cpp +++ b/be/src/exec/scan/olap_scanner.cpp @@ -105,6 +105,8 @@ OlapScanner::OlapScanner(ScanLocalStateBase* parent, OlapScanner::Params&& param .binlog_scan_type = params.binlog_scan_type}), _start_tso(params.start_tso), _end_tso(params.end_tso), + _bucket_seq(params.bucket_seq), + _bucket_num(params.bucket_num), _initial_file_cache_stats(std::move(params.initial_file_cache_stats)) { _tablet_reader_params.set_read_source(std::move(params.read_source), _state->skip_delete_bitmap()); @@ -650,11 +652,24 @@ Status OlapScanner::_init_read_schema() { return Status::OK(); } -bool OlapScanner::check_partition_pruned() const { - if (!_local_state) { - return false; - } - return _local_state->is_partition_pruned(_tablet_reader_params.tablet->partition_id()); +bool OlapScanner::is_pruned_by_runtime_filter() const { + DCHECK(_local_state != nullptr); + auto* olap_local_state = assert_cast(_local_state); + return olap_local_state->_is_tablet_pruned_by_runtime_filter( + _tablet_reader_params.tablet->partition_id(), _bucket_seq, _bucket_num); +} + +void OlapScanner::release_unopened_resources() { + DORIS_CHECK(!_is_open); + + _tablet_reader.reset(); + _tablet_reader_params = TabletReader::ReaderParams {}; + _common_expr_ctxs_push_down.clear(); + _virtual_column_exprs.clear(); + _score_runtime.reset(); + _ann_topn_runtime.reset(); + + Scanner::release_unopened_resources(); } doris::TabletStorageType OlapScanner::get_storage_type() { diff --git a/be/src/exec/scan/olap_scanner.h b/be/src/exec/scan/olap_scanner.h index 337c2c2cfea2ff..9eb5f867be7a46 100644 --- a/be/src/exec/scan/olap_scanner.h +++ b/be/src/exec/scan/olap_scanner.h @@ -80,6 +80,8 @@ class OlapScanner : public Scanner { bool aggregation; bool read_row_binlog = false; TBinlogScanType::type binlog_scan_type = TBinlogScanType::NONE; + int32_t bucket_seq = 0; + int32_t bucket_num = 0; std::optional start_tso; std::optional end_tso; }; @@ -94,7 +96,9 @@ class OlapScanner : public Scanner { doris::TabletStorageType get_storage_type() override; - bool check_partition_pruned() const override; + bool is_pruned_by_runtime_filter() const override; + + void release_unopened_resources() override; void update_realtime_counters() override; @@ -121,6 +125,8 @@ class OlapScanner : public Scanner { std::unique_ptr _tablet_reader; std::optional _start_tso; std::optional _end_tso; + int32_t _bucket_seq; + int32_t _bucket_num; public: io::FileCacheStatistics _initial_file_cache_stats; diff --git a/be/src/exec/scan/parallel_scanner_builder.cpp b/be/src/exec/scan/parallel_scanner_builder.cpp index 9ae9a8bb40342d..288c51e1802a49 100644 --- a/be/src/exec/scan/parallel_scanner_builder.cpp +++ b/be/src/exec/scan/parallel_scanner_builder.cpp @@ -80,7 +80,9 @@ Status ParallelScannerBuilder::build_scanners(std::list& scanners) Status ParallelScannerBuilder::_build_scanners_by_rowid(std::list& scanners) { DCHECK_GE(_rows_per_scanner, _min_rows_per_scanner); - for (auto&& [tablet, version] : _tablets) { + for (size_t tablet_idx = 0; tablet_idx < _tablets.size(); ++tablet_idx) { + auto&& [tablet, version] = _tablets[tablet_idx]; + const auto& scan_range = *_scan_ranges[tablet_idx]; DCHECK(_all_read_sources.contains(tablet->tablet_id())); auto& entire_read_source = _all_read_sources[tablet->tablet_id()]; @@ -142,7 +144,7 @@ Status ParallelScannerBuilder::_build_scanners_by_rowid(std::list& partitial_read_source.rs_splits.emplace_back(std::move(split)); scanners.emplace_back(_build_scanner( - tablet, version, _key_ranges, + tablet, version, _key_ranges, scan_range, {.rs_splits = std::move(partitial_read_source.rs_splits), .delete_predicates = entire_read_source.delete_predicates, .delete_bitmap = entire_read_source.delete_bitmap}, @@ -189,7 +191,7 @@ Status ParallelScannerBuilder::_build_scanners_by_rowid(std::list& } #endif scanners.emplace_back( - _build_scanner(tablet, version, _key_ranges, + _build_scanner(tablet, version, _key_ranges, scan_range, {.rs_splits = std::move(partitial_read_source.rs_splits), .delete_predicates = entire_read_source.delete_predicates, .delete_bitmap = entire_read_source.delete_bitmap}, @@ -208,7 +210,9 @@ Status ParallelScannerBuilder::_build_scanners_by_rowid(std::list& Status ParallelScannerBuilder::_build_scanners_by_per_segment(std::list& scanners) { DCHECK_GE(_rows_per_scanner, _min_rows_per_scanner); - for (auto&& [tablet, version] : _tablets) { + for (size_t tablet_idx = 0; tablet_idx < _tablets.size(); ++tablet_idx) { + auto&& [tablet, version] = _tablets[tablet_idx]; + const auto& scan_range = *_scan_ranges[tablet_idx]; DCHECK(_all_read_sources.contains(tablet->tablet_id())); auto& entire_read_source = _all_read_sources[tablet->tablet_id()]; @@ -240,7 +244,7 @@ Status ParallelScannerBuilder::_build_scanners_by_per_segment(std::list ParallelScannerBuilder::_build_scanner( BaseTabletSPtr tablet, int64_t version, const std::vector& key_ranges, - TabletReadSource&& read_source, io::FileCacheStatistics&& initial_file_cache_stats) { + const TPaloScanRange& scan_range, TabletReadSource&& read_source, + io::FileCacheStatistics&& initial_file_cache_stats) { OlapScanner::Params params { .state = _state, .profile = _scanner_profile.get(), @@ -308,6 +313,8 @@ std::shared_ptr ParallelScannerBuilder::_build_scanner( .aggregation = _is_preaggregation, .read_row_binlog = false, .binlog_scan_type = TBinlogScanType::NONE, + .bucket_seq = scan_range.bucket_seq, + .bucket_num = scan_range.bucket_num, .start_tso = std::nullopt, .end_tso = std::nullopt, }; diff --git a/be/src/exec/scan/parallel_scanner_builder.h b/be/src/exec/scan/parallel_scanner_builder.h index 82b63b07824c3a..60ee95fb267e82 100644 --- a/be/src/exec/scan/parallel_scanner_builder.h +++ b/be/src/exec/scan/parallel_scanner_builder.h @@ -43,6 +43,7 @@ class ParallelScannerBuilder { ParallelScannerBuilder(OlapScanLocalState* parent, const std::vector& tablets, std::vector& read_sources, + const std::vector>& scan_ranges, const std::shared_ptr& profile, const std::vector& key_ranges, RuntimeState* state, int64_t limit, bool is_dup_mow_key, bool is_preaggregation) @@ -54,7 +55,14 @@ class ParallelScannerBuilder { _is_preaggregation(is_preaggregation), _tablets(tablets.cbegin(), tablets.cend()), _key_ranges(key_ranges.cbegin(), key_ranges.cend()), - _read_sources(read_sources) {} + _scan_ranges(scan_ranges), + _read_sources(read_sources) { + DORIS_CHECK_EQ(_tablets.size(), scan_ranges.size()); + for (size_t i = 0; i < _tablets.size(); ++i) { + DORIS_CHECK(scan_ranges[i] != nullptr); + DORIS_CHECK_EQ(_tablets[i].tablet->tablet_id(), scan_ranges[i]->tablet_id); + } + } Status build_scanners(std::list& scanners); @@ -76,6 +84,7 @@ class ParallelScannerBuilder { std::shared_ptr _build_scanner(BaseTabletSPtr tablet, int64_t version, const std::vector& key_ranges, + const TPaloScanRange& scan_range, TabletReadSource&& read_source, io::FileCacheStatistics&& initial_file_cache_stats); @@ -111,6 +120,7 @@ class ParallelScannerBuilder { bool _is_preaggregation; std::vector _tablets; std::vector _key_ranges; + const std::vector>& _scan_ranges; std::unordered_map _all_read_sources; std::vector& _read_sources; }; diff --git a/be/src/exec/scan/scanner.h b/be/src/exec/scan/scanner.h index d53fad8c80d264..927472b2fa207c 100644 --- a/be/src/exec/scan/scanner.h +++ b/be/src/exec/scan/scanner.h @@ -211,9 +211,15 @@ class Scanner { return doris::TabletStorageType::STORAGE_TYPE_REMOTE; } - // Returns true if this scanner's partition has been pruned by a runtime filter. - // Overridden by OlapScanner to check partition pruning state. - virtual bool check_partition_pruned() const { return false; } + // Returns true if this scanner's scan range has been pruned by a runtime filter. + virtual bool is_pruned_by_runtime_filter() const { return false; } + + // Releases resources owned by a scanner that runtime-filter pruning makes unnecessary before + // open(). The scanner will not be scheduled again after this call. + virtual void release_unopened_resources() { + DORIS_CHECK(!_is_open); + _has_prepared = false; + } bool need_to_close() const { return _need_to_close; } diff --git a/be/src/exec/scan/scanner_scheduler.cpp b/be/src/exec/scan/scanner_scheduler.cpp index 3036cbdf6ee099..bc53d83e850d58 100644 --- a/be/src/exec/scan/scanner_scheduler.cpp +++ b/be/src/exec/scan/scanner_scheduler.cpp @@ -176,6 +176,13 @@ void ScannerScheduler::_scanner_scan(std::shared_ptr ctx, Status status = Status::OK(); bool eos = false; + auto append_late_arrival_runtime_filter = [&] { + Status rf_status = scanner->try_append_late_arrival_runtime_filter(); + if (!rf_status.ok()) { + LOG(WARNING) << "Failed to append late arrival runtime filter: " + << rf_status.to_string(); + } + }; ASSIGN_STATUS_IF_CATCH_EXCEPTION( RuntimeState* state = ctx->state(); DCHECK(nullptr != state); @@ -183,7 +190,12 @@ void ScannerScheduler::_scanner_scan(std::shared_ptr ctx, // so better to also check low memory and clear free blocks here. if (ctx->low_memory_mode()) { ctx->clear_free_blocks(); } - if (scanner->check_partition_pruned()) { eos = true; } + if (scanner->is_pruned_by_runtime_filter()) { + if (!scanner->is_open()) { + scanner->release_unopened_resources(); + } + eos = true; + } if (!eos && !scanner->has_prepared()) { status = scanner->prepare(); @@ -192,6 +204,16 @@ void ScannerScheduler::_scanner_scan(std::shared_ptr ctx, } } + // A filter may become ready while prepare() is doing tablet setup. Apply it before + // open() so a newly pruned OLAP scanner never initializes its reader or eagerly reads. + if (!eos && !scanner->is_open()) { + append_late_arrival_runtime_filter(); + if (scanner->is_pruned_by_runtime_filter()) { + scanner->release_unopened_resources(); + eos = true; + } + } + if (!eos && !scanner->is_open()) { status = scanner->open(state); if (!status.ok()) { @@ -200,16 +222,10 @@ void ScannerScheduler::_scanner_scan(std::shared_ptr ctx, scanner->set_opened(); } - if (!eos) { - Status rf_status = scanner->try_append_late_arrival_runtime_filter(); - if (!rf_status.ok()) { - LOG(WARNING) << "Failed to append late arrival runtime filter: " - << rf_status.to_string(); - } - } + if (!eos) { append_late_arrival_runtime_filter(); } - // After processing late RFs, check if this scanner's partition was pruned. - if (!eos && scanner->check_partition_pruned()) { eos = true; } + // After processing late RFs, check whether this scanner's scan range was pruned. + if (!eos && scanner->is_pruned_by_runtime_filter()) { eos = true; } size_t raw_bytes_threshold = config::doris_scanner_row_bytes; if (ctx->low_memory_mode()) { diff --git a/be/src/exprs/runtime_filter_expr.cpp b/be/src/exprs/runtime_filter_expr.cpp index 1e729ed5e45f88..e491b4329f3d76 100644 --- a/be/src/exprs/runtime_filter_expr.cpp +++ b/be/src/exprs/runtime_filter_expr.cpp @@ -30,6 +30,7 @@ #include "core/data_type/data_type.h" #include "core/types.h" #include "exec/common/util.hpp" +#include "exec/runtime_filter/runtime_filter_wrapper.h" #include "exprs/vslot_ref.h" #include "runtime/runtime_profile.h" #include "storage/index/zone_map/zonemap_eval_context.h" @@ -58,13 +59,15 @@ namespace doris { class VExprContext; RuntimeFilterExpr::RuntimeFilterExpr(const TExprNode& node, VExprSPtr impl, double ignore_thredhold, - bool null_aware, int filter_id, int sampling_frequency) + bool null_aware, int filter_id, int sampling_frequency, + std::shared_ptr runtime_filter_wrapper) : VExpr(node), _impl(std::move(impl)), _ignore_thredhold(ignore_thredhold), _null_aware(null_aware), _filter_id(filter_id), - _sampling_frequency(sampling_frequency) { + _sampling_frequency(sampling_frequency), + _runtime_filter_wrapper(std::move(runtime_filter_wrapper)) { DORIS_CHECK(_impl != nullptr); } @@ -75,13 +78,19 @@ Status RuntimeFilterExpr::clone_node(VExprSPtr* cloned_expr) const { RETURN_IF_ERROR(_impl->deep_clone(&cloned_impl)); auto cloned_runtime_filter = RuntimeFilterExpr::create_shared( clone_texpr_node(), std::move(cloned_impl), _ignore_thredhold, _null_aware, _filter_id, - _sampling_frequency); + _sampling_frequency, _runtime_filter_wrapper); cloned_runtime_filter->attach_profile_counter(_rf_input_rows, _rf_filter_rows, _always_true_filter_rows); *cloned_expr = std::move(cloned_runtime_filter); return Status::OK(); } +std::shared_ptr> RuntimeFilterExpr::get_bucket_prune_hashes( + const DataTypePtr& target_type) const { + DORIS_CHECK(_runtime_filter_wrapper != nullptr); + return _runtime_filter_wrapper->get_or_compute_bucket_prune_hashes(target_type); +} + Status RuntimeFilterExpr::prepare(RuntimeState* state, const RowDescriptor& desc, VExprContext* context) { RETURN_IF_ERROR_OR_PREPARED(_impl->prepare(state, desc, context)); diff --git a/be/src/exprs/runtime_filter_expr.h b/be/src/exprs/runtime_filter_expr.h index 6deb123668932e..32004500f9ca1f 100644 --- a/be/src/exprs/runtime_filter_expr.h +++ b/be/src/exprs/runtime_filter_expr.h @@ -46,6 +46,7 @@ double get_bloom_filter_ignore_thredhold(); namespace doris { class Block; +class RuntimeFilterWrapper; class VExprContext; class RuntimeFilterExpr final : public VExpr { @@ -54,7 +55,8 @@ class RuntimeFilterExpr final : public VExpr { public: RuntimeFilterExpr(const TExprNode& node, VExprSPtr impl, double ignore_thredhold, bool null_aware, int filter_id, - int sampling_frequency = RuntimeFilterSelectivity::DISABLE_SAMPLING); + int sampling_frequency = RuntimeFilterSelectivity::DISABLE_SAMPLING, + std::shared_ptr runtime_filter_wrapper = nullptr); ~RuntimeFilterExpr() override = default; Status execute_column_impl(VExprContext* context, const Block* block, const Selector* selector, size_t count, ColumnPtr& result_column) const override; @@ -124,6 +126,9 @@ class RuntimeFilterExpr final : public VExpr { int filter_id() const { return _filter_id; } + std::shared_ptr> get_bucket_prune_hashes( + const DataTypePtr& target_type) const; + std::shared_ptr predicate_filtered_rows_counter() const { return _rf_filter_rows; } @@ -152,6 +157,7 @@ class RuntimeFilterExpr final : public VExpr { bool _null_aware; int _filter_id; int _sampling_frequency; + std::shared_ptr _runtime_filter_wrapper; }; using RuntimeFilterExprPtr = std::shared_ptr; diff --git a/be/test/exec/runtime_filter/runtime_filter_bucket_pruner_test.cpp b/be/test/exec/runtime_filter/runtime_filter_bucket_pruner_test.cpp new file mode 100644 index 00000000000000..59737c0c3d6de1 --- /dev/null +++ b/be/test/exec/runtime_filter/runtime_filter_bucket_pruner_test.cpp @@ -0,0 +1,377 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "exec/runtime_filter/runtime_filter_bucket_pruner.h" + +#include +#include + +#include +#include +#include +#include +#include + +#include "core/column/column_vector.h" +#include "core/data_type/data_type_nullable.h" +#include "core/data_type/data_type_number.h" +#include "exec/runtime_filter/runtime_filter_definitions.h" +#include "exec/runtime_filter/runtime_filter_wrapper.h" +#include "exprs/create_predicate_function.h" +#include "exprs/runtime_filter_expr.h" +#include "exprs/vdirect_in_predicate.h" +#include "exprs/vexpr_context.h" +#include "exprs/vslot_ref.h" +#include "util/hash_util.hpp" +#include "util/raw_value.h" + +namespace doris { + +class RuntimeFilterBucketPrunerTest : public testing::Test { +protected: + static constexpr int SCAN_NODE_ID = 10; + using BucketPruneRanges = std::vector>; + + std::shared_ptr make_in_wrapper(int filter_id, + const std::vector& values, + bool null_aware = false) { + RuntimeFilterParams params {.filter_id = filter_id, + .filter_type = RuntimeFilterType::IN_FILTER, + .column_return_type = TYPE_INT, + .null_aware = null_aware, + .max_in_num = 1024}; + auto wrapper = std::make_shared(¶ms); + for (const int32_t value : values) { + wrapper->hybrid_set()->insert(&value); + } + if (null_aware) { + wrapper->hybrid_set()->insert(static_cast(nullptr)); + } + wrapper->set_state(RuntimeFilterWrapper::State::READY); + return wrapper; + } + + VExprContextSPtr make_in_conjunct( + int filter_id, const std::vector& values, + std::shared_ptr runtime_filter_wrapper = nullptr) { + if (runtime_filter_wrapper == nullptr) { + runtime_filter_wrapper = make_in_wrapper(filter_id, values); + } + + TExprNode node; + node.__set_type(create_type_desc(TYPE_BOOLEAN)); + node.__set_node_type(TExprNodeType::IN_PRED); + node.in_predicate.__set_is_not_in(false); + node.__set_opcode(TExprOpcode::FILTER_IN); + node.__set_is_nullable(false); + auto impl = + VDirectInPredicate::create_shared(node, runtime_filter_wrapper->hybrid_set(), true); + impl->add_child(VSlotRef::create_shared(/*slot_id=*/1, /*column_id=*/0, + /*column_uniq_id=*/1, + std::make_shared(), "dist_col")); + auto wrapper = RuntimeFilterExpr::create_shared(node, impl, 0, false, filter_id, + RuntimeFilterSelectivity::DISABLE_SAMPLING, + std::move(runtime_filter_wrapper)); + return std::make_shared(wrapper); + } + + VExprContextSPtr make_non_exact_conjunct(int filter_id) { + TExprNode node; + node.__set_type(create_type_desc(TYPE_BOOLEAN)); + node.__set_node_type(TExprNodeType::BLOOM_PRED); + node.__set_opcode(TExprOpcode::RT_FILTER); + node.__set_is_nullable(false); + auto impl = VDirectInPredicate::create_shared(node, nullptr, true); + impl->add_child(VSlotRef::create_shared(/*slot_id=*/1, /*column_id=*/0, + /*column_uniq_id=*/1, + std::make_shared(), "dist_col")); + auto wrapper = RuntimeFilterExpr::create_shared(node, impl, 0, false, filter_id); + return std::make_shared(wrapper); + } + + VExprContextSPtr make_null_aware_in_conjunct(int filter_id) { + auto runtime_filter_wrapper = make_in_wrapper(filter_id, {}, true); + + TExprNode node; + node.__set_type(create_type_desc(TYPE_BOOLEAN)); + node.__set_node_type(TExprNodeType::NULL_AWARE_IN_PRED); + node.in_predicate.__set_is_not_in(false); + node.__set_opcode(TExprOpcode::FILTER_IN); + node.__set_is_nullable(false); + auto impl = + VDirectInPredicate::create_shared(node, runtime_filter_wrapper->hybrid_set(), true); + impl->add_child(VSlotRef::create_shared( + /*slot_id=*/1, /*column_id=*/0, /*column_uniq_id=*/1, + std::make_shared(std::make_shared()), "dist_col")); + auto wrapper = RuntimeFilterExpr::create_shared(node, impl, 0, false, filter_id, + RuntimeFilterSelectivity::DISABLE_SAMPLING, + std::move(runtime_filter_wrapper)); + return std::make_shared(wrapper); + } + + TRuntimeFilterDesc bucket_prune_desc(int filter_id) { + TRuntimeFilterDesc desc; + desc.__set_filter_id(filter_id); + desc.__set_bucket_pruning_target_ids({SCAN_NODE_ID}); + return desc; + } + + void add_range(BucketPruneRanges* ranges, int64_t tablet_id, int32_t bucket_seq, + int32_t bucket_num) { + auto range = std::make_unique(); + range->__set_tablet_id(tablet_id); + range->__set_bucket_seq(bucket_seq); + range->__set_bucket_num(bucket_num); + ranges->push_back(std::move(range)); + } + + BucketPruneRanges four_bucket_ranges() { + BucketPruneRanges ranges; + for (int32_t bucket_seq = 0; bucket_seq < 4; ++bucket_seq) { + add_range(&ranges, 100 + bucket_seq, bucket_seq, 4); + } + return ranges; + } + + int32_t bucket_for_value(int32_t value, int32_t bucket_num) { + uint32_t hash = RawValue::zlib_crc32(&value, sizeof(value), TYPE_INT, 0); + return static_cast(hash % static_cast(bucket_num)); + } + + int32_t bucket_for_null(int32_t bucket_num) { + uint32_t hash = HashUtil::zlib_crc_hash_null(0); + return static_cast(hash % static_cast(bucket_num)); + } +}; + +TEST_F(RuntimeFilterBucketPrunerTest, ExactSetHashesSharedAcrossConsumers) { + constexpr int filter_id = 13; + auto runtime_filter_wrapper = make_in_wrapper(filter_id, {1, 2, 3}, true); + auto first = make_in_conjunct(filter_id, {}, runtime_filter_wrapper); + auto second = make_in_conjunct(filter_id, {}, runtime_filter_wrapper); + auto target_type = first->root()->get_impl()->children()[0]->data_type(); + + auto first_hashes = assert_cast(first->root().get()) + ->get_bucket_prune_hashes(target_type); + auto second_hashes = assert_cast(second->root().get()) + ->get_bucket_prune_hashes(target_type); + auto nullable_hashes = assert_cast(first->root().get()) + ->get_bucket_prune_hashes(std::make_shared( + std::make_shared())); + + EXPECT_EQ(first_hashes.get(), second_hashes.get()); + EXPECT_EQ(first_hashes.get(), nullable_hashes.get()); + ASSERT_EQ(first_hashes->size(), 4); + EXPECT_EQ(first_hashes->back(), HashUtil::zlib_crc_hash_null(0)); +} + +TEST_F(RuntimeFilterBucketPrunerTest, RejectsMergeAfterBucketHashesStart) { + constexpr int filter_id = 15; + auto wrapper = make_in_wrapper(filter_id, {1}); + auto other = make_in_wrapper(filter_id, {2}); + + static_cast( + wrapper->get_or_compute_bucket_prune_hashes(std::make_shared())); + + EXPECT_DEATH({ static_cast(wrapper->merge(other.get())); }, "Check failed"); +} + +TEST_F(RuntimeFilterBucketPrunerTest, ExactInKeepsOnlyMatchingBucket) { + constexpr int filter_id = 7; + constexpr int32_t value = 10; + VExprContextSPtrs conjuncts {make_in_conjunct(filter_id, {value})}; + std::vector rf_descs {bucket_prune_desc(filter_id)}; + + RuntimeFilterBucketPruner pruner; + int64_t newly_pruned = 0; + ASSERT_TRUE(pruner.prune_by_runtime_filters(four_bucket_ranges(), conjuncts, rf_descs, + SCAN_NODE_ID, /*max_in_num=*/1024, &newly_pruned) + .ok()); + + EXPECT_EQ(newly_pruned, 3); + EXPECT_EQ(pruner.pruned_tablet_count(), 3); + int32_t selected_bucket = bucket_for_value(value, 4); + for (int32_t bucket_seq = 0; bucket_seq < 4; ++bucket_seq) { + EXPECT_EQ(pruner.is_bucket_pruned(bucket_seq, 4), bucket_seq != selected_bucket); + } + + ASSERT_TRUE(pruner.prune_by_runtime_filters(four_bucket_ranges(), conjuncts, rf_descs, + SCAN_NODE_ID, /*max_in_num=*/1024, &newly_pruned) + .ok()); + EXPECT_EQ(newly_pruned, 0); +} + +TEST_F(RuntimeFilterBucketPrunerTest, MultipleFiltersCountEachPrunedBucketOnce) { + constexpr int first_filter_id = 17; + constexpr int second_filter_id = 18; + constexpr int32_t first_value = 10; + int32_t second_value = first_value + 1; + while (bucket_for_value(second_value, 4) == bucket_for_value(first_value, 4)) { + ++second_value; + } + VExprContextSPtrs conjuncts {make_in_conjunct(first_filter_id, {first_value}), + make_in_conjunct(second_filter_id, {second_value})}; + std::vector rf_descs {bucket_prune_desc(first_filter_id), + bucket_prune_desc(second_filter_id)}; + + RuntimeFilterBucketPruner pruner; + int64_t newly_pruned = 0; + ASSERT_TRUE(pruner.prune_by_runtime_filters(four_bucket_ranges(), conjuncts, rf_descs, + SCAN_NODE_ID, /*max_in_num=*/1024, &newly_pruned) + .ok()); + + EXPECT_EQ(newly_pruned, 4); + EXPECT_EQ(pruner.pruned_tablet_count(), 4); +} + +TEST_F(RuntimeFilterBucketPrunerTest, NonNullableTargetConservativelyKeepsNullBucket) { + constexpr int filter_id = 14; + constexpr int32_t value = 1; + auto runtime_filter_wrapper = make_in_wrapper(filter_id, {value}, true); + VExprContextSPtrs conjuncts { + make_in_conjunct(filter_id, {}, std::move(runtime_filter_wrapper))}; + std::vector rf_descs {bucket_prune_desc(filter_id)}; + + std::set selected_buckets {bucket_for_value(value, 4), bucket_for_null(4)}; + ASSERT_EQ(selected_buckets.size(), 2); + + RuntimeFilterBucketPruner pruner; + int64_t newly_pruned = 0; + ASSERT_TRUE(pruner.prune_by_runtime_filters(four_bucket_ranges(), conjuncts, rf_descs, + SCAN_NODE_ID, /*max_in_num=*/1024, &newly_pruned) + .ok()); + + EXPECT_EQ(newly_pruned, 2); + for (int32_t bucket_seq = 0; bucket_seq < 4; ++bucket_seq) { + EXPECT_EQ(pruner.is_bucket_pruned(bucket_seq, 4), !selected_buckets.contains(bucket_seq)); + } +} + +TEST_F(RuntimeFilterBucketPrunerTest, SupportsDifferentBucketCountsAcrossPartitions) { + constexpr int filter_id = 11; + constexpr int32_t value = 10; + VExprContextSPtrs conjuncts {make_in_conjunct(filter_id, {value})}; + std::vector rf_descs {bucket_prune_desc(filter_id)}; + BucketPruneRanges ranges; + for (int32_t bucket_seq = 0; bucket_seq < 4; ++bucket_seq) { + add_range(&ranges, 100 + bucket_seq, bucket_seq, 4); + } + for (int32_t bucket_seq = 0; bucket_seq < 7; ++bucket_seq) { + add_range(&ranges, 200 + bucket_seq, bucket_seq, 7); + } + + RuntimeFilterBucketPruner pruner; + int64_t newly_pruned = 0; + ASSERT_TRUE(pruner.prune_by_runtime_filters(ranges, conjuncts, rf_descs, SCAN_NODE_ID, + /*max_in_num=*/1024, &newly_pruned) + .ok()); + + EXPECT_EQ(newly_pruned, 9); + EXPECT_FALSE(pruner.is_bucket_pruned(bucket_for_value(value, 4), 4)); + EXPECT_FALSE(pruner.is_bucket_pruned(bucket_for_value(value, 7), 7)); +} + +TEST_F(RuntimeFilterBucketPrunerTest, EmptyExactInPrunesAllBuckets) { + constexpr int filter_id = 8; + VExprContextSPtrs conjuncts {make_in_conjunct(filter_id, {})}; + std::vector rf_descs {bucket_prune_desc(filter_id)}; + + RuntimeFilterBucketPruner pruner; + int64_t newly_pruned = 0; + ASSERT_TRUE(pruner.prune_by_runtime_filters(four_bucket_ranges(), conjuncts, rf_descs, + SCAN_NODE_ID, /*max_in_num=*/1024, &newly_pruned) + .ok()); + EXPECT_EQ(newly_pruned, 4); + EXPECT_EQ(pruner.pruned_tablet_count(), 4); +} + +TEST_F(RuntimeFilterBucketPrunerTest, NullAwareInKeepsNullBucket) { + constexpr int filter_id = 12; + VExprContextSPtrs conjuncts {make_null_aware_in_conjunct(filter_id)}; + std::vector rf_descs {bucket_prune_desc(filter_id)}; + + RuntimeFilterBucketPruner pruner; + int64_t newly_pruned = 0; + ASSERT_TRUE(pruner.prune_by_runtime_filters(four_bucket_ranges(), conjuncts, rf_descs, + SCAN_NODE_ID, /*max_in_num=*/1024, &newly_pruned) + .ok()); + + EXPECT_EQ(newly_pruned, 3); + EXPECT_EQ(pruner.pruned_tablet_count(), 3); + int32_t null_bucket = bucket_for_null(4); + for (int32_t bucket_seq = 0; bucket_seq < 4; ++bucket_seq) { + EXPECT_EQ(pruner.is_bucket_pruned(bucket_seq, 4), bucket_seq != null_bucket); + } +} + +TEST_F(RuntimeFilterBucketPrunerTest, HighRangeCountRetainsBucketState) { + constexpr int filter_id = 16; + constexpr int32_t value = 10; + constexpr int32_t partition_count = 128; + constexpr int32_t bucket_num = 256; + BucketPruneRanges ranges; + ranges.reserve(partition_count * bucket_num); + for (int32_t partition = 0; partition < partition_count; ++partition) { + for (int32_t bucket_seq = 0; bucket_seq < bucket_num; ++bucket_seq) { + add_range(&ranges, static_cast(partition) * bucket_num + bucket_seq, + bucket_seq, bucket_num); + } + } + VExprContextSPtrs conjuncts {make_in_conjunct(filter_id, {value})}; + std::vector rf_descs {bucket_prune_desc(filter_id)}; + + RuntimeFilterBucketPruner pruner; + int64_t newly_pruned = 0; + ASSERT_TRUE(pruner.prune_by_runtime_filters(ranges, conjuncts, rf_descs, SCAN_NODE_ID, + /*max_in_num=*/1024, &newly_pruned) + .ok()); + + EXPECT_EQ(newly_pruned, static_cast(partition_count) * (bucket_num - 1)); + EXPECT_EQ(pruner.pruned_tablet_count(), newly_pruned); + EXPECT_FALSE(pruner.is_bucket_pruned(bucket_for_value(value, bucket_num), bucket_num)); +} + +TEST_F(RuntimeFilterBucketPrunerTest, NonExactRuntimeRepresentationIsIgnored) { + constexpr int filter_id = 9; + VExprContextSPtrs conjuncts {make_non_exact_conjunct(filter_id)}; + std::vector rf_descs {bucket_prune_desc(filter_id)}; + + RuntimeFilterBucketPruner pruner; + int64_t newly_pruned = 0; + ASSERT_TRUE(pruner.prune_by_runtime_filters(four_bucket_ranges(), conjuncts, rf_descs, + SCAN_NODE_ID, /*max_in_num=*/1024, &newly_pruned) + .ok()); + EXPECT_EQ(newly_pruned, 0); + EXPECT_EQ(pruner.pruned_tablet_count(), 0); +} + +TEST_F(RuntimeFilterBucketPrunerTest, DescriptorMustMarkScanAsEligible) { + constexpr int filter_id = 10; + VExprContextSPtrs conjuncts {make_in_conjunct(filter_id, {10})}; + TRuntimeFilterDesc desc; + desc.__set_filter_id(filter_id); + + RuntimeFilterBucketPruner pruner; + int64_t newly_pruned = 0; + ASSERT_TRUE(pruner.prune_by_runtime_filters(four_bucket_ranges(), conjuncts, {desc}, + SCAN_NODE_ID, /*max_in_num=*/1024, &newly_pruned) + .ok()); + EXPECT_EQ(newly_pruned, 0); + EXPECT_EQ(pruner.pruned_tablet_count(), 0); +} + +} // namespace doris diff --git a/be/test/exec/scan/scanner_late_arrival_rf_test.cpp b/be/test/exec/scan/scanner_late_arrival_rf_test.cpp index 8327aaffc26556..8428a3d629063d 100644 --- a/be/test/exec/scan/scanner_late_arrival_rf_test.cpp +++ b/be/test/exec/scan/scanner_late_arrival_rf_test.cpp @@ -18,23 +18,37 @@ #include #include +#include +#include #include +#include #include "common/object_pool.h" #include "core/data_type/data_type_factory.hpp" #include "core/data_type/data_type_number.h" #include "exec/operator/mock_scan_operator.h" +#include "exec/operator/olap_scan_operator.h" #include "exec/runtime_filter/runtime_filter_consumer.h" #include "exec/runtime_filter/runtime_filter_consumer_helper.h" #include "exec/runtime_filter/runtime_filter_producer.h" #include "exec/runtime_filter/runtime_filter_test_utils.h" +#include "exec/scan/parallel_scanner_builder.h" #include "exec/scan/scanner.h" +#include "exec/scan/scanner_context.h" +#include "exec/scan/scanner_scheduler.h" #include "runtime/descriptors.h" #include "runtime/exec_env.h" +#include "storage/iterator/block_reader.h" +#include "storage/rowset/rowset_meta.h" +#include "storage/rowset/rowset_writer.h" +#include "storage/tablet/tablet_meta.h" #include "testutil/column_helper.h" +#include "testutil/desc_tbl_builder.h" #include "testutil/mock/mock_descriptors.h" #include "testutil/mock/mock_runtime_state.h" #include "testutil/mock/mock_slot_ref.h" +#include "util/defer_op.h" +#include "util/raw_value.h" namespace doris { @@ -70,6 +84,130 @@ class TestScanner final : public Scanner { std::list _blocks; }; +class FakeTablet final : public BaseTablet { +public: + FakeTablet(int64_t partition_id, int64_t tablet_id) + : BaseTablet(create_meta(partition_id, tablet_id)) {} + + std::string tablet_path() const override { return ""; } + + bool exceed_version_limit(int32_t /*limit*/) override { return false; } + + Result> create_rowset_writer(RowsetWriterContext& /*context*/, + bool /*vertical*/) override { + return ResultError(Status::NotSupported("fake tablet")); + } + + Result> create_transient_rowset_writer( + const Rowset& /*rowset*/, std::shared_ptr /*partial_update_info*/, + int64_t /*txn_expiration*/ = 0) override { + return ResultError(Status::NotSupported("fake tablet")); + } + + Status capture_rs_readers(const Version& /*spec_version*/, + std::vector* /*rs_splits*/, + const CaptureRowsetOps& /*opts*/) override { + return Status::NotSupported("fake tablet"); + } + + Status save_delete_bitmap(const TabletTxnInfo* /*txn_info*/, int64_t /*txn_id*/, + DeleteBitmapPtr /*delete_bitmap*/, RowsetWriter* /*rowset_writer*/, + const RowsetIdUnorderedSet& /*cur_rowset_ids*/, + int64_t /*lock_id*/ = -1, + int64_t /*next_visible_version*/ = -1) override { + return Status::NotSupported("fake tablet"); + } + + CalcDeleteBitmapExecutor* calc_delete_bitmap_executor() override { return nullptr; } + + void clear_cache() override {} + + Versions calc_missed_versions(int64_t /*spec_version*/, + Versions /*existing_versions*/) const override { + return {}; + } + + size_t tablet_footprint() override { return 0; } + +private: + static TabletMetaSharedPtr create_meta(int64_t partition_id, int64_t tablet_id) { + auto meta = std::make_shared(std::make_shared()); + meta->_partition_id = partition_id; + meta->_tablet_id = tablet_id; + return meta; + } +}; + +class LateBucketScanner final : public Scanner { +public: + LateBucketScanner(RuntimeState* state, OlapScanLocalState* local_state, int32_t bucket_seq, + int32_t bucket_num, bool has_matching_row, RuntimeProfile* profile, + std::latch* prepare_started, std::latch* filter_published) + : Scanner(state, local_state, -1, profile), + _olap_local_state(local_state), + _bucket_seq(bucket_seq), + _bucket_num(bucket_num), + _has_matching_row(has_matching_row), + _prepare_started(prepare_started), + _filter_published(filter_published) {} + + bool is_pruned_by_runtime_filter() const override { + return _olap_local_state->_is_tablet_pruned_by_runtime_filter(1, _bucket_seq, _bucket_num); + } + + int read_calls() const { return _read_calls.load(); } + int open_calls() const { return _open_calls.load(); } + int prepare_calls() const { return _prepare_calls.load(); } + int release_calls() const { return _release_calls.load(); } + + void release_unopened_resources() override { + ++_release_calls; + Scanner::release_unopened_resources(); + } + +protected: + Status _prepare_impl() override { + ++_prepare_calls; + _prepare_started->count_down(); + _filter_published->wait(); + return Scanner::_prepare_impl(); + } + + Status _open_impl(RuntimeState* state) override { + ++_open_calls; + return Scanner::_open_impl(state); + } + + Status _get_block_impl(RuntimeState* /*state*/, Block* block, bool* eof) override { + ++_read_calls; + if (_returned) { + *eof = true; + return Status::OK(); + } + _returned = true; + *eof = false; + if (_has_matching_row) { + *block = ColumnHelper::create_block({7}); + } else { + *block = ColumnHelper::create_block(std::vector {}); + } + return Status::OK(); + } + +private: + OlapScanLocalState* _olap_local_state; + int32_t _bucket_seq; + int32_t _bucket_num; + bool _has_matching_row; + std::latch* _prepare_started; + std::latch* _filter_published; + std::atomic _read_calls {0}; + std::atomic _open_calls {0}; + std::atomic _prepare_calls {0}; + std::atomic _release_calls {0}; + bool _returned = false; +}; + class ScannerLateArrivalRfTest : public RuntimeFilterTest { public: void SetUp() override { @@ -140,6 +278,441 @@ TEST_F(ScannerLateArrivalRfTest, applied_rf_num_advances_after_late_arrival) { ASSERT_TRUE(scanner->_conjuncts.empty()); } +TEST_F(ScannerLateArrivalRfTest, bucket_pruning_after_probe_tasks_start) { + constexpr int scan_node_id = 0; + constexpr int bucket_num = 4; + constexpr int filter_value = 7; + + auto desc = TRuntimeFilterDescBuilder().add_planId_to_target_expr(scan_node_id).build(); + desc.__set_bucket_pruning_target_ids({scan_node_id}); + + ObjectPool pool; + DescriptorTblBuilder desc_builder(&pool); + desc_builder.declare_tuple() << TupleDescBuilder::SlotType {std::make_shared(), + "dist_col"}; + DescriptorTbl* desc_tbl = desc_builder.build(); + ASSERT_NE(desc_tbl, nullptr); + + TOlapScanNode olap_scan_node; + olap_scan_node.__set_tuple_id(0); + olap_scan_node.__set_keyType(TKeysType::DUP_KEYS); + olap_scan_node.__set_key_column_name({"dist_col"}); + olap_scan_node.__set_key_column_type({TPrimitiveType::INT}); + + TPlanNode plan_node; + plan_node.__set_node_id(scan_node_id); + plan_node.__set_node_type(TPlanNodeType::OLAP_SCAN_NODE); + plan_node.__set_num_children(0); + plan_node.__set_limit(-1); + plan_node.__set_row_tuples({0}); + plan_node.__set_runtime_filters({desc}); + plan_node.__set_olap_scan_node(olap_scan_node); + + auto op = std::make_shared(&pool, plan_node, 0, *desc_tbl, bucket_num, + TQueryCacheParam {}); + auto* state = _runtime_states[0].get(); + state->set_desc_tbl(desc_tbl); + TQueryOptions query_options = + TQueryOptionsBuilder().set_runtime_filter_max_in_num(1024).build(); + query_options.__set_enable_runtime_filter_bucket_prune(true); + state->set_query_options(query_options); + + auto local_state = OlapScanLocalState::create_shared(state, op.get()); + std::vector> rf_dependencies; + ASSERT_TRUE(local_state->_helper.init(state, true, 0, 0, rf_dependencies, "").ok()); + ASSERT_TRUE(local_state->_helper + .acquire_runtime_filter(state, local_state->_conjuncts, + op->operator_row_desc_before_projection()) + .ok()); + ASSERT_TRUE(local_state->_conjuncts.empty()); + auto task_exec_ctx = std::make_shared(); + state->set_task_execution_context(task_exec_ctx); + for (int bucket_seq = 0; bucket_seq < bucket_num; ++bucket_seq) { + auto scan_range = std::make_unique(); + scan_range->__set_tablet_id(100 + bucket_seq); + scan_range->__set_bucket_seq(bucket_seq); + scan_range->__set_bucket_num(bucket_num); + local_state->_scan_ranges.push_back(std::move(scan_range)); + } + local_state->_has_rf_bucket_prune_metadata = true; + RuntimeProfile scan_profile("late bucket scan"); + local_state->_buckets_pruned_by_rf_counter = + ADD_COUNTER(&scan_profile, "BucketsPrunedByRuntimeFilter", TUnit::UNIT); + local_state->_scan_timer = ADD_TIMER(&scan_profile, "ScannerGetBlockTime"); + local_state->_scan_cpu_timer = ADD_TIMER(&scan_profile, "ScannerCpuTime"); + local_state->_filter_timer = ADD_TIMER(&scan_profile, "ScannerFilterTime"); + local_state->_rows_read_counter = ADD_COUNTER(&scan_profile, "RowsRead", TUnit::UNIT); + + uint32_t hash = RawValue::zlib_crc32(&filter_value, sizeof(filter_value), TYPE_INT, 0); + int selected_bucket = static_cast(hash % bucket_num); + std::latch prepare_started(bucket_num); + std::latch filter_published(1); + std::list> scanner_delegates; + std::vector> scanners; + for (int bucket_seq = 0; bucket_seq < bucket_num; ++bucket_seq) { + auto scanner = std::make_shared( + state, local_state.get(), bucket_seq, bucket_num, bucket_seq == selected_bucket, + &scan_profile, &prepare_started, &filter_published); + ASSERT_TRUE(scanner->init(state, {}).ok()); + scanners.push_back(scanner); + ScannerSPtr scanner_base = scanner; + scanner_delegates.push_back(std::make_shared(scanner_base)); + } + + auto dependency = Dependency::create_shared(0, 0, "late bucket scan dependency"); + std::atomic shared_limit {-1}; + auto scanner_context = ScannerContext::create_shared( + state, local_state.get(), desc_tbl->get_tuple_descriptor(0), false, scanner_delegates, + -1, dependency, &shared_limit, nullptr, nullptr, 0, false, bucket_num); + scanner_context->_newly_create_free_blocks_num = + ADD_COUNTER(&scan_profile, "NewlyCreatedFreeBlocks", TUnit::UNIT); + scanner_context->_scanner_memory_used_counter = + ADD_COUNTER(&scan_profile, "ScannerMemoryUsed", TUnit::BYTES); + scanner_context->_max_bytes_in_queue = 10 * 1024 * 1024; + std::vector> tasks; + for (const auto& scanner_delegate : scanner_delegates) { + auto task = std::make_shared(scanner_delegate); + task->set_state(ScanTask::State::IN_FLIGHT); + tasks.push_back(std::move(task)); + } + scanner_context->_in_flight_tasks_num = bucket_num; + + std::shared_ptr producer; + ASSERT_TRUE(RuntimeFilterProducer::create(_query_ctx.get(), &desc, &producer).ok()); + ASSERT_TRUE(producer->init(1).ok()); + auto filter_column = ColumnInt32::create(); + filter_column->insert_value(filter_value); + ASSERT_TRUE(producer->insert(std::move(filter_column), 0).ok()); + producer->set_wrapper_state_and_ready_to_publish(RuntimeFilterWrapper::State::READY); + + std::vector probe_threads; + bool filter_released = false; + Defer probe_thread_guard {[&] { + if (!filter_released) { + filter_published.count_down(); + } + for (auto& thread : probe_threads) { + if (thread.joinable()) { + thread.join(); + } + } + }}; + for (const auto& task : tasks) { + probe_threads.emplace_back([scanner_context, task] { + ScannerScheduler::_scanner_scan(scanner_context, task); + }); + } + + // Every task has passed the scheduler's pre-prepare pruning check while the RF is not ready. + prepare_started.wait(); + EXPECT_EQ(local_state->_rf_bucket_pruner.pruned_tablet_count(), 0); + + local_state->_helper._consumers[0]->signal(producer.get()); + filter_published.count_down(); + filter_released = true; + + for (auto& thread : probe_threads) { + if (thread.joinable()) { + thread.join(); + } + } + + for (int bucket_seq = 0; bucket_seq < bucket_num; ++bucket_seq) { + if (bucket_seq == selected_bucket) { + EXPECT_GT(scanners[bucket_seq]->open_calls(), 0); + EXPECT_GT(scanners[bucket_seq]->read_calls(), 0); + EXPECT_EQ(scanners[bucket_seq]->release_calls(), 0); + EXPECT_NE(tasks[bucket_seq]->cached_block, nullptr); + } else { + EXPECT_EQ(scanners[bucket_seq]->open_calls(), 0); + EXPECT_EQ(scanners[bucket_seq]->read_calls(), 0); + EXPECT_EQ(scanners[bucket_seq]->release_calls(), 1); + } + } + int64_t result_rows = 0; + for (const auto& task : tasks) { + if (task->cached_block != nullptr) { + result_rows += task->cached_block->rows(); + } + } + ASSERT_EQ(result_rows, 1); + ASSERT_GT(local_state->_buckets_pruned_by_rf_counter->value(), 0); + ASSERT_GT(local_state->_rf_bucket_pruner.pruned_tablet_count(), 0); + ASSERT_LT(local_state->_rf_bucket_pruner.pruned_tablet_count(), bucket_num); +} + +TEST_F(ScannerLateArrivalRfTest, bounded_concurrency_prunes_scanner_before_first_schedule) { + constexpr int scan_node_id = 0; + constexpr int32_t bucket_num = 2; + constexpr int32_t active_bucket = 0; + constexpr int32_t pending_bucket = 1; + + ObjectPool pool; + DescriptorTblBuilder desc_builder(&pool); + desc_builder.declare_tuple() << TupleDescBuilder::SlotType {std::make_shared(), + "dist_col"}; + DescriptorTbl* desc_tbl = desc_builder.build(); + ASSERT_NE(desc_tbl, nullptr); + + TOlapScanNode olap_scan_node; + olap_scan_node.__set_tuple_id(0); + olap_scan_node.__set_keyType(TKeysType::DUP_KEYS); + TPlanNode plan_node; + plan_node.__set_node_id(scan_node_id); + plan_node.__set_node_type(TPlanNodeType::OLAP_SCAN_NODE); + plan_node.__set_num_children(0); + plan_node.__set_limit(-1); + plan_node.__set_row_tuples({0}); + plan_node.__set_olap_scan_node(olap_scan_node); + + auto op = std::make_shared(&pool, plan_node, 0, *desc_tbl, bucket_num, + TQueryCacheParam {}); + auto* state = _runtime_states[0].get(); + state->set_desc_tbl(desc_tbl); + auto task_exec_ctx = std::make_shared(); + state->set_task_execution_context(task_exec_ctx); + auto local_state = OlapScanLocalState::create_shared(state, op.get()); + local_state->_has_rf_bucket_prune_metadata = true; + + RuntimeProfile scan_profile("bounded late bucket scan"); + local_state->_scan_timer = ADD_TIMER(&scan_profile, "ScannerGetBlockTime"); + local_state->_scan_cpu_timer = ADD_TIMER(&scan_profile, "ScannerCpuTime"); + local_state->_filter_timer = ADD_TIMER(&scan_profile, "ScannerFilterTime"); + local_state->_rows_read_counter = ADD_COUNTER(&scan_profile, "RowsRead", TUnit::UNIT); + + std::latch prepare_started(2); + std::latch filter_published(0); + auto active_scanner = std::make_shared( + state, local_state.get(), active_bucket, bucket_num, true, &scan_profile, + &prepare_started, &filter_published); + auto pending_scanner = std::make_shared( + state, local_state.get(), pending_bucket, bucket_num, false, &scan_profile, + &prepare_started, &filter_published); + ASSERT_TRUE(active_scanner->init(state, {}).ok()); + ASSERT_TRUE(pending_scanner->init(state, {}).ok()); + + ScannerSPtr pending_scanner_base = pending_scanner; + ScannerSPtr active_scanner_base = active_scanner; + std::list> scanner_delegates; + scanner_delegates.push_back(std::make_shared(pending_scanner_base)); + scanner_delegates.push_back(std::make_shared(active_scanner_base)); + + auto dependency = Dependency::create_shared(0, 0, "bounded late bucket scan dependency"); + std::atomic shared_limit {-1}; + auto scanner_context = ScannerContext::create_shared( + state, local_state.get(), desc_tbl->get_tuple_descriptor(0), false, scanner_delegates, + -1, dependency, &shared_limit, nullptr, nullptr, 0, false, 1); + scanner_context->_newly_create_free_blocks_num = + ADD_COUNTER(&scan_profile, "NewlyCreatedFreeBlocks", TUnit::UNIT); + scanner_context->_scanner_memory_used_counter = + ADD_COUNTER(&scan_profile, "ScannerMemoryUsed", TUnit::BYTES); + scanner_context->_max_bytes_in_queue = 10 * 1024 * 1024; + + auto active_task = scanner_context->_pull_next_scan_task(nullptr, 0); + ASSERT_NE(active_task, nullptr); + ASSERT_EQ(active_task->scanner.lock()->_scanner, active_scanner); + EXPECT_EQ(scanner_context->_pull_next_scan_task(nullptr, 1), nullptr); + + active_task->set_state(ScanTask::State::IN_FLIGHT); + scanner_context->_in_flight_tasks_num = 1; + ScannerScheduler::_scanner_scan(scanner_context, active_task); + ASSERT_EQ(active_scanner->prepare_calls(), 1); + ASSERT_EQ(active_scanner->open_calls(), 1); + ASSERT_EQ(scanner_context->_in_flight_tasks_num, 0); + ASSERT_EQ(scanner_context->_completed_tasks.size(), 1); + scanner_context->_completed_tasks.clear(); + + // Model the active scanner applying a newly published filter before the pending scanner gets + // its sole concurrency slot, so the shared pruner already excludes the pending bucket. + local_state->_rf_bucket_pruner._selected_buckets_by_num[bucket_num] = {active_bucket}; + + auto pending_task = scanner_context->_pull_next_scan_task(nullptr, 0); + ASSERT_NE(pending_task, nullptr); + ASSERT_EQ(pending_task->scanner.lock()->_scanner, pending_scanner); + pending_task->set_state(ScanTask::State::IN_FLIGHT); + scanner_context->_in_flight_tasks_num = 1; + ScannerScheduler::_scanner_scan(scanner_context, pending_task); + + EXPECT_EQ(pending_scanner->prepare_calls(), 0); + EXPECT_EQ(pending_scanner->open_calls(), 0); + EXPECT_EQ(pending_scanner->read_calls(), 0); + EXPECT_EQ(pending_scanner->release_calls(), 1); + EXPECT_TRUE(pending_task->is_eos()); +} + +TEST_F(ScannerLateArrivalRfTest, parallel_scanner_factory_preserves_bucket_identity) { + constexpr int scan_node_id = 0; + constexpr int32_t bucket_seq = 3; + constexpr int32_t bucket_num = 8; + constexpr int64_t partition_id = 10; + constexpr int64_t tablet_id = 20; + + ObjectPool pool; + DescriptorTblBuilder desc_builder(&pool); + desc_builder.declare_tuple() << TupleDescBuilder::SlotType {std::make_shared(), + "dist_col"}; + DescriptorTbl* desc_tbl = desc_builder.build(); + ASSERT_NE(desc_tbl, nullptr); + + TOlapScanNode olap_scan_node; + olap_scan_node.__set_tuple_id(0); + olap_scan_node.__set_keyType(TKeysType::DUP_KEYS); + TPlanNode plan_node; + plan_node.__set_node_id(scan_node_id); + plan_node.__set_node_type(TPlanNodeType::OLAP_SCAN_NODE); + plan_node.__set_num_children(0); + plan_node.__set_limit(-1); + plan_node.__set_row_tuples({0}); + plan_node.__set_olap_scan_node(olap_scan_node); + + auto op = std::make_shared(&pool, plan_node, 0, *desc_tbl, bucket_num, + TQueryCacheParam {}); + auto* state = _runtime_states[0].get(); + state->set_desc_tbl(desc_tbl); + auto local_state = OlapScanLocalState::create_shared(state, op.get()); + local_state->_has_rf_bucket_prune_metadata = true; + local_state->_rf_bucket_pruner._selected_buckets_by_num[bucket_num] = {bucket_seq - 1}; + + auto tablet = std::make_shared(partition_id, tablet_id); + std::vector tablets {{tablet, 1}}; + std::vector read_sources(1); + std::vector> scan_ranges; + auto scan_range = std::make_unique(); + scan_range->__set_tablet_id(tablet_id); + scan_range->__set_bucket_seq(bucket_seq); + scan_range->__set_bucket_num(bucket_num); + scan_ranges.push_back(std::move(scan_range)); + auto profile = std::make_shared("parallel scanner bucket identity"); + ParallelScannerBuilder builder(local_state.get(), tablets, read_sources, scan_ranges, profile, + {}, state, -1, true, true); + + auto scanner = builder._build_scanner(tablet, 1, {}, *scan_ranges.front(), TabletReadSource {}, + io::FileCacheStatistics {}); + EXPECT_EQ(scanner->_bucket_seq, bucket_seq); + EXPECT_EQ(scanner->_bucket_num, bucket_num); + EXPECT_TRUE(scanner->is_pruned_by_runtime_filter()); +} + +TEST_F(ScannerLateArrivalRfTest, high_cardinality_ineligible_parallel_builder_reuses_scan_ranges) { + constexpr size_t scan_range_count = 20'000; + constexpr int64_t first_tablet_id = 100; + + std::vector tablets; + std::vector read_sources(scan_range_count); + std::vector> scan_ranges; + tablets.reserve(scan_range_count); + scan_ranges.reserve(scan_range_count); + for (size_t i = 0; i < scan_range_count; ++i) { + int64_t tablet_id = first_tablet_id + static_cast(i); + tablets.push_back({std::make_shared(1, tablet_id), 1}); + auto scan_range = std::make_unique(); + scan_range->__set_tablet_id(tablet_id); + scan_ranges.push_back(std::move(scan_range)); + } + std::vector key_ranges; + std::shared_ptr profile; + + ParallelScannerBuilder builder(nullptr, tablets, read_sources, scan_ranges, profile, key_ranges, + nullptr, -1, true, true); + + EXPECT_EQ(&builder._scan_ranges, &scan_ranges); + EXPECT_EQ(builder._scan_ranges.size(), scan_range_count); + EXPECT_FALSE(builder._scan_ranges.front()->__isset.bucket_seq); + EXPECT_FALSE(builder._scan_ranges.front()->__isset.bucket_num); +} + +TEST_F(ScannerLateArrivalRfTest, olap_scanner_releases_resources_before_open) { + constexpr int scan_node_id = 0; + constexpr int64_t partition_id = 10; + constexpr int64_t tablet_id = 20; + + ObjectPool pool; + DescriptorTblBuilder desc_builder(&pool); + desc_builder.declare_tuple() << TupleDescBuilder::SlotType {std::make_shared(), + "dist_col"}; + DescriptorTbl* desc_tbl = desc_builder.build(); + ASSERT_NE(desc_tbl, nullptr); + + TOlapScanNode olap_scan_node; + olap_scan_node.__set_tuple_id(0); + olap_scan_node.__set_keyType(TKeysType::DUP_KEYS); + TPlanNode plan_node; + plan_node.__set_node_id(scan_node_id); + plan_node.__set_node_type(TPlanNodeType::OLAP_SCAN_NODE); + plan_node.__set_num_children(0); + plan_node.__set_limit(-1); + plan_node.__set_row_tuples({0}); + plan_node.__set_olap_scan_node(olap_scan_node); + + auto op = std::make_shared(&pool, plan_node, 0, *desc_tbl, 1, + TQueryCacheParam {}); + auto* state = _runtime_states[0].get(); + state->set_desc_tbl(desc_tbl); + auto local_state = OlapScanLocalState::create_shared(state, op.get()); + auto unprepared_tablet = std::make_shared(partition_id, tablet_id); + auto delete_predicate = std::make_shared(); + std::weak_ptr unprepared_tablet_ref = unprepared_tablet; + std::weak_ptr delete_predicate_ref = delete_predicate; + RuntimeProfile profile("unopened scanner cleanup"); + OlapScanner::Params params; + params.state = state; + params.profile = &profile; + params.tablet = unprepared_tablet; + params.version = 1; + params.read_source.rs_splits.emplace_back(); + params.read_source.delete_predicates.push_back(delete_predicate); + params.limit = -1; + params.aggregation = true; + auto unprepared_scanner = OlapScanner::create_shared(local_state.get(), std::move(params)); + unprepared_tablet.reset(); + delete_predicate.reset(); + + ASSERT_FALSE(unprepared_scanner->has_prepared()); + ASSERT_FALSE(unprepared_scanner->is_open()); + ASSERT_FALSE(unprepared_tablet_ref.expired()); + ASSERT_FALSE(delete_predicate_ref.expired()); + ASSERT_FALSE(unprepared_scanner->_tablet_reader_params.rs_splits.empty()); + ASSERT_FALSE(unprepared_scanner->_tablet_reader_params.delete_predicates.empty()); + + unprepared_scanner->release_unopened_resources(); + + EXPECT_FALSE(unprepared_scanner->has_prepared()); + EXPECT_FALSE(unprepared_scanner->is_open()); + EXPECT_TRUE(unprepared_tablet_ref.expired()); + EXPECT_TRUE(delete_predicate_ref.expired()); + EXPECT_EQ(unprepared_scanner->_tablet_reader_params.tablet, nullptr); + EXPECT_TRUE(unprepared_scanner->_tablet_reader_params.rs_splits.empty()); + EXPECT_TRUE(unprepared_scanner->_tablet_reader_params.delete_predicates.empty()); + + auto prepared_tablet = std::make_shared(partition_id, tablet_id + 1); + OlapScanner::Params prepared_params; + prepared_params.state = state; + prepared_params.profile = &profile; + prepared_params.tablet = prepared_tablet; + prepared_params.version = 1; + prepared_params.limit = -1; + prepared_params.aggregation = true; + auto prepared_scanner = + OlapScanner::create_shared(local_state.get(), std::move(prepared_params)); + ASSERT_TRUE(prepared_scanner->Scanner::_prepare_impl().ok()); + prepared_scanner->_tablet_reader = std::make_unique(); + prepared_scanner->_tablet_reader_params.rs_splits.emplace_back(); + + ASSERT_TRUE(prepared_scanner->has_prepared()); + ASSERT_FALSE(prepared_scanner->is_open()); + ASSERT_NE(prepared_scanner->_tablet_reader, nullptr); + ASSERT_FALSE(prepared_scanner->_tablet_reader_params.rs_splits.empty()); + + prepared_scanner->release_unopened_resources(); + + EXPECT_FALSE(prepared_scanner->has_prepared()); + EXPECT_FALSE(prepared_scanner->is_open()); + EXPECT_EQ(prepared_scanner->_tablet_reader, nullptr); + EXPECT_TRUE(prepared_scanner->_tablet_reader_params.rs_splits.empty()); + EXPECT_EQ(prepared_scanner->_tablet_reader_params.tablet, nullptr); + prepared_scanner->update_realtime_counters(); +} + TEST(ScannerProjectionTest, merges_padding_block_when_limit_eos_without_extra_flag) { ObjectPool pool; auto data_type = std::make_shared(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/RuntimeFilterPartitionPruneClassifier.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/RuntimeFilterPartitionPruneClassifier.java deleted file mode 100644 index 00c61916329e68..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/RuntimeFilterPartitionPruneClassifier.java +++ /dev/null @@ -1,322 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.glue.translator; - -import org.apache.doris.analysis.Expr; -import org.apache.doris.analysis.FunctionCallExpr; -import org.apache.doris.analysis.LiteralExpr; -import org.apache.doris.analysis.SlotRef; -import org.apache.doris.catalog.Column; -import org.apache.doris.catalog.ListPartitionItem; -import org.apache.doris.catalog.OlapTable; -import org.apache.doris.catalog.PartitionInfo; -import org.apache.doris.catalog.PartitionItem; -import org.apache.doris.catalog.PartitionKey; -import org.apache.doris.catalog.PartitionType; -import org.apache.doris.catalog.RangePartitionItem; -import org.apache.doris.nereids.exceptions.AnalysisException; -import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.expressions.Slot; -import org.apache.doris.nereids.trees.expressions.functions.Monotonic; -import org.apache.doris.nereids.trees.expressions.functions.NoneMovableFunction; -import org.apache.doris.nereids.trees.expressions.literal.Literal; -import org.apache.doris.planner.OlapScanNode; -import org.apache.doris.planner.PlanNode; -import org.apache.doris.thrift.TRuntimeFilterType; -import org.apache.doris.thrift.TTargetExprMonotonicity; - -import com.google.common.collect.Range; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** - * Classifies whether one runtime-filter target can safely drive BE-side partition pruning. - * - *

This is the single FE gate for RF partition pruning. It intentionally reasons about the - * final legacy target expression that will be sent to BE, so late-added casts and partition - * expression domains cannot bypass the safety checks. - */ -final class RuntimeFilterPartitionPruneClassifier { - private RuntimeFilterPartitionPruneClassifier() { - } - - static Classification classify(TRuntimeFilterType filterType, Expr targetExpr, - Expression nereidsTargetExpr, PlanNode scanNode) { - if (!(scanNode instanceof OlapScanNode)) { - return Classification.unsupported("target scan is not an OlapScanNode"); - } - - OlapScanNode olapScanNode = (OlapScanNode) scanNode; - OlapTable table = olapScanNode.getOlapTable(); - if (table == null) { - return Classification.unsupported("target scan has no OlapTable"); - } - - PartitionInfo partitionInfo = table.getPartitionInfo(); - PartitionType partType = partitionInfo.getType(); - if (partType != PartitionType.RANGE && partType != PartitionType.LIST) { - return Classification.unsupported("partition type is not RANGE or LIST"); - } - if (filterType == TRuntimeFilterType.BLOOM && partType == PartitionType.RANGE) { - return Classification.unsupported("BLOOM runtime filter does not support RANGE partition pruning"); - } - if (hasUnsupportedAutomaticPartitionExpression(partitionInfo)) { - return Classification.unsupported("automatic partition expression boundary is not modeled"); - } - if (nereidsTargetExpr.containsType(NoneMovableFunction.class)) { - return Classification.unsupported("target expression contains non-movable function"); - } - - if (targetExpr instanceof SlotRef) { - SlotRef slotRef = (SlotRef) targetExpr; - if (!isPartitionColumnSlot(slotRef, partitionInfo.getPartitionColumns())) { - return Classification.unsupported("target SlotRef is not a partition column"); - } - if (!hasSerializedBoundary(slotRef, partitionInfo, partType)) { - return Classification.unsupported("target SlotRef has no serialized partition boundary"); - } - Map partitionMonotonicity = - allSelectedPartitionsIncreasing(olapScanNode, partitionInfo); - if (partitionMonotonicity.isEmpty()) { - return Classification.unsupported("target SlotRef has no prunable selected partitions"); - } - return Classification.supportedPartitions(slotRef, partitionMonotonicity); - } - - SlotRef leafSlot = findUniqueSlotRef(targetExpr); - if (leafSlot == null || !isPartitionColumnSlot(leafSlot, partitionInfo.getPartitionColumns())) { - return Classification.unsupported("target expression is not rooted on one partition column"); - } - if (!hasSerializedBoundary(leafSlot, partitionInfo, partType)) { - return Classification.unsupported("target expression has no serialized partition boundary"); - } - if (partType == PartitionType.LIST) { - if (nereidsTargetExpr.containsNondeterministic()) { - return Classification.unsupported("target expression contains non-deterministic function"); - } - Map partitionMonotonicity = - allSelectedPartitionsIncreasing(olapScanNode, partitionInfo); - if (partitionMonotonicity.isEmpty()) { - return Classification.unsupported("target expression has no prunable selected partitions"); - } - return Classification.supportedPartitions(leafSlot, partitionMonotonicity); - } - - Map partitionMonotonicity = - classifyLocalMonotonicity(nereidsTargetExpr, olapScanNode, partitionInfo, leafSlot); - if (partitionMonotonicity.isEmpty()) { - return Classification.unsupported("target expression is not monotonic on selected partitions"); - } - return Classification.supportedPartitions(leafSlot, partitionMonotonicity); - } - - private static boolean hasUnsupportedAutomaticPartitionExpression(PartitionInfo partitionInfo) { - if (!partitionInfo.enableAutomaticPartition()) { - return false; - } - for (Expr partitionExpr : partitionInfo.getPartitionExprs()) { - if (containsFunctionCall(partitionExpr)) { - return true; - } - } - return false; - } - - private static boolean containsFunctionCall(Expr expr) { - if (expr instanceof FunctionCallExpr) { - return true; - } - for (Expr child : expr.getChildren()) { - if (containsFunctionCall(child)) { - return true; - } - } - return false; - } - - private static boolean hasSerializedBoundary(SlotRef slotRef, PartitionInfo partitionInfo, PartitionType partType) { - if (partType != PartitionType.RANGE) { - return true; - } - List partitionColumns = partitionInfo.getPartitionColumns(); - return !partitionColumns.isEmpty() && sameColumn(slotRef.getColumn(), partitionColumns.get(0)); - } - - private static boolean isPartitionColumnSlot(SlotRef slotRef, List partitionColumns) { - Column targetColumn = slotRef.getColumn(); - if (targetColumn == null) { - return false; - } - for (Column partitionColumn : partitionColumns) { - if (sameColumn(targetColumn, partitionColumn)) { - return true; - } - } - return false; - } - - private static boolean sameColumn(Column targetColumn, Column partitionColumn) { - if (targetColumn == partitionColumn) { - return true; - } - int targetUniqueId = targetColumn.getUniqueId(); - int partitionUniqueId = partitionColumn.getUniqueId(); - if (targetUniqueId != Column.COLUMN_UNIQUE_ID_INIT_VALUE - && partitionUniqueId != Column.COLUMN_UNIQUE_ID_INIT_VALUE - && targetUniqueId == partitionUniqueId) { - return true; - } - return targetColumn.equals(partitionColumn); - } - - private static SlotRef findUniqueSlotRef(Expr expr) { - if (expr instanceof SlotRef) { - return (SlotRef) expr; - } - SlotRef result = null; - for (Expr child : expr.getChildren()) { - SlotRef childSlot = findUniqueSlotRef(child); - if (childSlot == null) { - continue; - } - if (result != null && result.getSlotId().asInt() != childSlot.getSlotId().asInt()) { - return null; - } - result = childSlot; - } - return result; - } - - private static Map classifyLocalMonotonicity( - Expression nereidsTargetExpr, OlapScanNode scanNode, PartitionInfo partitionInfo, SlotRef leafSlot) { - Map result = new HashMap<>(); - if (!(nereidsTargetExpr instanceof Monotonic)) { - return result; - } - - Monotonic monotonic = (Monotonic) nereidsTargetExpr; - int childIndex = monotonic.getMonotonicFunctionChildIndex(); - if (childIndex < 0 || childIndex >= nereidsTargetExpr.arity() - || !(nereidsTargetExpr.child(childIndex) instanceof Slot) - || !hasInputSlotOnlyInMonotonicChild(nereidsTargetExpr, childIndex)) { - return result; - } - - Column partitionColumn = leafSlot.getColumn(); - for (Long partitionId : scanNode.getSelectedPartitionIds()) { - PartitionItem item = partitionInfo.getItem(partitionId); - if (!(item instanceof RangePartitionItem)) { - continue; - } - Range range = ((RangePartitionItem) item).getItems(); - Literal lower = null; - Literal upper = null; - if (range.hasLowerBound() && !range.lowerEndpoint().isMinValue()) { - lower = toNereidsLiteral(range.lowerEndpoint().getKeys().get(0), partitionColumn); - if (lower == null) { - continue; - } - } - if (range.hasUpperBound() && !range.upperEndpoint().isMaxValue()) { - upper = toNereidsLiteral(range.upperEndpoint().getKeys().get(0), partitionColumn); - if (upper == null) { - continue; - } - } - if (monotonic.isMonotonic(lower, upper)) { - result.put(partitionId, monotonic.isPositive() - ? TTargetExprMonotonicity.MONOTONIC_INCREASING - : TTargetExprMonotonicity.MONOTONIC_DECREASING); - } - } - return result; - } - - static boolean hasInputSlotOnlyInMonotonicChild(Expression expression, int monotonicChildIndex) { - for (int i = 0; i < expression.arity(); i++) { - if (i != monotonicChildIndex && !expression.child(i).getInputSlots().isEmpty()) { - return false; - } - } - return true; - } - - private static Map allSelectedPartitionsIncreasing( - OlapScanNode scanNode, PartitionInfo partitionInfo) { - Map result = new HashMap<>(); - for (Long partitionId : scanNode.getSelectedPartitionIds()) { - PartitionItem item = partitionInfo.getItem(partitionId); - if (item == null || (item instanceof ListPartitionItem - && ((ListPartitionItem) item).isDefaultPartition())) { - continue; - } - result.put(partitionId, TTargetExprMonotonicity.MONOTONIC_INCREASING); - } - return result; - } - - private static Literal toNereidsLiteral(LiteralExpr literalExpr, Column column) { - try { - return Literal.fromLegacyLiteral(literalExpr, column.getType()); - } catch (AnalysisException e) { - return null; - } - } - - static final class Classification { - private final boolean canPrunePartitions; - private final SlotRef partitionSlot; - private final Map partitionMonotonicity; - private final String unsupportedReason; - - private Classification(boolean canPrunePartitions, SlotRef partitionSlot, - Map partitionMonotonicity, String unsupportedReason) { - this.canPrunePartitions = canPrunePartitions; - this.partitionSlot = partitionSlot; - this.partitionMonotonicity = partitionMonotonicity; - this.unsupportedReason = unsupportedReason; - } - - static Classification supportedPartitions(SlotRef partitionSlot, - Map partitionMonotonicity) { - return new Classification(true, partitionSlot, partitionMonotonicity, ""); - } - - static Classification unsupported(String reason) { - return new Classification(false, null, new HashMap<>(), reason); - } - - boolean canPrunePartitions() { - return canPrunePartitions; - } - - SlotRef getPartitionSlot() { - return partitionSlot; - } - - Map getPartitionMonotonicity() { - return partitionMonotonicity; - } - - String getUnsupportedReason() { - return unsupportedReason; - } - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/RuntimeFilterTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/RuntimeFilterTranslator.java index 6558541f0d723e..8912502d1f43ed 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/RuntimeFilterTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/RuntimeFilterTranslator.java @@ -170,8 +170,6 @@ private void createLegacyRuntimeFilterFromGroup(List group, List targetExprList = new ArrayList<>(); List>> targetTupleIdMapList = new ArrayList<>(); List scanNodeList = new ArrayList<>(); - List nereidsTargetExprList = new ArrayList<>(); - List targetAdjustedAfterNonIdentityList = new ArrayList<>(); Map scanNodeTargetExprSql = new LinkedHashMap<>(); Map scanNodeHasDifferentTargets = new LinkedHashMap<>(); boolean hasInvalidTarget = false; @@ -200,16 +198,12 @@ private void createLegacyRuntimeFilterFromGroup(List group, new RuntimeFilterExpressionTranslator(targetSlotRef); targetExpr = curTargetExpression.accept(translator, ctx); } - boolean adjustedAfterNonIdentity = !src.getType().equals(targetExpr.getType()) - && !isIdentityTarget; targetExpr = castTargetToSourceTypeIfNeeded(src, targetExpr); TupleId targetTupleId = targetSlotRef.getDesc().getParentId(); SlotId targetSlotId = targetSlotRef.getSlotId(); scanNodeList.add(scanNode); targetExprList.add(targetExpr); - nereidsTargetExprList.add(curTargetExpression); targetTupleIdMapList.add(ImmutableMap.of(targetTupleId, ImmutableList.of(targetSlotId))); - targetAdjustedAfterNonIdentityList.add(adjustedAfterNonIdentity); int scanNodeId = scanNode.getId().asInt(); String targetExprSql = targetExpr.accept(ExprToSqlVisitor.INSTANCE, ToSqlParams.WITH_TABLE); String existingTargetExprSql = scanNodeTargetExprSql.putIfAbsent(scanNodeId, targetExprSql); @@ -241,14 +235,10 @@ private void createLegacyRuntimeFilterFromGroup(List group, // TRuntimeFilterDesc keys target expressions and pruning metadata by scan node ID. // If one grouped RF has different targets on the same scan, BE cannot match // metadata back to a specific target expression, so skip pruning metadata. - if (scanNodeHasDifferentTargets.getOrDefault(scanNode.getId().asInt(), false) - || targetAdjustedAfterNonIdentityList.get(i)) { + if (scanNodeHasDifferentTargets.getOrDefault(scanNode.getId().asInt(), false)) { continue; } - RuntimeFilterPartitionPruneClassifier.Classification classification = - RuntimeFilterPartitionPruneClassifier.classify( - head.getType(), targetExpr, nereidsTargetExprList.get(i), scanNode); - setPartitionPruningMetadata(origFilter, scanNode, classification); + setPruningMetadata(origFilter, scanNode, group.get(i)); } origFilter.setBloomFilterSizeCalculatedByNdv(head.isBloomFilterSizeCalculatedByNdv()); setWaitTimeMs(origFilter, head.isNonBlocking(), isLocalTarget); @@ -287,7 +277,6 @@ public void createLegacyRuntimeFilter(RuntimeFilter filter, PlanNode node, PlanT List targetExprList = new ArrayList<>(); List>> targetTupleIdMapList = new ArrayList<>(); List scanNodeList = new ArrayList<>(); - List targetAdjustedAfterNonIdentityList = new ArrayList<>(); boolean hasInvalidTarget = false; Slot curTargetSlot = filter.getTargetSlot(); Expression curTargetExpression = filter.getTargetExpression(); @@ -313,15 +302,12 @@ public void createLegacyRuntimeFilter(RuntimeFilter filter, PlanNode node, PlanT } // adjust data type - boolean adjustedAfterNonIdentity = !src.getType().equals(targetExpr.getType()) - && !isIdentityTarget; targetExpr = castTargetToSourceTypeIfNeeded(src, targetExpr); TupleId targetTupleId = targetSlotRef.getDesc().getParentId(); SlotId targetSlotId = targetSlotRef.getSlotId(); scanNodeList.add(scanNode); targetExprList.add(targetExpr); targetTupleIdMapList.add(ImmutableMap.of(targetTupleId, ImmutableList.of(targetSlotId))); - targetAdjustedAfterNonIdentityList.add(adjustedAfterNonIdentity); } if (!hasInvalidTarget) { org.apache.doris.planner.RuntimeFilter origFilter @@ -345,13 +331,7 @@ public void createLegacyRuntimeFilter(RuntimeFilter filter, PlanNode node, PlanT Expr targetExpr = targetExprList.get(i); origFilter.addTarget(new RuntimeFilterTarget( scanNode, targetExpr, true, isLocalTarget)); - if (targetAdjustedAfterNonIdentityList.get(i)) { - continue; - } - RuntimeFilterPartitionPruneClassifier.Classification classification = - RuntimeFilterPartitionPruneClassifier.classify( - filter.getType(), targetExpr, filter.getTargetExpressions().get(i), scanNode); - setPartitionPruningMetadata(origFilter, scanNode, classification); + setPruningMetadata(origFilter, scanNode, filter); } origFilter.setBloomFilterSizeCalculatedByNdv(filter.isBloomFilterSizeCalculatedByNdv()); setWaitTimeMs(origFilter, filter.isNonBlocking(), isLocalTarget); @@ -381,16 +361,19 @@ private org.apache.doris.planner.RuntimeFilter finalize(org.apache.doris.planner return origFilter; } - private void setPartitionPruningMetadata(org.apache.doris.planner.RuntimeFilter runtimeFilter, - ScanNode scanNode, RuntimeFilterPartitionPruneClassifier.Classification classification) { - if (classification.canPrunePartitions()) { + private void setPruningMetadata(org.apache.doris.planner.RuntimeFilter runtimeFilter, + ScanNode scanNode, RuntimeFilter nereidsFilter) { + if (nereidsFilter.canPrunePartitions()) { Preconditions.checkState(scanNode instanceof OlapScanNode, "partition-pruning runtime filter target must be an OlapScanNode"); runtimeFilter.markTargetCanPrunePartitions(scanNode.getId()); ((OlapScanNode) scanNode).snapshotPartitionBoundariesForRuntimeFilter(); } runtimeFilter.setTargetPartitionMonotonicity( - scanNode.getId(), classification.getPartitionMonotonicity()); + scanNode.getId(), nereidsFilter.getPartitionMonotonicity()); + if (nereidsFilter.canPruneBuckets()) { + runtimeFilter.markTargetCanPruneBuckets(scanNode.getId()); + } } private void setWaitTimeMs(org.apache.doris.planner.RuntimeFilter filter, diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterContext.java index 0336dd66431bb9..64a321ff03848f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterContext.java @@ -90,6 +90,14 @@ public FilterSizeLimits getLimits() { return limits; } + /** Generate pruning metadata once, while the runtime filter still owns its Nereids target scan. */ + public void generateRuntimeFilterPruneMetadata(RuntimeFilter filter) { + RuntimeFilterPruneClassifier.Classification classification = + RuntimeFilterPruneClassifier.classify(filter, sessionVariable); + filter.setPruningMetadata( + classification.canPruneBuckets(), classification.getPartitionMonotonicity()); + } + public void setTargetExprIdToFilter(ExprId id, RuntimeFilter filter) { Preconditions.checkArgument(filter.getTargetSlot().getExprId() == id); this.targetExprIdToFilter.computeIfAbsent(id, k -> Lists.newArrayList()).add(filter); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterPruneClassifier.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterPruneClassifier.java new file mode 100644 index 00000000000000..6aed407d96bdd0 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterPruneClassifier.java @@ -0,0 +1,457 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.processor.post; + +import org.apache.doris.analysis.Expr; +import org.apache.doris.analysis.FunctionCallExpr; +import org.apache.doris.analysis.LiteralExpr; +import org.apache.doris.analysis.SlotRef; +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.DistributionInfo; +import org.apache.doris.catalog.HashDistributionInfo; +import org.apache.doris.catalog.ListPartitionItem; +import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.Partition; +import org.apache.doris.catalog.PartitionInfo; +import org.apache.doris.catalog.PartitionItem; +import org.apache.doris.catalog.PartitionKey; +import org.apache.doris.catalog.PartitionType; +import org.apache.doris.catalog.RangePartitionItem; +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.functions.Monotonic; +import org.apache.doris.nereids.trees.expressions.functions.NoneMovableFunction; +import org.apache.doris.nereids.trees.expressions.literal.Literal; +import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapScan; +import org.apache.doris.nereids.trees.plans.physical.RuntimeFilter; +import org.apache.doris.qe.SessionVariable; +import org.apache.doris.thrift.TRuntimeFilterType; +import org.apache.doris.thrift.TTargetExprMonotonicity; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Range; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** Generates target-scoped partition and bucket pruning metadata with a Nereids runtime filter. */ +final class RuntimeFilterPruneClassifier { + private RuntimeFilterPruneClassifier() { + } + + static Classification classify(RuntimeFilter filter, SessionVariable sessionVariable) { + BucketClassification bucketClassification = sessionVariable.isEnableRuntimeFilterBucketPrune() + ? classifyBucketPruning(filter) + : BucketClassification.unsupported("runtime-filter bucket pruning is disabled"); + PartitionClassification partitionClassification = sessionVariable.isEnableRuntimeFilterPartitionPrune() + ? classifyPartitionPruning(filter) + : PartitionClassification.unsupported("runtime-filter partition pruning is disabled"); + return new Classification(bucketClassification, partitionClassification); + } + + private static BucketClassification classifyBucketPruning(RuntimeFilter filter) { + if (filter.getType() != TRuntimeFilterType.IN + && filter.getType() != TRuntimeFilterType.IN_OR_BLOOM) { + return BucketClassification.unsupported("runtime filter is not IN or IN_OR_BLOOM"); + } + if (!(filter.getTargetScan() instanceof PhysicalOlapScan)) { + return BucketClassification.unsupported("target scan is not a PhysicalOlapScan"); + } + if (!isFinalTargetDirectSlot(filter)) { + return BucketClassification.unsupported("target expression is not a direct slot"); + } + + Column targetColumn = targetColumn(filter.getTargetSlot()); + if (targetColumn == null) { + return BucketClassification.unsupported("target slot has no column"); + } + + PhysicalOlapScan scan = (PhysicalOlapScan) filter.getTargetScan(); + OlapTable table = scan.getTable(); + if (table == null || scan.getSelectedPartitionIds().isEmpty()) { + return BucketClassification.unsupported("target scan has no selected partitions"); + } + Column targetBaseColumn = directTargetBaseColumn(scan, targetColumn); + if (targetBaseColumn == null) { + return BucketClassification.unsupported( + "target has no direct base-column definition"); + } + + Column distributionColumn = null; + for (Long partitionId : scan.getSelectedPartitionIds()) { + Partition partition = table.getPartition(partitionId); + if (partition == null) { + return BucketClassification.unsupported("selected partition does not exist"); + } + DistributionInfo distributionInfo = partition.getDistributionInfo(); + if (!(distributionInfo instanceof HashDistributionInfo)) { + return BucketClassification.unsupported("distribution type is not HASH"); + } + HashDistributionInfo hashDistributionInfo = (HashDistributionInfo) distributionInfo; + if (hashDistributionInfo.getDistributionColumns().size() != 1) { + return BucketClassification.unsupported("HASH distribution is not single-column"); + } + Column currentDistributionColumn = hashDistributionInfo.getDistributionColumns().get(0); + if (!sameBucketColumn(targetColumn, currentDistributionColumn)) { + return BucketClassification.unsupported("target slot is not the HASH distribution column"); + } + if (distributionColumn != null + && !sameBucketColumn(distributionColumn, currentDistributionColumn)) { + return BucketClassification.unsupported( + "selected partitions use different distribution columns"); + } + distributionColumn = currentDistributionColumn; + } + return BucketClassification.supported(); + } + + private static PartitionClassification classifyPartitionPruning(RuntimeFilter filter) { + if (!(filter.getTargetScan() instanceof PhysicalOlapScan)) { + return PartitionClassification.unsupported("target scan is not a PhysicalOlapScan"); + } + if (requiresTargetTypeAdjustment(filter) + && !filter.getTargetSlot().equals(filter.getTargetExpression())) { + return PartitionClassification.unsupported( + "non-identity target expression requires a type adjustment"); + } + + PhysicalOlapScan scan = (PhysicalOlapScan) filter.getTargetScan(); + OlapTable table = scan.getTable(); + if (table == null) { + return PartitionClassification.unsupported("target scan has no OlapTable"); + } + + PartitionInfo partitionInfo = table.getPartitionInfo(); + PartitionType partitionType = partitionInfo.getType(); + if (partitionType != PartitionType.RANGE && partitionType != PartitionType.LIST) { + return PartitionClassification.unsupported("partition type is not RANGE or LIST"); + } + if (filter.getType() == TRuntimeFilterType.BLOOM && partitionType == PartitionType.RANGE) { + return PartitionClassification.unsupported( + "BLOOM runtime filter does not support RANGE partition pruning"); + } + if (hasUnsupportedAutomaticPartitionExpression(partitionInfo)) { + return PartitionClassification.unsupported( + "automatic partition expression boundary is not modeled"); + } + + Expression targetExpression = filter.getTargetExpression(); + if (targetExpression.containsType(NoneMovableFunction.class)) { + return PartitionClassification.unsupported( + "target expression contains non-movable function"); + } + Column targetColumn = targetColumn(filter.getTargetSlot()); + Column targetBaseColumn = directTargetBaseColumn(scan, targetColumn); + if (targetBaseColumn == null) { + return PartitionClassification.unsupported( + "target column has no direct base-column definition"); + } + if (!isPartitionColumn(targetColumn, targetBaseColumn, partitionInfo.getPartitionColumns())) { + return PartitionClassification.unsupported( + "target expression is not rooted on one partition column"); + } + if (!hasSerializedBoundary(targetColumn, targetBaseColumn, partitionInfo, partitionType)) { + return PartitionClassification.unsupported( + "target expression has no serialized partition boundary"); + } + + if (isFinalTargetDirectSlot(filter)) { + return supportedIncreasingPartitions(scan, partitionInfo, + "target slot has no prunable selected partitions"); + } + if (partitionType == PartitionType.LIST) { + if (targetExpression.containsNondeterministic()) { + return PartitionClassification.unsupported( + "target expression contains non-deterministic function"); + } + return supportedIncreasingPartitions(scan, partitionInfo, + "target expression has no prunable selected partitions"); + } + + Map partitionMonotonicity = + classifyLocalMonotonicity(targetExpression, scan, partitionInfo, targetColumn); + if (partitionMonotonicity.isEmpty()) { + return PartitionClassification.unsupported( + "target expression is not monotonic on selected partitions"); + } + return PartitionClassification.supported(partitionMonotonicity); + } + + private static PartitionClassification supportedIncreasingPartitions( + PhysicalOlapScan scan, PartitionInfo partitionInfo, String emptyReason) { + Map partitionMonotonicity = + allSelectedPartitionsIncreasing(scan, partitionInfo); + return partitionMonotonicity.isEmpty() + ? PartitionClassification.unsupported(emptyReason) + : PartitionClassification.supported(partitionMonotonicity); + } + + private static boolean isFinalTargetDirectSlot(RuntimeFilter filter) { + return filter.getTargetSlot().equals(filter.getTargetExpression()) + && !requiresTargetTypeAdjustment(filter); + } + + private static boolean requiresTargetTypeAdjustment(RuntimeFilter filter) { + return !filter.getSrcExpr().getDataType().toCatalogDataType().equals( + filter.getTargetExpression().getDataType().toCatalogDataType()); + } + + private static Column targetColumn(Slot slot) { + if (!(slot instanceof SlotReference)) { + return null; + } + return ((SlotReference) slot).getOriginalColumn().orElse(null); + } + + private static Column directTargetBaseColumn(PhysicalOlapScan scan, Column targetColumn) { + if (targetColumn == null) { + return null; + } + if (scan.getSelectedIndexId() != scan.getTable().getBaseIndexId() + && targetColumn.getDefineExpr() == null) { + return null; + } + return directBaseColumn(targetColumn); + } + + private static boolean sameBucketColumn(Column targetColumn, Column distributionColumn) { + Column targetBaseColumn = directBaseColumn(targetColumn); + Column distributionBaseColumn = directBaseColumn(distributionColumn); + if (targetBaseColumn == null || distributionBaseColumn == null) { + return false; + } + return targetBaseColumn.getName().equalsIgnoreCase(distributionBaseColumn.getName()) + && targetColumn.getType().equals(distributionColumn.getType()); + } + + private static Column directBaseColumn(Column column) { + Expr defineExpr = column.getDefineExpr(); + if (defineExpr == null) { + return column; + } + if (!(defineExpr instanceof SlotRef)) { + return null; + } + Column baseColumn = ((SlotRef) defineExpr).getColumn(); + if (baseColumn == null || baseColumn.isMaterializedViewColumn()) { + return null; + } + return baseColumn; + } + + private static boolean hasUnsupportedAutomaticPartitionExpression(PartitionInfo partitionInfo) { + if (!partitionInfo.enableAutomaticPartition()) { + return false; + } + for (Expr partitionExpr : partitionInfo.getPartitionExprs()) { + if (containsFunctionCall(partitionExpr)) { + return true; + } + } + return false; + } + + private static boolean containsFunctionCall(Expr expression) { + if (expression instanceof FunctionCallExpr) { + return true; + } + for (Expr child : expression.getChildren()) { + if (containsFunctionCall(child)) { + return true; + } + } + return false; + } + + private static boolean hasSerializedBoundary( + Column targetColumn, Column targetBaseColumn, + PartitionInfo partitionInfo, PartitionType partitionType) { + if (partitionType != PartitionType.RANGE) { + return true; + } + List partitionColumns = partitionInfo.getPartitionColumns(); + return !partitionColumns.isEmpty() + && samePartitionColumn(targetColumn, targetBaseColumn, partitionColumns.get(0)); + } + + private static boolean isPartitionColumn( + Column targetColumn, Column targetBaseColumn, List partitionColumns) { + for (Column partitionColumn : partitionColumns) { + if (samePartitionColumn(targetColumn, targetBaseColumn, partitionColumn)) { + return true; + } + } + return false; + } + + private static boolean samePartitionColumn( + Column targetColumn, Column targetBaseColumn, Column partitionColumn) { + return targetColumn.getName().equalsIgnoreCase(partitionColumn.getName()) + && targetColumn.getType().equals(partitionColumn.getType()) + && targetBaseColumn.getName().equalsIgnoreCase(partitionColumn.getName()) + && targetBaseColumn.getType().equals(partitionColumn.getType()); + } + + private static Map classifyLocalMonotonicity( + Expression targetExpression, PhysicalOlapScan scan, + PartitionInfo partitionInfo, Column partitionColumn) { + Map result = new HashMap<>(); + if (!(targetExpression instanceof Monotonic)) { + return result; + } + + Monotonic monotonic = (Monotonic) targetExpression; + int childIndex = monotonic.getMonotonicFunctionChildIndex(); + if (childIndex < 0 || childIndex >= targetExpression.arity() + || !(targetExpression.child(childIndex) instanceof Slot) + || !hasInputSlotOnlyInMonotonicChild(targetExpression, childIndex)) { + return result; + } + + for (Long partitionId : scan.getSelectedPartitionIds()) { + PartitionItem item = partitionInfo.getItem(partitionId); + if (!(item instanceof RangePartitionItem)) { + continue; + } + Range range = ((RangePartitionItem) item).getItems(); + Literal lower = null; + Literal upper = null; + if (range.hasLowerBound() && !range.lowerEndpoint().isMinValue()) { + lower = toNereidsLiteral(range.lowerEndpoint().getKeys().get(0), partitionColumn); + if (lower == null) { + continue; + } + } + if (range.hasUpperBound() && !range.upperEndpoint().isMaxValue()) { + upper = toNereidsLiteral(range.upperEndpoint().getKeys().get(0), partitionColumn); + if (upper == null) { + continue; + } + } + if (monotonic.isMonotonic(lower, upper)) { + result.put(partitionId, monotonic.isPositive() + ? TTargetExprMonotonicity.MONOTONIC_INCREASING + : TTargetExprMonotonicity.MONOTONIC_DECREASING); + } + } + return result; + } + + static boolean hasInputSlotOnlyInMonotonicChild(Expression expression, int monotonicChildIndex) { + for (int i = 0; i < expression.arity(); i++) { + if (i != monotonicChildIndex && !expression.child(i).getInputSlots().isEmpty()) { + return false; + } + } + return true; + } + + private static Map allSelectedPartitionsIncreasing( + PhysicalOlapScan scan, PartitionInfo partitionInfo) { + Map result = new HashMap<>(); + for (Long partitionId : scan.getSelectedPartitionIds()) { + PartitionItem item = partitionInfo.getItem(partitionId); + if (item == null || (item instanceof ListPartitionItem + && ((ListPartitionItem) item).isDefaultPartition())) { + continue; + } + result.put(partitionId, TTargetExprMonotonicity.MONOTONIC_INCREASING); + } + return result; + } + + private static Literal toNereidsLiteral(LiteralExpr literalExpr, Column column) { + try { + return Literal.fromLegacyLiteral(literalExpr, column.getType()); + } catch (AnalysisException e) { + return null; + } + } + + static final class Classification { + private final BucketClassification bucketClassification; + private final PartitionClassification partitionClassification; + + private Classification(BucketClassification bucketClassification, + PartitionClassification partitionClassification) { + this.bucketClassification = bucketClassification; + this.partitionClassification = partitionClassification; + } + + boolean canPruneBuckets() { + return bucketClassification.canPruneBuckets; + } + + boolean canPrunePartitions() { + return !partitionClassification.partitionMonotonicity.isEmpty(); + } + + Map getPartitionMonotonicity() { + return partitionClassification.partitionMonotonicity; + } + + String getBucketUnsupportedReason() { + return bucketClassification.unsupportedReason; + } + + String getPartitionUnsupportedReason() { + return partitionClassification.unsupportedReason; + } + } + + private static final class BucketClassification { + private final boolean canPruneBuckets; + private final String unsupportedReason; + + private BucketClassification(boolean canPruneBuckets, String unsupportedReason) { + this.canPruneBuckets = canPruneBuckets; + this.unsupportedReason = unsupportedReason; + } + + private static BucketClassification supported() { + return new BucketClassification(true, ""); + } + + private static BucketClassification unsupported(String reason) { + return new BucketClassification(false, reason); + } + } + + private static final class PartitionClassification { + private final Map partitionMonotonicity; + private final String unsupportedReason; + + private PartitionClassification( + Map partitionMonotonicity, String unsupportedReason) { + this.partitionMonotonicity = partitionMonotonicity; + this.unsupportedReason = unsupportedReason; + } + + private static PartitionClassification supported( + Map partitionMonotonicity) { + return new PartitionClassification(ImmutableMap.copyOf(partitionMonotonicity), ""); + } + + private static PartitionClassification unsupported(String reason) { + return new PartitionClassification(ImmutableMap.of(), reason); + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterPushDownVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterPushDownVisitor.java index b10b478aba64be..eb5441e2b474a7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterPushDownVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/RuntimeFilterPushDownVisitor.java @@ -197,6 +197,7 @@ public Boolean visitPhysicalRelation(PhysicalRelation scan, PushDownContext ctx) ctx.srcExpr, scanSlot, ctx.probeExpr, type, ctx.exprOrder, ctx.builderNode, ctx.buildSideNdv, !ctx.hasUnknownColStats, ctx.singleSideMinMax, scan); + ctx.rfContext.generateRuntimeFilterPruneMetadata(filter); scan.addAppliedRuntimeFilter(filter); ctx.rfContext.addJoinToTargetMap(ctx.builderNode, scanSlot.getExprId()); ctx.rfContext.setTargetExprIdToFilter(scanSlot.getExprId(), filter); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/RuntimeFilter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/RuntimeFilter.java index b0745ef2648310..9a83db27ba1113 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/RuntimeFilter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/RuntimeFilter.java @@ -23,11 +23,14 @@ import org.apache.doris.qe.ConnectContext; import org.apache.doris.thrift.TMinMaxRuntimeFilterType; import org.apache.doris.thrift.TRuntimeFilterType; +import org.apache.doris.thrift.TTargetExprMonotonicity; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import java.util.List; +import java.util.Map; /** * runtime filter @@ -53,6 +56,11 @@ public class RuntimeFilter { private boolean nonBlocking; + // Generated once with the runtime filter at its final target scan. Translation only + // maps this target-scoped metadata to the legacy scan node id. + private boolean canPruneBuckets; + private Map partitionMonotonicity = ImmutableMap.of(); + /** * constructor */ @@ -195,4 +203,22 @@ public void setNonBlocking(boolean nonBlocking) { public boolean isBloomFilterSizeCalculatedByNdv() { return bloomFilterSizeCalculatedByNdv; } + + public void setPruningMetadata(boolean canPruneBuckets, + Map partitionMonotonicity) { + this.canPruneBuckets = canPruneBuckets; + this.partitionMonotonicity = ImmutableMap.copyOf(partitionMonotonicity); + } + + public boolean canPruneBuckets() { + return canPruneBuckets; + } + + public boolean canPrunePartitions() { + return !partitionMonotonicity.isEmpty(); + } + + public Map getPartitionMonotonicity() { + return partitionMonotonicity; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java index 45ace61d3569c2..08f7c8efc92b0a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java @@ -65,6 +65,7 @@ import org.apache.doris.common.FeConstants; import org.apache.doris.common.Pair; import org.apache.doris.common.UserException; +import org.apache.doris.common.util.DebugPointUtil; import org.apache.doris.common.util.DebugUtil; import org.apache.doris.common.util.TimeUtils; import org.apache.doris.nereids.exceptions.ParseException; @@ -128,6 +129,9 @@ // Full scan of an Olap table. public class OlapScanNode extends ScanNode { private static final Logger LOG = LogManager.getLogger(OlapScanNode.class); + @VisibleForTesting + static final String MISSING_RF_BUCKET_METADATA_DEBUG_POINT = + "OlapScanNode.setRuntimeFilterBucketPruneParameters.missingBucketMetadata"; // average compression ratio in doris storage engine private static final int COMPRESSION_RATIO = 5; @@ -210,7 +214,9 @@ public class OlapScanNode extends ScanNode { private Set nereidsPrunedTabletIds = Sets.newHashSet(); private TableSample tableSample; - private Map tabletId2BucketSeq = Maps.newHashMap(); + // Pack bucket number and sequence into one value to avoid retaining two all-tablet maps. + private Map tabletId2BucketInfo = Maps.newHashMap(); + private boolean runtimeFilterBucketPruneParametersSet = false; // a bucket seq may map to many tablets, and each tablet has a // TScanRangeLocations. public ArrayListMultimap bucketSeq2locations = ArrayListMultimap.create(); @@ -797,7 +803,9 @@ private void addScanRangeLocations(Partition partition, private void addBucketSeqStatsIfNeeded(long tabletId, TScanRangeLocations locations, long oneReplicaBytes) { if (!isPointQuery()) { - Integer bucketSeq = tabletId2BucketSeq.get(tabletId); + long bucketInfo = Preconditions.checkNotNull(tabletId2BucketInfo.get(tabletId), + "missing bucket metadata for tablet %s", tabletId); + int bucketSeq = decodeBucketSeq(bucketInfo); bucketSeq2locations.put(bucketSeq, locations); bucketSeq2Bytes.merge(bucketSeq, oneReplicaBytes, Long::sum); } @@ -1063,8 +1071,9 @@ private void computeTabletInfo() throws UserException { } if (!isPointQuery()) { + int bucketNum = partition.getDistributionInfo().getBucketNum(); for (int i = 0; i < allTabletIds.size(); i++) { - tabletId2BucketSeq.put(allTabletIds.get(i), i); + tabletId2BucketInfo.put(allTabletIds.get(i), encodeBucketInfo(i, bucketNum)); } } @@ -1094,7 +1103,7 @@ public List lazyEvaluateRangeLocations() throws UserExcepti selectionHint = null; scanBackendOrderBySelection = false; scanTabletIds.clear(); - tabletId2BucketSeq.clear(); + tabletId2BucketInfo.clear(); bucketSeq2locations.clear(); bucketSeq2Bytes.clear(); scanReplicaIds.clear(); @@ -1475,14 +1484,12 @@ protected void toThrift(TPlanNode msg) { // filter whose target expression can drive partition pruning according // to the FE-side classifier, so we don't bloat thrift for tables with // many partitions but no usable RF target. - // Gated by session variable `enable_runtime_filter_partition_prune`. - ConnectContext rfPruneCtx = ConnectContext.get(); - if (rfPruneCtx != null - && rfPruneCtx.getSessionVariable().isEnableRuntimeFilterPartitionPrune() - && hasRfDrivingPartitionPruning()) { + if (hasRfDrivingPartitionPruning()) { setPartitionBoundariesForRuntimeFilter(msg.olap_scan_node); } - + if (hasRfDrivingBucketPruning()) { + setRuntimeFilterBucketPruneParameters(); + } super.toThrift(msg); } @@ -1532,6 +1539,58 @@ void setPartitionBoundariesForRuntimeFilter(TOlapScanNode olapScanNode) { } } + private boolean hasRfDrivingBucketPruning() { + PlanNodeId myId = this.getId(); + for (RuntimeFilter rf : runtimeFilters) { + if (rf.canPruneBucketsFor(myId)) { + return true; + } + } + return false; + } + + @VisibleForTesting + synchronized void setRuntimeFilterBucketPruneParameters() { + if (runtimeFilterBucketPruneParametersSet) { + return; + } + long debugMissingTabletId = DebugPointUtil.getDebugParamOrDefault( + MISSING_RF_BUCKET_METADATA_DEBUG_POINT, -1L); + for (TScanRangeLocations locations : scanRangeLocations) { + TPaloScanRange scanRange = locations.getScanRange().getPaloScanRange(); + Long bucketInfo = tabletId2BucketInfo.get(scanRange.getTabletId()); + if (scanRange.getTabletId() == debugMissingTabletId) { + bucketInfo = null; + } + if (bucketInfo == null || decodeBucketNum(bucketInfo) <= 0) { + LOG.warn("missing bucket metadata for runtime-filter bucket pruning, " + + "scanNode={}, tablet={}; disable pruning for this scan node", + getId(), scanRange.getTabletId()); + runtimeFilterBucketPruneParametersSet = true; + return; + } + } + for (TScanRangeLocations locations : scanRangeLocations) { + TPaloScanRange scanRange = locations.getScanRange().getPaloScanRange(); + long bucketInfo = tabletId2BucketInfo.get(scanRange.getTabletId()); + scanRange.setBucketSeq(decodeBucketSeq(bucketInfo)); + scanRange.setBucketNum(decodeBucketNum(bucketInfo)); + } + runtimeFilterBucketPruneParametersSet = true; + } + + private static long encodeBucketInfo(int bucketSeq, int bucketNum) { + return ((long) bucketNum << Integer.SIZE) | Integer.toUnsignedLong(bucketSeq); + } + + private static int decodeBucketSeq(long bucketInfo) { + return (int) bucketInfo; + } + + private static int decodeBucketNum(long bucketInfo) { + return (int) (bucketInfo >>> Integer.SIZE); + } + private List buildPartitionBoundariesForRuntimeFilter() { PartitionInfo partitionInfo = olapTable.getPartitionInfo(); PartitionType partType = partitionInfo.getType(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/RuntimeFilter.java b/fe/fe-core/src/main/java/org/apache/doris/planner/RuntimeFilter.java index 77bb4ffcdc1ae3..d98224d141acbf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/RuntimeFilter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/RuntimeFilter.java @@ -140,12 +140,12 @@ public FilterSizeLimits(SessionVariable sessionVariable) { private int waitTimeMs = -1; // Per-target monotonicity for BE-side runtime-filter partition pruning, - // keyed by the target scan node's plan id. Filled by the Nereids - // RuntimeFilterPartitionPruneClassifier at translation time from the final - // legacy target expression that will be sent to BE. + // keyed by the target scan node's plan id. Generated with the Nereids + // runtime filter and mapped to the legacy target scan id during translation. private final Map> targetPartitionMonotonicityByScanId = new HashMap<>(); private final Set partitionPruningTargetScanIds = new HashSet<>(); + private final Set bucketPruningTargetScanIds = new HashSet<>(); /** * Internal representation of a runtime filter target. @@ -358,37 +358,37 @@ public TRuntimeFilterDesc toThrift() { } // Per-target, per-partition monotonicity for BE-side partition pruning. Populated - // upstream by RuntimeFilterPartitionPruneClassifier; direct partition + // upstream by RuntimeFilterPruneClassifier; direct partition // column targets are monotonic increasing so BE can use one unified // partition-pruning path. - // Gated by session variable `enable_runtime_filter_partition_prune`. - ConnectContext rfPruneCtx = ConnectContext.get(); - boolean enableRfPartitionPrune = rfPruneCtx != null - && rfPruneCtx.getSessionVariable().isEnableRuntimeFilterPartitionPrune(); - if (enableRfPartitionPrune) { - if (!targetPartitionMonotonicityByScanId.isEmpty()) { - Map> partitionMonoMap = new HashMap<>(); - for (Map.Entry> e - : targetPartitionMonotonicityByScanId.entrySet()) { - List partitionMonoList = new ArrayList<>(); - for (Map.Entry partitionEntry : e.getValue().entrySet()) { - Preconditions.checkArgument( - partitionEntry.getValue() != TTargetExprMonotonicity.NON_MONOTONIC, - "partition pruning monotonicity must not be NON_MONOTONIC"); - TPartitionTargetExprMonotonicity partitionMono = - new TPartitionTargetExprMonotonicity(); - partitionMono.setPartitionId(partitionEntry.getKey()); - partitionMono.setMonotonicity(partitionEntry.getValue()); - partitionMonoList.add(partitionMono); - } - if (!partitionMonoList.isEmpty()) { - partitionMonoMap.put(e.getKey().asInt(), partitionMonoList); - } + if (!targetPartitionMonotonicityByScanId.isEmpty()) { + Map> partitionMonoMap = new HashMap<>(); + for (Map.Entry> e + : targetPartitionMonotonicityByScanId.entrySet()) { + List partitionMonoList = new ArrayList<>(); + for (Map.Entry partitionEntry : e.getValue().entrySet()) { + Preconditions.checkArgument( + partitionEntry.getValue() != TTargetExprMonotonicity.NON_MONOTONIC, + "partition pruning monotonicity must not be NON_MONOTONIC"); + TPartitionTargetExprMonotonicity partitionMono = + new TPartitionTargetExprMonotonicity(); + partitionMono.setPartitionId(partitionEntry.getKey()); + partitionMono.setMonotonicity(partitionEntry.getValue()); + partitionMonoList.add(partitionMono); } - if (!partitionMonoMap.isEmpty()) { - tFilter.setPlanIdToPartitionTargetMonotonicity(partitionMonoMap); + if (!partitionMonoList.isEmpty()) { + partitionMonoMap.put(e.getKey().asInt(), partitionMonoList); } } + if (!partitionMonoMap.isEmpty()) { + tFilter.setPlanIdToPartitionTargetMonotonicity(partitionMonoMap); + } + } + + if (!bucketPruningTargetScanIds.isEmpty()) { + tFilter.setBucketPruningTargetIds(bucketPruningTargetScanIds.stream() + .map(PlanNodeId::asInt) + .collect(Collectors.toSet())); } return tFilter; @@ -416,12 +416,20 @@ public void setTargetPartitionMonotonicity(PlanNodeId scanNodeId, * scan node. Used by OlapScanNode.toThrift to decide whether it is worth * serializing partition_boundaries to BE. The single source of truth for * that decision lives in - * RuntimeFilterPartitionPruneClassifier. + * RuntimeFilterPruneClassifier. */ public boolean canPrunePartitionsFor(PlanNodeId scanNodeId) { return partitionPruningTargetScanIds.contains(scanNodeId); } + public void markTargetCanPruneBuckets(PlanNodeId scanNodeId) { + bucketPruningTargetScanIds.add(scanNodeId); + } + + public boolean canPruneBucketsFor(PlanNodeId scanNodeId) { + return bucketPruningTargetScanIds.contains(scanNodeId); + } + public boolean hasTargets() { return !targets.isEmpty(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index 09990761712672..c5b503e9f144b0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -470,6 +470,9 @@ public String toString() { public static final String ENABLE_RUNTIME_FILTER_PARTITION_PRUNE = "enable_runtime_filter_partition_prune"; + public static final String ENABLE_RUNTIME_FILTER_BUCKET_PRUNE = + "enable_runtime_filter_bucket_prune"; + public static final String ENABLE_PRUNE_NESTED_COLUMN = "enable_prune_nested_column"; static final String SESSION_CONTEXT = "session_context"; @@ -2176,6 +2179,9 @@ public boolean isEnableHboNonStrictMatchingMode() { fuzzy = true) public boolean enableRuntimeFilterPartitionPrune = true; + @VarAttrDef.VarAttr(name = ENABLE_RUNTIME_FILTER_BUCKET_PRUNE, needForward = true, fuzzy = true) + public boolean enableRuntimeFilterBucketPrune = true; + /** * The client can pass some special information by setting this session variable in the format: "k1:v1;k2:v2". * For example, trace_id can be passed to trace the query request sent by the user. @@ -3710,6 +3716,7 @@ public void initFuzzyModeVariables() { this.enableParallelScan = random.nextInt(2) == 0; this.enableRuntimeFilterPrune = (randomInt % 10) == 0; this.enableRuntimeFilterPartitionPrune = (randomInt % 2) == 0; + this.enableRuntimeFilterBucketPrune = (randomInt % 2) == 0; this.runtimeFilterTreePublishMaxSendBytes = Util.getRandomLong(0, 64L * 1024L * 1024L, 128L * 1024L * 1024L, 256L * 1024L * 1024L); @@ -5291,6 +5298,14 @@ public void setEnableRuntimeFilterPartitionPrune(boolean enableRuntimeFilterPart this.enableRuntimeFilterPartitionPrune = enableRuntimeFilterPartitionPrune; } + public boolean isEnableRuntimeFilterBucketPrune() { + return enableRuntimeFilterBucketPrune; + } + + public void setEnableRuntimeFilterBucketPrune(boolean enableRuntimeFilterBucketPrune) { + this.enableRuntimeFilterBucketPrune = enableRuntimeFilterBucketPrune; + } + public void setFragmentTransmissionCompressionCodec(String codec) { this.fragmentTransmissionCompressionCodec = codec; } @@ -5669,6 +5684,7 @@ public TQueryOptions toThrift() { tResult.setIgnoreRuntimeFilterError(ignoreRuntimeFilterError); tResult.setProfileLevel(getProfileLevel()); tResult.setEnableRuntimeFilterPartitionPrune(enableRuntimeFilterPartitionPrune); + tResult.setEnableRuntimeFilterBucketPrune(enableRuntimeFilterBucketPrune); tResult.setMinimumOperatorMemoryRequiredKb(minimumOperatorMemoryRequiredKB); tResult.setExchangeMultiBlocksByteSize(exchangeMultiBlocksByteSize); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslatorTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslatorTest.java index eb885887ded140..68d2e3457f5254 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslatorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslatorTest.java @@ -404,7 +404,7 @@ public void testRfPartitionPruneSnapshotSurvivesEnablementChange() throws Except boolean oldDisableJoinReorder = connectContext.getSessionVariable().isDisableJoinReorder(); try { connectContext.getSessionVariable().setRuntimeFilterType(TRuntimeFilterType.MIN_MAX.getValue()); - connectContext.getSessionVariable().setEnableRuntimeFilterPartitionPrune(false); + connectContext.getSessionVariable().setEnableRuntimeFilterPartitionPrune(true); connectContext.getSessionVariable().setEnableRuntimeFilterPrune(false); connectContext.getSessionVariable().setDisableJoinReorder(true); @@ -423,7 +423,7 @@ public void testRfPartitionPruneSnapshotSurvivesEnablementChange() throws Except .orElseThrow(); Assertions.assertEquals(2, partitionedScan.getSelectedPartitionIds().size()); - connectContext.getSessionVariable().setEnableRuntimeFilterPartitionPrune(true); + connectContext.getSessionVariable().setEnableRuntimeFilterPartitionPrune(false); TPlanNode thriftScanNode = partitionedScan.treeToThrift().getNodes().get(0); Assertions.assertTrue(thriftScanNode.olap_scan_node.isSetPartitionBoundaries()); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/RuntimeFilterPartitionPruneClassifierTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/RuntimeFilterPartitionPruneClassifierTest.java deleted file mode 100644 index 58e32440364c49..00000000000000 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/RuntimeFilterPartitionPruneClassifierTest.java +++ /dev/null @@ -1,163 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.glue.translator; - -import org.apache.doris.analysis.BinaryPredicate; -import org.apache.doris.analysis.Expr; -import org.apache.doris.analysis.FunctionCallExpr; -import org.apache.doris.analysis.IntLiteral; -import org.apache.doris.analysis.SlotDescriptor; -import org.apache.doris.analysis.SlotId; -import org.apache.doris.analysis.SlotRef; -import org.apache.doris.analysis.StringLiteral; -import org.apache.doris.analysis.TupleId; -import org.apache.doris.catalog.Column; -import org.apache.doris.catalog.ListPartitionItem; -import org.apache.doris.catalog.OlapTable; -import org.apache.doris.catalog.PartitionInfo; -import org.apache.doris.catalog.PartitionItem; -import org.apache.doris.catalog.PartitionType; -import org.apache.doris.catalog.PrimitiveType; -import org.apache.doris.catalog.RangePartitionItem; -import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.expressions.GreaterThan; -import org.apache.doris.nereids.trees.expressions.SlotReference; -import org.apache.doris.nereids.trees.expressions.functions.Monotonic; -import org.apache.doris.nereids.trees.expressions.functions.scalar.AssertTrue; -import org.apache.doris.nereids.trees.expressions.functions.scalar.DateTrunc; -import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral; -import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral; -import org.apache.doris.nereids.types.DateTimeV2Type; -import org.apache.doris.nereids.types.IntegerType; -import org.apache.doris.planner.OlapScanNode; -import org.apache.doris.thrift.TRuntimeFilterType; -import org.apache.doris.thrift.TTargetExprMonotonicity; - -import com.google.common.collect.ImmutableList; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; -import org.mockito.Mockito; - -import java.util.Map; -import java.util.function.Function; - -class RuntimeFilterPartitionPruneClassifierTest { - @Test - void testRejectInputSlotOutsideMonotonicChild() { - SlotReference slot = new SlotReference("dt", DateTimeV2Type.SYSTEM_DEFAULT); - DateTrunc dateTrunc = new DateTrunc(slot, slot); - Monotonic monotonic = dateTrunc; - - Assertions.assertEquals(1, dateTrunc.getInputSlots().size()); - Assertions.assertFalse(RuntimeFilterPartitionPruneClassifier.hasInputSlotOnlyInMonotonicChild( - dateTrunc, monotonic.getMonotonicFunctionChildIndex())); - } - - @Test - void testAcceptInputSlotOnlyInMonotonicChild() { - SlotReference slot = new SlotReference("dt", DateTimeV2Type.SYSTEM_DEFAULT); - DateTrunc dateTrunc = new DateTrunc(slot, new VarcharLiteral("day")); - Monotonic monotonic = dateTrunc; - - Assertions.assertTrue(RuntimeFilterPartitionPruneClassifier.hasInputSlotOnlyInMonotonicChild( - dateTrunc, monotonic.getMonotonicFunctionChildIndex())); - } - - @Test - void testBloomRangePartitionUnsupported() { - RuntimeFilterPartitionPruneClassifier.Classification classification = classify( - TRuntimeFilterType.BLOOM, PartitionType.RANGE, RangePartitionItem.DUMMY_ITEM); - - Assertions.assertFalse(classification.canPrunePartitions()); - Assertions.assertTrue(classification.getUnsupportedReason().contains("BLOOM")); - Assertions.assertTrue(classification.getPartitionMonotonicity().isEmpty()); - } - - @Test - void testBloomListPartitionSupported() { - RuntimeFilterPartitionPruneClassifier.Classification classification = classify( - TRuntimeFilterType.BLOOM, PartitionType.LIST, ListPartitionItem.DUMMY_ITEM); - - assertSupportedIncreasingPartitions(classification); - } - - @Test - void testInOrBloomRangePartitionStillSupported() { - RuntimeFilterPartitionPruneClassifier.Classification classification = classify( - TRuntimeFilterType.IN_OR_BLOOM, PartitionType.RANGE, RangePartitionItem.DUMMY_ITEM); - - assertSupportedIncreasingPartitions(classification); - } - - @Test - void testRejectNoneMovableListTargetExpression() { - RuntimeFilterPartitionPruneClassifier.Classification classification = classify( - TRuntimeFilterType.IN, PartitionType.LIST, ListPartitionItem.DUMMY_ITEM, - targetSlot -> new FunctionCallExpr("assert_true", ImmutableList.of( - new BinaryPredicate(BinaryPredicate.Operator.NE, targetSlot, new IntLiteral(0)), - new StringLiteral("rfpp_expr_in_only_error")), false), - targetSlot -> new AssertTrue( - new GreaterThan(targetSlot, new IntegerLiteral(0)), - new VarcharLiteral("rfpp_expr_in_only_error"))); - - Assertions.assertFalse(classification.canPrunePartitions()); - Assertions.assertTrue(classification.getUnsupportedReason().contains("non-movable")); - Assertions.assertTrue(classification.getPartitionMonotonicity().isEmpty()); - } - - private RuntimeFilterPartitionPruneClassifier.Classification classify( - TRuntimeFilterType filterType, PartitionType partitionType, PartitionItem partitionItem) { - return classify(filterType, partitionType, partitionItem, targetSlot -> targetSlot, - targetSlot -> targetSlot); - } - - private RuntimeFilterPartitionPruneClassifier.Classification classify( - TRuntimeFilterType filterType, PartitionType partitionType, PartitionItem partitionItem, - Function legacyTargetFactory, - Function nereidsTargetFactory) { - Column partitionColumn = new Column("part_col", PrimitiveType.INT); - SlotDescriptor slotDescriptor = new SlotDescriptor(new SlotId(1), new TupleId(1)); - slotDescriptor.setColumn(partitionColumn); - slotDescriptor.setType(partitionColumn.getType()); - SlotRef targetSlot = new SlotRef(slotDescriptor); - SlotReference nereidsTarget = new SlotReference("part_col", IntegerType.INSTANCE); - - OlapTable table = Mockito.mock(OlapTable.class); - PartitionInfo partitionInfo = Mockito.mock(PartitionInfo.class); - OlapScanNode scanNode = Mockito.mock(OlapScanNode.class); - Mockito.when(scanNode.getOlapTable()).thenReturn(table); - Mockito.when(scanNode.getSelectedPartitionIds()).thenReturn(ImmutableList.of(1L, 2L)); - Mockito.when(table.getPartitionInfo()).thenReturn(partitionInfo); - Mockito.when(partitionInfo.getType()).thenReturn(partitionType); - Mockito.when(partitionInfo.getPartitionColumns()).thenReturn(ImmutableList.of(partitionColumn)); - Mockito.when(partitionInfo.getItem(1L)).thenReturn(partitionItem); - Mockito.when(partitionInfo.getItem(2L)).thenReturn(partitionItem); - - return RuntimeFilterPartitionPruneClassifier.classify(filterType, - legacyTargetFactory.apply(targetSlot), nereidsTargetFactory.apply(nereidsTarget), scanNode); - } - - private void assertSupportedIncreasingPartitions( - RuntimeFilterPartitionPruneClassifier.Classification classification) { - Assertions.assertTrue(classification.canPrunePartitions()); - Map monotonicity = classification.getPartitionMonotonicity(); - Assertions.assertEquals(2, monotonicity.size()); - Assertions.assertEquals(TTargetExprMonotonicity.MONOTONIC_INCREASING, monotonicity.get(1L)); - Assertions.assertEquals(TTargetExprMonotonicity.MONOTONIC_INCREASING, monotonicity.get(2L)); - } -} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/RuntimeFilterTranslatorBucketPruneTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/RuntimeFilterTranslatorBucketPruneTest.java new file mode 100644 index 00000000000000..43cd6d7bae709d --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/RuntimeFilterTranslatorBucketPruneTest.java @@ -0,0 +1,239 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.glue.translator; + +import org.apache.doris.analysis.SlotRef; +import org.apache.doris.analysis.TupleDescriptor; +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.HashDistributionInfo; +import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.Partition; +import org.apache.doris.catalog.PartitionInfo; +import org.apache.doris.catalog.PartitionType; +import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.common.IdGenerator; +import org.apache.doris.nereids.processor.post.RuntimeFilterContext; +import org.apache.doris.nereids.trees.expressions.Add; +import org.apache.doris.nereids.trees.expressions.ExprId; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral; +import org.apache.doris.nereids.trees.plans.physical.AbstractPhysicalPlan; +import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapScan; +import org.apache.doris.nereids.trees.plans.physical.RuntimeFilter; +import org.apache.doris.nereids.types.DataType; +import org.apache.doris.nereids.types.IntegerType; +import org.apache.doris.planner.OlapScanNode; +import org.apache.doris.planner.PlanFragment; +import org.apache.doris.planner.PlanFragmentId; +import org.apache.doris.planner.PlanNode; +import org.apache.doris.planner.PlanNodeId; +import org.apache.doris.planner.RuntimeFilterId; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.SessionVariable; +import org.apache.doris.thrift.TExprNodeType; +import org.apache.doris.thrift.TMinMaxRuntimeFilterType; +import org.apache.doris.thrift.TRuntimeFilterDesc; +import org.apache.doris.thrift.TRuntimeFilterType; + +import com.google.common.collect.ImmutableList; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import java.util.List; + +class RuntimeFilterTranslatorBucketPruneTest { + private static final int SCAN_NODE_ID = 7; + + private ConnectContext previousContext; + private SessionVariable sessionVariable; + + @BeforeEach + void setUp() { + previousContext = ConnectContext.get(); + sessionVariable = new SessionVariable(); + sessionVariable.setEnableRuntimeFilterPartitionPrune(false); + sessionVariable.setEnableRuntimeFilterBucketPrune(true); + ConnectContext connectContext = new ConnectContext(); + connectContext.setSessionVariable(sessionVariable); + connectContext.setThreadLocalInfo(); + } + + @AfterEach + void tearDown() { + ConnectContext.remove(); + if (previousContext != null) { + previousContext.setThreadLocalInfo(); + } + } + + @Test + void testGroupedSameTargetSerializesOneExpressionAndBucketTarget() { + TranslatorHarness harness = new TranslatorHarness(); + SlotReference target = harness.addTargetSlot("dist_col", harness.distributionColumn, + IntegerType.INSTANCE); + + TRuntimeFilterDesc desc = harness.translate(ImmutableList.of( + harness.newFilter(target, target), harness.newFilter(target, target))); + + Assertions.assertEquals(1, desc.planId_to_target_expr.size()); + Assertions.assertEquals(firstLegacySlotId(harness, target), + desc.planId_to_target_expr.get(SCAN_NODE_ID).nodes.get(0).slot_ref.slot_id); + Assertions.assertTrue(desc.isSetBucketPruningTargetIds()); + Assertions.assertEquals(ImmutableList.of(SCAN_NODE_ID), + desc.bucket_pruning_target_ids.stream().sorted().collect(java.util.stream.Collectors.toList())); + } + + @Test + void testGroupedDifferentTargetsKeepThriftMapButSuppressBucketTarget() { + TranslatorHarness harness = new TranslatorHarness(); + SlotReference distributionTarget = harness.addTargetSlot("dist_col", harness.distributionColumn, + IntegerType.INSTANCE); + Column valueColumn = new Column("value_col", PrimitiveType.INT); + SlotReference valueTarget = harness.addTargetSlot("value_col", valueColumn, IntegerType.INSTANCE); + + TRuntimeFilterDesc desc = harness.translate(ImmutableList.of( + harness.newFilter(distributionTarget, distributionTarget), + harness.newFilter(valueTarget, valueTarget))); + + Assertions.assertEquals(1, desc.planId_to_target_expr.size()); + Assertions.assertEquals(firstLegacySlotId(harness, valueTarget), + desc.planId_to_target_expr.get(SCAN_NODE_ID).nodes.get(0).slot_ref.slot_id); + Assertions.assertFalse(desc.isSetBucketPruningTargetIds()); + } + + @Test + void testCastAfterNonIdentityTargetSuppressesBucketTarget() { + TranslatorHarness harness = new TranslatorHarness(PrimitiveType.BIGINT); + SlotReference target = harness.addTargetSlot("dist_col", harness.distributionColumn, + IntegerType.INSTANCE); + + TRuntimeFilterDesc desc = harness.translate(ImmutableList.of( + harness.newFilter(target, new Add(target, new IntegerLiteral(1))))); + + Assertions.assertEquals(TExprNodeType.CAST_EXPR, + desc.planId_to_target_expr.get(SCAN_NODE_ID).nodes.get(0).node_type); + Assertions.assertFalse(desc.isSetBucketPruningTargetIds()); + } + + @Test + void testDisabledFeatureSkipsClassificationAndSerialization() { + sessionVariable.setEnableRuntimeFilterBucketPrune(false); + TranslatorHarness harness = new TranslatorHarness(); + SlotReference target = harness.addTargetSlot("dist_col", harness.distributionColumn, + IntegerType.INSTANCE); + Mockito.clearInvocations(harness.targetRelation); + + TRuntimeFilterDesc desc = harness.translate(ImmutableList.of(harness.newFilter(target, target))); + + Mockito.verify(harness.targetRelation, Mockito.never()).getTable(); + Mockito.verify(harness.targetRelation, Mockito.never()).getSelectedPartitionIds(); + Assertions.assertEquals(1, desc.planId_to_target_expr.size()); + Assertions.assertFalse(desc.isSetBucketPruningTargetIds()); + } + + private static int firstLegacySlotId(TranslatorHarness harness, SlotReference target) { + return harness.translatorContext.findSlotRef(target.getExprId()).getSlotId().asInt(); + } + + private class TranslatorHarness { + private final Column distributionColumn = new Column("dist_col", PrimitiveType.INT); + private final OlapTable table = Mockito.mock(OlapTable.class); + private final Partition partition = Mockito.mock(Partition.class); + private final OlapScanNode scanNode = Mockito.mock(OlapScanNode.class); + private final PlanNode builderNode = Mockito.mock(PlanNode.class); + private final AbstractPhysicalPlan nereidsBuilder = Mockito.mock(AbstractPhysicalPlan.class); + private final PhysicalOlapScan targetRelation = Mockito.mock(PhysicalOlapScan.class); + private final PlanTranslatorContext translatorContext = new PlanTranslatorContext(); + private final RuntimeFilterContext runtimeFilterContext; + private final RuntimeFilterTranslator translator; + private final IdGenerator filterIdGenerator = RuntimeFilterId.createGenerator(); + private final SlotReference source; + private final TupleDescriptor targetTuple = translatorContext.generateTupleDesc(); + private int nextExprId = 1; + + TranslatorHarness() { + this(PrimitiveType.INT); + } + + TranslatorHarness(PrimitiveType sourceType) { + runtimeFilterContext = new RuntimeFilterContext(sessionVariable, filterIdGenerator); + translator = new RuntimeFilterTranslator(runtimeFilterContext); + + PartitionInfo partitionInfo = Mockito.mock(PartitionInfo.class); + Mockito.when(partitionInfo.getType()).thenReturn(PartitionType.UNPARTITIONED); + Mockito.when(table.getPartitionInfo()).thenReturn(partitionInfo); + Mockito.when(table.getPartition(1L)).thenReturn(partition); + Mockito.when(partition.getDistributionInfo()).thenReturn( + new HashDistributionInfo(8, ImmutableList.of(distributionColumn))); + Mockito.when(scanNode.getOlapTable()).thenReturn(table); + Mockito.when(scanNode.getSelectedPartitionIds()).thenReturn(ImmutableList.of(1L)); + Mockito.when(scanNode.getId()).thenReturn(new PlanNodeId(SCAN_NODE_ID)); + Mockito.when(targetRelation.getTable()).thenReturn(table); + Mockito.when(targetRelation.getSelectedPartitionIds()).thenReturn(ImmutableList.of(1L)); + + PlanFragment fragment = Mockito.mock(PlanFragment.class); + PlanFragmentId fragmentId = new PlanFragmentId(3); + Mockito.when(fragment.getFragmentId()).thenReturn(fragmentId); + Mockito.doCallRealMethod().when(builderNode).setFragment(fragment); + Mockito.doCallRealMethod().when(scanNode).setFragment(fragment); + builderNode.setFragment(fragment); + scanNode.setFragment(fragment); + Mockito.when(builderNode.getFragment()).thenReturn(fragment); + Mockito.when(scanNode.getFragment()).thenReturn(fragment); + Mockito.when(builderNode.getFragmentId()).thenReturn(fragmentId); + Mockito.when(scanNode.getFragmentId()).thenReturn(fragmentId); + + Column sourceColumn = new Column("src", sourceType); + source = addSlot("src", sourceColumn, DataType.fromCatalogType(sourceColumn.getType()), + translatorContext.generateTupleDesc()); + } + + SlotReference addTargetSlot(String name, Column column, DataType dataType) { + SlotReference target = addSlot(name, column, dataType, targetTuple); + SlotRef targetSlotRef = translatorContext.findSlotRef(target.getExprId()); + runtimeFilterContext.getExprIdToOlapScanNodeSlotRef().put(target.getExprId(), targetSlotRef); + runtimeFilterContext.getScanNodeOfLegacyRuntimeFilterTarget().put(target, scanNode); + return target; + } + + RuntimeFilter newFilter(SlotReference target, Expression targetExpression) { + RuntimeFilter filter = new RuntimeFilter(filterIdGenerator.getNextId(), source, target, targetExpression, + TRuntimeFilterType.IN, 0, nereidsBuilder, 10, false, + TMinMaxRuntimeFilterType.MIN_MAX, targetRelation); + runtimeFilterContext.generateRuntimeFilterPruneMetadata(filter); + return filter; + } + + TRuntimeFilterDesc translate(List filters) { + translator.createLegacyRuntimeFilters(filters, builderNode, translatorContext); + Assertions.assertEquals(1, runtimeFilterContext.getLegacyFilters().size()); + return runtimeFilterContext.getLegacyFilters().get(0).toThrift(); + } + + private SlotReference addSlot(String name, Column column, DataType dataType, TupleDescriptor tuple) { + SlotReference slot = new SlotReference(new ExprId(nextExprId++), name, dataType, + false, ImmutableList.of("t"), table, column, table, column); + translatorContext.createSlotDesc(tuple, slot); + return slot; + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/processor/post/RuntimeFilterPruneClassifierTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/processor/post/RuntimeFilterPruneClassifierTest.java new file mode 100644 index 00000000000000..02157543a6ccb7 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/processor/post/RuntimeFilterPruneClassifierTest.java @@ -0,0 +1,489 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.processor.post; + +import org.apache.doris.analysis.FunctionCallExpr; +import org.apache.doris.analysis.SlotDescriptor; +import org.apache.doris.analysis.SlotId; +import org.apache.doris.analysis.SlotRef; +import org.apache.doris.analysis.TupleId; +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.DistributionInfo; +import org.apache.doris.catalog.HashDistributionInfo; +import org.apache.doris.catalog.ListPartitionItem; +import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.Partition; +import org.apache.doris.catalog.PartitionInfo; +import org.apache.doris.catalog.PartitionItem; +import org.apache.doris.catalog.PartitionType; +import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.catalog.RandomDistributionInfo; +import org.apache.doris.catalog.RangePartitionItem; +import org.apache.doris.nereids.trees.expressions.Add; +import org.apache.doris.nereids.trees.expressions.ExprId; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.GreaterThan; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.functions.Monotonic; +import org.apache.doris.nereids.trees.expressions.functions.scalar.AssertTrue; +import org.apache.doris.nereids.trees.expressions.functions.scalar.DateTrunc; +import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral; +import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral; +import org.apache.doris.nereids.trees.plans.physical.AbstractPhysicalPlan; +import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapScan; +import org.apache.doris.nereids.trees.plans.physical.RuntimeFilter; +import org.apache.doris.nereids.types.DataType; +import org.apache.doris.nereids.types.DateTimeV2Type; +import org.apache.doris.planner.RuntimeFilterId; +import org.apache.doris.qe.SessionVariable; +import org.apache.doris.thrift.TMinMaxRuntimeFilterType; +import org.apache.doris.thrift.TRuntimeFilterType; +import org.apache.doris.thrift.TTargetExprMonotonicity; + +import com.google.common.collect.ImmutableList; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import java.util.Map; +import java.util.function.Function; + +class RuntimeFilterPruneClassifierTest { + @Test + void testSingleColumnHashInSupported() { + Column distributionColumn = new Column("dist_col", PrimitiveType.INT); + RuntimeFilterPruneClassifier.Classification classification = classifyBucket( + TRuntimeFilterType.IN, distributionColumn, + new HashDistributionInfo(8, ImmutableList.of(distributionColumn))); + + Assertions.assertTrue(classification.canPruneBuckets()); + } + + @Test + void testInOrBloomSupportedAtPlanTime() { + Column distributionColumn = new Column("dist_col", PrimitiveType.INT); + RuntimeFilterPruneClassifier.Classification classification = classifyBucket( + TRuntimeFilterType.IN_OR_BLOOM, distributionColumn, + new HashDistributionInfo(8, ImmutableList.of(distributionColumn))); + + Assertions.assertTrue(classification.canPruneBuckets()); + } + + @Test + void testBloomAndNonHashDistributionsRejected() { + Column distributionColumn = new Column("dist_col", PrimitiveType.INT); + RuntimeFilterPruneClassifier.Classification bloomClassification = classifyBucket( + TRuntimeFilterType.BLOOM, distributionColumn, + new HashDistributionInfo(8, ImmutableList.of(distributionColumn))); + RuntimeFilterPruneClassifier.Classification randomClassification = classifyBucket( + TRuntimeFilterType.IN, distributionColumn, new RandomDistributionInfo(8)); + + Assertions.assertFalse(bloomClassification.canPruneBuckets()); + Assertions.assertTrue(bloomClassification.getBucketUnsupportedReason().contains("IN")); + Assertions.assertFalse(randomClassification.canPruneBuckets()); + Assertions.assertTrue(randomClassification.getBucketUnsupportedReason().contains("not HASH")); + } + + @Test + void testCompositeHashAndNonDistributionTargetsRejected() { + Column distributionColumn = new Column("dist_col", PrimitiveType.INT); + Column valueColumn = new Column("value_col", PrimitiveType.INT); + RuntimeFilterPruneClassifier.Classification compositeClassification = classifyBucket( + TRuntimeFilterType.IN, distributionColumn, + new HashDistributionInfo(8, ImmutableList.of(distributionColumn, valueColumn))); + RuntimeFilterPruneClassifier.Classification nonDistributionClassification = classifyBucket( + TRuntimeFilterType.IN, valueColumn, + new HashDistributionInfo(8, ImmutableList.of(distributionColumn))); + + Assertions.assertFalse(compositeClassification.canPruneBuckets()); + Assertions.assertTrue(compositeClassification.getBucketUnsupportedReason().contains("single-column")); + Assertions.assertFalse(nonDistributionClassification.canPruneBuckets()); + Assertions.assertTrue(nonDistributionClassification.getBucketUnsupportedReason() + .contains("distribution column")); + } + + @Test + void testDirectMvAliasSupportedButComputedAliasRejected() { + Column baseColumn = new Column("base_col", PrimitiveType.INT); + SlotDescriptor baseSlotDescriptor = new SlotDescriptor(new SlotId(2), new TupleId(2)); + baseSlotDescriptor.setColumn(baseColumn); + baseSlotDescriptor.setType(baseColumn.getType()); + + Column directMvColumn = new Column("mv_base_col", PrimitiveType.INT); + directMvColumn.setDefineExpr(new SlotRef(baseSlotDescriptor)); + RuntimeFilterPruneClassifier.Classification directClassification = classifyBucket( + TRuntimeFilterType.IN, directMvColumn, + new HashDistributionInfo(8, ImmutableList.of(baseColumn)), 1L, 2L); + + Column computedMvColumn = new Column("base_col", PrimitiveType.INT); + computedMvColumn.setDefineExpr(new FunctionCallExpr("abs", + ImmutableList.of(new SlotRef(baseSlotDescriptor)), true)); + RuntimeFilterPruneClassifier.Classification computedClassification = classifyBucket( + TRuntimeFilterType.IN, computedMvColumn, + new HashDistributionInfo(8, ImmutableList.of(baseColumn)), 1L, 2L); + + Assertions.assertTrue(directClassification.canPruneBuckets()); + Assertions.assertFalse(computedClassification.canPruneBuckets()); + } + + @Test + void testNonBaseIndexWithoutDirectDefinitionRejected() { + Column distributionColumn = new Column("dist_col", PrimitiveType.INT); + RuntimeFilterPruneClassifier.Classification classification = classifyBucket( + TRuntimeFilterType.IN, distributionColumn, + new HashDistributionInfo(8, ImmutableList.of(distributionColumn)), 1L, 2L); + + Assertions.assertFalse(classification.canPruneBuckets()); + Assertions.assertTrue(classification.getBucketUnsupportedReason().contains("no direct base-column")); + } + + @Test + void testBaseColumnNamesComparedSymmetrically() { + Column baseColumn = new Column("base_col", PrimitiveType.INT); + SlotDescriptor baseSlotDescriptor = new SlotDescriptor(new SlotId(2), new TupleId(2)); + baseSlotDescriptor.setColumn(baseColumn); + baseSlotDescriptor.setType(baseColumn.getType()); + + Column distributionColumn = new Column("mv_dist_col", PrimitiveType.INT); + distributionColumn.setDefineExpr(new SlotRef(baseSlotDescriptor)); + RuntimeFilterPruneClassifier.Classification classification = classifyBucket( + TRuntimeFilterType.IN, baseColumn, + new HashDistributionInfo(8, ImmutableList.of(distributionColumn))); + + Assertions.assertTrue(classification.canPruneBuckets()); + } + + @Test + void testDifferentUniqueIdsAllowedForSameBaseColumn() { + Column distributionColumn = new Column("dist_col", PrimitiveType.INT); + distributionColumn.setUniqueId(1); + Column targetColumn = new Column("dist_col", PrimitiveType.INT); + targetColumn.setUniqueId(2); + + RuntimeFilterPruneClassifier.Classification classification = classifyBucket( + TRuntimeFilterType.IN, targetColumn, + new HashDistributionInfo(8, ImmutableList.of(distributionColumn))); + + Assertions.assertTrue(classification.canPruneBuckets()); + } + + @Test + void testRollupUniqueIdCollisionRejectedForDifferentBaseColumns() { + Column baseDistributionColumn = new Column("k2", PrimitiveType.INT); + baseDistributionColumn.setUniqueId(1); + + Column baseTargetColumn = new Column("k1", PrimitiveType.INT); + SlotDescriptor baseSlotDescriptor = new SlotDescriptor(new SlotId(2), new TupleId(2)); + baseSlotDescriptor.setColumn(baseTargetColumn); + baseSlotDescriptor.setType(baseTargetColumn.getType()); + Column rollupTargetColumn = new Column("mv_k1", PrimitiveType.INT); + rollupTargetColumn.setUniqueId(1); + rollupTargetColumn.setDefineExpr(new SlotRef(baseSlotDescriptor)); + + RuntimeFilterPruneClassifier.Classification classification = classifyBucket( + TRuntimeFilterType.IN, rollupTargetColumn, + new HashDistributionInfo(8, ImmutableList.of(baseDistributionColumn))); + + Assertions.assertFalse(classification.canPruneBuckets()); + Assertions.assertTrue(classification.getBucketUnsupportedReason().contains("distribution column")); + } + + @Test + void testRollupWithDifferentTypeRejected() { + Column baseDistributionColumn = new Column("dist_col", PrimitiveType.INT); + SlotDescriptor baseSlotDescriptor = new SlotDescriptor(new SlotId(2), new TupleId(2)); + baseSlotDescriptor.setColumn(baseDistributionColumn); + baseSlotDescriptor.setType(baseDistributionColumn.getType()); + Column rollupTargetColumn = new Column("mv_dist_col", PrimitiveType.BIGINT); + rollupTargetColumn.setDefineExpr(new SlotRef(baseSlotDescriptor)); + + RuntimeFilterPruneClassifier.Classification classification = classifyBucket( + TRuntimeFilterType.IN, rollupTargetColumn, + new HashDistributionInfo(8, ImmutableList.of(baseDistributionColumn))); + + Assertions.assertFalse(classification.canPruneBuckets()); + Assertions.assertTrue(classification.getBucketUnsupportedReason().contains("distribution column")); + } + + @Test + void testFeatureGatesAvoidCatalogClassification() { + SessionVariable sessionVariable = new SessionVariable(); + sessionVariable.setEnableRuntimeFilterBucketPrune(false); + sessionVariable.setEnableRuntimeFilterPartitionPrune(false); + Column targetColumn = new Column("dist_col", PrimitiveType.INT); + OlapTable table = Mockito.mock(OlapTable.class); + PhysicalOlapScan scan = Mockito.mock(PhysicalOlapScan.class); + SlotReference target = slot(targetColumn, table, 1); + RuntimeFilter filter = newFilter( + TRuntimeFilterType.IN, target, target, target.getDataType(), scan); + + RuntimeFilterPruneClassifier.Classification classification = + RuntimeFilterPruneClassifier.classify(filter, sessionVariable); + + Assertions.assertFalse(classification.canPruneBuckets()); + Assertions.assertFalse(classification.canPrunePartitions()); + Mockito.verifyNoInteractions(scan); + } + + @Test + void testDirectRangeAndListPartitionTargetsSupported() { + RuntimeFilterPruneClassifier.Classification rangeClassification = classifyPartition( + TRuntimeFilterType.IN_OR_BLOOM, PartitionType.RANGE, RangePartitionItem.DUMMY_ITEM, + target -> target); + RuntimeFilterPruneClassifier.Classification listClassification = classifyPartition( + TRuntimeFilterType.BLOOM, PartitionType.LIST, ListPartitionItem.DUMMY_ITEM, + target -> target); + + assertSupportedIncreasingPartitions(rangeClassification); + assertSupportedIncreasingPartitions(listClassification); + } + + @Test + void testPartitionUniqueIdCollisionRejected() { + Column partitionColumn = new Column("part_col", PrimitiveType.INT); + partitionColumn.setUniqueId(0); + Column baseTargetColumn = new Column("value_col", PrimitiveType.INT); + baseTargetColumn.setUniqueId(1); + Column rollupTargetColumn = directRollupColumn("part_col", baseTargetColumn); + rollupTargetColumn.setUniqueId(0); + + RuntimeFilterPruneClassifier.Classification classification = classifyDirectPartition( + rollupTargetColumn, partitionColumn, 1L, 2L); + + Assertions.assertFalse(classification.canPrunePartitions()); + } + + @Test + void testDirectRollupPartitionColumnSupported() { + Column partitionColumn = new Column("part_col", PrimitiveType.INT); + Column rollupTargetColumn = directRollupColumn("part_col", partitionColumn); + + RuntimeFilterPruneClassifier.Classification classification = classifyDirectPartition( + rollupTargetColumn, partitionColumn, 1L, 2L); + + assertSupportedIncreasingPartitions(classification); + } + + @Test + void testRenamedRollupPartitionColumnRejected() { + Column partitionColumn = new Column("part_col", PrimitiveType.INT); + Column rollupTargetColumn = directRollupColumn("mv_part_col", partitionColumn); + + RuntimeFilterPruneClassifier.Classification classification = classifyDirectPartition( + rollupTargetColumn, partitionColumn, 1L, 2L); + + Assertions.assertFalse(classification.canPrunePartitions()); + } + + @Test + void testNonBasePartitionTargetWithoutDefinitionRejected() { + Column partitionColumn = new Column("part_col", PrimitiveType.INT); + Column rollupTargetColumn = new Column("part_col", PrimitiveType.INT); + + RuntimeFilterPruneClassifier.Classification classification = classifyDirectPartition( + rollupTargetColumn, partitionColumn, 1L, 2L); + + Assertions.assertFalse(classification.canPrunePartitions()); + Assertions.assertTrue(classification.getPartitionUnsupportedReason() + .contains("no direct base-column")); + } + + @Test + void testComputedRollupPartitionTargetRejected() { + Column partitionColumn = new Column("part_col", PrimitiveType.INT); + Column rollupTargetColumn = new Column("part_col", PrimitiveType.INT); + rollupTargetColumn.setDefineExpr(new FunctionCallExpr("abs", + ImmutableList.of(directSlotRef(partitionColumn)), true)); + + RuntimeFilterPruneClassifier.Classification classification = classifyDirectPartition( + rollupTargetColumn, partitionColumn, 1L, 2L); + + Assertions.assertFalse(classification.canPrunePartitions()); + Assertions.assertTrue(classification.getPartitionUnsupportedReason() + .contains("no direct base-column")); + } + + @Test + void testBloomRangePartitionRejected() { + RuntimeFilterPruneClassifier.Classification classification = classifyPartition( + TRuntimeFilterType.BLOOM, PartitionType.RANGE, RangePartitionItem.DUMMY_ITEM, + target -> target); + + Assertions.assertFalse(classification.canPrunePartitions()); + Assertions.assertTrue(classification.getPartitionUnsupportedReason().contains("BLOOM")); + } + + @Test + void testNoneMovableListExpressionRejected() { + RuntimeFilterPruneClassifier.Classification classification = classifyPartition( + TRuntimeFilterType.IN, PartitionType.LIST, ListPartitionItem.DUMMY_ITEM, + target -> new AssertTrue( + new GreaterThan(target, new IntegerLiteral(0)), + new VarcharLiteral("rfpp_expr_in_only_error"))); + + Assertions.assertFalse(classification.canPrunePartitions()); + Assertions.assertTrue(classification.getPartitionUnsupportedReason().contains("non-movable")); + } + + @Test + void testTypeAdjustedNonIdentityTargetRejectedBeforeTranslation() { + Column partitionColumn = new Column("part_col", PrimitiveType.INT); + OlapTable table = Mockito.mock(OlapTable.class); + PartitionInfo partitionInfo = partitionInfo( + partitionColumn, PartitionType.LIST, ListPartitionItem.DUMMY_ITEM); + Mockito.when(table.getPartitionInfo()).thenReturn(partitionInfo); + PhysicalOlapScan scan = scan(table, ImmutableList.of(1L, 2L)); + SlotReference target = slot(partitionColumn, table, 1); + RuntimeFilter filter = newFilter(TRuntimeFilterType.IN, target, + new Add(target, new IntegerLiteral(1)), DateTimeV2Type.SYSTEM_DEFAULT, scan); + SessionVariable sessionVariable = partitionOnlySession(); + + RuntimeFilterPruneClassifier.Classification classification = + RuntimeFilterPruneClassifier.classify(filter, sessionVariable); + + Assertions.assertFalse(classification.canPrunePartitions()); + Assertions.assertTrue(classification.getPartitionUnsupportedReason().contains("type adjustment")); + } + + @Test + void testMonotonicChildMustOwnAllInputSlots() { + SlotReference slot = new SlotReference("dt", DateTimeV2Type.SYSTEM_DEFAULT); + DateTrunc rejected = new DateTrunc(slot, slot); + DateTrunc accepted = new DateTrunc(slot, new VarcharLiteral("day")); + + Assertions.assertFalse(RuntimeFilterPruneClassifier.hasInputSlotOnlyInMonotonicChild( + rejected, ((Monotonic) rejected).getMonotonicFunctionChildIndex())); + Assertions.assertTrue(RuntimeFilterPruneClassifier.hasInputSlotOnlyInMonotonicChild( + accepted, ((Monotonic) accepted).getMonotonicFunctionChildIndex())); + } + + private RuntimeFilterPruneClassifier.Classification classifyBucket( + TRuntimeFilterType filterType, Column targetColumn, DistributionInfo distributionInfo) { + return classifyBucket(filterType, targetColumn, distributionInfo, 1L, 1L); + } + + private RuntimeFilterPruneClassifier.Classification classifyBucket( + TRuntimeFilterType filterType, Column targetColumn, DistributionInfo distributionInfo, + long baseIndexId, long selectedIndexId) { + OlapTable table = Mockito.mock(OlapTable.class); + Partition partition = Mockito.mock(Partition.class); + Mockito.when(table.getBaseIndexId()).thenReturn(baseIndexId); + Mockito.when(table.getPartition(1L)).thenReturn(partition); + Mockito.when(partition.getDistributionInfo()).thenReturn(distributionInfo); + PhysicalOlapScan scan = scan(table, ImmutableList.of(1L)); + Mockito.when(scan.getSelectedIndexId()).thenReturn(selectedIndexId); + SlotReference target = slot(targetColumn, table, 1); + RuntimeFilter filter = newFilter( + filterType, target, target, target.getDataType(), scan); + SessionVariable sessionVariable = new SessionVariable(); + sessionVariable.setEnableRuntimeFilterBucketPrune(true); + sessionVariable.setEnableRuntimeFilterPartitionPrune(false); + return RuntimeFilterPruneClassifier.classify(filter, sessionVariable); + } + + private RuntimeFilterPruneClassifier.Classification classifyPartition( + TRuntimeFilterType filterType, PartitionType partitionType, PartitionItem partitionItem, + Function targetFactory) { + Column partitionColumn = new Column("part_col", PrimitiveType.INT); + OlapTable table = Mockito.mock(OlapTable.class); + PartitionInfo partitionInfo = partitionInfo(partitionColumn, partitionType, partitionItem); + Mockito.when(table.getPartitionInfo()).thenReturn(partitionInfo); + PhysicalOlapScan scan = scan(table, ImmutableList.of(1L, 2L)); + SlotReference target = slot(partitionColumn, table, 1); + Expression targetExpression = targetFactory.apply(target); + RuntimeFilter filter = newFilter(filterType, target, + targetExpression, targetExpression.getDataType(), scan); + return RuntimeFilterPruneClassifier.classify(filter, partitionOnlySession()); + } + + private RuntimeFilterPruneClassifier.Classification classifyDirectPartition( + Column targetColumn, Column partitionColumn, long baseIndexId, long selectedIndexId) { + OlapTable table = Mockito.mock(OlapTable.class); + PartitionInfo partitionInfo = partitionInfo( + partitionColumn, PartitionType.RANGE, RangePartitionItem.DUMMY_ITEM); + Mockito.when(table.getBaseIndexId()).thenReturn(baseIndexId); + Mockito.when(table.getPartitionInfo()).thenReturn(partitionInfo); + PhysicalOlapScan scan = scan(table, ImmutableList.of(1L, 2L)); + Mockito.when(scan.getSelectedIndexId()).thenReturn(selectedIndexId); + SlotReference target = slot(targetColumn, table, 1); + RuntimeFilter filter = newFilter( + TRuntimeFilterType.IN, target, target, target.getDataType(), scan); + return RuntimeFilterPruneClassifier.classify(filter, partitionOnlySession()); + } + + private Column directRollupColumn(String name, Column baseColumn) { + Column rollupColumn = new Column(name, baseColumn.getType()); + rollupColumn.setDefineExpr(directSlotRef(baseColumn)); + return rollupColumn; + } + + private SlotRef directSlotRef(Column column) { + SlotDescriptor slotDescriptor = new SlotDescriptor(new SlotId(2), new TupleId(2)); + slotDescriptor.setColumn(column); + slotDescriptor.setType(column.getType()); + return new SlotRef(slotDescriptor); + } + + private PartitionInfo partitionInfo( + Column partitionColumn, PartitionType partitionType, PartitionItem partitionItem) { + PartitionInfo partitionInfo = Mockito.mock(PartitionInfo.class); + Mockito.when(partitionInfo.getType()).thenReturn(partitionType); + Mockito.when(partitionInfo.getPartitionColumns()).thenReturn(ImmutableList.of(partitionColumn)); + Mockito.when(partitionInfo.getItem(1L)).thenReturn(partitionItem); + Mockito.when(partitionInfo.getItem(2L)).thenReturn(partitionItem); + return partitionInfo; + } + + private PhysicalOlapScan scan(OlapTable table, ImmutableList selectedPartitionIds) { + PhysicalOlapScan scan = Mockito.mock(PhysicalOlapScan.class); + Mockito.when(scan.getTable()).thenReturn(table); + Mockito.when(scan.getSelectedPartitionIds()).thenReturn(selectedPartitionIds); + return scan; + } + + private SlotReference slot(Column column, OlapTable table, int exprId) { + return new SlotReference(new ExprId(exprId), column.getName(), + DataType.fromCatalogType(column.getType()), column.isAllowNull(), ImmutableList.of("t"), + table, column, table, column); + } + + private RuntimeFilter newFilter(TRuntimeFilterType filterType, SlotReference target, + Expression targetExpression, DataType sourceType, PhysicalOlapScan scan) { + SlotReference source = new SlotReference("src", sourceType); + AbstractPhysicalPlan builder = Mockito.mock(AbstractPhysicalPlan.class); + return new RuntimeFilter(RuntimeFilterId.createGenerator().getNextId(), + source, target, targetExpression, filterType, 0, builder, 10, false, + TMinMaxRuntimeFilterType.MIN_MAX, scan); + } + + private SessionVariable partitionOnlySession() { + SessionVariable sessionVariable = new SessionVariable(); + sessionVariable.setEnableRuntimeFilterBucketPrune(false); + sessionVariable.setEnableRuntimeFilterPartitionPrune(true); + return sessionVariable; + } + + private void assertSupportedIncreasingPartitions( + RuntimeFilterPruneClassifier.Classification classification) { + Assertions.assertTrue(classification.canPrunePartitions()); + Map monotonicity = classification.getPartitionMonotonicity(); + Assertions.assertEquals(2, monotonicity.size()); + Assertions.assertEquals(TTargetExprMonotonicity.MONOTONIC_INCREASING, monotonicity.get(1L)); + Assertions.assertEquals(TTargetExprMonotonicity.MONOTONIC_INCREASING, monotonicity.get(2L)); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/OlapScanNodeBackendSelectionConfigTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/OlapScanNodeBackendSelectionConfigTest.java index df0d2a0de2cc2a..b6db673c0d8260 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/planner/OlapScanNodeBackendSelectionConfigTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/planner/OlapScanNodeBackendSelectionConfigTest.java @@ -366,7 +366,8 @@ private List createScanRanges(Tablet tablet, Mockito.when(partition.getId()).thenReturn(10L); Mockito.when(partition.getVisibleVersion()).thenReturn(10L); - Deencapsulation.setField(scanNode, "tabletId2BucketSeq", ImmutableMap.of(20L, 0)); + Deencapsulation.setField(scanNode, "tabletId2BucketInfo", + ImmutableMap.of(20L, 1L << Integer.SIZE)); Deencapsulation.invoke(scanNode, "addScanRangeLocations", partition, ImmutableList.of(tablet), ImmutableMap.of()); return scanNode.getScanRangeLocations(0); diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/OlapScanNodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/OlapScanNodeTest.java index 89d734e7d08e21..c18b821ec116e4 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/planner/OlapScanNodeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/planner/OlapScanNodeTest.java @@ -36,9 +36,14 @@ import org.apache.doris.catalog.RangePartitionItem; import org.apache.doris.catalog.info.TableNameInfo; import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.Config; +import org.apache.doris.common.util.DebugPointUtil; import org.apache.doris.datasource.InternalCatalog; import org.apache.doris.thrift.TOlapScanNode; +import org.apache.doris.thrift.TPaloScanRange; import org.apache.doris.thrift.TPartitionBoundary; +import org.apache.doris.thrift.TScanRange; +import org.apache.doris.thrift.TScanRangeLocations; import com.google.common.collect.Lists; import com.google.common.collect.Maps; @@ -277,6 +282,80 @@ public void testRuntimeFilterPartitionBoundariesUsePlanningSnapshot() throws Ana Assert.assertEquals("p_target,p_after", scanNode.getSelectedPartitionNamesForExplain()); } + @Test + public void testRuntimeFilterBucketMetadataAttachedOnceAcrossWorkers() throws Exception { + OlapScanNode scanNode = newBucketPruneScanNode(10L); + TPaloScanRange paloScanRange = scanNode.scanRangeLocations.get(0) + .getScanRange().getPaloScanRange(); + Map bucketInfo = getBucketInfo(scanNode); + bucketInfo.put(10L, ((long) 4 << Integer.SIZE) | 2L); + + scanNode.setRuntimeFilterBucketPruneParameters(); + bucketInfo.clear(); + scanNode.setRuntimeFilterBucketPruneParameters(); + + Assert.assertEquals(2, paloScanRange.getBucketSeq()); + Assert.assertEquals(4, paloScanRange.getBucketNum()); + } + + @Test + public void testMissingRuntimeFilterBucketMetadataDisablesScanPruning() throws Exception { + OlapScanNode scanNode = newBucketPruneScanNode(10L, 11L); + TPaloScanRange firstScanRange = scanNode.scanRangeLocations.get(0) + .getScanRange().getPaloScanRange(); + TPaloScanRange secondScanRange = scanNode.scanRangeLocations.get(1) + .getScanRange().getPaloScanRange(); + Map bucketInfo = getBucketInfo(scanNode); + bucketInfo.put(10L, ((long) 4 << Integer.SIZE) | 2L); + bucketInfo.put(11L, ((long) 4 << Integer.SIZE) | 3L); + + boolean previousEnableDebugPoints = Config.enable_debug_points; + try { + Config.enable_debug_points = true; + DebugPointUtil.addDebugPointWithValue( + OlapScanNode.MISSING_RF_BUCKET_METADATA_DEBUG_POINT, 11L); + + scanNode.setRuntimeFilterBucketPruneParameters(); + + Assert.assertFalse(firstScanRange.isSetBucketSeq()); + Assert.assertFalse(firstScanRange.isSetBucketNum()); + Assert.assertFalse(secondScanRange.isSetBucketSeq()); + Assert.assertFalse(secondScanRange.isSetBucketNum()); + } finally { + DebugPointUtil.removeDebugPoint(OlapScanNode.MISSING_RF_BUCKET_METADATA_DEBUG_POINT); + Config.enable_debug_points = previousEnableDebugPoints; + } + } + + private OlapScanNode newBucketPruneScanNode(long... tabletIds) { + OlapTable table = Mockito.mock(OlapTable.class); + Mockito.when(table.getName()).thenReturn("rf_bucket_fact"); + Mockito.when(table.getDistributionColumnNames()).thenReturn(Collections.emptySet()); + + TupleDescriptor tupleDescriptor = new TupleDescriptor(new TupleId(1)); + tupleDescriptor.setTable(table); + OlapScanNode scanNode = new OlapScanNode( + new PlanNodeId(1), tupleDescriptor, "rfBucketScanNode", ScanContext.EMPTY); + for (long tabletId : tabletIds) { + TPaloScanRange paloScanRange = new TPaloScanRange(); + paloScanRange.setTabletId(tabletId); + TScanRange scanRange = new TScanRange(); + scanRange.setPaloScanRange(paloScanRange); + TScanRangeLocations locations = new TScanRangeLocations(); + locations.setScanRange(scanRange); + scanNode.scanRangeLocations.add(locations); + } + return scanNode; + } + + @SuppressWarnings("unchecked") + private Map getBucketInfo(OlapScanNode scanNode) throws Exception { + java.lang.reflect.Field bucketInfoField = + OlapScanNode.class.getDeclaredField("tabletId2BucketInfo"); + bucketInfoField.setAccessible(true); + return (Map) bucketInfoField.get(scanNode); + } + private Partition mockPartition(String name) { Partition partition = Mockito.mock(Partition.class); Mockito.when(partition.getName()).thenReturn(name); diff --git a/gensrc/thrift/PaloInternalService.thrift b/gensrc/thrift/PaloInternalService.thrift index fd16b4b7598861..07944139285228 100644 --- a/gensrc/thrift/PaloInternalService.thrift +++ b/gensrc/thrift/PaloInternalService.thrift @@ -518,6 +518,7 @@ struct TQueryOptions { 230: optional bool supports_external_file_report_ack = false; // Fall back to RE2 when Hyperscan cannot compile a regular expression. 231: optional bool enable_hyperscan_fallback = true; + 232: optional bool enable_runtime_filter_bucket_prune = true; // For cloud, to control if the content would be written into file cache // In write path, to control if the content would be written into file cache. // In read path, read from file cache or remote storage when execute query. diff --git a/gensrc/thrift/PlanNodes.thrift b/gensrc/thrift/PlanNodes.thrift index b2702dd0ed4c83..65c70fd519e84c 100644 --- a/gensrc/thrift/PlanNodes.thrift +++ b/gensrc/thrift/PlanNodes.thrift @@ -102,6 +102,11 @@ struct TPaloScanRange { 10: optional i64 start_tso 11: optional i64 end_tso 12: optional TBinlogScanType binlog_scan_type + // Bucket metadata for BE-side runtime-filter bucket pruning. These fields + // are populated only when the scan has an eligible single-column HASH + // distribution runtime-filter target. + 13: optional i32 bucket_seq + 14: optional i32 bucket_num } enum TFileFormatType { @@ -1659,6 +1664,11 @@ struct TRuntimeFilterDesc { // slice and must be merged before being applied. Computed truthfully by FE after local // exchange planning; replaces inferring this from the target scan's is_serial_operator. 21: optional bool force_local_merge; + + // Scan node ids whose target is a direct SlotRef on the only HASH + // distribution column. BE still verifies that the delivered filter has an + // exact IN set before using it for bucket pruning. + 22: optional set bucket_pruning_target_ids; } diff --git a/regression-test/data/query_p0/runtime_filter/rf_bucket_pruning.out b/regression-test/data/query_p0/runtime_filter/rf_bucket_pruning.out new file mode 100644 index 00000000000000..9020c828089377 --- /dev/null +++ b/regression-test/data/query_p0/runtime_filter/rf_bucket_pruning.out @@ -0,0 +1,6 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !bucket_result -- +3 30 + +-- !nullable_bucket_result -- +\N 90 diff --git a/regression-test/suites/query_p0/runtime_filter/rf_bucket_pruning.groovy b/regression-test/suites/query_p0/runtime_filter/rf_bucket_pruning.groovy new file mode 100644 index 00000000000000..9419c00dffb6c2 --- /dev/null +++ b/regression-test/suites/query_p0/runtime_filter/rf_bucket_pruning.groovy @@ -0,0 +1,136 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.action.ProfileAction + +suite("rf_bucket_pruning", "nonConcurrent") { + sql "set enable_runtime_filter_prune=false" + sql "set enable_runtime_filter_partition_prune=false" + sql "set enable_runtime_filter_bucket_prune=true" + sql "set runtime_filter_wait_infinitely=true" + sql "set runtime_filter_type='IN'" + sql "set disable_join_reorder=true" + sql "set enable_profile=true" + sql "set profile_level=2" + sql "set parallel_pipeline_task_num=1" + + sql "drop table if exists rf_bucket_prune_fact" + sql """ + CREATE TABLE rf_bucket_prune_fact ( + k INT NOT NULL, + v INT NOT NULL + ) + DISTRIBUTED BY HASH(k) BUCKETS 8 + PROPERTIES("replication_num" = "1") + """ + sql "drop table if exists rf_bucket_prune_composite" + sql """ + CREATE TABLE rf_bucket_prune_composite ( + k INT NOT NULL, + v INT NOT NULL + ) + DISTRIBUTED BY HASH(k, v) BUCKETS 8 + PROPERTIES("replication_num" = "1") + """ + sql "drop table if exists rf_bucket_prune_dim" + sql """ + CREATE TABLE rf_bucket_prune_dim ( + k INT NOT NULL + ) + DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES("replication_num" = "1") + """ + sql "drop table if exists rf_bucket_prune_nullable" + sql """ + CREATE TABLE rf_bucket_prune_nullable ( + k INT NULL, + v INT NOT NULL + ) + DISTRIBUTED BY HASH(k) BUCKETS 8 + PROPERTIES("replication_num" = "1") + """ + sql "drop table if exists rf_bucket_prune_dim_nullable" + sql """ + CREATE TABLE rf_bucket_prune_dim_nullable ( + k INT NULL + ) + DISTRIBUTED BY HASH(k) BUCKETS 1 + PROPERTIES("replication_num" = "1") + """ + + sql """ + INSERT INTO rf_bucket_prune_fact VALUES + (1, 10), (2, 20), (3, 30), (4, 40), + (5, 50), (6, 60), (7, 70), (8, 80) + """ + sql """ + INSERT INTO rf_bucket_prune_composite VALUES + (1, 10), (2, 20), (3, 30), (4, 40), + (5, 50), (6, 60), (7, 70), (8, 80) + """ + sql "INSERT INTO rf_bucket_prune_dim VALUES (3)" + sql "INSERT INTO rf_bucket_prune_nullable VALUES (NULL, 90), (3, 30)" + sql "INSERT INTO rf_bucket_prune_dim_nullable VALUES (NULL)" + + order_qt_bucket_result """ + SELECT f.k, f.v + FROM rf_bucket_prune_fact f + JOIN [broadcast] rf_bucket_prune_dim d ON f.k = d.k + ORDER BY f.k, f.v + """ + + order_qt_nullable_bucket_result """ + SELECT f.k, f.v + FROM rf_bucket_prune_nullable f + JOIN [broadcast] rf_bucket_prune_dim_nullable d ON f.k <=> d.k + ORDER BY f.k, f.v + """ + + def profileAction = new ProfileAction(context) + def getProfileByToken = { String token -> + return profileAction.getProfileBySql(token, ["BucketsPrunedByRuntimeFilter"]) + } + def extractPrunedBuckets = { String profile -> + def values = (profile =~ /-\s*BucketsPrunedByRuntimeFilter:\s*(\d+)/) + .collect { it[1].toLong() } + return values.isEmpty() ? 0L : values.sum() + } + def runProfileQuery = { String tableName, String dimensionName, String joinOperator -> + def token = UUID.randomUUID().toString() + sql """ + SELECT "${token}", COUNT(*) + FROM ${tableName} f + JOIN [broadcast] ${dimensionName} d ON f.k ${joinOperator} d.k + """ + def profile = getProfileByToken(token) + assertTrue(profile != "", "Profile not found for ${token}") + assertTrue(profile.contains("BucketsPrunedByRuntimeFilter"), + "Bucket-pruning counter not found for ${token}") + return extractPrunedBuckets(profile) + } + + assertTrue(runProfileQuery("rf_bucket_prune_fact", "rf_bucket_prune_dim", "=") > 0, + "single-column HASH distribution should be pruned") + assertTrue(runProfileQuery("rf_bucket_prune_composite", "rf_bucket_prune_dim", "=") == 0, + "multi-column HASH distribution must not be pruned") + assertTrue(runProfileQuery("rf_bucket_prune_nullable", "rf_bucket_prune_dim_nullable", "<=>") > 0, + "null-aware IN filter should prune buckets while retaining the NULL bucket") + + sql "set enable_runtime_filter_bucket_prune=false" + assertTrue(runProfileQuery("rf_bucket_prune_fact", "rf_bucket_prune_dim", "=") == 0, + "disabled runtime-filter bucket pruning must not prune buckets") +}