[Feature]support dataSketches include Quantile Sketches、Theta Sketch、Frequency…

be/src/exprs/agg/factory/aggregate_factory.hpp

@@ -31,7 +31,7 @@
 #include "exprs/agg/count.h"
 #include "exprs/agg/covariance.h"
 #include "exprs/agg/distinct.h"
-#include "exprs/agg/ds_hll_count_distinct.h"
+#include "exprs/agg/ds_agg.h"
 #include "exprs/agg/exchange_perf.h"
 #include "exprs/agg/group_concat.h"
 #include "exprs/agg/histogram.h"
@@ -188,9 +188,6 @@ class AggregateFactory {
     template <LogicalType T>
     static AggregateFunctionPtr MakeHllNdvAggregateFunction();
 
-    template <LogicalType T>
-    static AggregateFunctionPtr MakeHllSketchAggregateFunction();
-
     template <LogicalType T>
     static AggregateFunctionPtr MakeHllRawAggregateFunction();
 
@@ -259,6 +256,9 @@ class AggregateFactory {
 
     template <LogicalType LT>
     static auto MakeRetractMaxAggregateFunction();
+
+    template <LogicalType LT, SketchType ST>
+    static AggregateFunctionPtr MakeDataSketchesAggregateFunction();
 };
 
 // The function should be placed by alphabetical order
@@ -394,11 +394,6 @@ AggregateFunctionPtr AggregateFactory::MakeHllNdvAggregateFunction() {
     return std::make_shared<HllNdvAggregateFunction<LT, false>>();
 }
 
-template <LogicalType LT>
-AggregateFunctionPtr AggregateFactory::MakeHllSketchAggregateFunction() {
-    return std::make_shared<HllSketchAggregateFunction<LT>>();
-}
-
 template <LogicalType LT>
 AggregateFunctionPtr AggregateFactory::MakeHllRawAggregateFunction() {
     return std::make_shared<HllNdvAggregateFunction<LT, true>>();
@@ -442,4 +437,9 @@ auto AggregateFactory::MakeRetractMaxAggregateFunction() {
                                                                MaxElement<LT, MaxAggregateDataRetractable<LT>>>>();
 }
 
+template <LogicalType LT, SketchType ST>
+AggregateFunctionPtr AggregateFactory::MakeDataSketchesAggregateFunction() {
+    return std::make_shared<DataSketchesAggregateFunction<LT, ST>>();
+}
+
 } // namespace starrocks

be/src/exprs/agg/factory/aggregate_resolver_approx.cpp

@@ -38,9 +38,6 @@ struct HLLUnionBuilder {
 
             resolver->add_aggregate_mapping<lt, TYPE_BIGINT, HyperLogLog>(
                     "approx_count_distinct", false, AggregateFactory::MakeHllNdvAggregateFunction<lt>());
-
-            resolver->add_aggregate_mapping_variadic<lt, TYPE_BIGINT, HLLSketchState>(
-                    "ds_hll_count_distinct", false, AggregateFactory::MakeHllSketchAggregateFunction<lt>());
         }
     }
 };
@@ -57,10 +54,34 @@ struct ApproxTopKBuilder {
     }
 };
 
+struct DataSketchesBuilder {
+    template <LogicalType lt>
+    void operator()(AggregateFuncResolver* resolver) {
+        if constexpr (lt_is_fixedlength<lt> || lt_is_string<lt>) {
+            resolver->add_aggregate_mapping<lt, TYPE_BIGINT, DSSketchState<lt, SketchType::HLL>>(
+                    "ds_hll_count_distinct", false,
+                    AggregateFactory::MakeDataSketchesAggregateFunction<lt, SketchType::HLL>());
+            resolver->add_aggregate_mapping<lt, TYPE_BIGINT, DSSketchState<lt, SketchType::THETA>>(
+                    "ds_theta", false, AggregateFactory::MakeDataSketchesAggregateFunction<lt, SketchType::THETA>());
+        }
+        if constexpr (lt_is_integer<lt> || lt_is_float<lt>) {
+            resolver->add_aggregate_mapping<lt, TYPE_ARRAY, DSSketchState<lt, SketchType::QUANTILE>>(
+                    "ds_quantile", false,
+                    AggregateFactory::MakeDataSketchesAggregateFunction<lt, SketchType::QUANTILE>());
+        }
+        if constexpr (lt_is_integer<lt> || lt_is_float<lt> || lt_is_string<lt>) {
+            resolver->add_aggregate_mapping<lt, TYPE_ARRAY, DSSketchState<lt, SketchType::FREQUENT>>(
+                    "ds_frequent", false,
+                    AggregateFactory::MakeDataSketchesAggregateFunction<lt, SketchType::FREQUENT>());
+        }
+    }
+};
+
 void AggregateFuncResolver::register_approx() {
     for (auto type : aggregate_types()) {
         type_dispatch_all(type, HLLUnionBuilder(), this);
         type_dispatch_all(type, ApproxTopKBuilder(), this);
+        type_dispatch_all(type, DataSketchesBuilder(), this);
     }
     add_aggregate_mapping<TYPE_HLL, TYPE_HLL, HyperLogLog>("hll_union", false,
                                                            AggregateFactory::MakeHllUnionAggregateFunction());

be/src/types/CMakeLists.txt

@@ -18,8 +18,8 @@ add_library(Types STATIC
     array_type_info.cpp
     bitmap_value.cpp
     date_value.cpp
+    ds_sketch.cpp
     hll.cpp
-    hll_sketch.cpp
     logical_type.cpp
     map_type_info.cpp
     struct_type_info.cpp

be/src/types/constexpr.h

@@ -31,6 +31,12 @@ constexpr int HLL_EMPTY_SIZE = 1;
 const static int MAX_HLL_LOG_K = 20;
 const static uint8_t DEFAULT_HLL_LOG_K = 17;
 
+const static uint16_t DEFAULT_QUANTILE_K = 128;
+const static uint64_t DEFAULT_COUNTER_NUM = 10;
+
+const static uint8_t DEFAULT_FREQUENT_LG_MIn_SIZE = 3;
+const static uint8_t DEFAULT_FREQUENT_LG_MAX_SIZE = 21;
+
 // For JSON type
 constexpr int kJsonDefaultSize = 128;
 constexpr int kJsonMetaDefaultFormatVersion = 1;

be/src/types/hll_sketch.cpp → be/src/types/ds_sketch.cpp

@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include "types/hll_sketch.h"
+#include "types/ds_sketch.h"
 
 #include "common/logging.h"
 #include "runtime/mem_pool.h"
@@ -54,9 +54,6 @@ void DataSketchesHll::merge(const DataSketchesHll& other) {
         _sketch_union = std::make_unique<hll_union_type>(other.get_lg_config_k(), alloc_type(_memory_usage));
     }
     auto o_sketch = other.get_hll_sketch();
-    if (o_sketch == nullptr) {
-        return;
-    }
     _sketch_union->update(*o_sketch);
     this->mark_changed();
 }