perf: use compact pruning for large string IN lists

datafusion/common/src/config.rs

@@ -1368,13 +1368,15 @@ config_namespace! {
         /// parquet reader setting. 0 means no caching.
         pub max_predicate_cache_size: Option<usize>, default = None
 
-        /// Maximum number of values in an `IN (...)` list for which pruning will
-        /// occur. Longer lists will not be used to prune files, row groups, or
-        /// data pages.
+        /// Maximum number of input values in an `IN (...)` list eligible for
+        /// min/max pruning. Lists above this cap, or a cap of 0, skip this
+        /// rewrite; other predicates and Bloom-filter pruning remain available.
         ///
-        /// Higher values help in cases such as filtering on a list of
-        /// ~25-100 identifiers, but also make the predicate more expensive to
-        /// evaluate. Set to 0 to disable `IN (...)` list pruning entirely.
+        /// Within the cap, nonempty lists of at most 20 values use the existing
+        /// per-value rewrite. Larger positive, non-null literal string lists
+        /// on a string column use a compact sorted domain. Other lists retain
+        /// the existing per-value rewrite, so raising the cap can make those
+        /// predicates expensive to build and evaluate.
         ///
         /// Defaults to 20.
         pub max_in_list_size: usize, default = 20

datafusion/common/src/pruning.rs

@@ -67,13 +67,24 @@ pub trait PruningStatistics {
     /// returned array should have `null` in that row. If the minimum value is
     /// not known for any row, return `None`.
     ///
+    /// Each non-null entry must be a conservative lower bound for all non-null
+    /// values in its container, using Arrow's comparison order for the column
+    /// type. For strings, this is unsigned lexicographic UTF-8 byte order.
+    /// Bounds that cannot satisfy this contract must be reported as unknown.
+    /// Pruning relies on providers to uphold this contract; it does not validate
+    /// the ordering or bound guarantees of arbitrary statistics sources.
+    ///
     /// Note: the returned array must contain [`Self::num_containers`] rows
     fn min_values(&self, column: &Column) -> Option<ArrayRef>;
 
     /// Return the maximum values for the named column, if known.
     ///
     /// See [`Self::min_values`] for when to return `None` and null values.
     ///
+    /// Each non-null entry must be a conservative upper bound for all non-null
+    /// values in its container, using the comparison order described in
+    /// [`Self::min_values`].
+    ///
     /// Note: the returned array must contain [`Self::num_containers`] rows
     fn max_values(&self, column: &Column) -> Option<ArrayRef>;
 

datafusion/core/tests/parquet/mod.rs

@@ -58,6 +58,7 @@ mod page_pruning;
 mod row_group_pruning;
 mod schema;
 mod schema_coercion;
+mod string_in_list_pruning;
 mod utils;
 
 #[cfg(test)]