Merge branch 'main' into add-support-for-scored-named-queries

Signed-off-by: Dharin Shah <Dharin-shah@users.noreply.github.com>

CHANGELOG.md

@@ -136,6 +136,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Add `org.opensearch.rest.MethodHandlers` and `RestController#getAllHandlers` ([11876](https://github.com/opensearch-project/OpenSearch/pull/11876))
 - New DateTime format for RFC3339 compatible date fields ([#11465](https://github.com/opensearch-project/OpenSearch/pull/11465))
 - Add support for Google Application Default Credentials in repository-gcs ([#8394](https://github.com/opensearch-project/OpenSearch/pull/8394))
+- Remove concurrent segment search feature flag for GA launch ([#12074](https://github.com/opensearch-project/OpenSearch/pull/12074))
 - Support for returning scores in matched queries ([#11626](https://github.com/opensearch-project/OpenSearch/pull/11626))
 
 
@@ -210,6 +211,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Change error message when per shard document limit is breached ([#11312](https://github.com/opensearch-project/OpenSearch/pull/11312))
 - Improve boolean parsing performance ([#11308](https://github.com/opensearch-project/OpenSearch/pull/11308))
 - Interpret byte array as primitive using VarHandles ([#11362](https://github.com/opensearch-project/OpenSearch/pull/11362))
+- Allow composite aggregation to run under a parent filter aggregation ([#11499](https://github.com/opensearch-project/OpenSearch/pull/11499))
 - Automatically add scheme to discovery.ec2.endpoint ([#11512](https://github.com/opensearch-project/OpenSearch/pull/11512))
 - Restore support for Java 8 for RestClient ([#11562](https://github.com/opensearch-project/OpenSearch/pull/11562))
 - Add deleted doc count in _cat/shards ([#11678](https://github.com/opensearch-project/OpenSearch/pull/11678))
@@ -252,6 +254,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Fix memory leak issue in ReorganizingLongHash ([#11953](https://github.com/opensearch-project/OpenSearch/issues/11953))
 - Prevent setting remote_snapshot store type on index creation ([#11867](https://github.com/opensearch-project/OpenSearch/pull/11867))
 - [BUG] Fix remote shards balancer when filtering throttled nodes ([#11724](https://github.com/opensearch-project/OpenSearch/pull/11724))
+- Add advance(int) for numeric values in order to allow point based optimization to kick in ([#12089](https://github.com/opensearch-project/OpenSearch/pull/12089))
 
 ### Security
 

build.gradle

@@ -545,6 +545,7 @@ subprojects {
         includeClasses.add("org.opensearch.snapshots.SnapshotStatusApisIT")
         includeClasses.add("org.opensearch.test.rest.ClientYamlTestSuiteIT")
         includeClasses.add("org.opensearch.upgrade.DetectEsInstallationTaskTests")
+        includeClasses.add("org.opensearch.cluster.MinimumClusterManagerNodesIT")
       }
     }
   }

distribution/src/config/opensearch.yml

@@ -117,12 +117,6 @@ ${path.logs}
 #opensearch.experimental.feature.extensions.enabled: false
 #
 #
-# Gates the concurrent segment search feature. This feature enables concurrent segment search in a separate
-# index searcher threadpool.
-#
-#opensearch.experimental.feature.concurrent_segment_search.enabled: false
-#
-#
 # Gates the optimization of datetime formatters caching along with change in default datetime formatter
 # Once there is no observed impact on performance, this feature flag can be removed.
 #

gradle/formatting.gradle

@@ -99,7 +99,9 @@ allprojects {
           }
         }
         format 'misc', {
-            target '*.md', '*.gradle', '**/*.yaml', '**/*.yml', '**/*.svg'   
+            target '*.md', '*.gradle', '**/*.json', '**/*.yaml', '**/*.yml', '**/*.svg'
+
+            targetExclude '**/simple-bulk11.json', '**/simple-msearch5.json'
 
             trimTrailingWhitespace()
             endWithNewline()

modules/analysis-common/src/internalClusterTest/java/org/opensearch/analysis/common/QueryStringWithAnalyzersIT.java

@@ -36,7 +36,6 @@
 
 import org.opensearch.action.search.SearchResponse;
 import org.opensearch.common.settings.Settings;
-import org.opensearch.common.util.FeatureFlags;
 import org.opensearch.index.query.Operator;
 import org.opensearch.plugins.Plugin;
 import org.opensearch.test.ParameterizedStaticSettingsOpenSearchIntegTestCase;
@@ -63,11 +62,6 @@ public static Collection<Object[]> parameters() {
         );
     }
 
-    @Override
-    protected Settings featureFlagSettings() {
-        return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build();
-    }
-
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
         return Arrays.asList(CommonAnalysisModulePlugin.class);

modules/analysis-common/src/test/java/org/opensearch/analysis/common/HighlighterWithAnalyzersTests.java

@@ -36,7 +36,6 @@
 
 import org.opensearch.action.search.SearchResponse;
 import org.opensearch.common.settings.Settings;
-import org.opensearch.common.util.FeatureFlags;
 import org.opensearch.common.xcontent.XContentFactory;
 import org.opensearch.core.xcontent.XContentBuilder;
 import org.opensearch.index.IndexSettings;
@@ -82,11 +81,6 @@ public static Collection<Object[]> parameters() {
         );
     }
 
-    @Override
-    protected Settings featureFlagSettings() {
-        return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build();
-    }
-
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
         return Arrays.asList(CommonAnalysisModulePlugin.class);

modules/geo/src/internalClusterTest/java/org/opensearch/geo/GeoModulePluginIntegTestCase.java

@@ -11,7 +11,6 @@
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
 import org.opensearch.common.settings.Settings;
-import org.opensearch.common.util.FeatureFlags;
 import org.opensearch.geometry.utils.StandardValidator;
 import org.opensearch.geometry.utils.WellKnownText;
 import org.opensearch.index.mapper.GeoShapeFieldMapper;
@@ -47,11 +46,6 @@ public static Collection<Object[]> parameters() {
         );
     }
 
-    @Override
-    protected Settings featureFlagSettings() {
-        return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build();
-    }
-
     /**
      * Returns a collection of plugins that should be loaded on each node for doing the integration tests. As this
      * geo plugin is not getting packaged in a zip, we need to load it before the tests run.

modules/lang-expression/src/internalClusterTest/java/org/opensearch/script/expression/MoreExpressionIT.java

@@ -41,7 +41,6 @@
 import org.opensearch.action.update.UpdateRequestBuilder;
 import org.opensearch.common.lucene.search.function.CombineFunction;
 import org.opensearch.common.settings.Settings;
-import org.opensearch.common.util.FeatureFlags;
 import org.opensearch.common.xcontent.XContentFactory;
 import org.opensearch.core.xcontent.XContentBuilder;
 import org.opensearch.index.query.QueryBuilders;
@@ -94,11 +93,6 @@ public static Collection<Object[]> parameters() {
         );
     }
 
-    @Override
-    protected Settings featureFlagSettings() {
-        return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build();
-    }
-
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
         return Collections.singleton(ExpressionModulePlugin.class);

modules/lang-expression/src/internalClusterTest/java/org/opensearch/script/expression/StoredExpressionIT.java

@@ -35,7 +35,6 @@
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
 import org.opensearch.common.settings.Settings;
-import org.opensearch.common.util.FeatureFlags;
 import org.opensearch.core.common.bytes.BytesArray;
 import org.opensearch.core.xcontent.MediaTypeRegistry;
 import org.opensearch.plugins.Plugin;
@@ -68,11 +67,6 @@ public static Collection<Object[]> parameters() {
         );
     }
 
-    @Override
-    protected Settings featureFlagSettings() {
-        return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build();
-    }
-
     @Override
     protected Settings nodeSettings(int nodeOrdinal) {
         Settings.Builder builder = Settings.builder().put(super.nodeSettings(nodeOrdinal));

modules/lang-mustache/src/internalClusterTest/java/org/opensearch/script/mustache/MultiSearchTemplateIT.java

@@ -37,7 +37,6 @@
 import org.opensearch.action.index.IndexRequestBuilder;
 import org.opensearch.action.search.SearchRequest;
 import org.opensearch.common.settings.Settings;
-import org.opensearch.common.util.FeatureFlags;
 import org.opensearch.index.IndexNotFoundException;
 import org.opensearch.plugins.Plugin;
 import org.opensearch.script.ScriptType;
@@ -72,11 +71,6 @@ public static Collection<Object[]> parameters() {
         );
     }
 
-    @Override
-    protected Settings featureFlagSettings() {
-        return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build();
-    }
-
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
         return Collections.singleton(MustacheModulePlugin.class);

modules/parent-join/src/internalClusterTest/java/org/opensearch/join/aggregations/ChildrenIT.java

@@ -41,7 +41,6 @@
 import org.opensearch.client.Requests;
 import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.common.settings.Settings;
-import org.opensearch.common.util.FeatureFlags;
 import org.opensearch.search.SearchHit;
 import org.opensearch.search.aggregations.AggregationBuilders;
 import org.opensearch.search.aggregations.InternalAggregation;
@@ -89,11 +88,6 @@ public static Collection<Object[]> parameters() {
         );
     }
 
-    @Override
-    protected Settings featureFlagSettings() {
-        return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build();
-    }
-
     public void testChildrenAggs() throws Exception {
         SearchResponse searchResponse = client().prepareSearch("test")
             .setQuery(matchQuery("randomized", true))

modules/parent-join/src/internalClusterTest/java/org/opensearch/join/aggregations/ParentIT.java

@@ -37,7 +37,6 @@
 import org.opensearch.action.search.SearchRequestBuilder;
 import org.opensearch.action.search.SearchResponse;
 import org.opensearch.common.settings.Settings;
-import org.opensearch.common.util.FeatureFlags;
 import org.opensearch.search.aggregations.Aggregation;
 import org.opensearch.search.aggregations.bucket.MultiBucketsAggregation;
 import org.opensearch.search.aggregations.bucket.terms.Terms;
@@ -76,11 +75,6 @@ public static Collection<Object[]> parameters() {
         );
     }
 
-    @Override
-    protected Settings featureFlagSettings() {
-        return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build();
-    }
-
     public void testSimpleParentAgg() throws Exception {
         final SearchRequestBuilder searchRequest = client().prepareSearch("test")
             .setSize(10000)

modules/parent-join/src/internalClusterTest/java/org/opensearch/join/query/ChildQuerySearchIT.java

@@ -44,7 +44,6 @@
 import org.opensearch.common.lucene.search.function.FunctionScoreQuery;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.unit.TimeValue;
-import org.opensearch.common.util.FeatureFlags;
 import org.opensearch.core.rest.RestStatus;
 import org.opensearch.index.query.BoolQueryBuilder;
 import org.opensearch.index.query.IdsQueryBuilder;
@@ -118,11 +117,6 @@ public static Collection<Object[]> parameters() {
         );
     }
 
-    @Override
-    protected Settings featureFlagSettings() {
-        return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build();
-    }
-
     public void testMultiLevelChild() throws Exception {
         assertAcked(
             prepareCreate("test").setMapping(

modules/parent-join/src/internalClusterTest/java/org/opensearch/join/query/InnerHitsIT.java

@@ -40,7 +40,6 @@
 import org.opensearch.action.search.SearchPhaseExecutionException;
 import org.opensearch.action.search.SearchResponse;
 import org.opensearch.common.settings.Settings;
-import org.opensearch.common.util.FeatureFlags;
 import org.opensearch.index.IndexSettings;
 import org.opensearch.index.query.BoolQueryBuilder;
 import org.opensearch.index.query.InnerHitBuilder;
@@ -105,11 +104,6 @@ public static Collection<Object[]> parameters() {
         );
     }
 
-    @Override
-    protected Settings featureFlagSettings() {
-        return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build();
-    }
-
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
         ArrayList<Class<? extends Plugin>> plugins = new ArrayList<>(super.nodePlugins());

modules/percolator/src/internalClusterTest/java/org/opensearch/percolator/PercolatorQuerySearchIT.java

@@ -41,7 +41,6 @@
 import org.opensearch.common.geo.GeoPoint;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.unit.DistanceUnit;
-import org.opensearch.common.util.FeatureFlags;
 import org.opensearch.common.xcontent.XContentFactory;
 import org.opensearch.common.xcontent.XContentType;
 import org.opensearch.core.common.bytes.BytesArray;
@@ -104,11 +103,6 @@ public static Collection<Object[]> parameters() {
         );
     }
 
-    @Override
-    protected Settings featureFlagSettings() {
-        return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build();
-    }
-
     @Override
     protected boolean addMockGeoShapeFieldMapper() {
         return false;

modules/rank-eval/src/internalClusterTest/java/org/opensearch/index/rankeval/RankEvalRequestIT.java

@@ -39,7 +39,6 @@
 import org.opensearch.action.search.SearchRequest;
 import org.opensearch.action.support.IndicesOptions;
 import org.opensearch.common.settings.Settings;
-import org.opensearch.common.util.FeatureFlags;
 import org.opensearch.index.IndexNotFoundException;
 import org.opensearch.index.query.MatchAllQueryBuilder;
 import org.opensearch.index.query.QueryBuilders;
@@ -80,11 +79,6 @@ public static Collection<Object[]> parameters() {
         );
     }
 
-    @Override
-    protected Settings featureFlagSettings() {
-        return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.CONCURRENT_SEGMENT_SEARCH, "true").build();
-    }
-
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
         return Arrays.asList(RankEvalModulePlugin.class);

plugins/repository-azure/src/internalClusterTest/java/org/opensearch/repositories/azure/AzureBlobStoreRepositoryTests.java

@@ -39,6 +39,8 @@
 import com.azure.storage.common.implementation.Constants;
 import com.azure.storage.common.policy.RequestRetryOptions;
 import com.azure.storage.common.policy.RetryPolicyType;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.opensearch.common.SuppressForbidden;
 import org.opensearch.common.regex.Regex;
 import org.opensearch.common.settings.MockSecureSettings;
@@ -188,6 +190,7 @@ protected String requestUniqueId(final HttpExchange exchange) {
     @SuppressForbidden(reason = "this test uses a HttpServer to emulate an Azure endpoint")
     private static class AzureHTTPStatsCollectorHandler extends HttpStatsCollectorHandler {
 
+        private static final Logger testLogger = LogManager.getLogger(AzureHTTPStatsCollectorHandler.class);
         private static final Pattern listPattern = Pattern.compile("GET /[a-zA-Z0-9]+\\??.+");
         private static final Pattern getPattern = Pattern.compile("GET /[^?/]+/[^?/]+\\??.*");
 
@@ -197,6 +200,7 @@ private AzureHTTPStatsCollectorHandler(HttpHandler delegate) {
 
         @Override
         protected void maybeTrack(String request, Headers headers) {
+            testLogger.info(request, headers);
             if (getPattern.matcher(request).matches()) {
                 trackRequest("GetBlob");
             } else if (Regex.simpleMatch("HEAD /*/*", request)) {

rest-api-spec/src/main/resources/rest-api-spec/test/cat.shards/10_basic.yml

@@ -1,7 +1,7 @@
 "Help":
   - skip:
       version: " - 2.11.99"
-      reason:  deleted docs added in 2.12.0
+      reason:  deleted docs and concurrent search are added in 2.12.0
       features: node_selector
   - do:
       cat.shards:
@@ -66,6 +66,10 @@
              search.query_current             .+   \n
              search.query_time                .+   \n
              search.query_total               .+   \n
+             search.concurrent_query_current  .+   \n
+             search.concurrent_query_time     .+   \n
+             search.concurrent_query_total    .+   \n
+             search.concurrent_avg_slice_count  .+   \n
              search.scroll_current            .+   \n
              search.scroll_time               .+   \n
              search.scroll_total              .+   \n

rest-api-spec/src/main/resources/rest-api-spec/test/cat.thread_pool/10_basic.yml

@@ -11,6 +11,33 @@
         /  #node_name     name     active     queue     rejected
         ^  (\S+       \s+ \S+  \s+ \d+    \s+ \d+   \s+ \d+      \n)+  $/
 
+  - do:
+      cat.thread_pool:
+        thread_pool_patterns: search,search_throttled,generic
+        h: name,total_wait_time,twt
+        v: true
+
+  - match:
+      $body: |
+        /^  name              \s+ total_wait_time \s+ twt              \n
+           (generic           \s+ -1              \s+ -1               \n
+            search            \s+ \d*\.*\d*\D+    \s+ \d*\.*\d*\D+     \n
+            search_throttled  \s+ \d*\.*\d*\D+    \s+ \d*\.*\d*\D+     \n)+  $/
+
+---
+"Test cat thread_pool total_wait_time output with concurrent search thread_pool":
+  - skip:
+      version: " - 2.11.99"
+      reason: index_search thread_pool was introduced in V_2.12.0
+
+  - do:
+      cat.thread_pool: {}
+
+  - match:
+      $body: |
+        /  #node_name     name     active     queue     rejected
+        ^  (\S+       \s+ \S+  \s+ \d+    \s+ \d+   \s+ \d+      \n)+  $/
+
   - do:
       cat.thread_pool:
         thread_pool_patterns: search,search_throttled,index_searcher,generic
@@ -21,6 +48,7 @@
       $body: |
         /^  name              \s+ total_wait_time \s+ twt              \n
            (generic           \s+ -1              \s+ -1               \n
+            index_searcher    \s+ \d*\.*\d*\D+    \s+ \d*\.*\d*\D+     \n
             search            \s+ \d*\.*\d*\D+    \s+ \d*\.*\d*\D+     \n
             search_throttled  \s+ \d*\.*\d*\D+    \s+ \d*\.*\d*\D+     \n)+  $/