From 9346a1c763230ae456dc111e2c8897e46c908661 Mon Sep 17 00:00:00 2001 From: Ganesh Krishna Ramadurai Date: Thu, 16 Nov 2023 09:59:44 -0800 Subject: [PATCH] Fix tests for concurrent search with IndexRandomForConcurrentSearch follow-up [#11130] (#11212) Signed-off-by: Ganesh Ramadurai Co-authored-by: Ganesh Ramadurai Signed-off-by: Shivansh Arora --- .../search/scroll/SearchScrollIT.java | 14 ++++++++-- .../opensearch/search/sort/FieldSortIT.java | 28 +++++++++++++++---- .../opensearch/search/sort/GeoDistanceIT.java | 3 ++ .../search/source/SourceFetchingIT.java | 6 ++-- .../search/suggest/SuggestSearchIT.java | 1 + 5 files changed, 43 insertions(+), 9 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/search/scroll/SearchScrollIT.java b/server/src/internalClusterTest/java/org/opensearch/search/scroll/SearchScrollIT.java index b2b6409580061..b82048ffc924e 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/scroll/SearchScrollIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/scroll/SearchScrollIT.java @@ -135,6 +135,7 @@ public void testSimpleScrollQueryThenFetch() throws Exception { } client().admin().indices().prepareRefresh().get(); + indexRandomForConcurrentSearch("test"); SearchResponse searchResponse = client().prepareSearch() .setQuery(matchAllQuery()) @@ -188,6 +189,7 @@ public void testSimpleScrollQueryThenFetchSmallSizeUnevenDistribution() throws E } client().admin().indices().prepareRefresh().get(); + indexRandomForConcurrentSearch("test"); SearchResponse searchResponse = client().prepareSearch() .setSearchType(SearchType.QUERY_THEN_FETCH) @@ -256,6 +258,7 @@ public void testScrollAndUpdateIndex() throws Exception { } client().admin().indices().prepareRefresh().get(); + indexRandomForConcurrentSearch("test"); assertThat(client().prepareSearch().setSize(0).setQuery(matchAllQuery()).get().getHits().getTotalHits().value, equalTo(500L)); assertThat( @@ -328,6 +331,7 @@ public void testSimpleScrollQueryThenFetch_clearScrollIds() throws Exception { } client().admin().indices().prepareRefresh().get(); + indexRandomForConcurrentSearch("test"); SearchResponse searchResponse1 = client().prepareSearch() .setQuery(matchAllQuery()) @@ -448,6 +452,7 @@ public void testSimpleScrollQueryThenFetchClearAllScrollIds() throws Exception { } client().admin().indices().prepareRefresh().get(); + indexRandomForConcurrentSearch("test"); SearchResponse searchResponse1 = client().prepareSearch() .setQuery(matchAllQuery()) @@ -526,6 +531,7 @@ public void testDeepScrollingDoesNotBlowUp() throws Exception { .prepareUpdateSettings("index") .setSettings(Settings.builder().put(IndexSettings.MAX_RESULT_WINDOW_SETTING.getKey(), Integer.MAX_VALUE)) .get(); + indexRandomForConcurrentSearch("index"); for (SearchType searchType : SearchType.values()) { SearchRequestBuilder builder = client().prepareSearch("index") @@ -567,6 +573,7 @@ public void testStringSortMissingAscTerminates() throws Exception { ); client().prepareIndex("test").setId("1").setSource("some_field", "test").get(); refresh(); + indexRandomForConcurrentSearch("test"); SearchResponse response = client().prepareSearch("test") .addSort(new FieldSortBuilder("no_field").order(SortOrder.ASC).missing("_last")) @@ -592,12 +599,13 @@ public void testStringSortMissingAscTerminates() throws Exception { assertThat(response.getHits().getHits().length, equalTo(0)); } - public void testCloseAndReopenOrDeleteWithActiveScroll() { + public void testCloseAndReopenOrDeleteWithActiveScroll() throws InterruptedException { createIndex("test"); for (int i = 0; i < 100; i++) { client().prepareIndex("test").setId(Integer.toString(i)).setSource("field", i).get(); } refresh(); + indexRandomForConcurrentSearch("test"); SearchResponse searchResponse = client().prepareSearch() .setQuery(matchAllQuery()) .setSize(35) @@ -683,7 +691,7 @@ public void testScrollInvalidDefaultKeepAlive() throws IOException { assertThat(exc.getMessage(), containsString("was (1m > 30s)")); } - public void testInvalidScrollKeepAlive() throws IOException { + public void testInvalidScrollKeepAlive() throws IOException, InterruptedException { createIndex("test"); for (int i = 0; i < 2; i++) { client().prepareIndex("test") @@ -692,6 +700,7 @@ public void testInvalidScrollKeepAlive() throws IOException { .get(); } refresh(); + indexRandomForConcurrentSearch("test"); assertAcked( client().admin() .cluster() @@ -794,6 +803,7 @@ public void testRestartDataNodesDuringScrollSearch() throws Exception { index("prod", "_doc", "prod-" + i, Collections.emptyMap()); } client().admin().indices().prepareRefresh().get(); + indexRandomForConcurrentSearch("demo", "prod"); SearchResponse respFromDemoIndex = client().prepareSearch("demo") .setSize(randomIntBetween(1, 10)) .setQuery(new MatchAllQueryBuilder()) diff --git a/server/src/internalClusterTest/java/org/opensearch/search/sort/FieldSortIT.java b/server/src/internalClusterTest/java/org/opensearch/search/sort/FieldSortIT.java index 9c788110af937..3c5574f823c3b 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/sort/FieldSortIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/sort/FieldSortIT.java @@ -154,7 +154,7 @@ protected Collection> nodePlugins() { return Arrays.asList(InternalSettingsPlugin.class, CustomScriptPlugin.class); } - public void testIssue8226() { + public void testIssue8226() throws InterruptedException { int numIndices = between(5, 10); final boolean useMapping = randomBoolean(); for (int i = 0; i < numIndices; i++) { @@ -168,6 +168,9 @@ public void testIssue8226() { } } refresh(); + for (int i = 0; i < numIndices; i++) { + indexRandomForConcurrentSearch("test_" + i); + } // sort DESC SearchResponse searchResponse = client().prepareSearch() .addSort(new FieldSortBuilder("entry").order(SortOrder.DESC).unmappedType(useMapping ? null : "long")) @@ -451,6 +454,7 @@ public void testScoreSortDirection() throws Exception { client().prepareIndex("test").setId("3").setSource("field", 0).get(); refresh(); + indexRandomForConcurrentSearch("test"); SearchResponse searchResponse = client().prepareSearch("test") .setQuery(QueryBuilders.functionScoreQuery(matchAllQuery(), ScoreFunctionBuilders.fieldValueFactorFunction("field"))) @@ -489,6 +493,7 @@ public void testScoreSortDirectionWithFunctionScore() throws Exception { client().prepareIndex("test").setId("3").setSource("field", 0).get(); refresh(); + indexRandomForConcurrentSearch("test"); SearchResponse searchResponse = client().prepareSearch("test") .setQuery(functionScoreQuery(matchAllQuery(), fieldValueFactorFunction("field"))) @@ -536,7 +541,7 @@ public void testIssue2986() { } } - public void testIssue2991() { + public void testIssue2991() throws InterruptedException { for (int i = 1; i < 4; i++) { try { client().admin().indices().prepareDelete("test").get(); @@ -557,6 +562,7 @@ public void testIssue2991() { client().prepareIndex("test").setId("2").setSource("tag", "beta").get(); refresh(); + indexRandomForConcurrentSearch("test"); SearchResponse resp = client().prepareSearch("test") .setSize(2) .setQuery(matchAllQuery()) @@ -655,6 +661,7 @@ public void testSimpleSorts() throws Exception { } refresh(); + indexRandomForConcurrentSearch("test"); // STRING int size = 1 + random.nextInt(10); @@ -927,6 +934,7 @@ public void testSortMissingNumbers() throws Exception { flush(); refresh(); + indexRandomForConcurrentSearch("test"); // DOUBLE logger.info("--> sort with no missing (same as missing _last)"); @@ -1088,6 +1096,7 @@ public void testSortMissingNumbersMinMax() throws Exception { flush(); refresh(); + indexRandomForConcurrentSearch("test"); // LONG logger.info("--> sort with no missing (same as missing _last)"); @@ -1207,7 +1216,7 @@ public void testSortMissingNumbersMinMax() throws Exception { assertThat(searchResponse.getHits().getAt(2).getId(), equalTo("3")); } - public void testSortMissingStrings() throws IOException { + public void testSortMissingStrings() throws IOException, InterruptedException { assertAcked( prepareCreate("test").setMapping( XContentFactory.jsonBuilder() @@ -1235,6 +1244,7 @@ public void testSortMissingStrings() throws IOException { flush(); refresh(); + indexRandomForConcurrentSearch("test"); // TODO: WTF? try { @@ -1423,6 +1433,7 @@ public void testSortMVField() throws Exception { .get(); refresh(); + indexRandomForConcurrentSearch("test"); SearchResponse searchResponse = client().prepareSearch() .setQuery(matchAllQuery()) @@ -1707,7 +1718,7 @@ public void testSortMVField() throws Exception { assertThat(searchResponse.getHits().getAt(2).getSortValues()[0], equalTo("03")); } - public void testSortOnRareField() throws IOException { + public void testSortOnRareField() throws IOException, InterruptedException { assertAcked( prepareCreate("test").setMapping( XContentFactory.jsonBuilder() @@ -1727,6 +1738,7 @@ public void testSortOnRareField() throws IOException { .get(); refresh(); + indexRandomForConcurrentSearch("test"); SearchResponse searchResponse = client().prepareSearch() .setQuery(matchAllQuery()) .setSize(3) @@ -1822,6 +1834,7 @@ public void testSortMetaField() throws Exception { indexReqs[i] = client().prepareIndex("test").setId(Integer.toString(i)).setSource(); } indexRandom(true, indexReqs); + indexRandomForConcurrentSearch("test"); SortOrder order = randomFrom(SortOrder.values()); SearchResponse searchResponse = client().prepareSearch() @@ -1924,6 +1937,7 @@ public void testNestedSort() throws IOException, InterruptedException, Execution ) .get(); refresh(); + indexRandomForConcurrentSearch("test"); // We sort on nested field SearchResponse searchResponse = client().prepareSearch() @@ -2109,6 +2123,7 @@ public void testFieldAlias() throws Exception { builders.add(client().prepareIndex("old_index").setSource("distance", 50.5)); builders.add(client().prepareIndex("new_index").setSource("route_length_miles", 100.2)); indexRandom(true, true, builders); + indexRandomForConcurrentSearch("old_index", "new_index"); SearchResponse response = client().prepareSearch() .setQuery(matchAllQuery()) @@ -2135,6 +2150,7 @@ public void testFieldAliasesWithMissingValues() throws Exception { builders.add(client().prepareIndex("old_index").setSource(Collections.emptyMap())); builders.add(client().prepareIndex("new_index").setSource("route_length_miles", 100.2)); indexRandom(true, true, builders); + indexRandomForConcurrentSearch("old_index", "new_index"); SearchResponse response = client().prepareSearch() .setQuery(matchAllQuery()) @@ -2204,6 +2220,7 @@ public void testCastDate() throws Exception { builders.add(client().prepareIndex("index_date").setSource("field", "2024-04-11T23:47:17")); builders.add(client().prepareIndex("index_date_nanos").setSource("field", "2024-04-11T23:47:16.854775807Z")); indexRandom(true, true, builders); + indexRandomForConcurrentSearch("index_date", "index_date_nanos"); { SearchResponse response = client().prepareSearch() @@ -2329,7 +2346,7 @@ public void testCastNumericTypeExceptions() throws Exception { } } - public void testLongSortOptimizationCorrectResults() { + public void testLongSortOptimizationCorrectResults() throws InterruptedException { assertAcked( prepareCreate("test1").setSettings(Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 2)) .setMapping("long_field", "type=long") @@ -2346,6 +2363,7 @@ public void testLongSortOptimizationCorrectResults() { bulkBuilder.add(client().prepareIndex("test1").setId(Integer.toString(i)).setSource(source, MediaTypeRegistry.JSON)); } refresh(); + indexRandomForConcurrentSearch("test1"); // *** 1. sort DESC on long_field SearchResponse searchResponse = client().prepareSearch() diff --git a/server/src/internalClusterTest/java/org/opensearch/search/sort/GeoDistanceIT.java b/server/src/internalClusterTest/java/org/opensearch/search/sort/GeoDistanceIT.java index ff24a1f962332..766ac6139b24b 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/sort/GeoDistanceIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/sort/GeoDistanceIT.java @@ -192,6 +192,7 @@ public void testDistanceSortingMVFields() throws Exception { .get(); client().admin().indices().prepareRefresh().get(); + indexRandomForConcurrentSearch("test"); // Order: Asc SearchResponse searchResponse = client().prepareSearch("test") @@ -324,6 +325,7 @@ public void testDistanceSortingWithMissingGeoPoint() throws Exception { .get(); refresh(); + indexRandomForConcurrentSearch("test"); // Order: Asc SearchResponse searchResponse = client().prepareSearch("test") @@ -669,6 +671,7 @@ public void testDistanceSortingWithUnmappedField() throws Exception { .get(); refresh(); + indexRandomForConcurrentSearch("test1", "test2"); // Order: Asc SearchResponse searchResponse = client().prepareSearch("test1", "test2") diff --git a/server/src/internalClusterTest/java/org/opensearch/search/source/SourceFetchingIT.java b/server/src/internalClusterTest/java/org/opensearch/search/source/SourceFetchingIT.java index 0e9932051141b..805e82dc9850b 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/source/SourceFetchingIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/source/SourceFetchingIT.java @@ -85,12 +85,13 @@ public void testSourceDefaultBehavior() throws InterruptedException { } - public void testSourceFiltering() { + public void testSourceFiltering() throws InterruptedException { createIndex("test"); ensureGreen(); client().prepareIndex("test").setId("1").setSource("field1", "value", "field2", "value2").get(); refresh(); + indexRandomForConcurrentSearch("test"); SearchResponse response = client().prepareSearch("test").setFetchSource(false).get(); assertThat(response.getHits().getAt(0).getSourceAsString(), nullValue()); @@ -118,12 +119,13 @@ public void testSourceFiltering() { * Test Case for #5132: Source filtering with wildcards broken when given multiple patterns * https://github.com/elastic/elasticsearch/issues/5132 */ - public void testSourceWithWildcardFiltering() { + public void testSourceWithWildcardFiltering() throws InterruptedException { createIndex("test"); ensureGreen(); client().prepareIndex("test").setId("1").setSource("field", "value").get(); refresh(); + indexRandomForConcurrentSearch("test"); SearchResponse response = client().prepareSearch("test").setFetchSource(new String[] { "*.notexisting", "field" }, null).get(); assertThat(response.getHits().getAt(0).getSourceAsString(), notNullValue()); diff --git a/server/src/internalClusterTest/java/org/opensearch/search/suggest/SuggestSearchIT.java b/server/src/internalClusterTest/java/org/opensearch/search/suggest/SuggestSearchIT.java index dd6179d3820be..bc6e49008bf6b 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/suggest/SuggestSearchIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/suggest/SuggestSearchIT.java @@ -370,6 +370,7 @@ public void testSimple() throws Exception { index("test", "type1", "3", "text", "abbd"); index("test", "type1", "4", "text", "abcc"); refresh(); + indexRandomForConcurrentSearch("test"); SearchResponse search = client().prepareSearch().setQuery(matchQuery("text", "spellcecker")).get(); assertThat("didn't ask for suggestions but got some", search.getSuggest(), nullValue());