From fe57058f0a2699380210ca1ea8c345885f719112 Mon Sep 17 00:00:00 2001 From: Austin Lee Date: Mon, 2 Oct 2023 13:43:52 -0700 Subject: [PATCH 01/32] Enable extensions. Signed-off-by: Austin Lee --- distribution/src/config/log4j2.properties | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/distribution/src/config/log4j2.properties b/distribution/src/config/log4j2.properties index d040afae82e53..87b875587952e 100644 --- a/distribution/src/config/log4j2.properties +++ b/distribution/src/config/log4j2.properties @@ -14,7 +14,9 @@ status = error appender.console.type = Console appender.console.name = console appender.console.layout.type = PatternLayout -appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker %m%n +appender.console.layout.pattern = %style{[%d{ISO8601}]}{FG_Red BG_White} %highlight{[%-5p][%-25c{1.}]} %style{[%node_name]}{FG_Magenta BG_White} %style{<%t>}{bright green} %style{[%M]}{bright magenta} : %marker %style{%m}{FG_Black BG_Yellow}%n + +#[%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker %m%n ######## Server JSON ############################ appender.rolling.type = RollingFile @@ -65,7 +67,7 @@ appender.rolling_old.strategy.action.condition.nested_condition.type = IfAccumul appender.rolling_old.strategy.action.condition.nested_condition.exceeds = 2GB ################################################ -rootLogger.level = info +rootLogger.level = debug rootLogger.appenderRef.console.ref = console rootLogger.appenderRef.rolling.ref = rolling rootLogger.appenderRef.rolling_old.ref = rolling_old From bcb2dec88c7c8b4db4aa4470f820bdfadbd659b7 Mon Sep 17 00:00:00 2001 From: Rishikesh Pasham <62345295+Rishikesh1159@users.noreply.github.com> Date: Thu, 5 Oct 2023 13:56:09 -0700 Subject: [PATCH 02/32] [Remote Store] Fix stats reporting for multistream downloads. (#10402) * Fix stats reporting for multistream downloads. Signed-off-by: Rishikesh1159 * rename tracker to fileTransferTracker. Signed-off-by: Rishikesh1159 --------- Signed-off-by: Rishikesh1159 --- server/src/main/java/org/opensearch/index/shard/IndexShard.java | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java index cbb246219546b..0b57272093ff5 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexShard.java @@ -160,6 +160,7 @@ import org.opensearch.index.seqno.SequenceNumbers; import org.opensearch.index.shard.PrimaryReplicaSyncer.ResyncTask; import org.opensearch.index.similarity.SimilarityService; +import org.opensearch.index.store.DirectoryFileTransferTracker; import org.opensearch.index.store.RemoteSegmentStoreDirectory; import org.opensearch.index.store.RemoteStoreFileDownloader; import org.opensearch.index.store.Store; From 245c056eb7f088ac73359678640c5a4391282d91 Mon Sep 17 00:00:00 2001 From: Jay Deng Date: Fri, 6 Oct 2023 14:31:14 -0700 Subject: [PATCH 03/32] Mute flaky concurrent segment search tests pending fixes (#10437) * DiversifiedSamplerIT.testNestedSamples * QueryProfilePhaseTests.testMaxScore * QueryProfilePhaseTests.testCollapseQuerySearchResults * HighlighterSearchIT.testHighlightQueryRewriteDatesWithNow * FieldCapabilitiesIT.testWithIndexFilter * QueryProfilePhaseTests.testDisableTopScoreCollection Signed-off-by: Jay Deng Signed-off-by: Jay Deng --- .../search/query/QueryProfilePhaseTests.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/server/src/test/java/org/opensearch/search/query/QueryProfilePhaseTests.java b/server/src/test/java/org/opensearch/search/query/QueryProfilePhaseTests.java index 6af04e15acef0..7dd3a89b2009d 100644 --- a/server/src/test/java/org/opensearch/search/query/QueryProfilePhaseTests.java +++ b/server/src/test/java/org/opensearch/search/query/QueryProfilePhaseTests.java @@ -1102,6 +1102,10 @@ public void testIndexSortScrollOptimization() throws Exception { } public void testDisableTopScoreCollection() throws Exception { + assumeFalse( + "Concurrent search case muted pending fix: https://github.com/opensearch-project/OpenSearch/issues/10469", + executor != null + ); Directory dir = newDirectory(); IndexWriterConfig iwc = newIndexWriterConfig(new StandardAnalyzer()); RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc); @@ -1273,6 +1277,10 @@ public void testMinScore() throws Exception { } public void testMaxScore() throws Exception { + assumeFalse( + "Concurrent search case muted pending fix: https://github.com/opensearch-project/OpenSearch/issues/9932", + executor != null + ); Directory dir = newDirectory(); final Sort sort = new Sort(new SortField("filter", SortField.Type.STRING)); IndexWriterConfig iwc = newIndexWriterConfig().setIndexSort(sort); @@ -1392,6 +1400,10 @@ public void testMaxScore() throws Exception { } public void testCollapseQuerySearchResults() throws Exception { + assumeFalse( + "Concurrent search case muted pending fix: https://github.com/opensearch-project/OpenSearch/issues/10139", + executor != null + ); Directory dir = newDirectory(); final Sort sort = new Sort(new SortField("user", SortField.Type.INT)); IndexWriterConfig iwc = newIndexWriterConfig().setIndexSort(sort); From 31efd2fc1a82caa1ec0b7719f247e8d432c7635b Mon Sep 17 00:00:00 2001 From: Andrew Ross Date: Tue, 10 Oct 2023 19:54:15 -0500 Subject: [PATCH 04/32] Replace multipart download with parallel file download (#10519) There are a few open issues with the multi-stream download approach: - Recovery stats are not being reported correctly - It is incompatible (short of reopening and re-reading the entire file) with the existing Lucene checksum validation logic - There are some issues with integrating it with the pending client side encryption work Given this, I attempted an experiment where I replaced with multi-stream-within-a-single-file approach with simply parallelizing downloads across files (this is how snapshot restore works). I actually got better results with this approach: recovering a ~52GiB shard took about 4.7 minutes with the multi-stream code versus 3.9 minutes with the parallel file approach (r7g.4xlarge EC2 instance, 500MiB/s EBS volume, S3 as remote repository). I think this is the right approach as it leverages the more battle-tested code path and addresses the three issues listed above. The multi-stream approach still has promise as it will allow us to download very large files faster (whereas this approach they can be the long poll on the transfer operation). However, given that 5GB segments (made up of multiple files in practice) are the norm, we generally aren't dealing with huge files. Signed-off-by: Andrew Ross --- server/src/main/java/org/opensearch/index/shard/IndexShard.java | 1 - 1 file changed, 1 deletion(-) diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java index 0b57272093ff5..cbb246219546b 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexShard.java @@ -160,7 +160,6 @@ import org.opensearch.index.seqno.SequenceNumbers; import org.opensearch.index.shard.PrimaryReplicaSyncer.ResyncTask; import org.opensearch.index.similarity.SimilarityService; -import org.opensearch.index.store.DirectoryFileTransferTracker; import org.opensearch.index.store.RemoteSegmentStoreDirectory; import org.opensearch.index.store.RemoteStoreFileDownloader; import org.opensearch.index.store.Store; From 9e919a99fe55ed47f5d9e3ffaca1ae5588b4917b Mon Sep 17 00:00:00 2001 From: Ticheng Lin <51488860+ticheng-aws@users.noreply.github.com> Date: Wed, 11 Oct 2023 07:31:18 -0700 Subject: [PATCH 05/32] Fix flaky query profile phase tests with concurrent search enabled (#10547) (#10547) Signed-off-by: Ticheng Lin --- .../search/query/QueryProfilePhaseTests.java | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/server/src/test/java/org/opensearch/search/query/QueryProfilePhaseTests.java b/server/src/test/java/org/opensearch/search/query/QueryProfilePhaseTests.java index 7dd3a89b2009d..6af04e15acef0 100644 --- a/server/src/test/java/org/opensearch/search/query/QueryProfilePhaseTests.java +++ b/server/src/test/java/org/opensearch/search/query/QueryProfilePhaseTests.java @@ -1102,10 +1102,6 @@ public void testIndexSortScrollOptimization() throws Exception { } public void testDisableTopScoreCollection() throws Exception { - assumeFalse( - "Concurrent search case muted pending fix: https://github.com/opensearch-project/OpenSearch/issues/10469", - executor != null - ); Directory dir = newDirectory(); IndexWriterConfig iwc = newIndexWriterConfig(new StandardAnalyzer()); RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc); @@ -1277,10 +1273,6 @@ public void testMinScore() throws Exception { } public void testMaxScore() throws Exception { - assumeFalse( - "Concurrent search case muted pending fix: https://github.com/opensearch-project/OpenSearch/issues/9932", - executor != null - ); Directory dir = newDirectory(); final Sort sort = new Sort(new SortField("filter", SortField.Type.STRING)); IndexWriterConfig iwc = newIndexWriterConfig().setIndexSort(sort); @@ -1400,10 +1392,6 @@ public void testMaxScore() throws Exception { } public void testCollapseQuerySearchResults() throws Exception { - assumeFalse( - "Concurrent search case muted pending fix: https://github.com/opensearch-project/OpenSearch/issues/10139", - executor != null - ); Directory dir = newDirectory(); final Sort sort = new Sort(new SortField("user", SortField.Type.INT)); IndexWriterConfig iwc = newIndexWriterConfig().setIndexSort(sort); From f93fab4bb47c4bb96f3562a9f575bfadb1c5ecc8 Mon Sep 17 00:00:00 2001 From: Louis Chu Date: Thu, 12 Oct 2023 17:31:25 -0400 Subject: [PATCH 06/32] Add release notes for 2.11 (#10594) Signed-off-by: Louis Chu --- release-notes/opensearch.release-notes-2.11.0.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release-notes/opensearch.release-notes-2.11.0.md b/release-notes/opensearch.release-notes-2.11.0.md index 040cc053469ed..297de2577f89e 100644 --- a/release-notes/opensearch.release-notes-2.11.0.md +++ b/release-notes/opensearch.release-notes-2.11.0.md @@ -5,7 +5,7 @@ ### Added - Add coordinator level stats for search latency ([#8386](https://github.com/opensearch-project/OpenSearch/issues/8386)) - Add metrics for thread_pool task wait time ([#9681](https://github.com/opensearch-project/OpenSearch/pull/9681)) -- Add parallel file download support for remote store based replication ([#8596](https://github.com/opensearch-project/OpenSearch/pull/8596)) +- Add parallel file download support for remote store based replication ([#8596](https://github.com/opensearch-project/OpenSearch/pull/8596)) - Async blob read support for S3 plugin ([#9694](https://github.com/opensearch-project/OpenSearch/pull/9694)) - [Telemetry-Otel] Added support for OtlpGrpcSpanExporter exporter ([#9666](https://github.com/opensearch-project/OpenSearch/pull/9666)) - Async blob read support for encrypted containers ([#10131](https://github.com/opensearch-project/OpenSearch/pull/10131)) From 508a7f8baae9cb79fd51f9721383635e99d87d88 Mon Sep 17 00:00:00 2001 From: David Z <38449481+dzane17@users.noreply.github.com> Date: Fri, 13 Oct 2023 14:12:46 -0700 Subject: [PATCH 07/32] Per request phase latency (#10351) Signed-off-by: David Zane --- .../java/org/opensearch/action/search/SearchRequestTests.java | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/test/java/org/opensearch/action/search/SearchRequestTests.java b/server/src/test/java/org/opensearch/action/search/SearchRequestTests.java index f025e3a63b9bf..2896c30cc49be 100644 --- a/server/src/test/java/org/opensearch/action/search/SearchRequestTests.java +++ b/server/src/test/java/org/opensearch/action/search/SearchRequestTests.java @@ -244,6 +244,7 @@ private SearchRequest mutate(SearchRequest searchRequest) { ); mutators.add(() -> mutation.source(randomValueOtherThan(searchRequest.source(), this::createSearchSourceBuilder))); mutators.add(() -> mutation.setCcsMinimizeRoundtrips(searchRequest.isCcsMinimizeRoundtrips() == false)); + mutators.add(() -> mutation.setPhaseTook(searchRequest.isPhaseTook() == false)); mutators.add( () -> mutation.setPhaseTook(searchRequest.isPhaseTook() == null ? randomBoolean() : searchRequest.isPhaseTook() == false) ); From 2f3a3d7cbddca36559b5d64d75bea70c5b8bc060 Mon Sep 17 00:00:00 2001 From: David Z <38449481+dzane17@users.noreply.github.com> Date: Tue, 17 Oct 2023 08:29:30 -0700 Subject: [PATCH 08/32] Fix flaky testEqualsAndHashcode in SearchRequestTests (#10649) Signed-off-by: David Zane --- .../java/org/opensearch/action/search/SearchRequestTests.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/server/src/test/java/org/opensearch/action/search/SearchRequestTests.java b/server/src/test/java/org/opensearch/action/search/SearchRequestTests.java index 2896c30cc49be..d153c57f6ef3b 100644 --- a/server/src/test/java/org/opensearch/action/search/SearchRequestTests.java +++ b/server/src/test/java/org/opensearch/action/search/SearchRequestTests.java @@ -244,7 +244,9 @@ private SearchRequest mutate(SearchRequest searchRequest) { ); mutators.add(() -> mutation.source(randomValueOtherThan(searchRequest.source(), this::createSearchSourceBuilder))); mutators.add(() -> mutation.setCcsMinimizeRoundtrips(searchRequest.isCcsMinimizeRoundtrips() == false)); - mutators.add(() -> mutation.setPhaseTook(searchRequest.isPhaseTook() == false)); + mutators.add( + () -> mutation.setPhaseTook(searchRequest.isPhaseTook() == null ? randomBoolean() : searchRequest.isPhaseTook() == false) + ); mutators.add( () -> mutation.setPhaseTook(searchRequest.isPhaseTook() == null ? randomBoolean() : searchRequest.isPhaseTook() == false) ); From b0929060711b4df5107700fa53435b1c506fd5ba Mon Sep 17 00:00:00 2001 From: Varun Bansal Date: Wed, 18 Oct 2023 17:11:06 +0530 Subject: [PATCH 09/32] revive remote cluster state auto restore integ tests (#10503) * revive remote cluster state auto restore integ tests Signed-off-by: bansvaru --- .../opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java | 1 - 1 file changed, 1 deletion(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java index c61e2ec6e4f6c..c9cd82326dff9 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java @@ -33,7 +33,6 @@ import java.util.Locale; import java.util.Map; import java.util.Objects; -import java.util.concurrent.ExecutionException; import static org.opensearch.cluster.coordination.ClusterBootstrapService.INITIAL_CLUSTER_MANAGER_NODES_SETTING; import static org.opensearch.cluster.metadata.IndexMetadata.INDEX_READ_ONLY_SETTING; From 52b8d1adadab1a76fc65bf861dd5f83b9454e14e Mon Sep 17 00:00:00 2001 From: Varun Bansal Date: Wed, 18 Oct 2023 23:11:53 +0530 Subject: [PATCH 10/32] Remote Restore Cluster Metadata if local disk state lost after quorum loss recovery (#10447) * Remote Restore Cluster Metadata if local disk state lost after quorum loss Signed-off-by: bansvaru --- .../opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java index c9cd82326dff9..c61e2ec6e4f6c 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java @@ -33,6 +33,7 @@ import java.util.Locale; import java.util.Map; import java.util.Objects; +import java.util.concurrent.ExecutionException; import static org.opensearch.cluster.coordination.ClusterBootstrapService.INITIAL_CLUSTER_MANAGER_NODES_SETTING; import static org.opensearch.cluster.metadata.IndexMetadata.INDEX_READ_ONLY_SETTING; From b242aa173de729dec5711e03f41e553d0982ef85 Mon Sep 17 00:00:00 2001 From: Austin Lee Date: Sun, 10 Sep 2023 20:54:35 -0700 Subject: [PATCH 11/32] Bump JNA version from 5.5 to 5.13. Signed-off-by: Austin Lee Update changelog. Signed-off-by: Austin Lee --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 822e4196adf5e..27c4009f93ac8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -240,4 +240,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Security [Unreleased 3.0]: https://github.com/opensearch-project/OpenSearch/compare/2.x...HEAD +<<<<<<< HEAD [Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.12...2.x +======= +[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.11...2.x +>>>>>>> 37a5dd041f2 (Bump JNA version from 5.5 to 5.13.) From d43c8656484996f265d557f6007ddb3c74faad57 Mon Sep 17 00:00:00 2001 From: Rishikesh Pasham <62345295+Rishikesh1159@users.noreply.github.com> Date: Thu, 5 Oct 2023 13:56:09 -0700 Subject: [PATCH 12/32] [Remote Store] Fix stats reporting for multistream downloads. (#10402) * Fix stats reporting for multistream downloads. Signed-off-by: Rishikesh1159 * rename tracker to fileTransferTracker. Signed-off-by: Rishikesh1159 --------- Signed-off-by: Rishikesh1159 --- server/src/main/java/org/opensearch/index/shard/IndexShard.java | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java index cbb246219546b..0b57272093ff5 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexShard.java @@ -160,6 +160,7 @@ import org.opensearch.index.seqno.SequenceNumbers; import org.opensearch.index.shard.PrimaryReplicaSyncer.ResyncTask; import org.opensearch.index.similarity.SimilarityService; +import org.opensearch.index.store.DirectoryFileTransferTracker; import org.opensearch.index.store.RemoteSegmentStoreDirectory; import org.opensearch.index.store.RemoteStoreFileDownloader; import org.opensearch.index.store.Store; From 8c2c617e3c6b19d4d5ef85ecbce8f50233b42b4a Mon Sep 17 00:00:00 2001 From: Jay Deng Date: Fri, 6 Oct 2023 14:31:14 -0700 Subject: [PATCH 13/32] Mute flaky concurrent segment search tests pending fixes (#10437) * DiversifiedSamplerIT.testNestedSamples * QueryProfilePhaseTests.testMaxScore * QueryProfilePhaseTests.testCollapseQuerySearchResults * HighlighterSearchIT.testHighlightQueryRewriteDatesWithNow * FieldCapabilitiesIT.testWithIndexFilter * QueryProfilePhaseTests.testDisableTopScoreCollection Signed-off-by: Jay Deng Signed-off-by: Jay Deng --- .../search/query/QueryProfilePhaseTests.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/server/src/test/java/org/opensearch/search/query/QueryProfilePhaseTests.java b/server/src/test/java/org/opensearch/search/query/QueryProfilePhaseTests.java index 6af04e15acef0..7dd3a89b2009d 100644 --- a/server/src/test/java/org/opensearch/search/query/QueryProfilePhaseTests.java +++ b/server/src/test/java/org/opensearch/search/query/QueryProfilePhaseTests.java @@ -1102,6 +1102,10 @@ public void testIndexSortScrollOptimization() throws Exception { } public void testDisableTopScoreCollection() throws Exception { + assumeFalse( + "Concurrent search case muted pending fix: https://github.com/opensearch-project/OpenSearch/issues/10469", + executor != null + ); Directory dir = newDirectory(); IndexWriterConfig iwc = newIndexWriterConfig(new StandardAnalyzer()); RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc); @@ -1273,6 +1277,10 @@ public void testMinScore() throws Exception { } public void testMaxScore() throws Exception { + assumeFalse( + "Concurrent search case muted pending fix: https://github.com/opensearch-project/OpenSearch/issues/9932", + executor != null + ); Directory dir = newDirectory(); final Sort sort = new Sort(new SortField("filter", SortField.Type.STRING)); IndexWriterConfig iwc = newIndexWriterConfig().setIndexSort(sort); @@ -1392,6 +1400,10 @@ public void testMaxScore() throws Exception { } public void testCollapseQuerySearchResults() throws Exception { + assumeFalse( + "Concurrent search case muted pending fix: https://github.com/opensearch-project/OpenSearch/issues/10139", + executor != null + ); Directory dir = newDirectory(); final Sort sort = new Sort(new SortField("user", SortField.Type.INT)); IndexWriterConfig iwc = newIndexWriterConfig().setIndexSort(sort); From 36633013c5fadbc62926a3b4edb162bff58aba22 Mon Sep 17 00:00:00 2001 From: Andrew Ross Date: Tue, 10 Oct 2023 19:54:15 -0500 Subject: [PATCH 14/32] Replace multipart download with parallel file download (#10519) There are a few open issues with the multi-stream download approach: - Recovery stats are not being reported correctly - It is incompatible (short of reopening and re-reading the entire file) with the existing Lucene checksum validation logic - There are some issues with integrating it with the pending client side encryption work Given this, I attempted an experiment where I replaced with multi-stream-within-a-single-file approach with simply parallelizing downloads across files (this is how snapshot restore works). I actually got better results with this approach: recovering a ~52GiB shard took about 4.7 minutes with the multi-stream code versus 3.9 minutes with the parallel file approach (r7g.4xlarge EC2 instance, 500MiB/s EBS volume, S3 as remote repository). I think this is the right approach as it leverages the more battle-tested code path and addresses the three issues listed above. The multi-stream approach still has promise as it will allow us to download very large files faster (whereas this approach they can be the long poll on the transfer operation). However, given that 5GB segments (made up of multiple files in practice) are the norm, we generally aren't dealing with huge files. Signed-off-by: Andrew Ross --- server/src/main/java/org/opensearch/index/shard/IndexShard.java | 1 - 1 file changed, 1 deletion(-) diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java index 0b57272093ff5..cbb246219546b 100644 --- a/server/src/main/java/org/opensearch/index/shard/IndexShard.java +++ b/server/src/main/java/org/opensearch/index/shard/IndexShard.java @@ -160,7 +160,6 @@ import org.opensearch.index.seqno.SequenceNumbers; import org.opensearch.index.shard.PrimaryReplicaSyncer.ResyncTask; import org.opensearch.index.similarity.SimilarityService; -import org.opensearch.index.store.DirectoryFileTransferTracker; import org.opensearch.index.store.RemoteSegmentStoreDirectory; import org.opensearch.index.store.RemoteStoreFileDownloader; import org.opensearch.index.store.Store; From 3524c85afd7890693241a6489b9682ae9e1df102 Mon Sep 17 00:00:00 2001 From: Ticheng Lin <51488860+ticheng-aws@users.noreply.github.com> Date: Wed, 11 Oct 2023 07:31:18 -0700 Subject: [PATCH 15/32] Fix flaky query profile phase tests with concurrent search enabled (#10547) (#10547) Signed-off-by: Ticheng Lin --- .../search/query/QueryProfilePhaseTests.java | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/server/src/test/java/org/opensearch/search/query/QueryProfilePhaseTests.java b/server/src/test/java/org/opensearch/search/query/QueryProfilePhaseTests.java index 7dd3a89b2009d..6af04e15acef0 100644 --- a/server/src/test/java/org/opensearch/search/query/QueryProfilePhaseTests.java +++ b/server/src/test/java/org/opensearch/search/query/QueryProfilePhaseTests.java @@ -1102,10 +1102,6 @@ public void testIndexSortScrollOptimization() throws Exception { } public void testDisableTopScoreCollection() throws Exception { - assumeFalse( - "Concurrent search case muted pending fix: https://github.com/opensearch-project/OpenSearch/issues/10469", - executor != null - ); Directory dir = newDirectory(); IndexWriterConfig iwc = newIndexWriterConfig(new StandardAnalyzer()); RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc); @@ -1277,10 +1273,6 @@ public void testMinScore() throws Exception { } public void testMaxScore() throws Exception { - assumeFalse( - "Concurrent search case muted pending fix: https://github.com/opensearch-project/OpenSearch/issues/9932", - executor != null - ); Directory dir = newDirectory(); final Sort sort = new Sort(new SortField("filter", SortField.Type.STRING)); IndexWriterConfig iwc = newIndexWriterConfig().setIndexSort(sort); @@ -1400,10 +1392,6 @@ public void testMaxScore() throws Exception { } public void testCollapseQuerySearchResults() throws Exception { - assumeFalse( - "Concurrent search case muted pending fix: https://github.com/opensearch-project/OpenSearch/issues/10139", - executor != null - ); Directory dir = newDirectory(); final Sort sort = new Sort(new SortField("user", SortField.Type.INT)); IndexWriterConfig iwc = newIndexWriterConfig().setIndexSort(sort); From 1f56c7c4f815407a296fa7051aab0a712c1a5a00 Mon Sep 17 00:00:00 2001 From: David Z <38449481+dzane17@users.noreply.github.com> Date: Fri, 13 Oct 2023 14:12:46 -0700 Subject: [PATCH 16/32] Per request phase latency (#10351) Signed-off-by: David Zane --- .../java/org/opensearch/action/search/SearchRequestTests.java | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/test/java/org/opensearch/action/search/SearchRequestTests.java b/server/src/test/java/org/opensearch/action/search/SearchRequestTests.java index d153c57f6ef3b..46d8c3f381d4f 100644 --- a/server/src/test/java/org/opensearch/action/search/SearchRequestTests.java +++ b/server/src/test/java/org/opensearch/action/search/SearchRequestTests.java @@ -244,6 +244,7 @@ private SearchRequest mutate(SearchRequest searchRequest) { ); mutators.add(() -> mutation.source(randomValueOtherThan(searchRequest.source(), this::createSearchSourceBuilder))); mutators.add(() -> mutation.setCcsMinimizeRoundtrips(searchRequest.isCcsMinimizeRoundtrips() == false)); + mutators.add(() -> mutation.setPhaseTook(searchRequest.isPhaseTook() == false)); mutators.add( () -> mutation.setPhaseTook(searchRequest.isPhaseTook() == null ? randomBoolean() : searchRequest.isPhaseTook() == false) ); From 0605136bbce0ff898e2cbfa01533217c0389f581 Mon Sep 17 00:00:00 2001 From: David Z <38449481+dzane17@users.noreply.github.com> Date: Tue, 17 Oct 2023 08:29:30 -0700 Subject: [PATCH 17/32] Fix flaky testEqualsAndHashcode in SearchRequestTests (#10649) Signed-off-by: David Zane --- .../java/org/opensearch/action/search/SearchRequestTests.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/server/src/test/java/org/opensearch/action/search/SearchRequestTests.java b/server/src/test/java/org/opensearch/action/search/SearchRequestTests.java index 46d8c3f381d4f..c3c6a52688973 100644 --- a/server/src/test/java/org/opensearch/action/search/SearchRequestTests.java +++ b/server/src/test/java/org/opensearch/action/search/SearchRequestTests.java @@ -244,7 +244,9 @@ private SearchRequest mutate(SearchRequest searchRequest) { ); mutators.add(() -> mutation.source(randomValueOtherThan(searchRequest.source(), this::createSearchSourceBuilder))); mutators.add(() -> mutation.setCcsMinimizeRoundtrips(searchRequest.isCcsMinimizeRoundtrips() == false)); - mutators.add(() -> mutation.setPhaseTook(searchRequest.isPhaseTook() == false)); + mutators.add( + () -> mutation.setPhaseTook(searchRequest.isPhaseTook() == null ? randomBoolean() : searchRequest.isPhaseTook() == false) + ); mutators.add( () -> mutation.setPhaseTook(searchRequest.isPhaseTook() == null ? randomBoolean() : searchRequest.isPhaseTook() == false) ); From aeeab2b6e0dd5d7feaa6d70534bbb8f44fdf9577 Mon Sep 17 00:00:00 2001 From: Varun Bansal Date: Wed, 18 Oct 2023 17:11:06 +0530 Subject: [PATCH 18/32] revive remote cluster state auto restore integ tests (#10503) * revive remote cluster state auto restore integ tests Signed-off-by: bansvaru --- .../opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java | 1 - 1 file changed, 1 deletion(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java index c61e2ec6e4f6c..c9cd82326dff9 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java @@ -33,7 +33,6 @@ import java.util.Locale; import java.util.Map; import java.util.Objects; -import java.util.concurrent.ExecutionException; import static org.opensearch.cluster.coordination.ClusterBootstrapService.INITIAL_CLUSTER_MANAGER_NODES_SETTING; import static org.opensearch.cluster.metadata.IndexMetadata.INDEX_READ_ONLY_SETTING; From d21f5f1032190c9d5446b2e2f1436f08f02e6371 Mon Sep 17 00:00:00 2001 From: Varun Bansal Date: Wed, 18 Oct 2023 23:11:53 +0530 Subject: [PATCH 19/32] Remote Restore Cluster Metadata if local disk state lost after quorum loss recovery (#10447) * Remote Restore Cluster Metadata if local disk state lost after quorum loss Signed-off-by: bansvaru --- .../opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java index c9cd82326dff9..c61e2ec6e4f6c 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java @@ -33,6 +33,7 @@ import java.util.Locale; import java.util.Map; import java.util.Objects; +import java.util.concurrent.ExecutionException; import static org.opensearch.cluster.coordination.ClusterBootstrapService.INITIAL_CLUSTER_MANAGER_NODES_SETTING; import static org.opensearch.cluster.metadata.IndexMetadata.INDEX_READ_ONLY_SETTING; From 0567dd4352fa4637d4b0316833a0dc20ec451656 Mon Sep 17 00:00:00 2001 From: Austin Lee Date: Sun, 22 Oct 2023 15:25:29 -0700 Subject: [PATCH 20/32] Fix a flaky test that relies on terminate_after for an exact count match. (Issue # 10435) Signed-off-by: Austin Lee --- .../java/org/opensearch/search/simple/SimpleSearchIT.java | 8 ++++++-- .../opensearch/test/hamcrest/OpenSearchAssertions.java | 7 +++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/search/simple/SimpleSearchIT.java b/server/src/internalClusterTest/java/org/opensearch/search/simple/SimpleSearchIT.java index be666dbc304b5..cca0ce86418d1 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/simple/SimpleSearchIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/simple/SimpleSearchIT.java @@ -305,7 +305,12 @@ public void dotestSimpleTerminateAfterCountWithSize(int size, int max) throws Ex .setSize(size) .setTrackTotalHits(true) .get(); - assertHitCount(searchResponse, i); + + if (size == 0) { + assertHitCount(searchResponse, i, max); + } else { + assertHitCount(searchResponse, i); + } assertTrue(searchResponse.isTerminatedEarly()); assertEquals(Math.min(i, size), searchResponse.getHits().getHits().length); } @@ -319,7 +324,6 @@ public void dotestSimpleTerminateAfterCountWithSize(int size, int max) throws Ex assertFalse(searchResponse.isTerminatedEarly()); } - @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/10435") public void testSimpleTerminateAfterCountSize0() throws Exception { int max = randomIntBetween(3, 29); dotestSimpleTerminateAfterCountWithSize(0, max); diff --git a/test/framework/src/main/java/org/opensearch/test/hamcrest/OpenSearchAssertions.java b/test/framework/src/main/java/org/opensearch/test/hamcrest/OpenSearchAssertions.java index 183214c159c14..2f4a393b2f326 100644 --- a/test/framework/src/main/java/org/opensearch/test/hamcrest/OpenSearchAssertions.java +++ b/test/framework/src/main/java/org/opensearch/test/hamcrest/OpenSearchAssertions.java @@ -304,6 +304,13 @@ public static void assertHitCount(SearchResponse countResponse, long expectedHit } } + public static void assertHitCount(SearchResponse countResponse, long minHitCount, long maxHitCount) { + final TotalHits totalHits = countResponse.getHits().getTotalHits(); + if (!(totalHits.relation == TotalHits.Relation.EQUAL_TO && totalHits.value >= minHitCount && totalHits.value <= maxHitCount)) { + fail("Count is " + totalHits + " not between " + minHitCount + " and " + maxHitCount + " inclusive. " + formatShardStatus(countResponse)); + } + } + public static void assertExists(GetResponse response) { String message = String.format(Locale.ROOT, "Expected %s/%s to exist, but does not", response.getIndex(), response.getId()); assertThat(message, response.isExists(), is(true)); From 63c556d075adb28bb71f02ddb9a42bc28ace828a Mon Sep 17 00:00:00 2001 From: Austin Lee Date: Mon, 23 Oct 2023 06:48:15 -0700 Subject: [PATCH 21/32] Fix precommit failure, spotless check. Signed-off-by: Austin Lee --- .../test/hamcrest/OpenSearchAssertions.java | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/test/framework/src/main/java/org/opensearch/test/hamcrest/OpenSearchAssertions.java b/test/framework/src/main/java/org/opensearch/test/hamcrest/OpenSearchAssertions.java index 2f4a393b2f326..11515f079fd26 100644 --- a/test/framework/src/main/java/org/opensearch/test/hamcrest/OpenSearchAssertions.java +++ b/test/framework/src/main/java/org/opensearch/test/hamcrest/OpenSearchAssertions.java @@ -307,7 +307,16 @@ public static void assertHitCount(SearchResponse countResponse, long expectedHit public static void assertHitCount(SearchResponse countResponse, long minHitCount, long maxHitCount) { final TotalHits totalHits = countResponse.getHits().getTotalHits(); if (!(totalHits.relation == TotalHits.Relation.EQUAL_TO && totalHits.value >= minHitCount && totalHits.value <= maxHitCount)) { - fail("Count is " + totalHits + " not between " + minHitCount + " and " + maxHitCount + " inclusive. " + formatShardStatus(countResponse)); + fail( + "Count is " + + totalHits + + " not between " + + minHitCount + + " and " + + maxHitCount + + " inclusive. " + + formatShardStatus(countResponse) + ); } } From 60db2d053d096076136ed0ec2c7c62bba26e1694 Mon Sep 17 00:00:00 2001 From: Austin Lee Date: Wed, 25 Oct 2023 06:39:47 -0700 Subject: [PATCH 22/32] Fix flaky test for size = 0. Signed-off-by: Austin Lee --- .../search/simple/SimpleSearchIT.java | 80 +++++++++++++------ 1 file changed, 57 insertions(+), 23 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/search/simple/SimpleSearchIT.java b/server/src/internalClusterTest/java/org/opensearch/search/simple/SimpleSearchIT.java index cca0ce86418d1..ba853d00f9f80 100644 --- a/server/src/internalClusterTest/java/org/opensearch/search/simple/SimpleSearchIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/search/simple/SimpleSearchIT.java @@ -306,6 +306,9 @@ public void dotestSimpleTerminateAfterCountWithSize(int size, int max) throws Ex .setTrackTotalHits(true) .get(); + // Do not expect an exact match as an optimization introduced by https://issues.apache.org/jira/browse/LUCENE-10620 + // can produce a total hit count > terminated_after, but this only kicks in + // when size = 0 which is when TotalHitCountCollector is used. if (size == 0) { assertHitCount(searchResponse, i, max); } else { @@ -334,6 +337,24 @@ public void testSimpleTerminateAfterCountRandomSize() throws Exception { dotestSimpleTerminateAfterCountWithSize(randomIntBetween(1, max), max); } + /** + * Special cases when size = 0: + * + * If track_total_hits = true: + * Weight#count optimization can cause totalHits in the response to be up to the total doc count regardless of terminate_after. + * So, we will have to do a range check, not an equality check. + * + * If track_total_hits != true, but set to a value AND terminate_after is set: + * Again, due to the optimization, any count can be returned. + * Up to terminate_after, relation == EQUAL_TO. + * But if track_total_hits_up_to >= terminate_after, relation can be EQ _or_ GTE. + * This ambiguity is due to the fact that totalHits == track_total_hits_up_to + * or totalHits > track_total_hits_up_to and SearchPhaseController sets totalHits = track_total_hits_up_to when returning results + * in which case relation = GTE. + * + * @param size + * @throws Exception + */ public void doTestSimpleTerminateAfterTrackTotalHitsUpTo(int size) throws Exception { prepareCreate("test").setSettings(Settings.builder().put(SETTING_NUMBER_OF_SHARDS, 1).put(SETTING_NUMBER_OF_REPLICAS, 0)).get(); ensureGreen(); @@ -350,6 +371,7 @@ public void doTestSimpleTerminateAfterTrackTotalHitsUpTo(int size) throws Except refresh(); SearchResponse searchResponse; + searchResponse = client().prepareSearch("test") .setQuery(QueryBuilders.rangeQuery("field").gte(1).lte(numDocs)) .setTerminateAfter(10) @@ -360,25 +382,28 @@ public void doTestSimpleTerminateAfterTrackTotalHitsUpTo(int size) throws Except assertEquals(5, searchResponse.getHits().getTotalHits().value); assertEquals(GREATER_THAN_OR_EQUAL_TO, searchResponse.getHits().getTotalHits().relation); - searchResponse = client().prepareSearch("test") - .setQuery(QueryBuilders.rangeQuery("field").gte(1).lte(numDocs)) - .setTerminateAfter(5) - .setSize(size) - .setTrackTotalHitsUpTo(10) - .get(); - assertTrue(searchResponse.isTerminatedEarly()); - assertEquals(5, searchResponse.getHits().getTotalHits().value); - assertEquals(EQUAL_TO, searchResponse.getHits().getTotalHits().relation); + // For size = 0, the following queries terminate early, but hits and relation can vary. + if (size > 0) { + searchResponse = client().prepareSearch("test") + .setQuery(QueryBuilders.rangeQuery("field").gte(1).lte(numDocs)) + .setTerminateAfter(5) + .setSize(size) + .setTrackTotalHitsUpTo(10) + .get(); + assertTrue(searchResponse.isTerminatedEarly()); + assertEquals(5, searchResponse.getHits().getTotalHits().value); + assertEquals(EQUAL_TO, searchResponse.getHits().getTotalHits().relation); - searchResponse = client().prepareSearch("test") - .setQuery(QueryBuilders.rangeQuery("field").gte(1).lte(numDocs)) - .setTerminateAfter(5) - .setSize(size) - .setTrackTotalHitsUpTo(5) - .get(); - assertTrue(searchResponse.isTerminatedEarly()); - assertEquals(5, searchResponse.getHits().getTotalHits().value); - assertEquals(EQUAL_TO, searchResponse.getHits().getTotalHits().relation); + searchResponse = client().prepareSearch("test") + .setQuery(QueryBuilders.rangeQuery("field").gte(1).lte(numDocs)) + .setTerminateAfter(5) + .setSize(size) + .setTrackTotalHitsUpTo(5) + .get(); + assertTrue(searchResponse.isTerminatedEarly()); + assertEquals(5, searchResponse.getHits().getTotalHits().value); + assertEquals(EQUAL_TO, searchResponse.getHits().getTotalHits().relation); + } searchResponse = client().prepareSearch("test") .setQuery(QueryBuilders.rangeQuery("field").gte(1).lte(numDocs)) @@ -387,7 +412,12 @@ public void doTestSimpleTerminateAfterTrackTotalHitsUpTo(int size) throws Except .setTrackTotalHits(true) .get(); assertTrue(searchResponse.isTerminatedEarly()); - assertEquals(5, searchResponse.getHits().getTotalHits().value); + if (size == 0) { + // Since terminate_after < track_total_hits, we need to do a range check. + assertHitCount(searchResponse, 5, numDocs); + } else { + assertEquals(5, searchResponse.getHits().getTotalHits().value); + } assertEquals(EQUAL_TO, searchResponse.getHits().getTotalHits().relation); searchResponse = client().prepareSearch("test") @@ -397,7 +427,12 @@ public void doTestSimpleTerminateAfterTrackTotalHitsUpTo(int size) throws Except .setTrackTotalHits(true) .get(); assertFalse(searchResponse.isTerminatedEarly()); - assertEquals(numDocs, searchResponse.getHits().getTotalHits().value); + if (size == 0) { + // Since terminate_after < track_total_hits, we need to do a range check. + assertHitCount(searchResponse, 5, numDocs); + } else { + assertEquals(5, searchResponse.getHits().getTotalHits().value); + } assertEquals(EQUAL_TO, searchResponse.getHits().getTotalHits().relation); searchResponse = client().prepareSearch("test") @@ -409,12 +444,11 @@ public void doTestSimpleTerminateAfterTrackTotalHitsUpTo(int size) throws Except assertEquals(GREATER_THAN_OR_EQUAL_TO, searchResponse.getHits().getTotalHits().relation); } - @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/10435") - public void testSimpleTerminateAfterTrackTotalHitsUpToRandomSize() throws Exception { + public void testSimpleTerminateAfterTrackTotalHitsUpToRandomSize0() throws Exception { doTestSimpleTerminateAfterTrackTotalHitsUpTo(0); } - public void testSimpleTerminateAfterTrackTotalHitsUpToSize0() throws Exception { + public void testSimpleTerminateAfterTrackTotalHitsUpToSize() throws Exception { doTestSimpleTerminateAfterTrackTotalHitsUpTo(randomIntBetween(1, 29)); } From de84d82fd926d730deef678217b5b9ce3e744b46 Mon Sep 17 00:00:00 2001 From: Varun Bansal Date: Wed, 18 Oct 2023 17:11:06 +0530 Subject: [PATCH 23/32] revive remote cluster state auto restore integ tests (#10503) * revive remote cluster state auto restore integ tests Signed-off-by: bansvaru --- .../opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java | 1 - 1 file changed, 1 deletion(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java index c61e2ec6e4f6c..c9cd82326dff9 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java @@ -33,7 +33,6 @@ import java.util.Locale; import java.util.Map; import java.util.Objects; -import java.util.concurrent.ExecutionException; import static org.opensearch.cluster.coordination.ClusterBootstrapService.INITIAL_CLUSTER_MANAGER_NODES_SETTING; import static org.opensearch.cluster.metadata.IndexMetadata.INDEX_READ_ONLY_SETTING; From 8549d8ce4bc4537bc530d80ce58dd13077cb5a86 Mon Sep 17 00:00:00 2001 From: Varun Bansal Date: Wed, 18 Oct 2023 23:11:53 +0530 Subject: [PATCH 24/32] Remote Restore Cluster Metadata if local disk state lost after quorum loss recovery (#10447) * Remote Restore Cluster Metadata if local disk state lost after quorum loss Signed-off-by: bansvaru --- .../opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java index c9cd82326dff9..c61e2ec6e4f6c 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java @@ -33,6 +33,7 @@ import java.util.Locale; import java.util.Map; import java.util.Objects; +import java.util.concurrent.ExecutionException; import static org.opensearch.cluster.coordination.ClusterBootstrapService.INITIAL_CLUSTER_MANAGER_NODES_SETTING; import static org.opensearch.cluster.metadata.IndexMetadata.INDEX_READ_ONLY_SETTING; From 770ffbf7db06dbd1aa6ff79f4e4d0be2b5450b10 Mon Sep 17 00:00:00 2001 From: Varun Bansal Date: Wed, 18 Oct 2023 17:11:06 +0530 Subject: [PATCH 25/32] revive remote cluster state auto restore integ tests (#10503) * revive remote cluster state auto restore integ tests Signed-off-by: bansvaru --- .../opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java | 1 - 1 file changed, 1 deletion(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java index c61e2ec6e4f6c..c9cd82326dff9 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java @@ -33,7 +33,6 @@ import java.util.Locale; import java.util.Map; import java.util.Objects; -import java.util.concurrent.ExecutionException; import static org.opensearch.cluster.coordination.ClusterBootstrapService.INITIAL_CLUSTER_MANAGER_NODES_SETTING; import static org.opensearch.cluster.metadata.IndexMetadata.INDEX_READ_ONLY_SETTING; From d0aaca7bfc062ed45a3e59e30c77c2267db49b31 Mon Sep 17 00:00:00 2001 From: Varun Bansal Date: Wed, 18 Oct 2023 23:11:53 +0530 Subject: [PATCH 26/32] Remote Restore Cluster Metadata if local disk state lost after quorum loss recovery (#10447) * Remote Restore Cluster Metadata if local disk state lost after quorum loss Signed-off-by: bansvaru --- .../opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java index c9cd82326dff9..c61e2ec6e4f6c 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java @@ -33,6 +33,7 @@ import java.util.Locale; import java.util.Map; import java.util.Objects; +import java.util.concurrent.ExecutionException; import static org.opensearch.cluster.coordination.ClusterBootstrapService.INITIAL_CLUSTER_MANAGER_NODES_SETTING; import static org.opensearch.cluster.metadata.IndexMetadata.INDEX_READ_ONLY_SETTING; From 2c4fd193c20ed507edf66114a6246cd1b4cbff07 Mon Sep 17 00:00:00 2001 From: Varun Bansal Date: Wed, 18 Oct 2023 17:11:06 +0530 Subject: [PATCH 27/32] revive remote cluster state auto restore integ tests (#10503) * revive remote cluster state auto restore integ tests Signed-off-by: bansvaru --- .../opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java | 1 - 1 file changed, 1 deletion(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java index c61e2ec6e4f6c..c9cd82326dff9 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java @@ -33,7 +33,6 @@ import java.util.Locale; import java.util.Map; import java.util.Objects; -import java.util.concurrent.ExecutionException; import static org.opensearch.cluster.coordination.ClusterBootstrapService.INITIAL_CLUSTER_MANAGER_NODES_SETTING; import static org.opensearch.cluster.metadata.IndexMetadata.INDEX_READ_ONLY_SETTING; From 5ccd8f2a582bc2c943a46a5449bc3942d7c3dbff Mon Sep 17 00:00:00 2001 From: Varun Bansal Date: Wed, 18 Oct 2023 23:11:53 +0530 Subject: [PATCH 28/32] Remote Restore Cluster Metadata if local disk state lost after quorum loss recovery (#10447) * Remote Restore Cluster Metadata if local disk state lost after quorum loss Signed-off-by: bansvaru --- .../opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java index c9cd82326dff9..c61e2ec6e4f6c 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreClusterStateRestoreIT.java @@ -33,6 +33,7 @@ import java.util.Locale; import java.util.Map; import java.util.Objects; +import java.util.concurrent.ExecutionException; import static org.opensearch.cluster.coordination.ClusterBootstrapService.INITIAL_CLUSTER_MANAGER_NODES_SETTING; import static org.opensearch.cluster.metadata.IndexMetadata.INDEX_READ_ONLY_SETTING; From 803d1824005fb5c20e83fc7c39ea26a2d4042c95 Mon Sep 17 00:00:00 2001 From: Harsha Vamsi Kalluri Date: Wed, 10 Jan 2024 05:49:11 -0800 Subject: [PATCH 29/32] Update skip version to match branches (#11801) Signed-off-by: Harsha Vamsi Kalluri From 1fd4fdf5cd5d2b2862d77989672ac83f1265998d Mon Sep 17 00:00:00 2001 From: Harsha Vamsi Kalluri Date: Wed, 10 Jan 2024 05:49:11 -0800 Subject: [PATCH 30/32] Update skip version to match branches (#11801) Signed-off-by: Harsha Vamsi Kalluri From 2c2d4bd0c585206091cf032c1f5ae4bebe58704a Mon Sep 17 00:00:00 2001 From: Austin Lee Date: Fri, 19 Jan 2024 10:13:29 -0800 Subject: [PATCH 31/32] Fix bad rebase. Signed-off-by: Austin Lee --- CHANGELOG.md | 4 ---- distribution/src/config/log4j2.properties | 4 ++-- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 27c4009f93ac8..822e4196adf5e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -240,8 +240,4 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Security [Unreleased 3.0]: https://github.com/opensearch-project/OpenSearch/compare/2.x...HEAD -<<<<<<< HEAD [Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.12...2.x -======= -[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.11...2.x ->>>>>>> 37a5dd041f2 (Bump JNA version from 5.5 to 5.13.) diff --git a/distribution/src/config/log4j2.properties b/distribution/src/config/log4j2.properties index 87b875587952e..1f9935022c330 100644 --- a/distribution/src/config/log4j2.properties +++ b/distribution/src/config/log4j2.properties @@ -14,7 +14,7 @@ status = error appender.console.type = Console appender.console.name = console appender.console.layout.type = PatternLayout -appender.console.layout.pattern = %style{[%d{ISO8601}]}{FG_Red BG_White} %highlight{[%-5p][%-25c{1.}]} %style{[%node_name]}{FG_Magenta BG_White} %style{<%t>}{bright green} %style{[%M]}{bright magenta} : %marker %style{%m}{FG_Black BG_Yellow}%n +appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker %m%n #[%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker %m%n @@ -67,7 +67,7 @@ appender.rolling_old.strategy.action.condition.nested_condition.type = IfAccumul appender.rolling_old.strategy.action.condition.nested_condition.exceeds = 2GB ################################################ -rootLogger.level = debug +rootLogger.level = info rootLogger.appenderRef.console.ref = console rootLogger.appenderRef.rolling.ref = rolling rootLogger.appenderRef.rolling_old.ref = rolling_old From 5c190c2c7c0fca7c22370aae38687ee361bbe42f Mon Sep 17 00:00:00 2001 From: Austin Lee Date: Fri, 19 Jan 2024 10:25:28 -0800 Subject: [PATCH 32/32] Remove unintended change from log4j properties. Signed-off-by: Austin Lee --- distribution/src/config/log4j2.properties | 2 -- 1 file changed, 2 deletions(-) diff --git a/distribution/src/config/log4j2.properties b/distribution/src/config/log4j2.properties index 1f9935022c330..d040afae82e53 100644 --- a/distribution/src/config/log4j2.properties +++ b/distribution/src/config/log4j2.properties @@ -16,8 +16,6 @@ appender.console.name = console appender.console.layout.type = PatternLayout appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker %m%n -#[%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker %m%n - ######## Server JSON ############################ appender.rolling.type = RollingFile appender.rolling.name = rolling