diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 84e4461ab15..0fe7e1b8d02 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -118,6 +118,9 @@ Optimizations * SOLR-17102: The VersionBucket indexing lock mechanism was replaced with something just as fast yet that which consumes almost no memory, saving 1MB of memory per SolrCore. (David Smiley) +* SOLR-17381: Make CLUSTERSTATUS request configurable to improve performance by allowing retrieval of specific information, + reducing unnecessary data fetching. (Aparna Suresh, David Smiley) + * SOLR-17396: Reduce thread contention in ZkStateReader.getCollectionProperties(). (Aparna Suresh, David Smiley, Paul McArthur) Bug Fixes diff --git a/solr/core/src/java/org/apache/solr/handler/admin/ClusterStatus.java b/solr/core/src/java/org/apache/solr/handler/admin/ClusterStatus.java index f897aea129c..6c5998d17a2 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/ClusterStatus.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/ClusterStatus.java @@ -34,9 +34,9 @@ import org.apache.solr.common.cloud.PerReplicaStates; import org.apache.solr.common.cloud.Replica; import org.apache.solr.common.cloud.Slice; -import org.apache.solr.common.cloud.ZkNodeProps; import org.apache.solr.common.cloud.ZkStateReader; import org.apache.solr.common.params.ShardParams; +import org.apache.solr.common.params.SolrParams; import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.SimpleOrderedMap; import org.apache.solr.common.util.Utils; @@ -44,9 +44,14 @@ public class ClusterStatus { private final ZkStateReader zkStateReader; - private final ZkNodeProps message; + private final SolrParams solrParams; private final String collection; // maybe null + public static final String INCLUDE_ALL = "includeAll"; + public static final String LIVENODES_PROP = "liveNodes"; + public static final String CLUSTER_PROP = "clusterProperties"; + public static final String ALIASES_PROP = "aliases"; + /** Shard / collection health state. */ public enum Health { /** All replicas up, leader exists. */ @@ -89,16 +94,72 @@ public static Health combine(Collection states) { } } - public ClusterStatus(ZkStateReader zkStateReader, ZkNodeProps props) { + public ClusterStatus(ZkStateReader zkStateReader, SolrParams params) { this.zkStateReader = zkStateReader; - this.message = props; - collection = props.getStr(ZkStateReader.COLLECTION_PROP); + this.solrParams = params; + collection = params.get(ZkStateReader.COLLECTION_PROP); } public void getClusterStatus(NamedList results) throws KeeperException, InterruptedException { + NamedList clusterStatus = new SimpleOrderedMap<>(); + + boolean includeAll = solrParams.getBool(INCLUDE_ALL, true); + boolean withLiveNodes = solrParams.getBool(LIVENODES_PROP, includeAll); + boolean withClusterProperties = solrParams.getBool(CLUSTER_PROP, includeAll); + boolean withRoles = solrParams.getBool(ZkStateReader.ROLES_PROP, includeAll); + boolean withCollection = includeAll || (collection != null); + boolean withAliases = solrParams.getBool(ALIASES_PROP, includeAll); + + List liveNodes = null; + if (withLiveNodes || collection != null) { + liveNodes = + zkStateReader.getZkClient().getChildren(ZkStateReader.LIVE_NODES_ZKNODE, null, true); + // add live_nodes + if (withLiveNodes) clusterStatus.add("live_nodes", liveNodes); + } + + Aliases aliases = null; + if (withCollection || withAliases) { + aliases = zkStateReader.getAliases(); + } + + if (withCollection) { + assert liveNodes != null; + fetchClusterStatusForCollOrAlias(clusterStatus, liveNodes, aliases); + } + + if (withAliases) { + addAliasMap(aliases, clusterStatus); + } + + if (withClusterProperties) { + Map clusterProps = zkStateReader.getClusterProperties(); + if (clusterProps == null) { + clusterProps = Collections.emptyMap(); + } + clusterStatus.add("properties", clusterProps); + } + + // add the roles map + if (withRoles) { + Map roles = Collections.emptyMap(); + if (zkStateReader.getZkClient().exists(ZkStateReader.ROLES, true)) { + roles = + (Map) + Utils.fromJSON( + zkStateReader.getZkClient().getData(ZkStateReader.ROLES, null, null, true)); + } + clusterStatus.add("roles", roles); + } + + results.add("cluster", clusterStatus); + } + + private void fetchClusterStatusForCollOrAlias( + NamedList clusterStatus, List liveNodes, Aliases aliases) { + // read aliases - Aliases aliases = zkStateReader.getAliases(); Map> collectionVsAliases = new HashMap<>(); Map> aliasVsCollections = aliases.getCollectionAliasListMap(); for (Map.Entry> entry : aliasVsCollections.entrySet()) { @@ -112,18 +173,10 @@ public void getClusterStatus(NamedList results) } } - Map roles = null; - if (zkStateReader.getZkClient().exists(ZkStateReader.ROLES, true)) { - roles = - (Map) - Utils.fromJSON( - zkStateReader.getZkClient().getData(ZkStateReader.ROLES, null, null, true)); - } - ClusterState clusterState = zkStateReader.getClusterState(); - String routeKey = message.getStr(ShardParams._ROUTE_); - String shard = message.getStr(ZkStateReader.SHARD_ID_PROP); + String routeKey = solrParams.get(ShardParams._ROUTE_); + String shard = solrParams.get(ZkStateReader.SHARD_ID_PROP); Map collectionsMap = null; if (collection == null) { @@ -139,6 +192,7 @@ public void getClusterStatus(NamedList results) if (didNotFindCollection && isAlias) { // In this case this.collection is an alias name not a collection // get all collections and filter out collections not in the alias + // clusterState.getCollectionsMap() should be replaced with an inexpensive call collectionsMap = clusterState.getCollectionsMap().entrySet().stream() .filter((entry) -> aliasVsCollections.get(collection).contains(entry.getKey())) @@ -191,43 +245,25 @@ public void getClusterStatus(NamedList results) } String configName = clusterStateCollection.getConfigName(); collectionStatus.put("configName", configName); - if (message.getBool("prs", false) && clusterStateCollection.isPerReplicaState()) { + if (solrParams.getBool("prs", false) && clusterStateCollection.isPerReplicaState()) { PerReplicaStates prs = clusterStateCollection.getPerReplicaStates(); collectionStatus.put("PRS", prs); } collectionProps.add(name, collectionStatus); } - List liveNodes = - zkStateReader.getZkClient().getChildren(ZkStateReader.LIVE_NODES_ZKNODE, null, true); - // now we need to walk the collectionProps tree to cross-check replica state with live nodes crossCheckReplicaStateWithLiveNodes(liveNodes, collectionProps); - NamedList clusterStatus = new SimpleOrderedMap<>(); clusterStatus.add("collections", collectionProps); + } - // read cluster properties - Map clusterProps = zkStateReader.getClusterProperties(); - if (clusterProps != null && !clusterProps.isEmpty()) { - clusterStatus.add("properties", clusterProps); - } - + private void addAliasMap(Aliases aliases, NamedList clusterStatus) { // add the alias map too Map collectionAliasMap = aliases.getCollectionAliasMap(); // comma delim if (!collectionAliasMap.isEmpty()) { clusterStatus.add("aliases", collectionAliasMap); } - - // add the roles map - if (roles != null) { - clusterStatus.add("roles", roles); - } - - // add live_nodes - clusterStatus.add("live_nodes", liveNodes); - - results.add("cluster", clusterStatus); } /** diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java index 485670a4421..7ee6da6a0c1 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java @@ -100,7 +100,6 @@ import static org.apache.solr.common.params.CommonParams.VALUE_LONG; import static org.apache.solr.common.params.CoreAdminParams.BACKUP_LOCATION; import static org.apache.solr.common.params.CoreAdminParams.BACKUP_REPOSITORY; -import static org.apache.solr.common.params.ShardParams._ROUTE_; import static org.apache.solr.common.util.StrUtils.formatString; import java.lang.invoke.MethodHandles; @@ -975,10 +974,7 @@ public Map execute( CLUSTERSTATUS_OP( CLUSTERSTATUS, (req, rsp, h) -> { - Map all = - copy(req.getParams(), null, COLLECTION_PROP, SHARD_ID_PROP, _ROUTE_, "prs"); - new ClusterStatus( - h.coreContainer.getZkController().getZkStateReader(), new ZkNodeProps(all)) + new ClusterStatus(h.coreContainer.getZkController().getZkStateReader(), req.getParams()) .getClusterStatus(rsp.getValues()); return null; }), diff --git a/solr/solr-ref-guide/modules/deployment-guide/pages/cluster-node-management.adoc b/solr/solr-ref-guide/modules/deployment-guide/pages/cluster-node-management.adoc index 2b60644c42b..6765c1ca93b 100644 --- a/solr/solr-ref-guide/modules/deployment-guide/pages/cluster-node-management.adoc +++ b/solr/solr-ref-guide/modules/deployment-guide/pages/cluster-node-management.adoc @@ -99,6 +99,51 @@ Multiple shard names can be specified as a comma-separated list. + This can be used if you need the details of the shard where a particular document belongs to and you don't know which shard it falls under. +`aliases`:: ++ +[%autowidth,frame=none] +|=== +|Optional |Default: will default to the default value of `includeAll` parameter specified below +|=== ++ + +`liveNodes`:: ++ +[%autowidth,frame=none] +|=== +|Optional |Default: will default to the default value of `includeAll` parameter specified below +|=== ++ +If set to true, returns the status of live nodes in the cluster. + +`clusterProperties`:: ++ +[%autowidth,frame=none] +|=== +|Optional |Default: will default to the default value of `includeAll` parameter specified below +|=== ++ +If set to true, returns the properties of the cluster. + +`roles`:: ++ +[%autowidth,frame=none] +|=== +|Optional |Default: will default to the default value of `includeAll` parameter specified below +|=== ++ +If set to true, returns the roles within the cluster. + +`includeAll`:: ++ +[%autowidth,frame=none] +|=== +|Optional |Default: true +|=== ++ +If set to `true`, returns all information pertaining to live nodes, collections, aliases, cluster properties, roles, etc. +If set to `false`, the information returned is based on the other specified parameters. + === CLUSTERSTATUS Response The response will include the status of the request and the status of the cluster. diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/BaseHttpClusterStateProvider.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/BaseHttpClusterStateProvider.java index d5e2d188a75..2555a0cba4f 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/BaseHttpClusterStateProvider.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/BaseHttpClusterStateProvider.java @@ -84,6 +84,13 @@ public void init(List solrUrls) throws Exception { /** Create a SolrClient implementation that uses the specified Solr node URL */ protected abstract SolrClient getSolrClient(String baseUrl); + @Override + public DocCollection getCollection(String collection) { + // This change is to prevent BaseHttpCSP make a call to fetch the entire cluster state, as the + // default implementation calls getClusterState().getCollectionOrNull(name) + return getState(collection).get(); + } + @Override public ClusterState.CollectionRef getState(String collection) { for (String nodeName : liveNodes) { @@ -122,15 +129,9 @@ public ClusterState.CollectionRef getState(String collection) { private ClusterState fetchClusterState( SolrClient client, String collection, Map clusterProperties) throws SolrServerException, IOException, NotACollectionException { - ModifiableSolrParams params = new ModifiableSolrParams(); - if (collection != null) { - params.set("collection", collection); - } - params.set("action", "CLUSTERSTATUS"); - params.set("prs", "true"); - QueryRequest request = new QueryRequest(params); - request.setPath("/admin/collections"); - SimpleOrderedMap cluster = (SimpleOrderedMap) client.request(request).get("cluster"); + SimpleOrderedMap cluster = + submitClusterStateRequest(client, collection, ClusterStateRequestType.FETCH_COLLECTION); + Map collectionsMap; if (collection != null) { collectionsMap = @@ -149,10 +150,16 @@ private ClusterState fetchClusterState( } else { znodeVersion = -1; } - Set liveNodes = new HashSet<>((List) (cluster.get("live_nodes"))); - this.liveNodes = liveNodes; - liveNodesTimestamp = System.nanoTime(); - ClusterState cs = new ClusterState(liveNodes, new HashMap<>()); + + ClusterState cs = new ClusterState(this.liveNodes, new HashMap<>()); + List liveNodesList = (List) cluster.get("live_nodes"); + if (liveNodesList != null) { + Set liveNodes = new HashSet<>(liveNodesList); + this.liveNodes = liveNodes; + liveNodesTimestamp = System.nanoTime(); + cs = new ClusterState(liveNodes, new HashMap<>()); + } + for (Map.Entry e : collectionsMap.entrySet()) { @SuppressWarnings("rawtypes") Map m = (Map) e.getValue(); @@ -173,6 +180,30 @@ private ClusterState fetchClusterState( return cs; } + private SimpleOrderedMap submitClusterStateRequest( + SolrClient client, String collection, ClusterStateRequestType requestType) + throws SolrServerException, IOException { + + ModifiableSolrParams params = new ModifiableSolrParams(); + params.set("action", "CLUSTERSTATUS"); + + if (requestType == ClusterStateRequestType.FETCH_COLLECTION && collection != null) { + params.set("collection", collection); + } else if (requestType == ClusterStateRequestType.FETCH_LIVE_NODES) { + params.set("liveNodes", "true"); + } else if (requestType == ClusterStateRequestType.FETCH_CLUSTER_PROP) { + params.set("clusterProperties", "true"); + } else if (requestType == ClusterStateRequestType.FETCH_NODE_ROLES) { + params.set("roles", "true"); + } + + params.set("includeAll", "false"); + params.set("prs", "true"); + QueryRequest request = new QueryRequest(params); + request.setPath("/admin/collections"); + return (SimpleOrderedMap) client.request(request).get("cluster"); + } + @SuppressWarnings({"rawtypes", "unchecked"}) private DocCollection fillPrs( int znodeVersion, Map.Entry e, Instant creationTime, Map m) { @@ -228,12 +259,10 @@ > getCacheTimeout()) { } @SuppressWarnings({"rawtypes", "unchecked"}) - private static Set fetchLiveNodes(SolrClient client) throws Exception { - ModifiableSolrParams params = new ModifiableSolrParams(); - params.set("action", "CLUSTERSTATUS"); - QueryRequest request = new QueryRequest(params); - request.setPath("/admin/collections"); - NamedList cluster = (SimpleOrderedMap) client.request(request).get("cluster"); + private Set fetchLiveNodes(SolrClient client) throws Exception { + + SimpleOrderedMap cluster = + submitClusterStateRequest(client, null, ClusterStateRequestType.FETCH_LIVE_NODES); return (Set) new HashSet((List) (cluster.get("live_nodes"))); } @@ -335,21 +364,18 @@ public ClusterState getClusterState() { + " solrUrl(s) or zkHost(s)."); } + @SuppressWarnings("unchecked") @Override public Map getClusterProperties() { + // Map clusterPropertiesMap = new HashMap<>(); for (String nodeName : liveNodes) { String baseUrl = Utils.getBaseUrlForNodeName(nodeName, urlScheme); try (SolrClient client = getSolrClient(baseUrl)) { - Map clusterProperties = new HashMap<>(); - fetchClusterState(client, null, clusterProperties); - return clusterProperties; + SimpleOrderedMap cluster = + submitClusterStateRequest(client, null, ClusterStateRequestType.FETCH_CLUSTER_PROP); + return (Map) cluster.get("properties"); } catch (SolrServerException | BaseHttpSolrClient.RemoteSolrException | IOException e) { log.warn("Attempt to fetch cluster state from {} failed.", baseUrl, e); - } catch (NotACollectionException e) { - // not possible! (we passed in null for collection so it can't be an alias) - throw new RuntimeException( - "null should never cause NotACollectionException in " - + "fetchClusterState() Please report this as a bug!"); } } throw new RuntimeException( @@ -399,4 +425,11 @@ public String getQuorumHosts() { } return String.join(",", this.liveNodes); } + + private enum ClusterStateRequestType { + FETCH_LIVE_NODES, + FETCH_CLUSTER_PROP, + FETCH_NODE_ROLES, + FETCH_COLLECTION + } } diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/ClusterStateProvider.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/ClusterStateProvider.java index 271afca7461..b6afac3114f 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/ClusterStateProvider.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/ClusterStateProvider.java @@ -91,7 +91,11 @@ default boolean isRoutedAlias(String alias) { .anyMatch(e -> e.getKey().startsWith(CollectionAdminParams.ROUTER_PREFIX)); } - /** Obtain the current cluster state. */ + /** + * Obtain the current cluster state. WARNING: This method is quite expensive as it involves + * fetching remote information. Use with caution and be aware of the potential performance + * implications. + */ ClusterState getClusterState(); default DocCollection getCollection(String name) throws IOException { diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudHttp2SolrClientTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudHttp2SolrClientTest.java index 6f9cb774beb..2a0693c2360 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudHttp2SolrClientTest.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudHttp2SolrClientTest.java @@ -35,7 +35,6 @@ import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; -import java.util.regex.Pattern; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.lucene.tests.util.TestUtil; import org.apache.solr.client.solrj.SolrClient; @@ -93,15 +92,6 @@ public class CloudHttp2SolrClientTest extends SolrCloudTestCase { private static CloudSolrClient httpBasedCloudSolrClient = null; private static CloudSolrClient zkBasedCloudSolrClient = null; - private static final Pattern PATTERN_WITH_COLLECTION = - Pattern.compile( - "path=/admin/collections.*params=\\{[^}]*action=CLUSTERSTATUS" - + "[^}]*collection=[^&}]+[^}]*\\}"); - private static final Pattern PATTERN_WITHOUT_COLLECTION = - Pattern.compile( - "path=/admin/collections.*params=\\{[^}]*action=CLUSTERSTATUS" - + "(?![^}]*collection=)[^}]*\\}"); - @BeforeClass public static void setupCluster() throws Exception { System.setProperty("metricsEnabled", "true"); @@ -269,29 +259,40 @@ public void testHttpCspPerf() throws Exception { .process(cluster.getSolrClient()); cluster.waitForActiveCollection(collectionName, 2, 2); - try (LogListener entireClusterStateLogs = - LogListener.info(HttpSolrCall.class).regex(PATTERN_WITHOUT_COLLECTION); - LogListener collectionClusterStateLogs = - LogListener.info(HttpSolrCall.class).regex(PATTERN_WITH_COLLECTION); - LogListener adminRequestLogs = LogListener.info(HttpSolrCall.class).substring("[admin]"); + try (LogListener adminLogs = LogListener.info(HttpSolrCall.class).substring("[admin]"); CloudSolrClient solrClient = createHttpCSPBasedCloudSolrClient(); ) { + + assertEquals(1, adminLogs.getCount()); + assertTrue( + adminLogs + .pollMessage() + .contains( + "path=/admin/collections params={prs=true&liveNodes=true&action" + + "=CLUSTERSTATUS&includeAll=false")); + SolrInputDocument doc = new SolrInputDocument("id", "1", "title_s", "my doc"); solrClient.add(collectionName, doc); + + // getCount seems to return a cumulative count, but add() results in only 1 additional admin + // request to fetch CLUSTERSTATUS for the collection + assertEquals(2, adminLogs.getCount()); + assertTrue( + adminLogs + .pollMessage() + .contains( + "path=/admin/collections " + + "params={prs=true&action=CLUSTERSTATUS&includeAll=false")); + solrClient.commit(collectionName); + // No additional admin requests sent + assertEquals(2, adminLogs.getCount()); + for (int i = 0; i < 3; i++) { assertEquals( 1, solrClient.query(collectionName, params("q", "*:*")).getResults().getNumFound()); + // No additional admin requests sent + assertEquals(2, adminLogs.getCount()); } - - // 1 call to fetch entire cluster state via BaseHttpCSP.fetchLiveNodes() - // 1 call to fetch CLUSTERSTATUS for collection via getDocCollection() (first collection - // lookup) - assertLogCount(adminRequestLogs, 2); - // 1 call to fetch CLUSTERSTATUS for collection via getDocCollection() (first collection - // lookup) - assertLogCount(collectionClusterStateLogs, 1); - // 1 call to fetch entire cluster state from HttpCSP.fetchLiveNodes() - assertLogCount(entireClusterStateLogs, 1); } } @@ -301,16 +302,6 @@ private CloudSolrClient createHttpCSPBasedCloudSolrClient() { return new CloudHttp2SolrClient.Builder(solrUrls).build(); } - private void assertLogCount(LogListener logListener, int expectedCount) { - int logCount = logListener.getCount(); - assertEquals(expectedCount, logCount); - if (logCount > 0) { - for (int i = 0; i < logCount; i++) { - logListener.pollMessage(); - } - } - } - @Test public void testRouting() throws Exception { CollectionAdminRequest.createCollection("routing_collection", "conf", 2, 1) diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/HttpClusterStateSSLTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/HttpClusterStateSSLTest.java index db385e26b83..1242cde9945 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/HttpClusterStateSSLTest.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/HttpClusterStateSSLTest.java @@ -23,7 +23,6 @@ import java.util.Map; import org.apache.solr.client.solrj.request.CollectionAdminRequest; import org.apache.solr.cloud.SolrCloudTestCase; -import org.apache.solr.common.cloud.ClusterState; import org.apache.solr.common.cloud.DocCollection; import org.apache.solr.common.cloud.Replica; import org.apache.solr.common.util.Utils; @@ -88,7 +87,7 @@ public void testHttpClusterStateWithSSL() throws Exception { new CloudSolrClient.Builder(Collections.singletonList(url0.toExternalForm())).build()) { ClusterStateProvider csp = httpBasedCloudSolrClient.getClusterStateProvider(); assertTrue(csp instanceof Http2ClusterStateProvider); - verifyUrlSchemeInClusterState(csp.getClusterState(), collectionId, expectedReplicas); + verifyUrlSchemeInClusterState(csp.getCollection(collectionId), expectedReplicas); } // http2 @@ -97,20 +96,19 @@ public void testHttpClusterStateWithSSL() throws Exception { .build()) { ClusterStateProvider csp = http2BasedClient.getClusterStateProvider(); assertTrue(csp instanceof Http2ClusterStateProvider); - verifyUrlSchemeInClusterState(csp.getClusterState(), collectionId, expectedReplicas); + verifyUrlSchemeInClusterState(csp.getCollection(collectionId), expectedReplicas); } // Zk cluster state now ClusterStateProvider csp = cluster.getSolrClient().getClusterStateProvider(); assertTrue(csp instanceof ZkClientClusterStateProvider); - verifyUrlSchemeInClusterState(csp.getClusterState(), collectionId, expectedReplicas); + verifyUrlSchemeInClusterState(csp.getCollection(collectionId), expectedReplicas); } private void verifyUrlSchemeInClusterState( - final ClusterState cs, final String collectionId, final int expectedReplicas) { - DocCollection dc = cs.getCollection(collectionId); - assertNotNull(dc); - List replicas = dc.getReplicas(); + final DocCollection collection, final int expectedReplicas) { + assertNotNull(collection); + List replicas = collection.getReplicas(); assertNotNull(replicas); assertEquals(expectedReplicas, replicas.size()); for (Replica r : replicas) {