Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Replace CloudLegacySolrClient with CloudHttp2SolrClient in SolrClientCloudManager #2751

Merged
merged 12 commits into from
Oct 24, 2024
Merged

Replace CloudLegacySolrClient with CloudHttp2SolrClient in SolrClientCloudManager #2751

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
57e11d5
Replaced CloudLegacySolrClient with CloudHttp2SolrClient for SolrClie…
iamsanjay Oct 8, 2024
a2dc271
removed unusedDeclaredArtifacts httpclient, httpcore from solrj-zooke…
iamsanjay Oct 8, 2024
fd0fcbe
Switching to CloudSolrClient ref
iamsanjay Oct 11, 2024
7701aa6
Merge branch 'main' into jetty_solrClientCloudManager
iamsanjay Oct 11, 2024
cee40b9
Merge branch 'main' into jetty_solrClientCloudManager
iamsanjay Oct 17, 2024
140d5e0
Switch to CloudHttp2SolrClient to easily access Http2SolrClient
iamsanjay Oct 17, 2024
2763f69
Merge branch 'main' into jetty_solrClientCloudManager
iamsanjay Oct 21, 2024
8fc6129
Provided http2SolrClient with custom timeouts
iamsanjay Oct 21, 2024
40fbab7
Merge main
iamsanjay Oct 24, 2024
2d33669
Addresses gh comments
iamsanjay Oct 24, 2024
284b9ee
changes.txt
iamsanjay Oct 24, 2024
1040917
changes.txt
iamsanjay Oct 24, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 7 additions & 8 deletions solr/core/src/java/org/apache/solr/cloud/ZkController.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,11 +54,10 @@
import java.util.stream.Collectors;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.cloud.SolrCloudManager;
import org.apache.solr.client.solrj.impl.CloudLegacySolrClient;
import org.apache.solr.client.solrj.impl.CloudHttp2SolrClient;
import org.apache.solr.client.solrj.impl.HttpSolrClient.Builder;
import org.apache.solr.client.solrj.impl.SolrClientCloudManager;
import org.apache.solr.client.solrj.impl.SolrZkClientTimeout;
import org.apache.solr.client.solrj.impl.ZkClientClusterStateProvider;
import org.apache.solr.client.solrj.request.CoreAdminRequest.WaitForState;
import org.apache.solr.cloud.overseer.ClusterStateMutator;
import org.apache.solr.cloud.overseer.OverseerAction;
Expand Down Expand Up @@ -199,7 +198,8 @@ public String toString() {
private final SolrZkClient zkClient;
public final ZkStateReader zkStateReader;
private SolrCloudManager cloudManager;
private CloudLegacySolrClient cloudSolrClient;

private CloudHttp2SolrClient cloudSolrClient;

private final String zkServerAddress; // example: 127.0.0.1:54062/solr

Expand Down Expand Up @@ -866,12 +866,11 @@ public SolrCloudManager getSolrCloudManager() {
return cloudManager;
}
cloudSolrClient =
new CloudLegacySolrClient.Builder(new ZkClientClusterStateProvider(zkStateReader))
.withHttpClient(cc.getUpdateShardHandler().getDefaultHttpClient())
.withConnectionTimeout(15000, TimeUnit.MILLISECONDS)
.withSocketTimeout(30000, TimeUnit.MILLISECONDS)
new CloudHttp2SolrClient.Builder(
Collections.singletonList(getZkServerAddress()), Optional.empty())
.withHttpClient(cc.getDefaultHttpSolrClient())
.build();
cloudManager = new SolrClientCloudManager(cloudSolrClient, cc.getObjectCache());
cloudManager = new SolrClientCloudManager(cc.getObjectCache(), cloudSolrClient);
cloudManager.getClusterStateProvider().connect();
}
return cloudManager;
Expand Down
21 changes: 13 additions & 8 deletions solr/core/src/test/org/apache/solr/cloud/OverseerTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,8 +52,8 @@
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.cloud.DistributedQueue;
import org.apache.solr.client.solrj.cloud.SolrCloudManager;
import org.apache.solr.client.solrj.impl.CloudLegacySolrClient;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.impl.CloudHttp2SolrClient;
import org.apache.solr.client.solrj.impl.Http2SolrClient;
import org.apache.solr.client.solrj.impl.SolrClientCloudManager;
import org.apache.solr.cloud.overseer.NodeMutator;
import org.apache.solr.cloud.overseer.OverseerAction;
Expand Down Expand Up @@ -127,7 +127,7 @@ public class OverseerTest extends SolrTestCaseJ4 {
Collections.synchronizedList(new ArrayList<>());
private final List<UpdateShardHandler> updateShardHandlers =
Collections.synchronizedList(new ArrayList<>());
private final List<CloudSolrClient> solrClients = Collections.synchronizedList(new ArrayList<>());
private final List<SolrClient> solrClients = Collections.synchronizedList(new ArrayList<>());
private static final String COLLECTION = SolrTestCaseJ4.DEFAULT_TEST_COLLECTION_NAME;

public static class MockZKController {
Expand Down Expand Up @@ -1948,13 +1948,18 @@ public Void answer(InvocationOnMock invocation) {
}

private SolrCloudManager getCloudDataProvider(String zkAddress) {
var client =
new CloudLegacySolrClient.Builder(Collections.singletonList(zkAddress), Optional.empty())
.withSocketTimeout(30000, TimeUnit.MILLISECONDS)
var httpSolrClient =
new Http2SolrClient.Builder()
.withIdleTimeout(30000, TimeUnit.MILLISECONDS)
.withConnectionTimeout(15000, TimeUnit.MILLISECONDS)
.build();
solrClients.add(client);
SolrClientCloudManager sccm = new SolrClientCloudManager(client);
var cloudSolrClient =
new CloudHttp2SolrClient.Builder(Collections.singletonList(zkAddress), Optional.empty())
.withHttpClient(httpSolrClient)
.build();
solrClients.add(cloudSolrClient);
solrClients.add(httpSolrClient);
SolrClientCloudManager sccm = new SolrClientCloudManager(null, cloudSolrClient);
sccm.getClusterStateProvider().connect();
return sccm;
}
Expand Down
9 changes: 9 additions & 0 deletions solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.client.solrj.impl.Http2SolrClient;
import org.apache.solr.common.MapWriter;
import org.apache.solr.common.cloud.ClusterProperties;
import org.apache.solr.common.cloud.ClusterState;
Expand Down Expand Up @@ -514,6 +515,7 @@ public void tearDown() throws Exception {
private static class MockCoreContainer extends CoreContainer {
UpdateShardHandler updateShardHandler =
new UpdateShardHandler(UpdateShardHandlerConfig.DEFAULT);
Http2SolrClient solrClient;

public MockCoreContainer() {
super(SolrXmlConfig.fromString(TEST_PATH(), "<solr/>"));
Expand All @@ -522,6 +524,7 @@ public MockCoreContainer() {
this.shardHandlerFactory = httpShardHandlerFactory;
this.coreAdminHandler = new CoreAdminHandler();
this.metricManager = mock(SolrMetricManager.class);
this.solrClient = new Http2SolrClient.Builder().build();
}

@Override
Expand All @@ -535,9 +538,15 @@ public UpdateShardHandler getUpdateShardHandler() {
@Override
public void shutdown() {
updateShardHandler.close();
solrClient.close();
super.shutdown();
}

@Override
public Http2SolrClient getDefaultHttpSolrClient() {
return solrClient;
}

@Override
public SolrMetricManager getMetricManager() {
return metricManager;
Expand Down
2 changes: 0 additions & 2 deletions solr/solrj-zookeeper/build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,6 @@ dependencies {

// declare dependencies we use even though already declared by solrj-core
implementation 'org.slf4j:slf4j-api'
implementation 'org.apache.httpcomponents:httpclient'
implementation 'org.apache.httpcomponents:httpcore'

api('org.apache.zookeeper:zookeeper', {
exclude group: "org.apache.yetus", module: "audience-annotations"
Expand Down
13 changes: 0 additions & 13 deletions solr/solrj-zookeeper/src/java/org/apache/solr/client/solrj/cloud/DelegatingCloudManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@
package org.apache.solr.client.solrj.cloud;

import java.io.IOException;
import java.util.Map;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrResponse;
import org.apache.solr.client.solrj.impl.ClusterStateProvider;
Expand Down Expand Up @@ -69,18 +68,6 @@ public <T extends SolrResponse> T request(SolrRequest<T> req) throws IOException
return delegate.request(req);
}

@Override
public byte[] httpRequest(
String url,
SolrRequest.METHOD method,
Map<String, String> headers,
String payload,
int timeout,
boolean followRedirects)
throws IOException {
return delegate.httpRequest(url, method, headers, payload, timeout, followRedirects);
}

@Override
public void close() throws IOException {
delegate.close();
Expand Down
10 changes: 0 additions & 10 deletions solr/solrj-zookeeper/src/java/org/apache/solr/client/solrj/cloud/SolrCloudManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@
package org.apache.solr.client.solrj.cloud;

import java.io.IOException;
import java.util.Map;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrResponse;
import org.apache.solr.client.solrj.impl.ClusterStateProvider;
Expand Down Expand Up @@ -53,13 +52,4 @@ default ClusterState getClusterState() throws IOException {
// Solr-like methods

<T extends SolrResponse> T request(SolrRequest<T> req) throws IOException;

byte[] httpRequest(
String url,
SolrRequest.METHOD method,
Map<String, String> headers,
String payload,
int timeout,
boolean followRedirects)
throws IOException;
}
94 changes: 7 additions & 87 deletions solr/solrj-zookeeper/src/java/org/apache/solr/client/solrj/impl/SolrClientCloudManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,20 +19,6 @@

import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.nio.charset.StandardCharsets;
import java.util.Map;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.HttpDelete;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.client.methods.HttpPut;
import org.apache.http.client.methods.HttpRequestBase;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.entity.StringEntity;
import org.apache.http.util.EntityUtils;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrResponse;
import org.apache.solr.client.solrj.SolrServerException;
Expand All @@ -51,21 +37,17 @@
public class SolrClientCloudManager implements SolrCloudManager {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());

protected final CloudLegacySolrClient solrClient;
private final CloudHttp2SolrClient cloudSolrClient;
private final ZkDistribStateManager stateManager;
private final ZkStateReader zkStateReader;
private final SolrZkClient zkClient;
private final ObjectCache objectCache;
private final boolean closeObjectCache;
private volatile boolean isClosed;

public SolrClientCloudManager(CloudLegacySolrClient solrClient) {
this(solrClient, null);
}

public SolrClientCloudManager(CloudLegacySolrClient solrClient, ObjectCache objectCache) {
this.solrClient = solrClient;
this.zkStateReader = ZkStateReader.from(solrClient);
public SolrClientCloudManager(ObjectCache objectCache, CloudHttp2SolrClient client) {
iamsanjay marked this conversation as resolved.
Show resolved Hide resolved
this.cloudSolrClient = client;
this.zkStateReader = ZkStateReader.from(client);
this.zkClient = zkStateReader.getZkClient();
this.stateManager = new ZkDistribStateManager(zkClient);
this.isClosed = false;
Expand Down Expand Up @@ -103,12 +85,12 @@ public TimeSource getTimeSource() {

@Override
public ClusterStateProvider getClusterStateProvider() {
return solrClient.getClusterStateProvider();
return cloudSolrClient.getClusterStateProvider();
}

@Override
public NodeStateProvider getNodeStateProvider() {
return new SolrClientNodeStateProvider(solrClient);
return new SolrClientNodeStateProvider(cloudSolrClient, cloudSolrClient.getHttpClient());
}

@Override
Expand All @@ -119,76 +101,14 @@ public DistribStateManager getDistribStateManager() {
@Override
public <T extends SolrResponse> T request(SolrRequest<T> req) throws IOException {
try {
return req.process(solrClient);
return req.process(cloudSolrClient);
} catch (SolrServerException e) {
throw new IOException(e);
}
}

private static final byte[] EMPTY = new byte[0];

@Override
public byte[] httpRequest(
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

a reminder -- call this out in the commit message; we removed a public method.

String url,
SolrRequest.METHOD method,
Map<String, String> headers,
String payload,
int timeout,
boolean followRedirects)
throws IOException {
HttpClient client = solrClient.getHttpClient();
final HttpRequestBase req;
HttpEntity entity = null;
if (payload != null) {
entity = new StringEntity(payload, StandardCharsets.UTF_8);
}
switch (method) {
case GET:
req = new HttpGet(url);
break;
case POST:
req = new HttpPost(url);
if (entity != null) {
((HttpPost) req).setEntity(entity);
}
break;
case PUT:
req = new HttpPut(url);
if (entity != null) {
((HttpPut) req).setEntity(entity);
}
break;
case DELETE:
req = new HttpDelete(url);
break;
default:
throw new IOException("Unsupported method " + method);
}
if (headers != null) {
headers.forEach((k, v) -> req.addHeader(k, v));
}
RequestConfig.Builder requestConfigBuilder = HttpClientUtil.createDefaultRequestConfigBuilder();
if (timeout > 0) {
requestConfigBuilder.setSocketTimeout(timeout);
requestConfigBuilder.setConnectTimeout(timeout);
}
requestConfigBuilder.setRedirectsEnabled(followRedirects);
req.setConfig(requestConfigBuilder.build());
HttpClientContext httpClientRequestContext = HttpClientUtil.createNewHttpClientRequestContext();
HttpResponse rsp = client.execute(req, httpClientRequestContext);
int statusCode = rsp.getStatusLine().getStatusCode();
if (statusCode != 200) {
throw new IOException(
"Error sending request to " + url + ", HTTP response: " + rsp.toString());
}
HttpEntity responseEntity = rsp.getEntity();
if (responseEntity != null && responseEntity.getContent() != null) {
return EntityUtils.toByteArray(responseEntity);
} else {
return EMPTY;
}
}

public SolrZkClient getZkClient() {
return zkClient;
}
Expand Down
Loading
Loading