From 30b99d0962568d5934401783e357fd68616c8cf8 Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Mon, 21 Oct 2024 14:46:21 -0400 Subject: [PATCH 1/6] SOLR-17467: Finish migration to --user-managed mode (#2781) * Tighten test up to show that while we started, we aren't in cloud mode until this fix was added. --- solr/core/src/java/org/apache/solr/cli/RunExampleTool.java | 2 +- solr/packaging/test/test_example_noprompt.bats | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/cli/RunExampleTool.java b/solr/core/src/java/org/apache/solr/cli/RunExampleTool.java index 4ede96e9a70..788a9b86dfc 100644 --- a/solr/core/src/java/org/apache/solr/cli/RunExampleTool.java +++ b/solr/core/src/java/org/apache/solr/cli/RunExampleTool.java @@ -537,7 +537,7 @@ protected void runCloudExample(CommandLine cli) throws Exception { // start the other nodes for (int n = 1; n < numNodes; n++) startSolr( - new File(cloudDir, "node" + (n + 1) + "/solr"), false, cli, cloudPorts[n], zkHost, 30); + new File(cloudDir, "node" + (n + 1) + "/solr"), true, cli, cloudPorts[n], zkHost, 30); } String solrUrl = (String) nodeStatus.get("baseUrl"); diff --git a/solr/packaging/test/test_example_noprompt.bats b/solr/packaging/test/test_example_noprompt.bats index 7d5817a8847..19329a8041a 100644 --- a/solr/packaging/test/test_example_noprompt.bats +++ b/solr/packaging/test/test_example_noprompt.bats @@ -28,8 +28,8 @@ teardown() { solr stop --all >/dev/null 2>&1 } -@test "SOLR-16755 test works with --no-prompt" { +@test "start -e cloud works with --no-prompt" { solr start -e cloud --no-prompt - solr assert --started http://localhost:${SOLR_PORT} --timeout 10000 - solr assert --started http://localhost:${SOLR2_PORT} --timeout 10000 + solr assert --started http://localhost:${SOLR_PORT} --cloud http://localhost:${SOLR_PORT} --timeout 10000 + solr assert --started http://localhost:${SOLR2_PORT} --cloud http://localhost:${SOLR2_PORT} --timeout 10000 } From 943e02236ad8f9ae5a842189a135046d6f2b3bfa Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Mon, 21 Oct 2024 15:55:28 -0400 Subject: [PATCH 2/6] Update docs to reflect SolrCloud is now default mode. (#2780) --- README.md | 3 +-- .../deployment-guide/pages/enabling-ssl.adoc | 4 +-- .../pages/installing-solr.adoc | 6 ++--- .../pages/solr-control-script-reference.adoc | 26 +++++++++---------- .../deployment-guide/pages/solr-on-hdfs.adoc | 2 +- .../pages/tutorial-solrcloud.adoc | 14 +++++----- .../pages/tutorial-techproducts.adoc | 4 +-- 7 files changed, 29 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index 920a4fa3005..cf08fa329a5 100644 --- a/README.md +++ b/README.md @@ -94,7 +94,7 @@ Solr uses [Gradle](https://gradle.org/) for its build system. Here are some usef ``` cd ./solr/packaging/build/dev -bin/solr start -c +bin/solr start ``` - Open a web browser and go to http://localhost:8983/solr/ to access the Solr Admin interface. You can also use the `bin/solr` script to create and manage Solr collections. For example use the `bin/solr post` tool to index some sample data. @@ -108,4 +108,3 @@ To get involved in the developer community: - Slack: `#solr-dev` in the `the-asf` organization. Sign up at https://the-asf.slack.com/messages/CE70MDPMF - [Issue Tracker (JIRA)](https://issues.apache.org/jira/browse/SOLR) - IRC: `#solr-dev` on [libera.chat](https://web.libera.chat/?channels=#solr-dev) - diff --git a/solr/solr-ref-guide/modules/deployment-guide/pages/enabling-ssl.adoc b/solr/solr-ref-guide/modules/deployment-guide/pages/enabling-ssl.adoc index 55aa2bde610..34d540975a6 100644 --- a/solr/solr-ref-guide/modules/deployment-guide/pages/enabling-ssl.adoc +++ b/solr/solr-ref-guide/modules/deployment-guide/pages/enabling-ssl.adoc @@ -286,7 +286,7 @@ If you created the SSL key without all DNS names or IP addresses on which Solr n ==== [source,terminal] ---- -$ bin/solr start --cloud --solr-home cloud/node1 -z server1:2181,server2:2181,server3:2181 -p 8984 +$ bin/solr start --solr-home cloud/node1 -z server1:2181,server2:2181,server3:2181 -p 8984 ---- ==== @@ -295,7 +295,7 @@ Windows:: ==== [source,powershell] ---- -C:\> bin\solr.cmd --cloud --solr-home cloud\node1 -z server1:2181,server2:2181,server3:2181 +C:\> bin\solr.cmd --solr-home cloud\node1 -z server1:2181,server2:2181,server3:2181 ---- ==== diff --git a/solr/solr-ref-guide/modules/deployment-guide/pages/installing-solr.adoc b/solr/solr-ref-guide/modules/deployment-guide/pages/installing-solr.adoc index 8dceba13dbe..497006c4de2 100644 --- a/solr/solr-ref-guide/modules/deployment-guide/pages/installing-solr.adoc +++ b/solr/solr-ref-guide/modules/deployment-guide/pages/installing-solr.adoc @@ -171,14 +171,14 @@ To use it to start Solr you can simply enter: [source,bash] ---- -bin/solr start --cloud +bin/solr start ---- If you are running Windows, you can start Solr by running `bin\solr.cmd` instead. [source,plain] ---- -bin\solr.cmd start --cloud +bin\solr.cmd start ---- This will start Solr in the background, listening on port 8983. @@ -195,7 +195,7 @@ For instance, to launch the "techproducts" example, you would do: [source,bash] ---- -bin/solr start --cloud -e techproducts +bin/solr start -e techproducts ---- Currently, the available examples you can run are: techproducts, schemaless, and cloud. diff --git a/solr/solr-ref-guide/modules/deployment-guide/pages/solr-control-script-reference.adoc b/solr/solr-ref-guide/modules/deployment-guide/pages/solr-control-script-reference.adoc index 7dd6bd418ce..e66b591280b 100644 --- a/solr/solr-ref-guide/modules/deployment-guide/pages/solr-control-script-reference.adoc +++ b/solr/solr-ref-guide/modules/deployment-guide/pages/solr-control-script-reference.adoc @@ -80,21 +80,18 @@ Additional parameters to pass to Jetty when starting Solr. For example, to add configuration folder that Jetty should read you could pass: `-j "--include-jetty-dir=/etc/jetty/custom/server/"`. In most cases, you should wrap the additional parameters in double quotes. -`--cloud` or `-c`:: +`--user-managed`:: + [%autowidth,frame=none] |=== |Optional |Default: none |=== + -Start Solr in SolrCloud mode, which will also launch the embedded ZooKeeper instance included with Solr. -The embedded ZooKeeper instance is started on Solr port+1000, so 9983 if Solr is bound to 8983. +Start Solr in User Managed mode (AKA Standalone mode). This is the style of Solr that supports only cores, not collections. + -If you are already running a ZooKeeper ensemble that you want to use instead of the embedded (single-node) ZooKeeper, you should also either specify `ZK_HOST` in `solr.in.sh`/`solr.in.cmd` (see xref:zookeeper-ensemble.adoc#updating-solr-include-files[Updating Solr Include Files]) or pass the `-z` parameter. +For more details, see the section <> below. + -For more details, see the section <> below. -+ -*Example*: `bin/solr start -c` +*Example*: `bin/solr start --user-managed` `--server-dir `:: + @@ -303,11 +300,7 @@ the `SOLR_TOOL_OPTS` environment variable should be used. ==== SolrCloud Mode -The `-c` and `--cloud` options are equivalent: - -`bin/solr start -c` - -`bin/solr start --cloud` +This is the default mode for running Solr. If you specify a ZooKeeper connection string, such as `-z 192.168.1.4:2181`, then Solr will connect to ZooKeeper and join the cluster. @@ -327,6 +320,13 @@ When starting in SolrCloud mode, the interactive script session will prompt you For more information about starting Solr in SolrCloud mode, see also the section xref:getting-started:tutorial-solrcloud.adoc[]. +==== User Managed Mode + +`bin/solr start --user-managed` starts Solr in User Managed mode (AKA Standalone mode). This was the default mode up until Solr 10x. + +For more information about the different modes, see the section xref:deployment-guide:cluster-types.adoc[]. + + ==== Running with Example Configurations `bin/solr start -e ` @@ -337,7 +337,7 @@ Each example launches Solr with a managed schema, which allows use of the xref:i If you would prefer to manually modify a `schema.xml` file directly, you can change this default as described in the section xref:configuration-guide:schema-factory.adoc[]. -Unless otherwise noted in the descriptions below, the examples do not enable SolrCloud nor xref:indexing-guide:schemaless-mode.adoc[]. +Unless otherwise noted in the descriptions below, the examples do not enable xref:indexing-guide:schemaless-mode.adoc[]. The following examples are provided: diff --git a/solr/solr-ref-guide/modules/deployment-guide/pages/solr-on-hdfs.adoc b/solr/solr-ref-guide/modules/deployment-guide/pages/solr-on-hdfs.adoc index f2b7f5a4f63..25548bfac2c 100644 --- a/solr/solr-ref-guide/modules/deployment-guide/pages/solr-on-hdfs.adoc +++ b/solr/solr-ref-guide/modules/deployment-guide/pages/solr-on-hdfs.adoc @@ -46,7 +46,7 @@ All dynamically created collections will create the appropriate directories auto [source,bash] ---- -bin/solr start --cloud -Dsolr.directoryFactory=HdfsDirectoryFactory +bin/solr start -Dsolr.directoryFactory=HdfsDirectoryFactory -Dsolr.lock.type=hdfs -Dsolr.hdfs.home=hdfs://host:port/path ---- diff --git a/solr/solr-ref-guide/modules/getting-started/pages/tutorial-solrcloud.adoc b/solr/solr-ref-guide/modules/getting-started/pages/tutorial-solrcloud.adoc index b1dd0b2c04f..3e966f929ec 100644 --- a/solr/solr-ref-guide/modules/getting-started/pages/tutorial-solrcloud.adoc +++ b/solr/solr-ref-guide/modules/getting-started/pages/tutorial-solrcloud.adoc @@ -1,4 +1,4 @@ -= Getting Started with SolrCloud += Exercise 6: Getting Deeper with SolrCloud :tabs-sync-option: // Licensed to the Apache Software Foundation (ASF) under one // or more contributor license agreements. See the NOTICE file @@ -17,7 +17,7 @@ // specific language governing permissions and limitations // under the License. -SolrCloud is designed to provide a highly available, fault tolerant environment for distributing your indexed content and query requests across multiple servers. +SolrCloud is designed to provide a highly available, fault tolerant environment for distributing your indexed content and query requests across multiple servers. It is the default mode for running Solr, even with a single node! It's a system in which data is organized into multiple pieces, or shards, that can be hosted on multiple machines, with replicas providing redundancy for both scalability and fault tolerance, and a ZooKeeper server that helps manage the overall structure so that both indexing and search requests can be routed properly. @@ -82,7 +82,7 @@ The script will start each node in order and show you the command it uses to sta [,console] ---- -$ bin/solr start --cloud --solr-home example/cloud/node1/solr -p 8983 +$ bin/solr start --solr-home example/cloud/node1/solr -p 8983 ---- The first node will also start an embedded ZooKeeper server bound to port 9983. @@ -176,14 +176,14 @@ For instance, to restart node1 running on port 8983 (with an embedded ZooKeeper [,console] ---- -$ bin/solr restart -c -p 8983 --solr-home example/cloud/node1/solr +$ bin/solr restart -p 8983 --solr-home example/cloud/node1/solr ---- To restart node2 running on port 7574, you can do: [,console] ---- -$ bin/solr restart -c -p 7574 -z localhost:9983 --solr-home example/cloud/node2/solr +$ bin/solr restart -p 7574 -z localhost:9983 --solr-home example/cloud/node2/solr ---- Notice that you need to specify the ZooKeeper address (`-z localhost:9983`) when starting node2 so that it can join the cluster with node1. @@ -196,7 +196,7 @@ Once you start a SolrCloud cluster using the startup scripts, you can add a new [,console] ---- $ mkdir -$ bin/solr start --cloud --solr-home /solr -p -z +$ bin/solr start --solr-home /solr -p -z ---- Notice that the above requires you to create a Solr home directory. @@ -206,7 +206,7 @@ Example (with directory structure) that adds a node to an example started with " [,console] ---- $ mkdir -p example/cloud/node3/solr -$ bin/solr start --cloud --solr-home example/cloud/node3/solr -p 8987 -z localhost:9983 +$ bin/solr start --solr-home example/cloud/node3/solr -p 8987 -z localhost:9983 ---- The previous command will start another Solr node on port 8987 with Solr home set to `example/cloud/node3/solr`. diff --git a/solr/solr-ref-guide/modules/getting-started/pages/tutorial-techproducts.adoc b/solr/solr-ref-guide/modules/getting-started/pages/tutorial-techproducts.adoc index ef9fab4d0cc..e397ccf9db9 100644 --- a/solr/solr-ref-guide/modules/getting-started/pages/tutorial-techproducts.adoc +++ b/solr/solr-ref-guide/modules/getting-started/pages/tutorial-techproducts.adoc @@ -71,14 +71,14 @@ Cloning /solr-{solr-full-version}/example/cloud/node1 into /solr-{solr-full-version}/example/cloud/node2 Starting up Solr on port 8983 using command: -"bin/solr" start --cloud -p 8983 --solr-home "example/cloud/node1/solr" +"bin/solr" start -p 8983 --solr-home "example/cloud/node1/solr" Waiting up to 180 seconds to see Solr running on port 8983 [\] Started Solr server on port 8983 (pid=34942). Happy searching! Starting up Solr on port 7574 using command: -"bin/solr" start --cloud -p 7574 --solr-home "example/cloud/node2/solr" -z localhost:9983 +"bin/solr" start -p 7574 --solr-home "example/cloud/node2/solr" -z localhost:9983 Waiting up to 180 seconds to see Solr running on port 7574 [\] Started Solr server on port 7574 (pid=35036). Happy searching! From 018e4f445fe558d4383ea5bc9f84400da74f9517 Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Mon, 21 Oct 2024 16:16:26 -0400 Subject: [PATCH 3/6] SOLR-17494: Remove language specific writer types (#2758) wt=python, ruby, php, and phps response writers are removed. --- dev-tools/scripts/smokeTestRelease.py | 2 - solr/CHANGES.txt | 2 + .../bench/search/QueryResponseWriters.java | 12 +- .../java/org/apache/solr/core/SolrCore.java | 8 - .../solr/response/PHPResponseWriter.java | 124 -------- .../response/PHPSerializedResponseWriter.java | 282 ------------------ .../solr/response/PythonResponseWriter.java | 158 ---------- .../solr/response/RubyResponseWriter.java | 92 ------ .../apache/solr/response/JSONWriterTest.java | 18 +- .../TestPHPSerializedResponseWriter.java | 97 ------ solr/example/exampledocs/test_utf8.sh | 93 ------ .../configsets/_default/conf/solrconfig.xml | 4 - .../conf/solrconfig.xml | 4 - .../deployment-guide/pages/python.adoc | 42 +-- .../modules/deployment-guide/pages/ruby.adoc | 115 ++----- .../pages/tutorial-techproducts.adoc | 1 - .../pages/indexing-with-tika.adoc | 2 +- .../query-guide/pages/response-writers.adoc | 66 +--- .../pages/major-changes-in-solr-10.adoc | 3 + 19 files changed, 46 insertions(+), 1079 deletions(-) delete mode 100644 solr/core/src/java/org/apache/solr/response/PHPResponseWriter.java delete mode 100644 solr/core/src/java/org/apache/solr/response/PHPSerializedResponseWriter.java delete mode 100644 solr/core/src/java/org/apache/solr/response/PythonResponseWriter.java delete mode 100644 solr/core/src/java/org/apache/solr/response/RubyResponseWriter.java delete mode 100644 solr/core/src/test/org/apache/solr/response/TestPHPSerializedResponseWriter.java delete mode 100755 solr/example/exampledocs/test_utf8.sh diff --git a/dev-tools/scripts/smokeTestRelease.py b/dev-tools/scripts/smokeTestRelease.py index 97ca684ff56..acfa109f2fc 100755 --- a/dev-tools/scripts/smokeTestRelease.py +++ b/dev-tools/scripts/smokeTestRelease.py @@ -774,8 +774,6 @@ def testSolrExample(binaryDistPath, javaPath): raise RuntimeError('Failed to run the techproducts example, check log for previous errors.') os.chdir('example') - print(' test utf8...') - run('sh ./exampledocs/test_utf8.sh http://localhost:8983/solr/techproducts', 'utf8.log') print(' run query...') s = load('http://localhost:8983/solr/techproducts/select/?q=video') if s.find('"numFound":3,') == -1: diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 1b60d044ad0..1eff932d698 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -73,6 +73,8 @@ Deprecation Removals * SOLR-17400: Remove deprecated script snapshotcli.sh. bin/solr snapshot-* commands have replaced this. (Eric Pugh) +* SOLR-17494: Remove language specific writer types (i.e wt= ruby, python, php, and phps). (Eric Pugh) + Dependency Upgrades --------------------- (No changes) diff --git a/solr/benchmark/src/java/org/apache/solr/bench/search/QueryResponseWriters.java b/solr/benchmark/src/java/org/apache/solr/bench/search/QueryResponseWriters.java index 15b4526d49e..3c3217cc085 100644 --- a/solr/benchmark/src/java/org/apache/solr/bench/search/QueryResponseWriters.java +++ b/solr/benchmark/src/java/org/apache/solr/bench/search/QueryResponseWriters.java @@ -59,17 +59,7 @@ public class QueryResponseWriters { public static class BenchState { /** See {@link SolrCore#DEFAULT_RESPONSE_WRITERS} */ - @Param({ - CommonParams.JAVABIN, - CommonParams.JSON, - "cbor", - "smile", - "xml", - "python", - "phps", - "ruby", - "raw" - }) + @Param({CommonParams.JAVABIN, CommonParams.JSON, "cbor", "smile", "xml", "raw"}) String wt; private int docs = 100; diff --git a/solr/core/src/java/org/apache/solr/core/SolrCore.java b/solr/core/src/java/org/apache/solr/core/SolrCore.java index da78f08fdc4..f56cb846767 100644 --- a/solr/core/src/java/org/apache/solr/core/SolrCore.java +++ b/solr/core/src/java/org/apache/solr/core/SolrCore.java @@ -128,13 +128,9 @@ import org.apache.solr.response.GeoJSONResponseWriter; import org.apache.solr.response.GraphMLResponseWriter; import org.apache.solr.response.JacksonJsonWriter; -import org.apache.solr.response.PHPResponseWriter; -import org.apache.solr.response.PHPSerializedResponseWriter; import org.apache.solr.response.PrometheusResponseWriter; -import org.apache.solr.response.PythonResponseWriter; import org.apache.solr.response.QueryResponseWriter; import org.apache.solr.response.RawResponseWriter; -import org.apache.solr.response.RubyResponseWriter; import org.apache.solr.response.SchemaXmlResponseWriter; import org.apache.solr.response.SmileResponseWriter; import org.apache.solr.response.SolrQueryResponse; @@ -3015,10 +3011,6 @@ public PluginBag getResponseWriters() { m.put("standard", m.get(CommonParams.JSON)); m.put("geojson", new GeoJSONResponseWriter()); m.put("graphml", new GraphMLResponseWriter()); - m.put("python", new PythonResponseWriter()); - m.put("php", new PHPResponseWriter()); - m.put("phps", new PHPSerializedResponseWriter()); - m.put("ruby", new RubyResponseWriter()); m.put("raw", new RawResponseWriter()); m.put(CommonParams.JAVABIN, new BinaryResponseWriter()); m.put("cbor", new CborResponseWriter()); diff --git a/solr/core/src/java/org/apache/solr/response/PHPResponseWriter.java b/solr/core/src/java/org/apache/solr/response/PHPResponseWriter.java deleted file mode 100644 index b9f4c3417fd..00000000000 --- a/solr/core/src/java/org/apache/solr/response/PHPResponseWriter.java +++ /dev/null @@ -1,124 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.response; - -import java.io.IOException; -import java.io.Writer; -import java.util.List; -import org.apache.solr.common.util.NamedList; -import org.apache.solr.request.SolrQueryRequest; - -public class PHPResponseWriter implements QueryResponseWriter { - static String CONTENT_TYPE_PHP_UTF8 = "text/x-php;charset=UTF-8"; - - private String contentType = CONTENT_TYPE_PHP_UTF8; - - @Override - public void init(NamedList namedList) { - String contentType = (String) namedList.get("content-type"); - if (contentType != null) { - this.contentType = contentType; - } - } - - @Override - public void write(Writer writer, SolrQueryRequest req, SolrQueryResponse rsp) throws IOException { - PHPWriter w = new PHPWriter(writer, req, rsp); - try { - w.writeResponse(); - } finally { - w.close(); - } - } - - @Override - public String getContentType(SolrQueryRequest request, SolrQueryResponse response) { - return contentType; - } -} - -class PHPWriter extends JSONWriter { - public PHPWriter(Writer writer, SolrQueryRequest req, SolrQueryResponse rsp) { - super(writer, req, rsp); - } - - @Override - public void writeNamedList(String name, NamedList val) throws IOException { - writeNamedListAsMapMangled(name, val); - } - - @Override - public void writeMapOpener(int size) throws IOException { - writer.write("array("); - } - - @Override - public void writeMapCloser() throws IOException { - writer.write(')'); - } - - @Override - public void writeArrayOpener(int size) throws IOException { - writer.write("array("); - } - - @Override - public void writeArray(String name, List l, boolean raw) throws IOException { - writeArray(name, l.iterator(), raw); - } - - @Override - public void writeArrayCloser() throws IOException { - writer.write(')'); - } - - @Override - public void writeNull(String name) throws IOException { - writer.write("null"); - } - - @Override - public void writeKey(String fname, boolean needsEscaping) throws IOException { - writeStr(null, fname, needsEscaping); - writer.write('='); - writer.write('>'); - } - - @Override - public void writeStr(String name, String val, boolean needsEscaping) throws IOException { - if (needsEscaping) { - writer.write('\''); - for (int i = 0; i < val.length(); i++) { - char ch = val.charAt(i); - switch (ch) { - case '\'': - case '\\': - writer.write('\\'); - writer.write(ch); - break; - default: - writer.write(ch); - } - } - writer.write('\''); - } else { - writer.write('\''); - writer.write(val); - writer.write('\''); - } - } -} diff --git a/solr/core/src/java/org/apache/solr/response/PHPSerializedResponseWriter.java b/solr/core/src/java/org/apache/solr/response/PHPSerializedResponseWriter.java deleted file mode 100644 index 0a6f828936f..00000000000 --- a/solr/core/src/java/org/apache/solr/response/PHPSerializedResponseWriter.java +++ /dev/null @@ -1,282 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.response; - -import java.io.IOException; -import java.io.Writer; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Iterator; -import java.util.LinkedHashMap; -import java.util.Map; -import org.apache.lucene.util.ArrayUtil; -import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.UnicodeUtil; -import org.apache.solr.common.SolrDocument; -import org.apache.solr.common.params.CommonParams; -import org.apache.solr.common.util.NamedList; -import org.apache.solr.request.SolrQueryRequest; -import org.apache.solr.search.ReturnFields; - -/** - * A description of the PHP serialization format can be found here: - * http://www.hurring.com/scott/code/perl/serialize/ - */ -public class PHPSerializedResponseWriter implements QueryResponseWriter { - static String CONTENT_TYPE_PHP_UTF8 = "text/x-php-serialized;charset=UTF-8"; - - private String contentType = CONTENT_TYPE_PHP_UTF8; - - @Override - public void init(NamedList namedList) { - String contentType = (String) namedList.get("content-type"); - if (contentType != null) { - this.contentType = contentType; - } - } - - @Override - public void write(Writer writer, SolrQueryRequest req, SolrQueryResponse rsp) throws IOException { - PHPSerializedWriter w = new PHPSerializedWriter(writer, req, rsp); - try { - w.writeResponse(); - } finally { - w.close(); - } - } - - @Override - public String getContentType(SolrQueryRequest request, SolrQueryResponse response) { - return contentType; - } -} - -class PHPSerializedWriter extends JSONWriter { - byte[] utf8; - - public PHPSerializedWriter(Writer writer, SolrQueryRequest req, SolrQueryResponse rsp) { - super(writer, req, rsp); - this.utf8 = BytesRef.EMPTY_BYTES; - // never indent serialized PHP data - doIndent = false; - } - - @Override - public void writeResponse() throws IOException { - Boolean omitHeader = req.getParams().getBool(CommonParams.OMIT_HEADER); - if (omitHeader != null && omitHeader) rsp.removeResponseHeader(); - writeNamedList(null, rsp.getValues()); - } - - @Override - public void writeNamedList(String name, NamedList val) throws IOException { - writeNamedListAsMapMangled(name, val); - } - - @Override - public void writeStartDocumentList( - String name, long start, int size, long numFound, Float maxScore, Boolean numFoundExact) - throws IOException { - writeMapOpener(headerSize(maxScore, numFoundExact)); - writeKey("numFound", false); - writeLong(null, numFound); - writeKey("start", false); - writeLong(null, start); - - if (maxScore != null) { - writeKey("maxScore", false); - writeFloat(null, maxScore); - } - if (numFoundExact != null) { - writeKey("numFoundExact", false); - writeBool(null, numFoundExact); - } - writeKey("docs", false); - writeArrayOpener(size); - } - - @Override - public void writeEndDocumentList() throws IOException { - writeArrayCloser(); // doc list - writeMapCloser(); - } - - @Override - public void writeSolrDocument(String name, SolrDocument doc, ReturnFields returnFields, int idx) - throws IOException { - writeKey(idx, false); - - LinkedHashMap single = new LinkedHashMap<>(); - LinkedHashMap multi = new LinkedHashMap<>(); - - for (String fname : doc.getFieldNames()) { - if (returnFields != null && !returnFields.wantsField(fname)) { - continue; - } - - Object val = doc.getFieldValue(fname); - if (val instanceof Collection) { - multi.put(fname, val); - } else { - single.put(fname, val); - } - } - - writeMapOpener(single.size() + multi.size()); - for (Map.Entry entry : single.entrySet()) { - String fname = entry.getKey(); - Object val = entry.getValue(); - writeKey(fname, true); - writeVal(fname, val); - } - - for (Map.Entry entry : multi.entrySet()) { - String fname = entry.getKey(); - writeKey(fname, true); - - Object val = entry.getValue(); - if (!(val instanceof Collection)) { - // should never be reached if multivalued fields are stored as a Collection - // so I'm assuming a size of 1 just to wrap the single value - writeArrayOpener(1); - writeVal(fname, val); - writeArrayCloser(); - } else { - writeVal(fname, val); - } - } - - writeMapCloser(); - } - - @Override - public void writeArray(String name, Object[] val, boolean raw) throws IOException { - assert !raw; - writeMapOpener(val.length); - for (int i = 0; i < val.length; i++) { - writeKey(i, false); - writeVal(String.valueOf(i), val[i]); - } - writeMapCloser(); - } - - @Override - public void writeArray(String name, Iterator val, boolean raw) throws IOException { - assert !raw; - ArrayList vals = new ArrayList<>(); - while (val.hasNext()) { - vals.add(val.next()); - } - writeArray(name, vals.toArray(), false); - } - - @Override - public void writeMapOpener(int size) throws IOException, IllegalArgumentException { - // negative size value indicates that something has gone wrong - if (size < 0) { - throw new IllegalArgumentException("Map size must not be negative"); - } - writer.write("a:" + size + ":{"); - } - - @Override - public void writeMapSeparator() throws IOException { - /* NOOP */ - } - - @Override - public void writeMapCloser() throws IOException { - writer.write('}'); - } - - @Override - public void writeArrayOpener(int size) throws IOException, IllegalArgumentException { - // negative size value indicates that something has gone wrong - if (size < 0) { - throw new IllegalArgumentException("Array size must not be negative"); - } - writer.write("a:" + size + ":{"); - } - - @Override - public void writeArraySeparator() throws IOException { - /* NOOP */ - } - - @Override - public void writeArrayCloser() throws IOException { - writer.write('}'); - } - - @Override - public void writeNull(String name) throws IOException { - writer.write("N;"); - } - - @Override - public void writeKey(String fname, boolean needsEscaping) throws IOException { - writeStr(null, fname, needsEscaping); - } - - void writeKey(int val, boolean needsEscaping) throws IOException { - writeInt(null, String.valueOf(val)); - } - - @Override - public void writeBool(String name, boolean val) throws IOException { - writer.write(val ? "b:1;" : "b:0;"); - } - - @Override - public void writeBool(String name, String val) throws IOException { - writeBool(name, val.charAt(0) == 't'); - } - - @Override - public void writeInt(String name, String val) throws IOException { - writer.write("i:" + val + ";"); - } - - @Override - public void writeLong(String name, String val) throws IOException { - writeInt(name, val); - } - - @Override - public void writeFloat(String name, String val) throws IOException { - writeDouble(name, val); - } - - @Override - public void writeDouble(String name, String val) throws IOException { - writer.write("d:" + val + ";"); - } - - @Override - public void writeStr(String name, String val, boolean needsEscaping) throws IOException { - // serialized PHP strings don't need to be escaped at all, however the - // string size reported needs be the number of bytes rather than chars. - utf8 = ArrayUtil.grow(utf8, val.length() * UnicodeUtil.MAX_UTF8_BYTES_PER_CHAR); - final int nBytes = UnicodeUtil.UTF16toUTF8(val, 0, val.length(), utf8); - - writer.write("s:"); - writer.write(Integer.toString(nBytes)); - writer.write(":\""); - writer.write(val); - writer.write("\";"); - } -} diff --git a/solr/core/src/java/org/apache/solr/response/PythonResponseWriter.java b/solr/core/src/java/org/apache/solr/response/PythonResponseWriter.java deleted file mode 100644 index d50fb95e287..00000000000 --- a/solr/core/src/java/org/apache/solr/response/PythonResponseWriter.java +++ /dev/null @@ -1,158 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.response; - -import java.io.IOException; -import java.io.Writer; -import org.apache.solr.request.SolrQueryRequest; - -public class PythonResponseWriter implements QueryResponseWriter { - static String CONTENT_TYPE_PYTHON_ASCII = "text/x-python;charset=US-ASCII"; - - @Override - public void write(Writer writer, SolrQueryRequest req, SolrQueryResponse rsp) throws IOException { - PythonWriter w = new PythonWriter(writer, req, rsp); - try { - w.writeResponse(); - } finally { - w.close(); - } - } - - @Override - public String getContentType(SolrQueryRequest request, SolrQueryResponse response) { - return CONTENT_TYPE_TEXT_ASCII; - } -} - -class PythonWriter extends JSONResponseWriter.NaNFloatWriter { - @Override - protected String getNaN() { - return "float('NaN')"; - } - - @Override - protected String getInf() { - return "float('Inf')"; - } - - public PythonWriter(Writer writer, SolrQueryRequest req, SolrQueryResponse rsp) { - super(writer, req, rsp); - } - - @Override - public void writeNull(String name) throws IOException { - writer.write("None"); - } - - @Override - public void writeBool(String name, boolean val) throws IOException { - writer.write(val ? "True" : "False"); - } - - @Override - public void writeBool(String name, String val) throws IOException { - writeBool(name, val.charAt(0) == 't'); - } - - /* optionally use a unicode python string if necessary */ - @Override - public void writeStr(String name, String val, boolean needsEscaping) throws IOException { - if (!needsEscaping) { - writer.write('\''); - writer.write(val); - writer.write('\''); - return; - } - - // use python unicode strings... - // python doesn't tolerate newlines in strings in its eval(), so we must escape them. - - StringBuilder sb = new StringBuilder(val.length()); - boolean needUnicode = false; - - for (int i = 0; i < val.length(); i++) { - char ch = val.charAt(i); - switch (ch) { - case '\'': - case '\\': - sb.append('\\'); - sb.append(ch); - break; - case '\r': - sb.append("\\r"); - break; - case '\n': - sb.append("\\n"); - break; - case '\t': - sb.append("\\t"); - break; - default: - // we don't strictly have to escape these chars, but it will probably increase - // portability to stick to visible ascii - if (ch < ' ' || ch > 127) { - unicodeEscape(sb, ch); - needUnicode = true; - } else { - sb.append(ch); - } - } - } - - if (needUnicode) { - writer.write('u'); - } - writer.write('\''); - writer.append(sb); - writer.write('\''); - } - - /* - old version that always used unicode - public void writeStr(String name, String val, boolean needsEscaping) throws IOException { - // use python unicode strings... - // python doesn't tolerate newlines in strings in its eval(), so we must escape them. - writer.write("u'"); - // it might be more efficient to use a stringbuilder or write substrings - // if writing chars to the stream is slow. - if (needsEscaping) { - for (int i=0; i127) { - unicodeChar(ch); - } else { - writer.write(ch); - } - } - } - } else { - writer.write(val); - } - writer.write('\''); - } - */ - -} diff --git a/solr/core/src/java/org/apache/solr/response/RubyResponseWriter.java b/solr/core/src/java/org/apache/solr/response/RubyResponseWriter.java deleted file mode 100644 index bce2c2c9498..00000000000 --- a/solr/core/src/java/org/apache/solr/response/RubyResponseWriter.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.response; - -import java.io.IOException; -import java.io.Writer; -import org.apache.solr.request.SolrQueryRequest; - -public class RubyResponseWriter implements QueryResponseWriter { - static String CONTENT_TYPE_RUBY_UTF8 = "text/x-ruby;charset=UTF-8"; - - @Override - public void write(Writer writer, SolrQueryRequest req, SolrQueryResponse rsp) throws IOException { - RubyWriter w = new RubyWriter(writer, req, rsp); - try { - w.writeResponse(); - } finally { - w.close(); - } - } - - @Override - public String getContentType(SolrQueryRequest request, SolrQueryResponse response) { - return CONTENT_TYPE_TEXT_UTF8; - } -} - -class RubyWriter extends JSONResponseWriter.NaNFloatWriter { - - @Override - protected String getNaN() { - return "(0.0/0.0)"; - } - - @Override - protected String getInf() { - return "(1.0/0.0)"; - } - - public RubyWriter(Writer writer, SolrQueryRequest req, SolrQueryResponse rsp) { - super(writer, req, rsp); - } - - @Override - public void writeNull(String name) throws IOException { - writer.write("nil"); - } - - @Override - public void writeKey(String fname, boolean needsEscaping) throws IOException { - writeStr(null, fname, needsEscaping); - writer.write('='); - writer.write('>'); - } - - @Override - public void writeStr(String name, String val, boolean needsEscaping) throws IOException { - // Ruby doesn't do unicode escapes... so let the servlet container write raw UTF-8 - // bytes into the string. - // - // Use single quoted strings for safety since no evaluation is done within them. - // Also, there are very few escapes recognized in a single quoted string, so - // only escape the backslash and single quote. - writer.write('\''); - if (needsEscaping) { - for (int i = 0; i < val.length(); i++) { - char ch = val.charAt(i); - if (ch == '\'' || ch == '\\') { - writer.write('\\'); - } - writer.write(ch); - } - } else { - writer.write(val); - } - writer.write('\''); - } -} diff --git a/solr/core/src/test/org/apache/solr/response/JSONWriterTest.java b/solr/core/src/test/org/apache/solr/response/JSONWriterTest.java index 6d45634da78..5ccf19cdecf 100644 --- a/solr/core/src/test/org/apache/solr/response/JSONWriterTest.java +++ b/solr/core/src/test/org/apache/solr/response/JSONWriterTest.java @@ -39,7 +39,7 @@ import org.junit.BeforeClass; import org.junit.Test; -/** Test some aspects of JSON/python writer output (very incomplete) */ +/** Test some aspects of JSON writer output */ public class JSONWriterTest extends SolrTestCaseJ4 { @BeforeClass public static void beforeClass() throws Exception { @@ -53,25 +53,17 @@ private void jsonEq(String expected, String received) { } @Test - public void testTypes() throws IOException { + public void testSimpleJson() throws IOException { SolrQueryRequest req = req("q", "dummy", "indent", "off"); SolrQueryResponse rsp = new SolrQueryResponse(); - QueryResponseWriter w = new PythonResponseWriter(); + QueryResponseWriter w = new JSONResponseWriter(); StringWriter buf = new StringWriter(); + rsp.add("data1", Float.NaN); rsp.add("data2", Double.NEGATIVE_INFINITY); rsp.add("data3", Float.POSITIVE_INFINITY); - w.write(buf, req, rsp); - jsonEq(buf.toString(), "{'data1':float('NaN'),'data2':-float('Inf'),'data3':float('Inf')}"); - - w = new RubyResponseWriter(); - buf = new StringWriter(); - w.write(buf, req, rsp); - jsonEq(buf.toString(), "{'data1'=>(0.0/0.0),'data2'=>-(1.0/0.0),'data3'=>(1.0/0.0)}"); - w = new JSONResponseWriter(); - buf = new StringWriter(); w.write(buf, req, rsp); jsonEq(buf.toString(), "{\"data1\":\"NaN\",\"data2\":\"-Infinity\",\"data3\":\"Infinity\"}"); req.close(); @@ -326,7 +318,7 @@ public void testWfrJacksonJsonWriter() throws IOException { JacksonJsonWriter w = new JacksonJsonWriter(); ByteArrayOutputStream baos = new ByteArrayOutputStream(); w.write(baos, req, rsp); - String received = new String(baos.toByteArray(), StandardCharsets.UTF_8); + String received = baos.toString(StandardCharsets.UTF_8); String expected = "testFun( {\n \"param0\":\"v0\",\n \"param1\":42\n} )"; jsonEq(expected, received); req.close(); diff --git a/solr/core/src/test/org/apache/solr/response/TestPHPSerializedResponseWriter.java b/solr/core/src/test/org/apache/solr/response/TestPHPSerializedResponseWriter.java deleted file mode 100644 index de6339650a3..00000000000 --- a/solr/core/src/test/org/apache/solr/response/TestPHPSerializedResponseWriter.java +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.response; - -import java.io.IOException; -import java.io.StringWriter; -import java.util.Arrays; -import java.util.LinkedHashMap; -import org.apache.solr.SolrTestCaseJ4; -import org.apache.solr.common.SolrDocument; -import org.apache.solr.common.SolrDocumentList; -import org.apache.solr.request.SolrQueryRequest; -import org.junit.BeforeClass; -import org.junit.Test; - -/** Basic PHPS tests based on JSONWriterTest */ -public class TestPHPSerializedResponseWriter extends SolrTestCaseJ4 { - @BeforeClass - public static void beforeClass() throws Exception { - initCore("solrconfig.xml", "schema.xml"); - } - - @Test - public void testSimple() throws IOException { - SolrQueryRequest req = req("dummy"); - SolrQueryResponse rsp = new SolrQueryResponse(); - QueryResponseWriter w = new PHPSerializedResponseWriter(); - - StringWriter buf = new StringWriter(); - rsp.add("data1", "hello"); - rsp.add("data2", 42); - rsp.add("data3", true); - w.write(buf, req, rsp); - assertEquals( - "a:3:{s:5:\"data1\";s:5:\"hello\";s:5:\"data2\";i:42;s:5:\"data3\";b:1;}", buf.toString()); - req.close(); - } - - @Test - public void testSolrDocuments() throws IOException { - SolrQueryRequest req = req("q", "*:*"); - SolrQueryResponse rsp = new SolrQueryResponse(); - QueryResponseWriter w = new PHPSerializedResponseWriter(); - StringWriter buf = new StringWriter(); - - SolrDocument d = new SolrDocument(); - - SolrDocument d1 = d; - d.addField("id", "1"); - d.addField("data1", "hello"); - d.addField("data2", 42); - d.addField("data3", true); - - // multivalued fields: - - // extremely odd edge case: value is a map - - // we use LinkedHashMap because we are doing a string comparison - // later, and we need predictable ordering - LinkedHashMap nl = new LinkedHashMap<>(); - nl.put("data4.1", "hashmap"); - nl.put("data4.2", "hello"); - d.addField("data4", nl); - // array value - d.addField("data5", Arrays.asList("data5.1", "data5.2", "data5.3")); - - // adding one more document to test array indexes - d = new SolrDocument(); - SolrDocument d2 = d; - d.addField("id", "2"); - - SolrDocumentList sdl = new SolrDocumentList(); - sdl.add(d1); - sdl.add(d2); - rsp.addResponse(sdl); - - w.write(buf, req, rsp); - assertEquals( - "a:1:{s:8:\"response\";a:4:{s:8:\"numFound\";i:0;s:5:\"start\";i:0;s:13:\"numFoundExact\";b:1;s:4:\"docs\";a:2:{i:0;a:6:{s:2:\"id\";s:1:\"1\";s:5:\"data1\";s:5:\"hello\";s:5:\"data2\";i:42;s:5:\"data3\";b:1;s:5:\"data4\";a:2:{s:7:\"data4.1\";s:7:\"hashmap\";s:7:\"data4.2\";s:5:\"hello\";}s:5:\"data5\";a:3:{i:0;s:7:\"data5.1\";i:1;s:7:\"data5.2\";i:2;s:7:\"data5.3\";}}i:1;a:1:{s:2:\"id\";s:1:\"2\";}}}}", - buf.toString()); - req.close(); - } -} diff --git a/solr/example/exampledocs/test_utf8.sh b/solr/example/exampledocs/test_utf8.sh deleted file mode 100755 index 9032e12ffe3..00000000000 --- a/solr/example/exampledocs/test_utf8.sh +++ /dev/null @@ -1,93 +0,0 @@ -#!/bin/sh -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -#Test script to tell if the server is accepting UTF-8 -#The python writer currently escapes non-ascii chars, so it's good for testing - -SOLR_URL=http://localhost:8983/solr - -if [ ! -z $1 ]; then - SOLR_URL=$1 -fi - -curl "$SOLR_URL/select?q=hello¶ms=explicit&wt=python" 2> /dev/null | grep 'hello' > /dev/null 2>&1 -if [ $? = 0 ]; then - echo "Solr server is up." -else - echo "ERROR: Could not curl to Solr - is curl installed? Is Solr not running?" - exit 1 -fi - -curl "$SOLR_URL/select?q=h%C3%A9llo&echoParams=explicit&wt=python" 2> /dev/null | grep 'h\\u00e9llo' > /dev/null 2>&1 -if [ $? = 0 ]; then - echo "HTTP GET is accepting UTF-8" -else - echo "ERROR: HTTP GET is not accepting UTF-8" -fi - -curl $SOLR_URL/select --data-binary 'q=h%C3%A9llo&echoParams=explicit&wt=python' -H 'Content-type:application/x-www-form-urlencoded; charset=UTF-8' 2> /dev/null | grep 'h\\u00e9llo' > /dev/null 2>&1 -if [ $? = 0 ]; then - echo "HTTP POST is accepting UTF-8" -else - echo "ERROR: HTTP POST is not accepting UTF-8" -fi - -curl $SOLR_URL/select --data-binary 'q=h%C3%A9llo&echoParams=explicit&wt=python' 2> /dev/null | grep 'h\\u00e9llo' > /dev/null 2>&1 -if [ $? = 0 ]; then - echo "HTTP POST defaults to UTF-8" -else - echo "HTTP POST does not default to UTF-8" -fi - - -#A unicode character outside of the BMP (a circle with an x inside) -CHAR="𐌈" -CODEPOINT='0x10308' -#URL encoded UTF8 of the codepoint -UTF8_Q='%F0%90%8C%88' -#expected return of the python writer (currently uses UTF-16 surrogates) -EXPECTED='\\ud800\\udf08' - -curl "$SOLR_URL/select?q=$UTF8_Q&echoParams=explicit&wt=python" 2> /dev/null | grep $EXPECTED > /dev/null 2>&1 -if [ $? = 0 ]; then - echo "HTTP GET is accepting UTF-8 beyond the basic multilingual plane" -else - echo "ERROR: HTTP GET is not accepting UTF-8 beyond the basic multilingual plane" -fi - -curl $SOLR_URL/select --data-binary "q=$UTF8_Q&echoParams=explicit&wt=python" -H 'Content-type:application/x-www-form-urlencoded; charset=UTF-8' 2> /dev/null | grep $EXPECTED > /dev/null 2>&1 -if [ $? = 0 ]; then - echo "HTTP POST is accepting UTF-8 beyond the basic multilingual plane" -else - echo "ERROR: HTTP POST is not accepting UTF-8 beyond the basic multilingual plane" -fi - -curl "$SOLR_URL/select?q=$UTF8_Q&echoParams=explicit&wt=python" --data-binary '' 2> /dev/null | grep $EXPECTED > /dev/null 2>&1 -if [ $? = 0 ]; then - echo "HTTP POST + URL params is accepting UTF-8 beyond the basic multilingual plane" -else - echo "ERROR: HTTP POST + URL params is not accepting UTF-8 beyond the basic multilingual plane" -fi - -#curl "$SOLR_URL/select?q=$UTF8_Q&echoParams=explicit" 2> /dev/null | od -tx1 -w1000 | sed 's/ //g' | grep 'f4808198' > /dev/null 2>&1 -curl "$SOLR_URL/select?q=$UTF8_Q&echoParams=explicit" 2> /dev/null | grep "$CHAR" > /dev/null 2>&1 -if [ $? = 0 ]; then - echo "Response correctly returns UTF-8 beyond the basic multilingual plane" -else - echo "ERROR: Response can't return UTF-8 beyond the basic multilingual plane" -fi - - diff --git a/solr/server/solr/configsets/_default/conf/solrconfig.xml b/solr/server/solr/configsets/_default/conf/solrconfig.xml index 825993191ae..5bc2cf13415 100644 --- a/solr/server/solr/configsets/_default/conf/solrconfig.xml +++ b/solr/server/solr/configsets/_default/conf/solrconfig.xml @@ -999,10 +999,6 @@ default="true" class="solr.XMLResponseWriter" /> - - - - --> diff --git a/solr/server/solr/configsets/sample_techproducts_configs/conf/solrconfig.xml b/solr/server/solr/configsets/sample_techproducts_configs/conf/solrconfig.xml index c8250755dc7..8350831eeb9 100644 --- a/solr/server/solr/configsets/sample_techproducts_configs/conf/solrconfig.xml +++ b/solr/server/solr/configsets/sample_techproducts_configs/conf/solrconfig.xml @@ -1243,10 +1243,6 @@ via parameters. The below configuration supports hl.method=original and fastVec default="true" class="solr.XMLResponseWriter" /> - - - - --> diff --git a/solr/solr-ref-guide/modules/deployment-guide/pages/python.adoc b/solr/solr-ref-guide/modules/deployment-guide/pages/python.adoc index f2fbd3bf5c6..0c94ef5606d 100644 --- a/solr/solr-ref-guide/modules/deployment-guide/pages/python.adoc +++ b/solr/solr-ref-guide/modules/deployment-guide/pages/python.adoc @@ -16,45 +16,9 @@ // specific language governing permissions and limitations // under the License. -Solr includes an output format specifically for xref:query-guide:response-writers.adoc#python-response-writer[Python Response Writer], but the xref:query-guide:response-writers.adoc#json-response-writer[JSON Response Writer] is a little more robust. +== Python -== Simple Python - -Making a query is a simple matter. -First, tell Python you will need to make HTTP connections. - -[source,python] ----- -from urllib2 import * ----- - -Now open a connection to the server and get a response. -The `wt` query parameter tells Solr to return results in a format that Python can understand. - -[source,python] ----- -connection = urlopen('http://localhost:8983/solr/collection_name/select?q=cheese&wt=python') -response = eval(connection.read()) ----- - -Now interpreting the response is just a matter of pulling out the information that you need. - -[source,python] ----- -print response['response']['numFound'], "documents found." - -# Print the name of each document. - -for document in response['response']['docs']: - print " Name =", document['name'] ----- - -== Python with JSON - -JSON is a more robust response format, and Python has support for it in its standard library since version 2.6. - -Making a query is nearly the same as before. -However, notice that the `wt` query parameter is now `json` (which is also the default if no `wt` parameter is specified), and the response is now digested by `json.load()`. +JSON is the preferred interchange format for Python users, and Python has support for it in its standard library since version 2.6. [source,python] ---- @@ -69,3 +33,5 @@ print response['response']['numFound'], "documents found." for document in response['response']['docs']: print " Name =", document['name'] ---- + +You can even skip the `wt=json` as it's the default writer type. diff --git a/solr/solr-ref-guide/modules/deployment-guide/pages/ruby.adoc b/solr/solr-ref-guide/modules/deployment-guide/pages/ruby.adoc index 92b71b5a708..7f82e552e84 100644 --- a/solr/solr-ref-guide/modules/deployment-guide/pages/ruby.adoc +++ b/solr/solr-ref-guide/modules/deployment-guide/pages/ruby.adoc @@ -16,100 +16,43 @@ // specific language governing permissions and limitations // under the License. -Solr has an optional Ruby response format that extends the xref:query-guide:response-writers.adoc#json-response-writer[JSON Response Writer] to allow the response to be safely eval'd by Ruby's interpreter -This Ruby response format differs from JSON in the following ways: +== Ruby -* Ruby's single quoted strings are used to prevent possible string exploits -** `\` and `'` are the only two characters escaped... -** unicode escapes not used... data is written as raw UTF-8 -* nil used for null -* `\=>` used as the key/value separator in maps - -Here's an example Ruby response from Solr, for a request like `\http://localhost:8983/solr/techproducts/select?q=iPod&wt=ruby&indent=on` (with Solr launching using `bin/solr start -e techproducts`): - -[source,ruby] ----- -{ - 'responseHeader'=>{ - 'status'=>0, - 'QTime'=>0, - 'params'=>{ - 'q'=>'iPod', - 'indent'=>'on', - 'wt'=>'ruby'}}, - 'response'=>{'numFound'=>3,'start'=>0,'docs'=>[ - { - 'id'=>'IW-02', - 'name'=>'iPod & iPod Mini USB 2.0 Cable', - 'manu'=>'Belkin', - 'manu_id_s'=>'belkin', - 'cat'=>['electronics', - 'connector'], - 'features'=>['car power adapter for iPod, white'], - 'weight'=>2.0, - 'price'=>11.5, - 'price_c'=>'11.50,USD', - 'popularity'=>1, - 'inStock'=>false, - 'store'=>'37.7752,-122.4232', - 'manufacturedate_dt'=>'2006-02-14T23:55:59Z', - '_version_'=>1491038048794705920}, - { - 'id'=>'F8V7067-APL-KIT', - 'name'=>'Belkin Mobile Power Cord for iPod w/ Dock', - 'manu'=>'Belkin', - 'manu_id_s'=>'belkin', - 'cat'=>['electronics', - 'connector'], - 'features'=>['car power adapter, white'], - 'weight'=>4.0, - 'price'=>19.95, - 'price_c'=>'19.95,USD', - 'popularity'=>1, - 'inStock'=>false, - 'store'=>'45.18014,-93.87741', - 'manufacturedate_dt'=>'2005-08-01T16:30:25Z', - '_version_'=>1491038048792608768}, - { - 'id'=>'MA147LL/A', - 'name'=>'Apple 60 GB iPod with Video Playback Black', - 'manu'=>'Apple Computer Inc.', - 'manu_id_s'=>'apple', - 'cat'=>['electronics', - 'music'], - 'features'=>['iTunes, Podcasts, Audiobooks', - 'Stores up to 15,000 songs, 25,000 photos, or 150 hours of video', - '2.5-inch, 320x240 color TFT LCD display with LED backlight', - 'Up to 20 hours of battery life', - 'Plays AAC, MP3, WAV, AIFF, Audible, Apple Lossless, H.264 video', - 'Notes, Calendar, Phone book, Hold button, Date display, Photo wallet, Built-in games, JPEG photo playback, Upgradeable firmware, USB 2.0 compatibility, Playback speed control, Rechargeable capability, Battery level indication'], - 'includes'=>'earbud headphones, USB cable', - 'weight'=>5.5, - 'price'=>399.0, - 'price_c'=>'399.00,USD', - 'popularity'=>10, - 'inStock'=>true, - 'store'=>'37.7752,-100.0232', - 'manufacturedate_dt'=>'2005-10-12T08:00:00Z', - '_version_'=>1491038048799948800}] - }} ----- - -Here is a simple example of how one may query Solr using the Ruby response format: +JSON is the preferred interchange format for Ruby users. [source,ruby] ---- require 'net/http' +require 'json' h = Net::HTTP.new('localhost', 8983) -http_response = h.get('/solr/techproducts/select?q=iPod&wt=ruby') -rsp = eval(http_response.body) +http_response = h.get('/solr/techproducts/select?q=iPod&wt=json') -puts 'number of matches = ' + rsp['response']['numFound'].to_s -#print out the name field for each returned document -rsp['response']['docs'].each { |doc| puts 'name field = ' + doc['name'] } +# Parse the JSON response +rsp = JSON.parse(http_response.body) + +puts 'Number of matches = ' + rsp['response']['numFound'].to_s + +# Print out the name field for each returned document +rsp['response']['docs'].each do |doc| + puts 'Name field = ' + doc['name'] +end ---- -For simple interactions with Solr, this may be all you need! -If you are building complex interactions with Solr, then consider the libraries mentioned in the https://cwiki.apache.org/confluence/display/solr/Ruby+Response+Format#RubyResponseFormat-Libraries[Solr Wiki]. +You can even skip the `wt=json` as it's the default writer type. + +If you are building complex interactions with Solr, then consider these libraries: + +=== Active +* link:http://rubygems.org/gems/rsolr[RSolr]: A lightweight, general purpose client library for Solr. + ** link:https://github.com/sunspot/sunspot[Sunspot]: A nice DSL framework for integrating Solr into your models. Built on top of RSolr. +* link:http://projectblacklight.org[Blacklight]: A popular and well-maintained Ruby on Rails framework for building flexible and attractive front-ends to Solr. + +=== Inactive +* link:http://rubygems.org/gems/solr-ruby[Solr-ruby]: Low-level access to Solr from Ruby. +* link:http://rubygems.org/gems/flare[Flare]: A plugin adding faceted browsing, AJAX suggest, and more to Rails controllers. +* link:http://acts_as_solr.railsfreaks.com/[acts_as_solr]: A plugin to add full-text search capabilities using Solr to ActiveRecord models. link:http://www.quarkruby.com/2007/8/12/acts_as_solr-for-search-and-faceting/[nice acts_as_solr tutorial] +* link:http://rubyforge.org/projects/background-solr/[acts_as_background_solr]: An extension to acts_as_solr moving more processing to the background. +* link:http://mojodna.net/searchable/ruby/[Searchable]: A Rails plugin that provides search integration with ActiveRecord. (an alternative to acts_as_solr) +* link:http://delsolr.rubyforge.org/[DelSolr]: A Solr wrapper designed to simplify facet and filter queries. diff --git a/solr/solr-ref-guide/modules/getting-started/pages/tutorial-techproducts.adoc b/solr/solr-ref-guide/modules/getting-started/pages/tutorial-techproducts.adoc index e397ccf9db9..be9060e6230 100644 --- a/solr/solr-ref-guide/modules/getting-started/pages/tutorial-techproducts.adoc +++ b/solr/solr-ref-guide/modules/getting-started/pages/tutorial-techproducts.adoc @@ -195,7 +195,6 @@ POSTing file sample.html (text/html) to [base]/extract POSTing file sd500.xml (application/xml) to [base] POSTing file solr-word.pdf (application/pdf) to [base]/extract POSTing file solr.xml (application/xml) to [base] -POSTing file test_utf8.sh (application/octet-stream) to [base]/extract POSTing file utf8-example.xml (application/xml) to [base] POSTing file vidcard.xml (application/xml) to [base] 20 files indexed. diff --git a/solr/solr-ref-guide/modules/indexing-guide/pages/indexing-with-tika.adoc b/solr/solr-ref-guide/modules/indexing-guide/pages/indexing-with-tika.adoc index 14b26ed556d..3d4e1509fbe 100644 --- a/solr/solr-ref-guide/modules/indexing-guide/pages/indexing-with-tika.adoc +++ b/solr/solr-ref-guide/modules/indexing-guide/pages/indexing-with-tika.adoc @@ -603,7 +603,7 @@ The output includes XML generated by Tika (and further escaped by Solr's XML) us [,console] ---- -$ bin/solr post -c gettingstarted --params "extractOnly=true&wt=ruby&indent=true" --out yes example/exampledocs/sample.html +$ bin/solr post -c gettingstarted --params "extractOnly=true&wt=json&indent=true" --out yes example/exampledocs/sample.html ---- === Using Solr Cell with a POST Request diff --git a/solr/solr-ref-guide/modules/query-guide/pages/response-writers.adoc b/solr/solr-ref-guide/modules/query-guide/pages/response-writers.adoc index d4961312f34..84e938da5fb 100644 --- a/solr/solr-ref-guide/modules/query-guide/pages/response-writers.adoc +++ b/solr/solr-ref-guide/modules/query-guide/pages/response-writers.adoc @@ -24,14 +24,11 @@ Solr supports a variety of Response Writers to ensure that query responses can b The `wt` parameter selects the Response Writer to be used. The list below describe shows the most common settings for the `wt` parameter, with links to further sections that discuss them in more detail. +* <> * <> * <> * <> * <> -* <> -* <> -* <> -* <> * <> * <> * <> @@ -285,67 +282,6 @@ Returns Solr results in http://geojson.org[GeoJSON] augmented with Solr-specific To use this, set `wt=geojson` and `geojson.field` to the name of a spatial Solr field. Not all spatial fields types are supported, and you'll get an error if you use an unsupported one. -== Python Response Writer - -Solr has an optional Python response format that extends its JSON output in the following ways to allow the response to be safely evaluated by the python interpreter: - -* true and false changed to True and False -* Python unicode strings are used where needed -* ASCII output (with unicode escapes) is used for less error-prone interoperability -* newlines are escaped -* null changed to None - -[[php-writer]] -== PHP Response Writer and PHP Serialized Response Writer - -Solr has a PHP response format that outputs an array (as PHP code) which can be evaluated. -Setting the `wt` parameter to `php` invokes the PHP Response Writer. - -Example usage: - -[source,php] ----- -$code = file_get_contents('http://localhost:8983/solr/techproducts/select?q=iPod&wt=php'); -eval("$result = " . $code . ";"); -print_r($result); ----- - -Solr also includes a PHP Serialized Response Writer that formats output in a serialized array. -Setting the `wt` parameter to `phps` invokes the PHP Serialized Response Writer. - -Example usage: - -[source,php] ----- -$serializedResult = file_get_contents('http://localhost:8983/solr/techproducts/select?q=iPod&wt=phps'); -$result = unserialize($serializedResult); -print_r($result); ----- - -== Ruby Response Writer - -Solr has an optional Ruby response format that extends its JSON output in the following ways to allow the response to be safely evaluated by Ruby's interpreter: - -* Ruby's single quoted strings are used to prevent possible string exploits. -* \ and ' are the only two characters escaped. -* Unicode escapes are not used. -Data is written as raw UTF-8. -* nil used for null. -* \=> is used as the key/value separator in maps. - -Here is a simple example of how one may query Solr using the Ruby response format: - -[source,ruby] ----- -require 'net/http' -h = Net::HTTP.new('localhost', 8983) -hresp, data = h.get('/solr/techproducts/select?q=iPod&wt=ruby', nil) -rsp = eval(data) -puts 'number of matches = ' + rsp['response']['numFound'].to_s -#print out the name field for each returned document -rsp['response']['docs'].each { |doc| puts 'name field = ' + doc['name'\] } ----- - == CSV Response Writer The CSV response writer returns a list of documents in comma-separated values (CSV) format. diff --git a/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-10.adoc b/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-10.adoc index 532c13fabe0..0f86500fac7 100644 --- a/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-10.adoc +++ b/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-10.adoc @@ -64,3 +64,6 @@ has been removed. Please use `-Dsolr.hiddenSysProps` or the envVar `SOLR_HIDDEN_ * The `BlobRepository`, which was deprecated in 8x in favour of the `FileStore` approach is removed. Users should migrate to the `FileStore` implementation (per node stored file) and the still existing `BlobHandler` (across the cluster storage backed by `.system` collection). Please note this also removes the ability to share resource intensive objects across multiple cores as this feature was tied to the `BlobRepository` implementation. + +* The language specific Response Writers, which were deprecated in 9.8 in favour of more widely used formats like JSON have been removed. +The removed writer types (invoked as part of the `wt` parameter) include `python`, `ruby`, `php`, and `phps`. From 1dba3931c7ccb3870464fdad1f1e1a4a5f727209 Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Mon, 21 Oct 2024 16:47:28 -0400 Subject: [PATCH 4/6] SOLR-17505: Revamp tutorials in Ref Guide to deemphasize SolrCloud since it's just default mode. (#2779) --- .../modules/deployment-guide/pages/cluster-types.adoc | 2 +- .../modules/getting-started/pages/tutorial-five-minutes.adoc | 4 ++-- .../modules/getting-started/pages/tutorial-solrcloud.adoc | 2 +- .../modules/getting-started/pages/tutorial-techproducts.adoc | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/solr/solr-ref-guide/modules/deployment-guide/pages/cluster-types.adoc b/solr/solr-ref-guide/modules/deployment-guide/pages/cluster-types.adoc index d8583085adb..44a03d3e805 100644 --- a/solr/solr-ref-guide/modules/deployment-guide/pages/cluster-types.adoc +++ b/solr/solr-ref-guide/modules/deployment-guide/pages/cluster-types.adoc @@ -90,7 +90,7 @@ As long as one replica of each relevant shard is available, a user query or inde == User-Managed Mode -Solr's user-managed mode requires that cluster coordination activities that SolrCloud normally uses ZooKeeper for tp be performed manually or with local scripts. +Solr's user-managed mode requires that cluster coordination activities that SolrCloud normally uses ZooKeeper for to be performed manually or with local scripts. If the corpus of documents is too large for a single-sharded index, the logic to create shards is entirely left to the user. There are no automated or programmatic ways for Solr to create shards during indexing. diff --git a/solr/solr-ref-guide/modules/getting-started/pages/tutorial-five-minutes.adoc b/solr/solr-ref-guide/modules/getting-started/pages/tutorial-five-minutes.adoc index d4048a6d1d2..b5f5b3fd534 100644 --- a/solr/solr-ref-guide/modules/getting-started/pages/tutorial-five-minutes.adoc +++ b/solr/solr-ref-guide/modules/getting-started/pages/tutorial-five-minutes.adoc @@ -19,8 +19,8 @@ This exercise will walk you through how to start using Solr in just 5 Minutes! -== Launch Solr in SolrCloud Mode -To launch Solr, run: `bin/solr start -c` on Unix or MacOS; `bin\solr.cmd start -c` on Windows. +== Launch Solr +To launch Solr, run: `bin/solr start` on Unix or MacOS; `bin\solr.cmd start` on Windows. To start another Solr node and have it join the cluster alongside the first node, diff --git a/solr/solr-ref-guide/modules/getting-started/pages/tutorial-solrcloud.adoc b/solr/solr-ref-guide/modules/getting-started/pages/tutorial-solrcloud.adoc index 3e966f929ec..37bf8711b9d 100644 --- a/solr/solr-ref-guide/modules/getting-started/pages/tutorial-solrcloud.adoc +++ b/solr/solr-ref-guide/modules/getting-started/pages/tutorial-solrcloud.adoc @@ -121,7 +121,7 @@ $ curl http://host:8983/solr/mycollection/config -d '{"set-user-property": {"upd ---- ==== -V2 API SolrCloud:: +V2 API:: + ==== [,console] diff --git a/solr/solr-ref-guide/modules/getting-started/pages/tutorial-techproducts.adoc b/solr/solr-ref-guide/modules/getting-started/pages/tutorial-techproducts.adoc index be9060e6230..bfd05668588 100644 --- a/solr/solr-ref-guide/modules/getting-started/pages/tutorial-techproducts.adoc +++ b/solr/solr-ref-guide/modules/getting-started/pages/tutorial-techproducts.adoc @@ -20,7 +20,7 @@ This exercise will walk you through how to start Solr as a two-node cluster (both nodes on the same machine) and create a collection during startup. Then you will index some sample data that ships with Solr and do some basic searches. -== Launch Solr in SolrCloud Mode +== Launch Solr To launch Solr, run: `bin/solr start -e cloud` on Unix or MacOS; `bin\solr.cmd start -e cloud` on Windows. This will start an interactive session that will start two Solr "servers" on your machine. From 577e7012ca476b2ea2c7102d5301bfbec128f048 Mon Sep 17 00:00:00 2001 From: Christos Malliaridis Date: Tue, 22 Oct 2024 14:27:16 +0200 Subject: [PATCH 5/6] SOLR-17467: Fix Windows start script for default / cloud mode (#2784) --- solr/bin/solr.cmd | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/solr/bin/solr.cmd b/solr/bin/solr.cmd index 94a973236b1..db94bc8691a 100755 --- a/solr/bin/solr.cmd +++ b/solr/bin/solr.cmd @@ -904,7 +904,7 @@ if !JAVA_MAJOR_VERSION! LSS 9 ( ) IF NOT "%ZK_HOST%"=="" set SOLR_MODE=solrcloud -IF NOT "%SOLR_MODE%"=="" set SOLR_MODE=solrcloud +IF "%SOLR_MODE%"=="" set SOLR_MODE=solrcloud IF "%SOLR_MODE%"=="solrcloud" ( IF "%ZK_CLIENT_TIMEOUT%"=="" set "ZK_CLIENT_TIMEOUT=30000" @@ -941,7 +941,7 @@ IF "%SOLR_MODE%"=="solrcloud" ( IF EXIST "%SOLR_HOME%\collection1\core.properties" set "CLOUD_MODE_OPTS=!CLOUD_MODE_OPTS! -Dbootstrap_confdir=./solr/collection1/conf -Dcollection.configName=myconf -DnumShards=1" ) ELSE ( REM change Cloud mode to User Managed mode with flag - set "CLOUD_MODE_OPTS=--user-managed" + set "CLOUD_MODE_OPTS=" IF NOT EXIST "%SOLR_HOME%\solr.xml" ( IF "%SOLR_SOLRXML_REQUIRED%"=="true" ( set "SCRIPT_ERROR=Solr home directory %SOLR_HOME% must contain solr.xml!" From 457ca3668a8906277f8f638af9c740b57f2a2a20 Mon Sep 17 00:00:00 2001 From: Gus Heck <46900717+gus-asf@users.noreply.github.com> Date: Wed, 23 Oct 2024 10:11:28 -0400 Subject: [PATCH 6/6] Add an option in the gradle.properties for html junit reports (#2777) --- gradle/template.gradle.properties | 8 ++++++++ gradle/testing/defaults-tests.gradle | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/gradle/template.gradle.properties b/gradle/template.gradle.properties index c52e3048a77..79b18753f43 100644 --- a/gradle/template.gradle.properties +++ b/gradle/template.gradle.properties @@ -49,6 +49,11 @@ # tests.minheapsize=512m # tests.jvmargs=-XX:+UseParallelGC -XX:TieredStopAtLevel=1 -XX:ActiveProcessorCount=1 # +# If you want tests to produce an html report (which intellij provides a clickable link for +# at the end of a failed build) set this to true, defaults to false to save a few seconds. +# +# tests.html=false +# ################# # Gradle Daemon # ################# @@ -98,5 +103,8 @@ org.gradle.workers.max=@MAX_WORKERS@ # Maximum number of test JVMs forked per test task. tests.jvms=@TEST_JVMS@ +# By default skip html generation +tests.html=false + # Disable auto JVM provisioning (we don't use toolchains yet but want no surprises). org.gradle.java.installations.auto-download=false diff --git a/gradle/testing/defaults-tests.gradle b/gradle/testing/defaults-tests.gradle index f0af07c81de..6c551df103a 100644 --- a/gradle/testing/defaults-tests.gradle +++ b/gradle/testing/defaults-tests.gradle @@ -153,7 +153,7 @@ allprojects { } // Disable HTML report generation. The reports are big and slow to generate. - reports.html.required = false + reports.html.required = Boolean.parseBoolean(providers.gradleProperty("tests.html").getOrElse("false")) // Set up logging. testLogging {