diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 213846bf2c8..85a375a07f2 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -142,6 +142,8 @@ led to the suppression of exceptions. (Andrey Bozhko) * SOLR-17577: Remove "solr.indexfetcher.sotimeout" system property that was for optimizing replication tests. It was disabled, but not removed. (Eric Pugh) +* SOLR-16781: "" tags in solrconfig.xml are now quietly ignored by default unless explicitly enabled with the `SOLR_CONFIG_LIB_ENABLED=true` enviroment variable (or corresponding sysprop). These tags are now considered deprecated and will be removed in Solr 10. + ================== 9.7.1 ================== Bug Fixes --------------------- diff --git a/solr/core/src/java/org/apache/solr/cli/RunExampleTool.java b/solr/core/src/java/org/apache/solr/cli/RunExampleTool.java index 7ef74c5f298..0b278d13e60 100644 --- a/solr/core/src/java/org/apache/solr/cli/RunExampleTool.java +++ b/solr/core/src/java/org/apache/solr/cli/RunExampleTool.java @@ -620,10 +620,15 @@ protected Map startSolr( if (!isWindows && cwdPath.length() > 1 && solrHome.startsWith(cwdPath)) solrHome = solrHome.substring(cwdPath.length() + 1); + final var syspropArg = + ("techproducts".equals(cli.getOptionValue("example"))) + ? "-Dsolr.modules=clustering,extraction,langid,ltr,scripting -Dsolr.ltr.enabled=true -Dsolr.clustering.enabled=true" + : ""; + String startCmd = String.format( Locale.ROOT, - "\"%s\" start %s -p %d --solr-home \"%s\" %s %s %s %s %s %s %s", + "\"%s\" start %s -p %d --solr-home \"%s\" %s %s %s %s %s %s %s %s", callScript, cloudModeArg, port, @@ -634,7 +639,8 @@ protected Map startSolr( forceArg, verboseArg, extraArgs, - jvmOptsArg); + jvmOptsArg, + syspropArg); startCmd = startCmd.replaceAll("\\s+", " ").trim(); // for pretty printing echo("\nStarting up Solr on port " + port + " using command:"); diff --git a/solr/core/src/java/org/apache/solr/core/SolrConfig.java b/solr/core/src/java/org/apache/solr/core/SolrConfig.java index 173a44b9ba2..b6e177d92a0 100644 --- a/solr/core/src/java/org/apache/solr/core/SolrConfig.java +++ b/solr/core/src/java/org/apache/solr/core/SolrConfig.java @@ -65,6 +65,7 @@ import org.apache.solr.common.MapSerializable; import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException.ErrorCode; +import org.apache.solr.common.util.EnvUtils; import org.apache.solr.common.util.IOUtils; import org.apache.solr.common.util.Utils; import org.apache.solr.handler.component.SearchComponent; @@ -937,6 +938,8 @@ public PluginInfo getPluginInfo(String type) { SolrException.ErrorCode.SERVER_ERROR, "Multiple plugins configured for type: " + type); } + public static final String LIB_ENABLED_SYSPROP = "solr.config.lib.enabled"; + private void initLibs(SolrResourceLoader loader, boolean isConfigsetTrusted) { // TODO Want to remove SolrResourceLoader.getInstancePath; it can be on a Standalone subclass. // For Zk subclass, it's needed for the time being as well. We could remove that one if we @@ -954,6 +957,7 @@ private void initLibs(SolrResourceLoader loader, boolean isConfigsetTrusted) { } } + boolean libDirectiveAllowed = EnvUtils.getPropertyAsBool(LIB_ENABLED_SYSPROP, false); List nodes = root.getAll("lib"); if (nodes != null && nodes.size() > 0) { if (!isConfigsetTrusted) { @@ -964,30 +968,12 @@ private void initLibs(SolrResourceLoader loader, boolean isConfigsetTrusted) { + " after enabling authentication and authorization."); } - for (int i = 0; i < nodes.size(); i++) { - ConfigNode node = nodes.get(i); - String baseDir = node.attr("dir"); - String path = node.attr(PATH); - if (null != baseDir) { - // :TODO: add support for a simpler 'glob' mutually exclusive of regex - Path dir = instancePath.resolve(baseDir); - String regex = node.attr("regex"); - try { - if (regex == null) urls.addAll(SolrResourceLoader.getURLs(dir)); - else urls.addAll(SolrResourceLoader.getFilteredURLs(dir, regex)); - } catch (IOException e) { - log.warn("Couldn't add files from {} filtered by {} to classpath: {}", dir, regex, e); - } - } else if (null != path) { - final Path dir = instancePath.resolve(path); - try { - urls.add(dir.toUri().toURL()); - } catch (MalformedURLException e) { - log.warn("Couldn't add file {} to classpath: {}", dir, e); - } - } else { - throw new RuntimeException("lib: missing mandatory attributes: 'dir' or 'path'"); - } + if (!libDirectiveAllowed) { + log.warn( + "Configset references one or more directives, but usage is disabled on this Solr node. Either remove all tags from the relevant configset, or enable use of this feature by setting '{}=true'", + LIB_ENABLED_SYSPROP); + } else { + urls.addAll(processLibDirectives(nodes, instancePath)); } } @@ -997,6 +983,38 @@ private void initLibs(SolrResourceLoader loader, boolean isConfigsetTrusted) { } } + private List processLibDirectives(List nodes, Path instancePath) { + final var urls = new ArrayList(); + + for (int i = 0; i < nodes.size(); i++) { + ConfigNode node = nodes.get(i); + String baseDir = node.attr("dir"); + String path = node.attr(PATH); + if (null != baseDir) { + // :TODO: add support for a simpler 'glob' mutually exclusive of regex + Path dir = instancePath.resolve(baseDir); + String regex = node.attr("regex"); + try { + if (regex == null) urls.addAll(SolrResourceLoader.getURLs(dir)); + else urls.addAll(SolrResourceLoader.getFilteredURLs(dir, regex)); + } catch (IOException e) { + log.warn("Couldn't add files from {} filtered by {} to classpath: {}", dir, regex, e); + } + } else if (null != path) { + final Path dir = instancePath.resolve(path); + try { + urls.add(dir.toUri().toURL()); + } catch (MalformedURLException e) { + log.warn("Couldn't add file {} to classpath: {}", dir, e); + } + } else { + throw new RuntimeException("lib: missing mandatory attributes: 'dir' or 'path'"); + } + } + + return urls; + } + public int getMultipartUploadLimitKB() { return multipartUploadLimitKB; } diff --git a/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java b/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java index 68142392df1..eb28fdced07 100644 --- a/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java +++ b/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java @@ -19,6 +19,7 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static org.apache.solr.common.params.CommonParams.NAME; import static org.apache.solr.core.ConfigSetProperties.DEFAULT_FILENAME; +import static org.apache.solr.core.SolrConfig.LIB_ENABLED_SYSPROP; import static org.hamcrest.CoreMatchers.containsString; import java.io.ByteArrayInputStream; @@ -122,6 +123,7 @@ public class TestConfigSetsAPI extends SolrCloudTestCase { @BeforeClass public static void setUpClass() throws Exception { + System.setProperty(LIB_ENABLED_SYSPROP, "true"); System.setProperty("managed.schema.mutable", "true"); configureCluster(1).withSecurityJson(getSecurityJson()).configure(); } diff --git a/solr/core/src/test/org/apache/solr/core/TestConfig.java b/solr/core/src/test/org/apache/solr/core/TestConfig.java index 2c17cbf1e1a..ea5f1267c15 100644 --- a/solr/core/src/test/org/apache/solr/core/TestConfig.java +++ b/solr/core/src/test/org/apache/solr/core/TestConfig.java @@ -16,6 +16,8 @@ */ package org.apache.solr.core; +import static org.apache.solr.core.SolrConfig.LIB_ENABLED_SYSPROP; + import java.io.IOException; import java.io.InputStream; import java.util.Collections; @@ -38,6 +40,7 @@ public class TestConfig extends SolrTestCaseJ4 { @BeforeClass public static void beforeClass() throws Exception { + System.setProperty(LIB_ENABLED_SYSPROP, "true"); initCore("solrconfig-test-misc.xml", "schema-reversed.xml"); } diff --git a/solr/core/src/test/org/apache/solr/core/TestConfigWithLibDisabled.java b/solr/core/src/test/org/apache/solr/core/TestConfigWithLibDisabled.java new file mode 100644 index 00000000000..00c74d629b5 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/core/TestConfigWithLibDisabled.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.core; + +import static org.apache.solr.core.SolrConfig.LIB_ENABLED_SYSPROP; +import static org.hamcrest.core.StringContains.containsString; + +import java.io.IOException; +import org.apache.solr.SolrTestCaseJ4; +import org.junit.BeforeClass; +import org.junit.Test; + +// TODO - replace 'initCore' usage and merge with TestConfig +/** + * Unit test verifying that "lib" tags are quietly ignored when not explicitly enabled. + * + *

Based on code from {@link TestConfig}. + */ +public class TestConfigWithLibDisabled extends SolrTestCaseJ4 { + + @BeforeClass + public static void beforeClass() throws Exception { + System.setProperty(LIB_ENABLED_SYSPROP, "false"); // tags disabled! + initCore("solrconfig-test-misc.xml", "schema-reversed.xml"); + } + + // solrconfig-test-misc has lib tags referencing various files + // This test ensures that none of those files are loadable when + // tags are disabled + @Test + public void testLibFilesShouldntBeVisible() throws IOException { + SolrResourceLoader loader = h.getCore().getResourceLoader(); + String[] filesReferencedByLib = + new String[] { + "empty-file-a1.txt", + "empty-file-a2.txt", + "empty-file-b1.txt", + "empty-file-b2.txt", + "empty-file-c1.txt" + }; + for (String f : filesReferencedByLib) { + final var e = + expectThrows( + SolrResourceNotFoundException.class, + () -> { + loader.openResource(f); + }); + assertThat(e.getMessage(), containsString("Can't find resource")); + assertThat(e.getMessage(), containsString(f)); + } + } +} diff --git a/solr/solr-ref-guide/modules/configuration-guide/pages/libs.adoc b/solr/solr-ref-guide/modules/configuration-guide/pages/libs.adoc index 269e56f4169..02910b1199b 100644 --- a/solr/solr-ref-guide/modules/configuration-guide/pages/libs.adoc +++ b/solr/solr-ref-guide/modules/configuration-guide/pages/libs.adoc @@ -50,6 +50,13 @@ Solr plugins won't work in these locations. == Lib Directives in SolrConfig +[WARNING] +==== +`` directives are deprecated and will be removed in Solr 10.0. +In the interim, Solr disables this feature by default in order to minimize the security exposure of our users. +Expert users who wish to enable `` directives in their deployment may do so by specifying setting the `solr.config.lib.enabled` sysprop to `true`. +==== + _Both_ plugin and xref:resource-loading.adoc[resource] file paths are configurable via `` directives in `solrconfig.xml`. When a directive matches a directory, then resources can be resolved from it. When a directive matches a `.jar` file, Solr plugins and their dependencies are resolved from it. diff --git a/solr/solr-ref-guide/modules/configuration-guide/pages/resource-loading.adoc b/solr/solr-ref-guide/modules/configuration-guide/pages/resource-loading.adoc index bcdbe17c5ef..1eb5e566ee4 100644 --- a/solr/solr-ref-guide/modules/configuration-guide/pages/resource-loading.adoc +++ b/solr/solr-ref-guide/modules/configuration-guide/pages/resource-loading.adoc @@ -37,7 +37,7 @@ Prefer to put resources here. == Resources in Other Places -Resources can also be placed in an arbitrary directory and xref:libs.adoc#lib-directives-in-solrconfig[referenced] from a `` directive in `solrconfig.xml`, provided the directive refers to a directory and not the actual resource file. +Users who have enabled Solr's `` directive feature may place resources in an arbitrary directory xref:libs.adoc#lib-directives-in-solrconfig[referenced] from a `` directive in `solrconfig.xml`, provided the directive refers to a directory and not the actual resource file. Example: `` This choice may make sense if the resource is too large for a configset in ZooKeeper. However it's up to you to somehow ensure that all nodes in your cluster have access to these resources. diff --git a/solr/solr-ref-guide/modules/indexing-guide/pages/indexing-with-tika.adoc b/solr/solr-ref-guide/modules/indexing-guide/pages/indexing-with-tika.adoc index b1344ed5519..b0cdb7eba30 100644 --- a/solr/solr-ref-guide/modules/indexing-guide/pages/indexing-with-tika.adoc +++ b/solr/solr-ref-guide/modules/indexing-guide/pages/indexing-with-tika.adoc @@ -421,13 +421,8 @@ Also see the section <> for an example. If you have started Solr with one of the supplied xref:configuration-guide:config-sets.adoc[example configsets], you may already have the `ExtractingRequestHandler` configured by default. -First, you must enable the xref:#module[Module]. -If `solrconfig.xml` is not already configured, you will need to modify it to find the `ExtractingRequestHandler` and its dependencies: - -[source,xml] ----- - ----- +First, the `extraction` xref:#module[module] must be enabled. +This can be done by specifying the environment variable `SOLR_MODULES=extraction` in your startup configuration. You can then configure the `ExtractingRequestHandler` in `solrconfig.xml`. The following is the default configuration found in Solr's `sample_techproducts_configs` configset, which you can modify as needed: diff --git a/solr/solr-ref-guide/modules/query-guide/pages/learning-to-rank.adoc b/solr/solr-ref-guide/modules/query-guide/pages/learning-to-rank.adoc index 1d3f575ba03..e40deb89cc1 100644 --- a/solr/solr-ref-guide/modules/query-guide/pages/learning-to-rank.adoc +++ b/solr/solr-ref-guide/modules/query-guide/pages/learning-to-rank.adoc @@ -125,13 +125,8 @@ Learning-To-Rank is a module and therefore its plugins must be configured in `so === Minimum Requirements -* Include the required module JARs. -Note that by default paths are relative to the Solr core, so they may need adjustments to your configuration, or an explicit specification of the `$solr.install.dir`. -+ -[source,xml] ----- - ----- +* Enable the `ltr` module to make the LTR classes available on Solr's classpath. +See xref:configuration-guide:solr-modules.adoc[Solr Module] for more details. * Declaration of the `ltr` query parser. + diff --git a/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-9.adoc b/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-9.adoc index 1c7bbb93c7e..834c76fccac 100644 --- a/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-9.adoc +++ b/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-9.adoc @@ -71,6 +71,13 @@ Due to changes in Lucene 9, that isn't possible any more. === Configuration In solrconfig.xml, the `numVersionBuckets` and `versionBucketLockTimeoutMs` settings are now obsolete and ignored; a warning will be logged if specified. +`` directives have been made into an "opt-in" feature in Solr 9.8 and must be explicitly enabled if desired (using a system property or environment variable like `-Dsolr.config.lib.enabled=true`) +If not explicitly enabled, Solr will log a warning but otherwise quietly ignore any `` tags in configsets. +This is a breaking change, and we apologize for the disruption, but it's a necessary response to a number of security gaps that have come to light around this feature. +`` directives should be considered deprecated and will be removed entirely in Solr 10.0. +Current users of `` have a number of options to ensure their libraries remain on Solr's classpath, including: `SOLR_MODULES`, core "lib" directories, the solr.xml `` tag, Solr's package manager, etc. + + === Partial Results When query limits are in use and partial results are not desirable (i.e. reporting or quantitative usages of search) users may pass `partialResults=false`.