SOLR-16781: Disable <lib/> by default on 9.x (#2894)

`<lib/>` usage will now log a warning by default for future 9.x
releases.  Wary users can re-enabled the feature by specifying a
sysprop: `solr.config.lib.enabled=true`.

(This commit is for 9.x branches only, and not 'main')

solr/CHANGES.txt

@@ -142,6 +142,8 @@ led to the suppression of exceptions. (Andrey Bozhko)
 
 * SOLR-17577: Remove "solr.indexfetcher.sotimeout" system property that was for optimizing replication tests.  It was disabled, but not removed. (Eric Pugh)
 
+* SOLR-16781: "<lib/>" tags in solrconfig.xml are now quietly ignored by default unless explicitly enabled with the `SOLR_CONFIG_LIB_ENABLED=true` enviroment variable (or corresponding sysprop).  These tags are now considered deprecated and will be removed in Solr 10.
+
 ==================  9.7.1 ==================
 Bug Fixes
 ---------------------

solr/core/src/java/org/apache/solr/cli/RunExampleTool.java

@@ -620,10 +620,15 @@ protected Map<String, Object> startSolr(
     if (!isWindows && cwdPath.length() > 1 && solrHome.startsWith(cwdPath))
       solrHome = solrHome.substring(cwdPath.length() + 1);
 
+    final var syspropArg =
+        ("techproducts".equals(cli.getOptionValue("example")))
+            ? "-Dsolr.modules=clustering,extraction,langid,ltr,scripting -Dsolr.ltr.enabled=true -Dsolr.clustering.enabled=true"
+            : "";
+
     String startCmd =
         String.format(
             Locale.ROOT,
-            "\"%s\" start %s -p %d --solr-home \"%s\" %s %s %s %s %s %s %s",
+            "\"%s\" start %s -p %d --solr-home \"%s\" %s %s %s %s %s %s %s %s",
             callScript,
             cloudModeArg,
             port,
@@ -634,7 +639,8 @@ protected Map<String, Object> startSolr(
             forceArg,
             verboseArg,
             extraArgs,
-            jvmOptsArg);
+            jvmOptsArg,
+            syspropArg);
     startCmd = startCmd.replaceAll("\\s+", " ").trim(); // for pretty printing
 
     echo("\nStarting up Solr on port " + port + " using command:");

solr/core/src/java/org/apache/solr/core/SolrConfig.java

@@ -65,6 +65,7 @@
 import org.apache.solr.common.MapSerializable;
 import org.apache.solr.common.SolrException;
 import org.apache.solr.common.SolrException.ErrorCode;
+import org.apache.solr.common.util.EnvUtils;
 import org.apache.solr.common.util.IOUtils;
 import org.apache.solr.common.util.Utils;
 import org.apache.solr.handler.component.SearchComponent;
@@ -937,6 +938,8 @@ public PluginInfo getPluginInfo(String type) {
         SolrException.ErrorCode.SERVER_ERROR, "Multiple plugins configured for type: " + type);
   }
 
+  public static final String LIB_ENABLED_SYSPROP = "solr.config.lib.enabled";
+
   private void initLibs(SolrResourceLoader loader, boolean isConfigsetTrusted) {
     // TODO Want to remove SolrResourceLoader.getInstancePath; it can be on a Standalone subclass.
     // For Zk subclass, it's needed for the time being as well.  We could remove that one if we
@@ -954,6 +957,7 @@ private void initLibs(SolrResourceLoader loader, boolean isConfigsetTrusted) {
       }
     }
 
+    boolean libDirectiveAllowed = EnvUtils.getPropertyAsBool(LIB_ENABLED_SYSPROP, false);
     List<ConfigNode> nodes = root.getAll("lib");
     if (nodes != null && nodes.size() > 0) {
       if (!isConfigsetTrusted) {
@@ -964,30 +968,12 @@ private void initLibs(SolrResourceLoader loader, boolean isConfigsetTrusted) {
                 + " after enabling authentication and authorization.");
       }
 
-      for (int i = 0; i < nodes.size(); i++) {
-        ConfigNode node = nodes.get(i);
-        String baseDir = node.attr("dir");
-        String path = node.attr(PATH);
-        if (null != baseDir) {
-          // :TODO: add support for a simpler 'glob' mutually exclusive of regex
-          Path dir = instancePath.resolve(baseDir);
-          String regex = node.attr("regex");
-          try {
-            if (regex == null) urls.addAll(SolrResourceLoader.getURLs(dir));
-            else urls.addAll(SolrResourceLoader.getFilteredURLs(dir, regex));
-          } catch (IOException e) {
-            log.warn("Couldn't add files from {} filtered by {} to classpath: {}", dir, regex, e);
-          }
-        } else if (null != path) {
-          final Path dir = instancePath.resolve(path);
-          try {
-            urls.add(dir.toUri().toURL());
-          } catch (MalformedURLException e) {
-            log.warn("Couldn't add file {} to classpath: {}", dir, e);
-          }
-        } else {
-          throw new RuntimeException("lib: missing mandatory attributes: 'dir' or 'path'");
-        }
+      if (!libDirectiveAllowed) {
+        log.warn(
+            "Configset references one or more <lib/> directives, but <lib/> usage is disabled on this Solr node.  Either remove all <lib/> tags from the relevant configset, or enable use of this feature by setting '{}=true'",
+            LIB_ENABLED_SYSPROP);
+      } else {
+        urls.addAll(processLibDirectives(nodes, instancePath));
       }
     }
 
@@ -997,6 +983,38 @@ private void initLibs(SolrResourceLoader loader, boolean isConfigsetTrusted) {
     }
   }
 
+  private List<URL> processLibDirectives(List<ConfigNode> nodes, Path instancePath) {
+    final var urls = new ArrayList<URL>();
+
+    for (int i = 0; i < nodes.size(); i++) {
+      ConfigNode node = nodes.get(i);
+      String baseDir = node.attr("dir");
+      String path = node.attr(PATH);
+      if (null != baseDir) {
+        // :TODO: add support for a simpler 'glob' mutually exclusive of regex
+        Path dir = instancePath.resolve(baseDir);
+        String regex = node.attr("regex");
+        try {
+          if (regex == null) urls.addAll(SolrResourceLoader.getURLs(dir));
+          else urls.addAll(SolrResourceLoader.getFilteredURLs(dir, regex));
+        } catch (IOException e) {
+          log.warn("Couldn't add files from {} filtered by {} to classpath: {}", dir, regex, e);
+        }
+      } else if (null != path) {
+        final Path dir = instancePath.resolve(path);
+        try {
+          urls.add(dir.toUri().toURL());
+        } catch (MalformedURLException e) {
+          log.warn("Couldn't add file {} to classpath: {}", dir, e);
+        }
+      } else {
+        throw new RuntimeException("lib: missing mandatory attributes: 'dir' or 'path'");
+      }
+    }
+
+    return urls;
+  }
+
   public int getMultipartUploadLimitKB() {
     return multipartUploadLimitKB;
   }

solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java

@@ -19,6 +19,7 @@
 import static java.nio.charset.StandardCharsets.UTF_8;
 import static org.apache.solr.common.params.CommonParams.NAME;
 import static org.apache.solr.core.ConfigSetProperties.DEFAULT_FILENAME;
+import static org.apache.solr.core.SolrConfig.LIB_ENABLED_SYSPROP;
 import static org.hamcrest.CoreMatchers.containsString;
 
 import java.io.ByteArrayInputStream;
@@ -122,6 +123,7 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {
 
   @BeforeClass
   public static void setUpClass() throws Exception {
+    System.setProperty(LIB_ENABLED_SYSPROP, "true");
     System.setProperty("managed.schema.mutable", "true");
     configureCluster(1).withSecurityJson(getSecurityJson()).configure();
   }

solr/core/src/test/org/apache/solr/core/TestConfig.java

@@ -16,6 +16,8 @@
  */
 package org.apache.solr.core;
 
+import static org.apache.solr.core.SolrConfig.LIB_ENABLED_SYSPROP;
+
 import java.io.IOException;
 import java.io.InputStream;
 import java.util.Collections;
@@ -38,6 +40,7 @@ public class TestConfig extends SolrTestCaseJ4 {
 
   @BeforeClass
   public static void beforeClass() throws Exception {
+    System.setProperty(LIB_ENABLED_SYSPROP, "true");
     initCore("solrconfig-test-misc.xml", "schema-reversed.xml");
   }
 

solr/core/src/test/org/apache/solr/core/TestConfigWithLibDisabled.java

@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.core;
+
+import static org.apache.solr.core.SolrConfig.LIB_ENABLED_SYSPROP;
+import static org.hamcrest.core.StringContains.containsString;
+
+import java.io.IOException;
+import org.apache.solr.SolrTestCaseJ4;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+// TODO - replace 'initCore' usage and merge with TestConfig
+/**
+ * Unit test verifying that "lib" tags are quietly ignored when not explicitly enabled.
+ *
+ * <p>Based on code from {@link TestConfig}.
+ */
+public class TestConfigWithLibDisabled extends SolrTestCaseJ4 {
+
+  @BeforeClass
+  public static void beforeClass() throws Exception {
+    System.setProperty(LIB_ENABLED_SYSPROP, "false"); // <lib> tags disabled!
+    initCore("solrconfig-test-misc.xml", "schema-reversed.xml");
+  }
+
+  // solrconfig-test-misc has lib tags referencing various files
+  // This test ensures that none of those files are loadable when
+  // <lib> tags are disabled
+  @Test
+  public void testLibFilesShouldntBeVisible() throws IOException {
+    SolrResourceLoader loader = h.getCore().getResourceLoader();
+    String[] filesReferencedByLib =
+        new String[] {
+          "empty-file-a1.txt",
+          "empty-file-a2.txt",
+          "empty-file-b1.txt",
+          "empty-file-b2.txt",
+          "empty-file-c1.txt"
+        };
+    for (String f : filesReferencedByLib) {
+      final var e =
+          expectThrows(
+              SolrResourceNotFoundException.class,
+              () -> {
+                loader.openResource(f);
+              });
+      assertThat(e.getMessage(), containsString("Can't find resource"));
+      assertThat(e.getMessage(), containsString(f));
+    }
+  }
+}

solr/solr-ref-guide/modules/configuration-guide/pages/libs.adoc

@@ -50,6 +50,13 @@ Solr plugins won't work in these locations.
 
 == Lib Directives in SolrConfig
 
+[WARNING]
+====
+`<lib/>` directives are deprecated and will be removed in Solr 10.0.
+In the interim, Solr disables this feature by default in order to minimize the security exposure of our users.
+Expert users who wish to enable `<lib/>` directives in their deployment may do so by specifying setting the `solr.config.lib.enabled` sysprop to `true`.
+====
+
 _Both_ plugin and xref:resource-loading.adoc[resource] file paths are configurable via `<lib/>` directives in `solrconfig.xml`.
 When a directive matches a directory, then resources can be resolved from it.
 When a directive matches a `.jar` file, Solr plugins and their dependencies are resolved from it.

solr/solr-ref-guide/modules/configuration-guide/pages/resource-loading.adoc

@@ -37,7 +37,7 @@ Prefer to put resources here.
 
 == Resources in Other Places
 
-Resources can also be placed in an arbitrary directory and xref:libs.adoc#lib-directives-in-solrconfig[referenced] from a `<lib />` directive in `solrconfig.xml`, provided the directive refers to a directory and not the actual resource file.
+Users who have enabled Solr's `<lib/>` directive feature may place resources in an arbitrary directory xref:libs.adoc#lib-directives-in-solrconfig[referenced] from a `<lib />` directive in `solrconfig.xml`, provided the directive refers to a directory and not the actual resource file.
 Example: `<lib path="/volume/models/" />`
 This choice may make sense if the resource is too large for a configset in ZooKeeper.
 However it's up to you to somehow ensure that all nodes in your cluster have access to these resources.

solr/solr-ref-guide/modules/indexing-guide/pages/indexing-with-tika.adoc

@@ -421,13 +421,8 @@ Also see the section <<Defining XPath Expressions>> for an example.
 
 If you have started Solr with one of the supplied xref:configuration-guide:config-sets.adoc[example configsets], you may already have the `ExtractingRequestHandler` configured by default.
 
-First, you must enable the xref:#module[Module].
-If `solrconfig.xml` is not already configured, you will need to modify it to find the `ExtractingRequestHandler` and its dependencies:
-
-[source,xml]
-----
-  <lib dir="${solr.install.dir:../../..}/modules/extraction/lib" regex=".*\.jar" />
-----
+First, the `extraction` xref:#module[module] must be enabled.
+This can be done by specifying the environment variable `SOLR_MODULES=extraction` in your startup configuration.
 
 You can then configure the `ExtractingRequestHandler` in `solrconfig.xml`.
 The following is the default configuration found in Solr's `sample_techproducts_configs` configset, which you can modify as needed:

solr/solr-ref-guide/modules/query-guide/pages/learning-to-rank.adoc

@@ -125,13 +125,8 @@ Learning-To-Rank is a module and therefore its plugins must be configured in `so
 
 === Minimum Requirements
 
-* Include the required module JARs.
-Note that by default paths are relative to the Solr core, so they may need adjustments to your configuration, or an explicit specification of the `$solr.install.dir`.
-+
-[source,xml]
-----
-<lib dir="${solr.install.dir:../../../..}/modules/ltr/lib/" regex=".*\.jar" />
-----
+* Enable the `ltr` module to make the LTR classes available on Solr's classpath.
+See xref:configuration-guide:solr-modules.adoc[Solr Module] for more details.
 
 * Declaration of the `ltr` query parser.
 +

solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-9.adoc

@@ -71,6 +71,13 @@ Due to changes in Lucene 9, that isn't possible any more.
 === Configuration
 In solrconfig.xml, the `numVersionBuckets` and `versionBucketLockTimeoutMs` settings are now obsolete and ignored; a warning will be logged if specified.
 
+`<lib/>` directives have been made into an "opt-in" feature in Solr 9.8 and must be explicitly enabled if desired (using a system property or environment variable like `-Dsolr.config.lib.enabled=true`)
+If not explicitly enabled, Solr will log a warning but otherwise quietly ignore any `<lib/>` tags in configsets.
+This is a breaking change, and we apologize for the disruption, but it's a necessary response to a number of security gaps that have come to light around this feature.
+`<lib/>` directives should be considered deprecated and will be removed entirely in Solr 10.0.
+Current users of `<lib/>` have a number of options to ensure their libraries remain on Solr's classpath, including: `SOLR_MODULES`, core "lib" directories, the solr.xml `<sharedLib>` tag, Solr's package manager, etc.
+
+
 === Partial Results
 When query limits are in use and partial results are not desirable (i.e. reporting or quantitative usages of search)
 users may pass `partialResults=false`.