Skip to content

Commit

Permalink
SOLR-14115: Allow bin/solr zk commands to have parity with zkcli.sh c…
Browse files Browse the repository at this point in the history
…ommands. (#2298)

Add updateacls, cluster, and linkconfig as standard Solr tools.  Introduce some better testing of these relatively unknown tools.   Keep zkcli.sh as is for the 9x branch of code.   Some challenges around how we handle compression of state.json dealt with.
  • Loading branch information
epugh authored Apr 29, 2024
1 parent f9f9905 commit 963b629
Show file tree
Hide file tree
Showing 23 changed files with 1,284 additions and 92 deletions.
15 changes: 7 additions & 8 deletions dev-tools/scripts/cloud.sh
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,9 @@
# that not using the embedded zookeeper is key to being able
# switch between testing setups and to test vs alternate versions
# of zookeeper if desired.
#
# An option is:
# docker run --name my-zookeeper -p 2181:2181 -d zookeeper
#
# SETUP: 1. Place this script in a directory intended to hold all your
# testing installations of solr.
Expand All @@ -79,7 +82,7 @@
#
# ./cloud.sh stop
#
# Compile and push new code to a running cluster (incl bounce the cluster)
# Compile and push new code to a running cluster (including bounce the cluster)
#
# ./cloud.sh restart -r
#
Expand Down Expand Up @@ -310,13 +313,9 @@ start(){
findSolr

echo "SOLR=$SOLR"
SOLR_ROOT=$("${SOLR}/server/scripts/cloud-scripts/zkcli.sh" -zkhost localhost:${ZK_PORT} -cmd getfile "/solr_${SAFE_DEST}" /dev/stdout);
if [[ -z ${SOLR_ROOT} ]]; then
# Need a fresh root in zookeeper...
"${SOLR}/server/scripts/cloud-scripts/zkcli.sh" -zkhost localhost:${ZK_PORT} -cmd makepath "/solr_${SAFE_DEST}";
"${SOLR}/server/scripts/cloud-scripts/zkcli.sh" -zkhost localhost:${ZK_PORT} -cmd put "/solr_${SAFE_DEST}" "created by cloud.sh"; # so we can test for existence next time
fi

# Create the root if it doesn't already exist
${SOLR}/bin/solr zk mkroot "/solr_${SAFE_DEST}" -z localhost:${ZK_PORT}

ACTUAL_NUM_NODES=$(ls -1 -d ${CLUSTER_WD}/n* | wc -l )
if [[ "$NUM_NODES" -eq 0 ]]; then
NUM_NODES=${ACTUAL_NUM_NODES}
Expand Down
2 changes: 2 additions & 0 deletions solr/CHANGES.txt
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,8 @@ Improvements
---------------------
* SOLR-16921: use -solrUrl to derive the zk host connection for bin/solr zk subcommands (Eric Pugh)

* SOLR-14115: Add linkconfig, cluster, and updateacls as commands to SolrCLI. Allow bin/solr commands to have parity with zkcli.sh commands. (Eric Pugh)

Optimizations
---------------------
(No changes)
Expand Down
47 changes: 34 additions & 13 deletions solr/bin/solr
Original file line number Diff line number Diff line change
Expand Up @@ -591,6 +591,8 @@ function print_short_zk_usage() {
echo " solr zk mv <src> <dest> [-z zkHost] [-s solrUrl]"
echo " solr zk ls [-r] <path> [-z zkHost] [-s solrUrl]"
echo " solr zk mkroot <path> [-z zkHost] [-s solrUrl]"
echo " solr zk linkconfig -confname <confname> -c <collection> [-z zkHost] [-s solrUrl]"
echo " solr zk updateacls <path> [-z zkHost] [-s solrUrl]"
echo ""

if [ "$1" == "" ]; then
Expand Down Expand Up @@ -876,7 +878,7 @@ if [[ "$SCRIPT_CMD" == "delete" ]]; then
fi

# Prevent any zk subcommands from going through with out invoking zk command
if [[ "$SCRIPT_CMD" == "upconfig" || $SCRIPT_CMD == "downconfig" || $SCRIPT_CMD == "cp" || $SCRIPT_CMD == "rm" || $SCRIPT_CMD == "mv" || $SCRIPT_CMD == "ls" || $SCRIPT_CMD == "mkroot" ]]; then
if [[ "$SCRIPT_CMD" == "upconfig" || $SCRIPT_CMD == "downconfig" || $SCRIPT_CMD == "cp" || $SCRIPT_CMD == "rm" || $SCRIPT_CMD == "mv" || $SCRIPT_CMD == "ls" || $SCRIPT_CMD == "mkroot" || $SCRIPT_CMD == "linkconfig" || $SCRIPT_CMD == "updateacls" ]]; then
print_short_zk_usage "You must invoke this subcommand using the zk command. bin/solr zk $SCRIPT_CMD."
exit $?
fi
Expand All @@ -893,7 +895,7 @@ if [[ "$SCRIPT_CMD" == "zk" ]]; then
if [ $# -gt 0 ]; then
while true; do
case "${1:-}" in
upconfig|downconfig|cp|rm|mv|ls|mkroot)
upconfig|downconfig|cp|rm|mv|ls|mkroot|linkconfig|updateacls)
ZK_OP=$1
shift 1
;;
Expand All @@ -910,25 +912,33 @@ if [[ "$SCRIPT_CMD" == "zk" ]]; then
fi
ZK_SOLR_URL="$2"
shift 2
;;
-n|-confname)
;;
-n|--confname)
if [[ -z "$2" || "${2:0:1}" == "-" ]]; then
print_short_zk_usage "$SCRIPT_CMD" "Configuration name is required when using the $1 option!"
fi
CONFIGSET_CONFNAME="$2"
shift 2
;;
-d|-confdir)
-d|--confdir)
if [[ -z "$2" || "${2:0:1}" == "-" ]]; then
print_short_zk_usage "$SCRIPT_CMD" "Configuration directory is required when using the $1 option!"
fi
CONFIGSET_CONFDIR="$2"
shift 2
;;
--solr-home)
ZK_SOLR_HOME="$2"
shift 2
;;
-r)
ZK_RECURSE="true"
shift
;;
-c|-collection)
ZK_COLLECTION="$2"
shift
;;
-V|-verbose)
VERBOSE="-verbose"
shift
Expand Down Expand Up @@ -961,9 +971,9 @@ if [[ "$SCRIPT_CMD" == "zk" ]]; then
fi

if [ -z "$ZK_OP" ]; then
print_short_zk_usage "Zookeeper operation (one of 'upconfig', 'downconfig', 'rm', 'mv', 'cp', 'ls', 'mkroot') is required!"
print_short_zk_usage "Zookeeper operation (one of 'upconfig', 'downconfig', 'rm', 'mv', 'cp', 'ls', 'mkroot', 'linkconfig', 'updateacls') is required!"
fi

if [[ "$ZK_OP" == "upconfig" || "$ZK_OP" == "downconfig" ]]; then
if [ -z "$CONFIGSET_CONFDIR" ]; then
print_short_zk_usage "Local directory of the configset (-d) argument is required!"
Expand All @@ -990,19 +1000,19 @@ if [[ "$SCRIPT_CMD" == "zk" ]]; then
fi

CONNECTION_PARAMS=""

if [[ -z "$ZK_HOST" ]]; then
CONNECTION_PARAMS="-solrUrl ${ZK_SOLR_URL}"
else
CONNECTION_PARAMS="-zkHost ${ZK_HOST}"
fi
fi

case "$ZK_OP" in
upconfig)
run_tool "$ZK_OP" -confname "$CONFIGSET_CONFNAME" -confdir "$CONFIGSET_CONFDIR" $CONNECTION_PARAMS -configsetsDir "$SOLR_TIP/server/solr/configsets" $VERBOSE
run_tool "$ZK_OP" --confname "$CONFIGSET_CONFNAME" -confdir "$CONFIGSET_CONFDIR" $CONNECTION_PARAMS -configsetsDir "$SOLR_TIP/server/solr/configsets" $VERBOSE
;;
downconfig)
run_tool "$ZK_OP" -confname "$CONFIGSET_CONFNAME" -confdir "$CONFIGSET_CONFDIR" $CONNECTION_PARAMS $VERBOSE
run_tool "$ZK_OP" --confname "$CONFIGSET_CONFNAME" -confdir "$CONFIGSET_CONFDIR" $CONNECTION_PARAMS $VERBOSE
;;
rm)
if [ -z "$ZK_SRC" ]; then
Expand All @@ -1014,7 +1024,12 @@ if [[ "$SCRIPT_CMD" == "zk" ]]; then
run_tool "$ZK_OP" -src "$ZK_SRC" -dst "$ZK_DST" $CONNECTION_PARAMS $VERBOSE
;;
cp)
run_tool "$ZK_OP" -src "$ZK_SRC" -dst "$ZK_DST" $CONNECTION_PARAMS -recurse "$ZK_RECURSE" $VERBOSE
if [ -z "$ZK_SOLR_HOME" ]; then
run_tool "$ZK_OP" -src "$ZK_SRC" -dst "$ZK_DST" $CONNECTION_PARAMS -recurse "$ZK_RECURSE" $VERBOSE
else
run_tool "$ZK_OP" -src "$ZK_SRC" -dst "$ZK_DST" $CONNECTION_PARAMS -recurse "$ZK_RECURSE" $VERBOSE --solr-home "$ZK_SOLR_HOME"
fi

;;
ls)
if [ -z "$ZK_SRC" ]; then
Expand All @@ -1026,7 +1041,13 @@ if [[ "$SCRIPT_CMD" == "zk" ]]; then
if [ -z "$ZK_SRC" ]; then
print_short_zk_usage "Zookeeper path to list must be specified when using the 'mkroot' command"
fi
run_tool "$ZK_OP" -path "$ZK_SRC" $CONNECTION_PARAMS $VERBOSE
run_tool "$ZK_OP" --path "$ZK_SRC" $CONNECTION_PARAMS $VERBOSE
;;
linkconfig)
run_tool "$ZK_OP" --confname "$CONFIGSET_CONFNAME" -c "$ZK_COLLECTION" $CONNECTION_PARAMS $VERBOSE
;;
updateacls)
run_tool "$ZK_OP" --path "$ZK_SRC" $CONNECTION_PARAMS $VERBOSE
;;
*)
print_short_zk_usage "Unrecognized Zookeeper operation $ZK_OP"
Expand Down
37 changes: 37 additions & 0 deletions solr/bin/solr.cmd
Original file line number Diff line number Diff line change
Expand Up @@ -298,6 +298,7 @@ IF "%SCRIPT_CMD%"=="stop" goto stop_usage
IF "%SCRIPT_CMD%"=="healthcheck" goto run_solrcli
IF "%SCRIPT_CMD%"=="create" goto run_solrcli
IF "%SCRIPT_CMD%"=="delete" goto run_solrcli
IF "%SCRIPT_CMD%"=="cluster" goto run_solrcli
IF "%SCRIPT_CMD%"=="zk" goto zk_usage
IF "%SCRIPT_CMD%"=="auth" goto auth_usage
IF "%SCRIPT_CMD%"=="status" goto run_solrcli
Expand Down Expand Up @@ -478,6 +479,8 @@ echo solr zk rm [-r] ^<path^> [-z zkHost] [-s solrUrl]
echo solr zk mv ^<src^> ^<dest^> [-z zkHost] [-s solrUrl]
echo solr zk ls [-r] ^<path^> [-z zkHost] [-s solrUrl]
echo solr zk mkroot ^<path^> [-z zkHost] [-s solrUrl]
echo solr zk linkconfig -confname ^<confname^> -c ^<collection^> [-z zkHost] [-s solrUrl]
echo solr zk updateacls ^<path^> [-z zkHost] [-s solrUrl]
echo.
IF "%ZK_FULL%"=="true" (
goto zk_full_usage
Expand Down Expand Up @@ -1412,6 +1415,10 @@ IF "%1"=="-V" (
goto set_zk_op
) ELSE IF "%1"=="mkroot" (
goto set_zk_op
) ELSE IF "%1"=="linkconfig" (
goto set_zk_op
) ELSE IF "%1"=="updateacls" (
goto set_zk_op
) ELSE IF "%1"=="-n" (
goto set_config_name
) ELSE IF "%1"=="-r" (
Expand All @@ -1422,6 +1429,8 @@ IF "%1"=="-V" (
goto set_configdir
) ELSE IF "%1"=="-confdir" (
goto set_configdir
) ELSE IF "%1"=="-c" (
goto set_collection_zk
) ELSE IF "%1"=="-z" (
goto set_config_zk
) ELSE IF "%1"=="/?" (
Expand Down Expand Up @@ -1470,6 +1479,12 @@ SHIFT
SHIFT
goto parse_zk_args

:set_collection_zk
set ZK_COLLECTION=%~2
SHIFT
SHIFT
goto parse_zk_args

:set_config_zk
set ZK_HOST=%~2
SHIFT
Expand Down Expand Up @@ -1533,6 +1548,28 @@ IF "!ZK_OP!"=="upconfig" (
-Dlog4j.configurationFile="file:///%DEFAULT_SERVER_DIR%\resources\log4j2-console.xml" ^
-classpath "%DEFAULT_SERVER_DIR%\solr-webapp\webapp\WEB-INF\lib\*;%DEFAULT_SERVER_DIR%\lib\ext\*" ^
org.apache.solr.cli.SolrCLI !ZK_OP! -confname !CONFIGSET_NAME! -confdir !CONFIGSET_DIR! -zkHost !ZK_HOST! %ZK_VERBOSE%
) ELSE IF "!ZK_OP!"=="linkconfig" (
IF "!CONFIGSET_NAME!"=="" (
set ERROR_MSG="-n option must be set for linkconfig"
goto zk_short_usage
)
IF "!ZK_COLLECTION!"=="" (
set ERROR_MSG="The -c option must be set for linkconfig."
goto zk_short_usage
)
"%JAVA%" %SOLR_SSL_OPTS% %AUTHC_OPTS% %SOLR_ZK_CREDS_AND_ACLS% %SOLR_TOOL_OPTS% -Dsolr.install.dir="%SOLR_TIP%" ^
-Dlog4j.configurationFile="file:///%DEFAULT_SERVER_DIR%\resources\log4j2-console.xml" ^
-classpath "%DEFAULT_SERVER_DIR%\solr-webapp\webapp\WEB-INF\lib\*;%DEFAULT_SERVER_DIR%\lib\ext\*" ^
org.apache.solr.cli.SolrCLI !ZK_OP! -confname !CONFIGSET_NAME! -c !ZK_COLLECTION! -zkHost !ZK_HOST! %ZK_VERBOSE%
) ELSE IF "!ZK_OP!"=="updateacls" (
IF "%ZK_SRC"=="" (
set ERROR_MSG="Zookeeper path to remove must be specified when using the 'ls' command"
goto zk_short_usage
)
"%JAVA%" %SOLR_SSL_OPTS% %AUTHC_OPTS% %SOLR_ZK_CREDS_AND_ACLS% %SOLR_TOOL_OPTS% -Dsolr.install.dir="%SOLR_TIP%" ^
-Dlog4j.configurationFile="file:///%DEFAULT_SERVER_DIR%\resources\log4j2-console.xml" ^
-classpath "%DEFAULT_SERVER_DIR%\solr-webapp\webapp\WEB-INF\lib\*;%DEFAULT_SERVER_DIR%\lib\ext\*" ^
org.apache.solr.cli.SolrCLI !ZK_OP! -path !ZK_SRC! -zkHost !ZK_HOST! %ZK_VERBOSE%
) ELSE IF "!ZK_OP!"=="cp" (
IF "%ZK_SRC%"=="" (
set ERROR_MSG="<src> must be specified for 'cp' command"
Expand Down
100 changes: 100 additions & 0 deletions solr/core/src/java/org/apache/solr/cli/ClusterTool.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,100 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.solr.cli;

import java.io.IOException;
import java.io.PrintStream;
import java.util.List;
import java.util.concurrent.TimeUnit;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.Option;
import org.apache.solr.client.solrj.impl.SolrZkClientTimeout;
import org.apache.solr.cloud.ZkController;
import org.apache.solr.common.cloud.ClusterProperties;
import org.apache.solr.common.cloud.SolrZkClient;

/**
* Supports cluster command in the bin/solr script.
*
* <p>Set cluster properties by directly manipulating ZooKeeper.
*/
public class ClusterTool extends ToolBase {
// It is a shame this tool doesn't more closely mimic how the ConfigTool works.

public ClusterTool() {
this(CLIO.getOutStream());
}

public ClusterTool(PrintStream stdout) {
super(stdout);
}

@Override
public String getName() {
return "cluster";
}

@Override
public List<Option> getOptions() {
return List.of(
Option.builder()
.longOpt("property")
.argName("PROP")
.hasArg()
.required(true)
.desc("Name of the Cluster property to apply the action to, such as: 'urlScheme'.")
.build(),
Option.builder()
.longOpt("value")
.argName("VALUE")
.hasArg()
.required(false)
.desc("Set the property to this value.")
.build(),
SolrCLI.OPTION_ZKHOST);
}

@Override
public void runImpl(CommandLine cli) throws Exception {

String propertyName = cli.getOptionValue("property");
String propertyValue = cli.getOptionValue("value");
String zkHost = SolrCLI.getZkHost(cli);

if (!ZkController.checkChrootPath(zkHost, true)) {
throw new IllegalStateException(
"A chroot was specified in zkHost but the znode doesn't exist.");
}

try (SolrZkClient zkClient =
new SolrZkClient.Builder()
.withUrl(zkHost)
.withTimeout(SolrZkClientTimeout.DEFAULT_ZK_CLIENT_TIMEOUT, TimeUnit.MILLISECONDS)
.build()) {

ClusterProperties props = new ClusterProperties(zkClient);
try {
props.setClusterProperty(propertyName, propertyValue);
} catch (IOException ex) {
throw new Exception(
"Unable to set the cluster property due to following error : "
+ ex.getLocalizedMessage());
}
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -45,14 +45,16 @@ public ConfigSetDownloadTool(PrintStream stdout) {
@Override
public List<Option> getOptions() {
return List.of(
Option.builder("confname")
.argName("confname")
Option.builder("n")
.longOpt("confname")
.argName("NAME")
.hasArg()
.required(true)
.desc("Configset name in ZooKeeper.")
.build(),
Option.builder("confdir")
.argName("confdir")
Option.builder("d")
.longOpt("confdir")
.argName("DIR")
.hasArg()
.required(true)
.desc("Local directory with configs.")
Expand Down
16 changes: 9 additions & 7 deletions solr/core/src/java/org/apache/solr/cli/ConfigSetUploadTool.java
Original file line number Diff line number Diff line change
Expand Up @@ -46,14 +46,16 @@ public ConfigSetUploadTool(PrintStream stdout) {
@Override
public List<Option> getOptions() {
return List.of(
Option.builder("confname")
.argName("confname") // Comes out in help message
.hasArg() // Has one sub-argument
.required(true) // confname argument must be present
Option.builder("n")
.longOpt("confname")
.argName("NAME")
.hasArg()
.required(true)
.desc("Configset name in ZooKeeper.")
.build(), // passed as -confname value
Option.builder("confdir")
.argName("confdir")
.build(),
Option.builder("d")
.longOpt("confdir")
.argName("DIR")
.hasArg()
.required(true)
.desc("Local directory with configs.")
Expand Down
Loading

0 comments on commit 963b629

Please sign in to comment.