From 244e740d87db648d0b258ee1526d9cf3e591c58d Mon Sep 17 00:00:00 2001 From: dev-mlb <19797865+dev-mlb@users.noreply.github.com> Date: Tue, 12 Dec 2023 11:10:00 -0500 Subject: [PATCH] remove deprecated runcommand and main-runner (#568) --- README.md | 14 - pom.xml | 10 - src/main/java/emissary/Emissary.java | 3 +- .../java/emissary/command/RunCommand.java | 76 -- src/main/java/emissary/id/SizeIdPlace.java | 3 - src/main/java/emissary/id/UnixFilePlace.java | 3 - .../java/emissary/output/DropOffPlace.java | 6 - .../pickup/file/FilePickUpClient.java | 4 - .../emissary/pickup/file/FilePickUpPlace.java | 3 - .../emissary/place/CoordinationPlace.java | 6 - src/main/java/emissary/place/Main.java | 1169 ----------------- .../place/MultiFileUnixCommandPlace.java | 6 - .../emissary/place/ServiceProviderPlace.java | 25 - .../java/emissary/place/UnixCommandPlace.java | 6 - .../emissary/place/sample/DelayPlace.java | 6 - .../emissary/place/sample/DevNullPlace.java | 6 - .../emissary/place/sample/TemplatePlace.java | 6 - .../emissary/place/sample/ToLowerPlace.java | 6 - .../emissary/place/sample/ToUpperPlace.java | 6 - .../emissary/transform/HtmlEscapePlace.java | 6 - .../transform/JavascriptEscapePlace.java | 6 - .../emissary/transform/JsonEscapePlace.java | 7 - .../java/emissary/command/RunCommandIT.java | 101 -- src/test/java/emissary/place/MainTest.java | 368 ------ 24 files changed, 1 insertion(+), 1851 deletions(-) delete mode 100644 src/main/java/emissary/command/RunCommand.java delete mode 100644 src/main/java/emissary/place/Main.java delete mode 100644 src/test/java/emissary/command/RunCommandIT.java delete mode 100644 src/test/java/emissary/place/MainTest.java diff --git a/README.md b/README.md index 8220b86d49..dc46076afa 100644 --- a/README.md +++ b/README.md @@ -191,20 +191,6 @@ or in offline mode: ./emissary config --place emissary.place.sample.ToLowerPlace --offline --detailed ``` -#### Run - -The Run command is a simple command to execute the main method of the given class. For example - -``` -./emissary run emissary.config.ConfigUtil -``` - -If you need to pass flags to the main method, use *--* to stop parsing flags and simply pass them along. - -``` -./emissary run emissary.config.ExtractResource -- -o outputdir somefile -``` - #### Server (Cluster) Emissary is fun in standalone, but running cluster is more appropriate for real work. The way to run clustered diff --git a/pom.xml b/pom.xml index 9e05e52b4d..b8a3ec41a6 100644 --- a/pom.xml +++ b/pom.xml @@ -46,7 +46,6 @@ ${project.version} ${project.basedir}/contrib/checkstyle.xml - 1.5.0 1.15 4.4 1.21 @@ -145,11 +144,6 @@ guava ${dep.guava.version} - - commons-cli - commons-cli - ${dep.commons-cli.version} - commons-codec commons-codec @@ -350,10 +344,6 @@ com.google.guava guava - - commons-cli - commons-cli - commons-codec commons-codec diff --git a/src/main/java/emissary/Emissary.java b/src/main/java/emissary/Emissary.java index 15824eae15..a70deec1cb 100644 --- a/src/main/java/emissary/Emissary.java +++ b/src/main/java/emissary/Emissary.java @@ -10,7 +10,6 @@ import emissary.command.HelpCommand; import emissary.command.PeersCommand; import emissary.command.PoolCommand; -import emissary.command.RunCommand; import emissary.command.ServerCommand; import emissary.command.TopologyCommand; import emissary.command.VersionCommand; @@ -56,7 +55,7 @@ public class Emissary { static { List> cmds = Arrays.asList(ServerCommand.class, HelpCommand.class, TopologyCommand.class, FeedCommand.class, - AgentsCommand.class, PoolCommand.class, VersionCommand.class, RunCommand.class, EnvCommand.class, + AgentsCommand.class, PoolCommand.class, VersionCommand.class, EnvCommand.class, PeersCommand.class, ConfigCommand.class, DirectoryCommand.class); Map staticCopy = new HashMap<>(); for (Class clz : cmds) { diff --git a/src/main/java/emissary/command/RunCommand.java b/src/main/java/emissary/command/RunCommand.java deleted file mode 100644 index 9a902cea0a..0000000000 --- a/src/main/java/emissary/command/RunCommand.java +++ /dev/null @@ -1,76 +0,0 @@ -package emissary.command; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import picocli.CommandLine; -import picocli.CommandLine.Command; -import picocli.CommandLine.Parameters; - -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -@Deprecated -@Command(description = "Run arbitrary class with optional args", subcommands = {HelpCommand.class}) -public class RunCommand extends BaseCommand { - - private static final Logger LOG = LoggerFactory.getLogger(RunCommand.class); - - @Parameters( - arity = "1..*", - description = "fully qualified class name to run with remaining arguments passed on as args to that classes main method. Use -- to stop processing strings as args and pass them along.") - public List args = new ArrayList<>(); - - @Override - public String getCommandName() { - return "run"; - } - - @Override - public void run(CommandLine c) { - setup(); - // make a class from whatever name - String clazzName = args.get(0); - String[] clazzArgs = null; - LOG.info("Class to run is {}", clazzName); - try { - Class clazz = Class.forName(clazzName); - Method meth = clazz.getMethod("main", String[].class); - if (args.size() > 1) { - // run with rests of args - clazzArgs = new String[args.size() - 1]; - for (int i = 1; i < args.size(); i++) { - clazzArgs[i - 1] = args.get(i); - } - LOG.info("Running with {}", Arrays.toString(clazzArgs)); - meth.invoke(null, (Object) clazzArgs); - } else { - LOG.info("Running no args"); - // run class - meth.invoke(null, (Object) new String[0]); - } - } catch (ClassNotFoundException e) { - throw new RuntimeException("Could not find fully qualified class named " + clazzName); - } catch (IllegalAccessException | IllegalArgumentException | InvocationTargetException | NoSuchMethodException | SecurityException e) { - String errorMsg = "Problem calling main from " + clazzName + " with " + Arrays.toString(clazzArgs); - LOG.error(errorMsg, e); - throw new RuntimeException(errorMsg + " : " + e.getMessage()); - } - } - - @Override - public void run() { - run(null); - } - - @Override - public void setupCommand() { - setupRun(); - } - - public void setupRun() { - setupConfig(); - } -} diff --git a/src/main/java/emissary/id/SizeIdPlace.java b/src/main/java/emissary/id/SizeIdPlace.java index d458883049..b2bbfc19ad 100644 --- a/src/main/java/emissary/id/SizeIdPlace.java +++ b/src/main/java/emissary/id/SizeIdPlace.java @@ -92,7 +92,4 @@ public String fileTypeBySize(int sz) { return LABELS[LABELS.length - 1]; } - public static void main(String[] args) { - mainRunner(SizeIdPlace.class, args); - } } diff --git a/src/main/java/emissary/id/UnixFilePlace.java b/src/main/java/emissary/id/UnixFilePlace.java index 8af90a3fbe..9dc29243af 100755 --- a/src/main/java/emissary/id/UnixFilePlace.java +++ b/src/main/java/emissary/id/UnixFilePlace.java @@ -145,7 +145,4 @@ public void process(final IBaseDataObject d) { } } - public static void main(final String[] args) { - mainRunner(UnixFilePlace.class.getName(), args); - } } diff --git a/src/main/java/emissary/output/DropOffPlace.java b/src/main/java/emissary/output/DropOffPlace.java index 8095b93b86..057c954aa8 100755 --- a/src/main/java/emissary/output/DropOffPlace.java +++ b/src/main/java/emissary/output/DropOffPlace.java @@ -491,10 +491,4 @@ public void addFilter(final IDropOffFilter filter) { this.outputFilters.add(filter); } - /** - * Run the command line interface - */ - public static void main(final String[] argv) throws Exception { - mainRunner(DropOffPlace.class.getName(), argv); - } } diff --git a/src/main/java/emissary/pickup/file/FilePickUpClient.java b/src/main/java/emissary/pickup/file/FilePickUpClient.java index 637691fe51..f6fdb276b1 100755 --- a/src/main/java/emissary/pickup/file/FilePickUpClient.java +++ b/src/main/java/emissary/pickup/file/FilePickUpClient.java @@ -473,8 +473,4 @@ protected void processDirectoryEntry(String root, String prefix, String caseid, } } - public static void main(String[] args) { - mainRunner(FilePickUpClient.class.getName(), args); - } - } diff --git a/src/main/java/emissary/pickup/file/FilePickUpPlace.java b/src/main/java/emissary/pickup/file/FilePickUpPlace.java index 36860bb73c..920ed2b813 100755 --- a/src/main/java/emissary/pickup/file/FilePickUpPlace.java +++ b/src/main/java/emissary/pickup/file/FilePickUpPlace.java @@ -129,7 +129,4 @@ public void startDataServer() { } } - public static void main(String[] args) { - mainRunner(FilePickUpPlace.class.getName(), args); - } } diff --git a/src/main/java/emissary/place/CoordinationPlace.java b/src/main/java/emissary/place/CoordinationPlace.java index e205ae2293..cdf9f238cb 100755 --- a/src/main/java/emissary/place/CoordinationPlace.java +++ b/src/main/java/emissary/place/CoordinationPlace.java @@ -278,10 +278,4 @@ public List processHeavyDuty(IBaseDataObject d) throws Resource return coordinate(d, true); } - /** - * Test standalone main - */ - public static void main(String[] argv) { - mainRunner(CoordinationPlace.class.getName(), argv); - } } diff --git a/src/main/java/emissary/place/Main.java b/src/main/java/emissary/place/Main.java deleted file mode 100644 index 7ae6b5e07c..0000000000 --- a/src/main/java/emissary/place/Main.java +++ /dev/null @@ -1,1169 +0,0 @@ -package emissary.place; - -import emissary.config.ConfigUtil; -import emissary.core.Factory; -import emissary.core.Family; -import emissary.core.Form; -import emissary.core.IBaseDataObject; -import emissary.directory.DirectoryPlace; -import emissary.directory.EmissaryNode; -import emissary.kff.KffDataObjectHandler; -import emissary.log.MDCConstants; -import emissary.parser.ParserEOFException; -import emissary.parser.ParserException; -import emissary.parser.SessionParser; -import emissary.parser.SessionProducer; -import emissary.parser.SimpleParser; -import emissary.util.Version; -import emissary.util.shell.Executrix; - -import org.apache.commons.cli.CommandLine; -import org.apache.commons.cli.CommandLineParser; -import org.apache.commons.cli.HelpFormatter; -import org.apache.commons.cli.Options; -import org.apache.commons.cli.ParseException; -import org.apache.commons.cli.PosixParser; -import org.apache.commons.lang3.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.slf4j.MDC; - -import java.io.File; -import java.io.IOException; -import java.io.PrintStream; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.NoSuchElementException; -import java.util.Set; -import java.util.regex.Pattern; -import javax.annotation.Nullable; - -/** - * This class handles running a the main method from a ServiceProviderPlace instance in a well defined but extensible - * way. We have common command line options from getStandardOptions that can be used as-is or can be retrieved - * and augmented with additional options and passed into the command line parser and runner. The normal flow is to use - * the scripts/run.sh script passing in the name of the class to run followed by options, followed by a file or set of - * files to process through that place. - */ -@Deprecated -public class Main { - /** Name of the ServiceProviderPlace class to run */ - protected String placeClass; - /** The arguments from the command line */ - protected String[] args; - /** File args from command line after options parsed */ - protected String[] fileArgs; - /** The instantiated service provider place */ - protected IServiceProviderPlace isp; - /** The location of the config file or stream resource */ - protected String configLocation; - /** Indicate if the default config has been overridden */ - private boolean configLocIsDefault = true; - /** Current form for input data */ - protected String currentForm = emissary.core.Form.UNKNOWN; - /** Where to send output */ - protected PrintStream outStream = System.out; - /** Output directory for split option */ - protected String baseOutputDir = "."; - /** True if output should be split to multiple files */ - protected boolean splitOutput = false; - /** Recurse on input directories */ - protected boolean recurseOnFileArgs = false; - /** Be extra quiet */ - protected boolean silent = false; - /** Be extra verbose */ - protected boolean verbose = false; - /** Print a short one-liner per input, good for ID places */ - protected boolean oneLiner = false; - /** Regex for which metadata to print out */ - protected String[] metaPatterns; - /** hash of parameters/values to set before process */ - protected HashMap params = new HashMap<>(); - /** Set to turn off Emissary node context and directory setup */ - protected boolean runWithoutContext = false; - /** The command line options */ - protected Options options = getStandardOptions(); - /** set of alt views to print after processing */ - protected Set viewsToPrint = new HashSet<>(); - /** Class name of parser to run on input data */ - protected String parserName = SimpleParser.class.getName(); - /** Number of threads to use to process input files */ - protected int numThreads = 1; - /** The workers that call the place */ - List workers = new ArrayList<>(); - /** Queue of payload object for workers to pull from */ - protected final LinkedList workQueue = new LinkedList<>(); - /** hashing support */ - protected KffDataObjectHandler kff = null; - /** loop on input */ - protected boolean loopOnInput = false; - - - // My logger (nb. not used for messages from the place being run) - protected static final Logger logger = LoggerFactory.getLogger(Main.class); - - /** - * Create an instance to run a job - * - * @param placeClass class name of the place to run - * @param args the commandline args - */ - public Main(String placeClass, String[] args) { - this.placeClass = placeClass; - this.configLocation = placeClass + ConfigUtil.CONFIG_FILE_ENDING; - this.args = args; - initKff(); - } - - /** - * Initialize the Kff Handler with our policy settings - */ - protected synchronized void initKff() { - kff = - new KffDataObjectHandler(KffDataObjectHandler.TRUNCATE_KNOWN_DATA, KffDataObjectHandler.SET_FORM_WHEN_KNOWN, - KffDataObjectHandler.SET_FILE_TYPE); - } - - /** - * Return the standard command line options for running a place - * - * @return a Jakarta Commons CLI Options object - */ - public static Options getStandardOptions() { - Options options = new Options(); - options.addOption("1", "short", false, "short one-line listing per input"); - options.addOption("i", "loop", false, "loop on input"); - options.addOption("a", "altview", true, "name of alternate view to print"); - options.addOption("c", "config", true, "specify config file, resource, or stream name"); - options.addOption("d", "directory", true, "specify output directory for split output"); - options.addOption("h", "help", false, "print usage information"); - options.addOption("l", "level", true, "level for class logger (DEBUG|WARN|INFO|ERROR|FATAL)"); - options.addOption("L", "LEVEL", true, "level for root logger (DEBUG|WARN|INFO|ERROR|FATAL)"); - options.addOption("m", "metadata", true, "print value of named metadata slot (regex ok)"); - options.addOption("o", "output", true, "specify output filename"); - options.addOption("P", "parser", true, "class name of parser to use when loading data"); - options.addOption("p", "parameter", true, "DataObject parameter to set (key=value)"); - options.addOption("R", "recursive", false, "recurse on input directories"); - options.addOption("s", "silent", false, "dont print anything to output"); - options.addOption("S", "split", false, "split the output into separate files"); - options.addOption("t", "type", true, "type of data (current form)"); - options.addOption("T", "threads", true, "number of threads to use (default is 1)"); - options.addOption("v", "verbose", false, "print verbose output"); - options.addOption("X", "nocontext", false, "run without initializing context"); - return options; - } - - /** - * Get the currently configured options in order to change them - * - * @return the Jakarta Commons CLI Options object currently configured - */ - public Options getOptions() { - return options; - } - - /** - * Set a new options object. To start with the defaults use getStandardOptions, add your stuff, and set it here. - * - * @param options the new options that will be accepted - */ - public void setOptions(Options options) { - this.options = options; - } - - /** - * Print help and usage info based on the options - */ - public void printUsage() { - // automatically generate the help statement - System.out.println("Emissary Version: " + new Version().toString()); - HelpFormatter formatter = new HelpFormatter(); - String header = null; - String footer = " [file1 [ file2 ... fileN]]"; - boolean autoUsage = false; - formatter.printHelp(placeClass, header, options, footer, autoUsage); - } - - /** - * Set the place config location to the specified file, resource, or stream - * - * @param s the string config location - */ - public void setConfigLocation(String s) { - configLocation = s; - configLocIsDefault = false; - } - - /** - * Get config location value - */ - public String getConfigLocation() { - return configLocation; - } - - /** - * Set the parser class name - */ - public void setParserName(String s) { - parserName = s; - } - - /** - * Get the parser class name - */ - public String getParserName() { - return parserName; - } - - /** - * Set the current form for payloads to process - * - * @param s the new current form value - */ - public void setCurrentForm(String s) { - currentForm = s; - } - - /** - * Get current form value in use - */ - public String getCurrentForm() { - return currentForm; - } - - /** - * Set number of threads - */ - public void setThreads(String s) { - setThreads(Integer.parseInt(s)); - } - - /** - * Set number of threads - */ - public void setThreads(int i) { - numThreads = i; - } - - /** - * Get the number of threads to use - */ - public int getThreads() { - return numThreads; - } - - /** - * Set to recurse on file args specified on command line - * - * @param value the new value for the recurse flag - */ - public void setRecursive(boolean value) { - logger.debug("Set recursive " + value); - recurseOnFileArgs = value; - } - - /** - * Get value of recursion field - */ - public boolean isRecursive() { - return recurseOnFileArgs; - } - - /** - * Set to loop on input - * - * @param value the new value for the loop flag - */ - public void setLoopOnInput(boolean value) { - logger.debug("Set loop " + value); - loopOnInput = value; - } - - /** - * Get value of loop on input field - */ - public boolean isLoopOnInput() { - return loopOnInput; - } - - /** - * Set value of verbose flag - * - * @param value the new value for verbose flag - */ - public void setVerbose(boolean value) { - logger.debug("Setting verbose " + value); - verbose = value; - } - - /** - * Get value of verbose flag - */ - public boolean isVerbose() { - return verbose; - } - - /** - * Set the value of the oneLiner flag - * - * @param value the new value for the oneLiner flag - */ - public void setOneLiner(boolean value) { - logger.debug("Setting one liner " + value); - oneLiner = value; - } - - /** - * Get the value of the oneLiner flag - */ - public boolean isOneLiner() { - return oneLiner; - } - - /** - * Get the value of the no context flag - */ - public boolean isRunWithoutContext() { - return runWithoutContext; - } - - /** - * Set the value of the no context flag - */ - public void setRunWithoutContext(boolean val) { - logger.debug("Set runWithoutContext " + val); - runWithoutContext = val; - } - - /** - * Get file args remaining after option processing - * - * @return the list of file arguments or an empty list - */ - public List getFileArgs() { - List l = new ArrayList<>(); - for (int i = 0; fileArgs != null && i < fileArgs.length; i++) { - l.add(fileArgs[i]); - } - return l; - } - - /** - * Set value of silent - * - * @param value the new value of the silent flag - */ - public void setSilent(boolean value) { - logger.debug("Set silent " + value); - silent = value; - } - - /** - * Get the value of the silent flag - */ - public boolean isSilent() { - return silent; - } - - /** - * Set the split output flag - */ - public void setSplitOutput(boolean value) { - logger.debug("Set split output " + value); - splitOutput = value; - } - - /** - * Get the value of the splitOutput flag - */ - public boolean isSplitOutput() { - return splitOutput; - } - - /** - * Set the value of the base directory for split output - */ - public void setBaseOutputDir(String value) { - baseOutputDir = value; - } - - /** - * Get the value of the base output directory for split output - */ - public String getBaseOutputDir() { - return baseOutputDir; - } - - /** - * Instantiate the parser when one is specified for the bytes of data This could be expanded to use RAF parsers, NIO - * parsers, etc. - */ - protected SessionParser createParser(byte[] data) { - if (getParserName() == null) { - return null; - } - return (SessionParser) Factory.create(getParserName(), data); - } - - /** - * Instantiate the specified processing place - * - * @return true if it is created - */ - protected boolean createPlace() { - String classOnlyName = placeClass.substring(placeClass.lastIndexOf(".") + 1); - - // See if we need to alter the default config location to use the old - // style class no-package config name - if (configLocIsDefault) { - String cf = ConfigUtil.getConfigFile(configLocation); - File fcf = new File(cf); - if (!fcf.exists()) { - cf = ConfigUtil.getConfigFile(classOnlyName + ConfigUtil.CONFIG_FILE_ENDING); - fcf = new File(cf); - if (fcf.exists()) { - // Save it if long name does not exist but short one does - setConfigLocation(classOnlyName + ConfigUtil.CONFIG_FILE_ENDING); - } - } - } - - boolean pseudoNodeCreated = false; - try { - EmissaryNode node = new EmissaryNode(); - if (!node.isValid()) { - System.setProperty(EmissaryNode.NODE_NAME_PROPERTY, "localhost"); - System.setProperty(EmissaryNode.NODE_PORT_PROPERTY, "0000"); - node = new EmissaryNode(); - pseudoNodeCreated = true; - - } - String placeLoc = "http://" + node.getNodeName() + ":" + node.getNodePort() + "/" + classOnlyName; - isp = - (IServiceProviderPlace) Factory.create(placeClass, configLocation, "EMISSARY_DIRECTORY_SERVICES.STUDY.DIRECTORY_PLACE.http://" - + node.getNodeName() + ":" + node.getNodePort() + "/DirectoryPlace", placeLoc); - - } catch (Throwable ex) { - logger.error("Cannot create " + placeClass + ": " + ex.getMessage()); - return false; - } finally { - if (pseudoNodeCreated) { - System.clearProperty(EmissaryNode.NODE_NAME_PROPERTY); - System.clearProperty(EmissaryNode.NODE_PORT_PROPERTY); - } - } - - return true; - } - - /** - * Run the main method in normal testing mode using the standard or configured options and taking the remaining - * arguments as data files or directories of data files to be processed - */ - public void run() { - parseArguments(); - - // Create the pool of workers - logger.debug("Setting up " + getThreads() + " worker threads"); - for (int i = 0; i < getThreads(); i++) { - Worker w = new Worker(); - new Thread(w, "MainWorker-" + i).start(); - workers.add(w); - } - - if (isp != null) { - do { - // Process the data, loading it into the work queue - processMainData(fileArgs, null); - - // Wait for the work queue to be empty - int size = -1; - synchronized (workQueue) { - size = workQueue.size(); - } - while (size > 0) { - int workQueueSize = -1; - synchronized (workQueue) { - workQueueSize = workQueue.size(); - } - if (workQueueSize < size) { - size = workQueueSize; - logger.debug("Waiting for work queue " + size + " remaining"); - } - try { - Thread.sleep(100); - } catch (InterruptedException ignore) { - Thread.currentThread().interrupt(); - } - } - } while (loopOnInput); - } - - // Shutdown the workers - logger.debug("Stopping " + workers.size() + " workers"); - for (int i = 0; i < getThreads(); i++) { - workers.get(i).stop(); - } - - synchronized (workQueue) { - workQueue.notifyAll(); - } - - // Wait for all workers to actually stop - for (int i = 0; i < getThreads(); i++) { - Worker w = workers.get(i); - while (!w.isIdle()) { - try { - Thread.sleep(10); - } catch (InterruptedException ignore) { - Thread.currentThread().interrupt(); - } - } - } - } - - /** - * Allow extenders to determine whether args are parsed here or not - * - * @param cmd the parsed command line - * @return true if normal argument processing should continue - */ - protected boolean preArgumentsHook(CommandLine cmd) { - // Do nothing in this impl but allow the normal stuff to go on - return true; - } - - /** - * Allow extenders to handle any additional arguments - * - * @param cmd the parsed command line - */ - protected void postArgumentsHook(CommandLine cmd) { - // Do nothing in this impl. - } - - /** - * Parse the command line arguments - */ - public void parseArguments() { - CommandLineParser parser = new PosixParser(); - CommandLine cmd = null; - try { - cmd = parser.parse(options, args); - fileArgs = cmd.getArgs(); - if (fileArgs != null) { - logger.debug("Parsed arguments and have " + fileArgs.length + " file args remaining"); - } - } catch (ParseException ex) { - // oops, something went wrong - logger.error(placeClass + ": Parsing failed. Reason: " + ex.getMessage()); - printUsage(); - return; - } - - boolean shouldContinue = preArgumentsHook(cmd); - - if (shouldContinue) { - processParsedArguments(cmd); - postArgumentsHook(cmd); - } - - } - - protected void processParsedArguments(CommandLine cmd) { - // Take care of the help option - if (cmd.hasOption("h")) { - printUsage(); - return; - } - - // Set boolean flags - setSilent(cmd.hasOption("s")); - setVerbose(cmd.hasOption("v")); - setRecursive(cmd.hasOption("R")); - setLoopOnInput(cmd.hasOption("i")); - setOneLiner(cmd.hasOption("1")); - setRunWithoutContext(cmd.hasOption("X")); - - // Override config if specified - if (cmd.hasOption("c")) { - setConfigLocation(cmd.getOptionValue("c")); - } - - // Set up a parser if one is specified - if (cmd.hasOption("P")) { - setParserName(cmd.getOptionValue("P")); - } - - // Override current form if specified - if (cmd.hasOption("t")) { - setCurrentForm(cmd.getOptionValue("t")); - } - - // Set up number of threads to use - if (cmd.hasOption("T")) { - logger.debug("Thread option is " + cmd.getOptionValue("T")); - setThreads(cmd.getOptionValue("T")); - } - - // save parameter key=value pairs to set - if (cmd.hasOption("p")) { - setParameters(cmd.getOptionValues("p")); - } - - // Save regex's for looking at metadata - setMetaPatterns(cmd.getOptionValues("m")); - - // Save alt view names to print in a set - String[] altViews = cmd.getOptionValues("a"); - for (int i = 0; altViews != null && i < altViews.length; i++) { - viewsToPrint.add(altViews[i]); - } - - // Do some of the things that the normal context initializer does - boolean pseudoNodeCreated = false; - if (!isRunWithoutContext()) { - try { - EmissaryNode node = new EmissaryNode(); - if (!node.isValid()) { - System.setProperty(EmissaryNode.NODE_NAME_PROPERTY, "localhost"); - System.setProperty(EmissaryNode.NODE_PORT_PROPERTY, "0000"); - node = new EmissaryNode(); - pseudoNodeCreated = true; - } - new DirectoryPlace("EMISSARY_DIRECTORY_SERVICES.STUDY.DIRECTORY_PLACE.http://" + node.getNodeName() + ":" + node.getNodePort() - + "/DirectoryPlace", node); - } catch (IOException iox) { - logger.debug("Could not create standalone directory", iox); - } finally { - if (pseudoNodeCreated) { - System.clearProperty(EmissaryNode.NODE_NAME_PROPERTY); - System.clearProperty(EmissaryNode.NODE_PORT_PROPERTY); - } - } - } - - // Create the place - if (!createPlace()) { - logger.warn("Unable to run main processing, no place created"); - return; - } - - // Grab the base output directory for split - if (cmd.hasOption("d")) { - setBaseOutputDir(cmd.getOptionValue("d")); - } - - // Turn on splitting of output if requested - if (cmd.hasOption("S")) { - setSplitOutput(true); - } - - - // Redirect the output stream if desired - if (cmd.hasOption("o")) { - try { - setOutputStream(cmd.getOptionValue("o")); - } catch (IOException iox) { - logger.error("Cannot redirect output to " + cmd.getOptionValue("o") + ": " + iox.getMessage()); - } - - } - } - - /** - * Set the output stream to the new value - * - * @param stream the PrintStream to use further output - */ - public void setOutputStream(PrintStream stream) { - outStream = stream; - } - - /** - * Set the output stream using the name provided - * - * @param name name to use in creating a new PrintStream - */ - public void setOutputStream(String name) throws IOException { - outStream = new PrintStream(name); - } - - /** - * Set parameters on the new DataObject. Format of pattern is key=value. - * - * @param keyval the params to use - */ - public void setParameters(String[] keyval) { - for (String p : keyval) { - if (p.indexOf("=") > 0) { - // limit to two, so key=val1=val2 just turns into - // "key" = "val1=val2" - String[] kv = p.split("=", 2); - params.put(kv[0], kv[1]); - } else { - logger.debug("param format must be key=value, skipping " + p); - } - } - } - - /** - * return parameters on the new DataObject. - */ - public HashMap getParameters() { - return params; - } - - /** - * Set the metadata print patterns - * - * @param patterns the patterns to use - */ - public void setMetaPatterns(String[] patterns) { - metaPatterns = patterns; - } - - /** - * Get a ref to the processing place being used - */ - public IServiceProviderPlace getProcessingPlace() { - return isp; - } - - /** - * Hook called before processHeavyDuty with every payload - * - * @param payload the just created payload - * @param attachments empty list that can be populated - * @return false to stop call to processHeavyDuty from continuing - */ - protected boolean preProcessHook(IBaseDataObject payload, List attachments) { - // Do nothing in this impl but allow processing to continue - return true; - } - - /** - * Hook called after every return from processHeavyDuty - * - * @param payload the completed payload - * @param att the list of attachments returned - */ - protected void postProcessHook(IBaseDataObject payload, List att) { - // Do nothing in this impl. - } - - /** - * Hook called after any exception thrown from the call to processHeavyDuty - * - * @param payload the completed payload - */ - protected void postProcessErrorHook(IBaseDataObject payload) { - // Do nothing in this impl. - } - - /** - * Hook called before printing the payload and attachments - * - * @param payload the processed payload - * @param att the list of attachments - * @return false to stop printing - */ - protected boolean prePrintHook(IBaseDataObject payload, List att) { - // Do nothing here but allow printing to continue - logger.debug("Preprint hook called"); - return true; - } - - /** - * Hook called after printing the payload and attachments - * - * @param payload the processed payload - * @param att the list of attachments - */ - protected void postPrintHook(IBaseDataObject payload, List att) { - // Do nothing in this impl - logger.debug("Postprint hook called"); - } - - /** - * Hook called before splitting the output to files - * - * @param payload the processed payload - * @param att the list of attachments - * @return false to stop the splitting - */ - protected boolean preSplitHook(IBaseDataObject payload, List att) { - // Do nothing here but allow the splitting to continue - return true; - } - - /** - * Hook called after splitting the output - * - * @param payload the processed payload - * @param att the list of attachments - */ - protected void postSplitHook(IBaseDataObject payload, List att) { - // Do nothing in this impl - } - - /** - * Process the file arguments from the command-line or recursed directory - * - * @param args the remaining (non-option) arguments - * @param path the path we are operating in, for appending when doing file recursion - */ - public void processMainData(String[] args, @Nullable String path) { - for (int i = 0; i < args.length; i++) { - String spath = (path != null ? (path + "/") : "") + args[i]; - File f = new File(spath); - if (f.isDirectory()) { - if (isRecursive()) { - String[] files = f.list(); - // recursion alert - logger.debug("Recursing into directory " + spath); - processMainData(files, spath); - logger.debug("Leaving recursed directory" + spath); - // end recursive call - } else { - outStream.println(spath + ": DIRECTORY"); - } - continue; - } - - if (!f.canRead()) { - outStream.println(spath + ": UNREADABLE"); - continue; - } - - // Process the file - processFile(spath); - } - } - - /** - * Process the single named file - */ - public void processFile(String spath) { - // This could be expanded to allow configuration to choose - // to use a RAF or NIO parser instead of forcing a byte array - // parser - byte[] content = Executrix.readDataFromFile(spath); - if (content == null) { - outStream.println(spath + ": UNREADABLE"); - return; - } - - SessionParser sp = null; - try { - sp = createParser(content); - } catch (Throwable t) { - logger.error("Cannot create parser " + getParserName() + ": " + t); - sp = new SimpleParser(content); - } - - try { - SessionProducer source = new SessionProducer(sp, currentForm); - int count = 1; - while (true) { - IBaseDataObject payload = source.getNextSession(spath + (count > 1 ? ("-" + count) : "")); - kff.hash(payload); - - // Set up filetype if non-default - if (!emissary.core.Form.UNKNOWN.equals(currentForm)) { - payload.setFileType(currentForm); - } - - // add command line metadata before processing - payload.setParameters(params); - - // Push this payload onto a queue for the Worker consumers to pull from - queuePayload(payload); - - if (count % numThreads == 0) { - Thread.yield(); - } - - count++; - } - } catch (ParserEOFException eof) { - // expected at end of file - } catch (ParserException ex) { - logger.error("File " + spath + " cannot be parsed by " + getParserName() + ": " + ex); - } - } - - /** - * Stuff the payload onto the queue for processing and notify workers - */ - public void queuePayload(IBaseDataObject payload) { - synchronized (workQueue) { - workQueue.addLast(payload); - workQueue.notifyAll(); - } - } - - /** - * Called by the Worker consumer to process the given payload - */ - public void processPayload(IBaseDataObject payload) { - List attachments = new ArrayList<>(); - MDC.put(MDCConstants.SHORT_NAME, payload.shortName()); - try { - boolean shouldContinue = preProcessHook(payload, attachments); - if (shouldContinue) { - attachments = isp.agentProcessHeavyDuty(payload); - } - postProcessHook(payload, attachments); - } catch (Throwable pex) { - payload.replaceCurrentForm(Form.ERROR); - attachments = Collections.emptyList(); - postProcessErrorHook(payload); - } finally { - MDC.remove(MDCConstants.SHORT_NAME); - } - - if (!isSilent()) { - boolean shouldPrint = prePrintHook(payload, attachments); - - if (shouldPrint) { - if (isOneLiner()) { - printPayloadOneLiner(payload, attachments.size() + payload.getExtractedRecordCount()); - } else { - printPayload(payload); - - for (IBaseDataObject att : attachments) { - printPayload(att); - } - - if (payload.hasExtractedRecords()) { - for (IBaseDataObject rec : payload.getExtractedRecords()) { - printPayload(rec); - } - } - } - } - - postPrintHook(payload, attachments); - } - - // Split output into files if specified - // This is independent of printStream output - // printing, isSlient, shouldPrint, etc. - if (isSplitOutput()) { - boolean shoudSplit = preSplitHook(payload, attachments); - if (shoudSplit) { - handleSplitOutput(payload, attachments); - } - postSplitHook(payload, attachments); - } - } - - /** - * Print a one-liner for the payload - * - * @param payload the processed object - * @param attachmentCount number of attachments - */ - public void printPayloadOneLiner(IBaseDataObject payload, int attachmentCount) { - String ft = payload.getFileType(); - String cf = payload.currentForm(); - outStream.println(payload.getFilename() + ": " + (payload.currentFormSize() > 1 ? payload.getAllCurrentForms() : payload.currentForm()) - + ((ft != null && !ft.equals(cf)) ? (" > " + payload.getFileType()) : "") + " " - + (attachmentCount > 0 ? ("(" + attachmentCount + ")") : "")); - } - - /** - * Print out stuff related to a command-line processed payload - * - * @param payload the processed object - */ - public void printPayload(@Nullable IBaseDataObject payload) { - if (payload == null) { - return; - } - outStream.println(payload.getFilename()); - outStream.println("Current form: " + payload.getAllCurrentForms()); - outStream.println("File type: " + payload.getFileType()); - outStream.println("Encoding: " + payload.getFontEncoding()); - outStream.println("Length: " + payload.dataLength()); - if (payload.getNumAlternateViews() > 0) { - outStream.println("Alt views: " + payload.getAlternateViewNames()); - } - if (payload.getNumChildren() > 0) { - outStream.println("Attachments: " + payload.getNumChildren()); - } - - if (metaPatterns != null && metaPatterns.length > 0) { - for (Map.Entry entry : payload.getCookedParameters().entrySet()) { - for (int i = 0; i < metaPatterns.length; i++) { - if (Pattern.matches(metaPatterns[i], entry.getKey())) { - outStream.println("Metadata " + entry.getKey() + ": " + entry.getValue()); - } - } - } - } - - boolean needTrailingCr = true; - - if (payload.dataLength() > 0 && (isVerbose() || (viewsToPrint.contains("MAIN") && payload.getFilename().indexOf(Family.SEP) == -1))) { - outStream.println("Data: < - * Note: Output will be suppressed if an IBDO form or filename would yield an output path outside the configured - * {@link #getBaseOutputDir() base output directory} path - *

- * - * @param payload the processed payload - * @param att the list of attachments - */ - public void handleSplitOutput(IBaseDataObject payload, List att) { - String fn = getBaseOutputDir() + "/" + payload.shortName() + "." + payload.currentForm(); - - String safePath; - try { - safePath = filePathIsWithinBaseDirectory(getBaseOutputDir(), fn); - if (Executrix.writeDataToFile(payload.data(), safePath)) { - logger.debug("Wrote output to {}", safePath); - } else { - logger.error("Could not write output to {}", safePath); - } - } catch (IllegalArgumentException e) { - logger.error("Could not write output to {}", fn, e); - } - for (IBaseDataObject part : att) { - fn = getBaseOutputDir() + "/" + part.shortName() + "." + part.currentForm(); - try { - safePath = filePathIsWithinBaseDirectory(getBaseOutputDir(), fn); - if (Executrix.writeDataToFile(payload.data(), safePath)) { - logger.debug("Wrote attachment output to {}", safePath); - } else { - logger.error("Could not write output to {}", safePath); - } - } catch (IllegalArgumentException e) { - logger.error("Could not write output to {}", fn, e); - } - } - } - - /** - * Normalizes the provided paths and tests to see whether the file path is within the required base directory. - * - * @param requiredBase required base directory - * @param filePath file path to be tested - * @return the normalized file path - * @throws IllegalArgumentException if the filePath contains illegal characters or is outside the required base - * directory - */ - static String filePathIsWithinBaseDirectory(final String requiredBase, final String filePath) throws IllegalArgumentException { - - if (StringUtils.isBlank(requiredBase)) { - throw new IllegalArgumentException("requiredBase must not be blank"); - } - - if (StringUtils.isBlank(filePath)) { - throw new IllegalArgumentException("filePath must not be blank"); - } - - // probably an overly simplistic test - if (filePath.contains("..")) { - throw new IllegalArgumentException("filePath contains illegal character sequence \"..\""); - } - Path normalizedBasePath = Paths.get(requiredBase).normalize().toAbsolutePath(); - Path normalizedFilePath = Paths.get(filePath).normalize().toAbsolutePath(); - - // append path separator to defeat traversal via a sibling directory with a similar name - if (!normalizedFilePath.startsWith(normalizedBasePath.toString() + "/")) { - throw new IllegalArgumentException("Normalized file path (\"" + filePath + "\") is outside the required base path (\"" - + requiredBase + "\")"); - } - return normalizedFilePath.toString(); - } - - /** - * Process an IBaseDataObject from teh work queue - */ - protected class Worker implements Runnable { - boolean idle = true; - boolean timeToStop = false; - - public boolean isIdle() { - return idle; - } - - public void stop() { - timeToStop = true; - } - - /** - * From the Runnable interface - */ - @Override - public void run() { - IBaseDataObject payload = null; - - while (!timeToStop) { - // Try to get a payload - synchronized (workQueue) { - try { - payload = workQueue.removeFirst(); - } catch (NoSuchElementException ignore) { - // empty catch block - } - } - - // Process a payload if there is one - if (payload != null) { - idle = false; - processPayload(payload); - payload = null; - idle = true; - } - - // Wait for a payload - else if (!timeToStop) { - synchronized (workQueue) { - try { - workQueue.wait(1000); - } catch (InterruptedException ignore) { - Thread.currentThread().interrupt(); - } - } - } - } - } - } -} diff --git a/src/main/java/emissary/place/MultiFileUnixCommandPlace.java b/src/main/java/emissary/place/MultiFileUnixCommandPlace.java index f0b8a0e73e..dc1e2de776 100755 --- a/src/main/java/emissary/place/MultiFileUnixCommandPlace.java +++ b/src/main/java/emissary/place/MultiFileUnixCommandPlace.java @@ -715,10 +715,4 @@ protected void cleanupDirectory(File dir) { } } - /** - * Test main - */ - public static void main(String[] argv) { - mainRunner(MultiFileUnixCommandPlace.class.getName(), argv); - } } diff --git a/src/main/java/emissary/place/ServiceProviderPlace.java b/src/main/java/emissary/place/ServiceProviderPlace.java index 021fecf8d3..24c2ed03e3 100755 --- a/src/main/java/emissary/place/ServiceProviderPlace.java +++ b/src/main/java/emissary/place/ServiceProviderPlace.java @@ -1028,31 +1028,6 @@ public MobileAgent getAgent() throws NamespaceException { return (MobileAgent) Namespace.lookup(Thread.currentThread().getName()); } - /** - * Convenience to access the Main runner from a subclass If more flexibility is desired, the subclass can instantiate - * and use the Main class directly. - * - * @param placeClass the class to instantiate - * @param args from command line - */ - public static void mainRunner(Class placeClass, String[] args) { - Main main = new Main(placeClass.getName(), args); - main.run(); - } - - /** - * Convenience to access the Main runner from a subclass If more flexibility is desired, the subclass can instantiate - * and use the Main class directly. - * - * @param placeClass the class name to instantiate - * @param args from command line - */ - public static void mainRunner(String placeClass, String[] args) { - Main main = new Main(placeClass, args); - main.run(); - } - - @Override public List getRunningConfig() { List runningConfigList = new ArrayList<>(); diff --git a/src/main/java/emissary/place/UnixCommandPlace.java b/src/main/java/emissary/place/UnixCommandPlace.java index 7eb717175a..b086d5341e 100755 --- a/src/main/java/emissary/place/UnixCommandPlace.java +++ b/src/main/java/emissary/place/UnixCommandPlace.java @@ -428,10 +428,4 @@ protected void processData(IBaseDataObject tData) throws ResourceException { return; } - /** - * Run the class - */ - public static void main(String[] argv) { - mainRunner(UnixCommandPlace.class, argv); - } } diff --git a/src/main/java/emissary/place/sample/DelayPlace.java b/src/main/java/emissary/place/sample/DelayPlace.java index 92b3e07491..00235a12be 100644 --- a/src/main/java/emissary/place/sample/DelayPlace.java +++ b/src/main/java/emissary/place/sample/DelayPlace.java @@ -51,10 +51,4 @@ public void process(IBaseDataObject tData) throws ResourceException { } } - /** - * Test run - */ - public static void main(String[] argv) { - mainRunner(DelayPlace.class.getName(), argv); - } } diff --git a/src/main/java/emissary/place/sample/DevNullPlace.java b/src/main/java/emissary/place/sample/DevNullPlace.java index 19e062e036..94f75178b1 100755 --- a/src/main/java/emissary/place/sample/DevNullPlace.java +++ b/src/main/java/emissary/place/sample/DevNullPlace.java @@ -39,10 +39,4 @@ public void process(IBaseDataObject tData) { logger.debug("Nuked {} of {} current form values leaving {}", (after - before), before, tData.getAllCurrentForms()); } - /** - * Test run - */ - public static void main(String[] argv) { - mainRunner(DevNullPlace.class.getName(), argv); - } } diff --git a/src/main/java/emissary/place/sample/TemplatePlace.java b/src/main/java/emissary/place/sample/TemplatePlace.java index afd68394da..ffb7165e56 100755 --- a/src/main/java/emissary/place/sample/TemplatePlace.java +++ b/src/main/java/emissary/place/sample/TemplatePlace.java @@ -105,10 +105,4 @@ public List processHeavyDuty(IBaseDataObject d) { return Collections.emptyList(); } - /** - * Test standalone main - */ - public static void main(String[] argv) { - mainRunner(TemplatePlace.class.getName(), argv); - } } diff --git a/src/main/java/emissary/place/sample/ToLowerPlace.java b/src/main/java/emissary/place/sample/ToLowerPlace.java index a1f6c14209..287abe0ca5 100755 --- a/src/main/java/emissary/place/sample/ToLowerPlace.java +++ b/src/main/java/emissary/place/sample/ToLowerPlace.java @@ -67,10 +67,4 @@ public void process(IBaseDataObject d) { } } - /** - * Test standalone main - */ - public static void main(String[] argv) { - mainRunner(ToLowerPlace.class.getName(), argv); - } } diff --git a/src/main/java/emissary/place/sample/ToUpperPlace.java b/src/main/java/emissary/place/sample/ToUpperPlace.java index c8e480ab47..9cd61b8e53 100755 --- a/src/main/java/emissary/place/sample/ToUpperPlace.java +++ b/src/main/java/emissary/place/sample/ToUpperPlace.java @@ -67,10 +67,4 @@ public void process(IBaseDataObject d) { } } - /** - * Test standalone main - */ - public static void main(String[] argv) { - mainRunner(ToUpperPlace.class.getName(), argv); - } } diff --git a/src/main/java/emissary/transform/HtmlEscapePlace.java b/src/main/java/emissary/transform/HtmlEscapePlace.java index 6e6231aa82..8c73e8202d 100755 --- a/src/main/java/emissary/transform/HtmlEscapePlace.java +++ b/src/main/java/emissary/transform/HtmlEscapePlace.java @@ -185,10 +185,4 @@ public static String makeString(byte[] s) { } - /** - * Test standalone main - */ - public static void main(String[] argv) { - mainRunner(HtmlEscapePlace.class.getName(), argv); - } } diff --git a/src/main/java/emissary/transform/JavascriptEscapePlace.java b/src/main/java/emissary/transform/JavascriptEscapePlace.java index 6469449e0b..395b86387a 100644 --- a/src/main/java/emissary/transform/JavascriptEscapePlace.java +++ b/src/main/java/emissary/transform/JavascriptEscapePlace.java @@ -89,10 +89,4 @@ public void process(IBaseDataObject d) { } - /** - * Test standalone main - */ - public static void main(String[] argv) { - mainRunner(JavascriptEscapePlace.class.getName(), argv); - } } diff --git a/src/main/java/emissary/transform/JsonEscapePlace.java b/src/main/java/emissary/transform/JsonEscapePlace.java index 9c3a4bc084..43715ead1f 100755 --- a/src/main/java/emissary/transform/JsonEscapePlace.java +++ b/src/main/java/emissary/transform/JsonEscapePlace.java @@ -89,11 +89,4 @@ public void process(IBaseDataObject d) { } } - - /** - * Test standalone main - */ - public static void main(String[] argv) { - mainRunner(JsonEscapePlace.class.getName(), argv); - } } diff --git a/src/test/java/emissary/command/RunCommandIT.java b/src/test/java/emissary/command/RunCommandIT.java deleted file mode 100644 index 3416bf397c..0000000000 --- a/src/test/java/emissary/command/RunCommandIT.java +++ /dev/null @@ -1,101 +0,0 @@ -package emissary.command; - -import emissary.test.core.junit5.UnitTest; - -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import java.io.ByteArrayOutputStream; -import java.io.PrintStream; -import java.util.Arrays; - -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; - -class RunCommandIT extends UnitTest { - - private ByteArrayOutputStream outContent; - private ByteArrayOutputStream errContent; - - @Override - @BeforeEach - public void setUp() throws Exception { - super.setUp(); - outContent = new ByteArrayOutputStream(); - errContent = new ByteArrayOutputStream(); - } - - @Override - @AfterEach - public void tearDown() throws Exception { - super.tearDown(); - outContent = null; - errContent = null; - } - - @Test - void testClassMustExist() { - String clazzName = "com.junk.Who"; - Exception e = assertThrows(Exception.class, () -> { - RunCommand cmd = RunCommand.parse(RunCommand.class, clazzName); - cmd.run(); - }); - assertTrue(e.getMessage().contains("Could not find fully qualified class named " + clazzName)); - } - - @Test - void testClassIsRunWhenFound() throws Exception { - String clazzName = "emissary.command.RunCommandIT"; - RunCommand cmd = RunCommand.parse(RunCommand.class, clazzName); - - captureStdOutAndStdErrAndRunCommand(cmd); - - assertTrue(outContent.toString().contains("I am a test that runs myself. My args are []")); - } - - @Test - void testClassIsRunWhenFoundWithArgs() throws Exception { - String clazzName = "emissary.command.RunCommandIT"; - String arg1 = "asdf"; - String arg2 = "dkdke"; - String arg3 = "k3k23k"; - RunCommand cmd = RunCommand.parse(RunCommand.class, clazzName, arg1, arg2, arg3); - - captureStdOutAndStdErrAndRunCommand(cmd); - - assertTrue(outContent.toString().contains("I am a test that runs myself. My args are [" + arg1 + ", " + arg2 + ", " + arg3 + "]")); - } - - @Test - void testFlagArgsPassedThrough() throws Exception { - String clazzName = "emissary.command.RunCommandIT"; - String stopPicocliProcessing = "--"; - String arg1 = "-f"; - String arg2 = "somefile"; - String arg3 = "--greatestArg"; - String arg4 = "ever"; - RunCommand cmd = RunCommand.parse(RunCommand.class, clazzName, stopPicocliProcessing, arg1, arg2, arg3, arg4); - - captureStdOutAndStdErrAndRunCommand(cmd); - - assertTrue(outContent.toString().contains("I am a test that runs myself. My args are [" + arg1 + ", " + arg2 + ", " + arg3 + ", " - + arg4 + "]")); - - } - - private void captureStdOutAndStdErrAndRunCommand(RunCommand cmd) { - PrintStream origOut = System.out; - PrintStream origErr = System.err; - System.setOut(new PrintStream(outContent)); - System.setErr(new PrintStream(errContent)); - cmd.run(); - System.setOut(origOut); - System.setErr(origErr); - } - - public static void main(String[] args) { - System.out.println("I am a test that runs myself. My args are " + Arrays.toString(args)); - } - -} diff --git a/src/test/java/emissary/place/MainTest.java b/src/test/java/emissary/place/MainTest.java deleted file mode 100644 index 08ed6eba34..0000000000 --- a/src/test/java/emissary/place/MainTest.java +++ /dev/null @@ -1,368 +0,0 @@ -package emissary.place; - -import emissary.core.DataObjectFactory; -import emissary.core.Form; -import emissary.core.IBaseDataObject; -import emissary.core.Namespace; -import emissary.core.ResourceException; -import emissary.test.core.junit5.UnitTest; - -import org.apache.commons.cli.CommandLine; -import org.apache.commons.cli.Options; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.io.TempDir; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.OutputStream; -import java.io.PrintStream; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.List; - -import static emissary.place.Main.filePathIsWithinBaseDirectory; -import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.fail; - -final class MainTest extends UnitTest { - private final String className = "emissary.place.sample.DevNullPlace"; - private final String[] defaultArgs = {"-s"}; - - @TempDir - public Path testOutputFolder; - - @Override - @BeforeEach - public void setUp() throws Exception { - Path dataFile = Paths.get(TMPDIR, "testmain.dat"); - try (OutputStream ros = Files.newOutputStream(dataFile)) { - ros.write("abcdefghijklmnopqrstuvwxyz".getBytes()); - } catch (IOException ex) { - fail("Unable to create test file", ex); - } - } - - @Override - @AfterEach - public void tearDown() throws Exception { - super.tearDown(); - // Clear out namespace - for (String key : Namespace.keySet()) { - Namespace.unbind(key); - } - Path dataFile = Paths.get(TMPDIR, "testmain.dat"); - Files.deleteIfExists(dataFile); - } - - @Test - void testStandardOptions() { - Options o = Main.getStandardOptions(); - assertNotNull(o, "Standard options should be produced"); - - Main m = new Main(className, defaultArgs); - Options mo = m.getOptions(); - assertNotNull(mo, "Standard options from instance should be produced"); - - assertEquals(o.getOptions().size(), mo.getOptions().size(), "Standard options are used by default"); - - } - - @Test - void testParseDefaultValues() { - Main m = new Main(className, defaultArgs); - m.parseArguments(); - assertTrue(m.isSilent(), "Default args sets silent"); - assertFalse(m.isRecursive(), "Default args are not recursive"); - assertFalse(m.isVerbose(), "Default args are not verbose"); - assertEquals(className + ".cfg", m.getConfigLocation(), "Default config location should stick"); - assertEquals(emissary.core.Form.UNKNOWN, m.getCurrentForm(), "Default current form should be there"); - assertEquals(0, m.getFileArgs().size(), "No files should be left"); - } - - @Test - void testParseReverseValues() { - String[] newArgs = {"-v", "-R"}; - Main m = new Main(className, newArgs); - m.parseArguments(); - assertFalse(m.isSilent(), "These args are not silent"); - assertTrue(m.isVerbose(), "These args are verbose"); - assertTrue(m.isRecursive(), "These args are recursive"); - assertEquals(0, m.getFileArgs().size(), "No files should be left"); - } - - @Test - void testParseConfigLoc() { - String[] newArgs = {"-c", "emissary.place.sample.ToLowerPlace.cfg"}; - Main m = new Main(className, newArgs); - m.parseArguments(); - assertEquals(newArgs[1], m.getConfigLocation(), "Specified config location should stick"); - assertEquals(0, m.getFileArgs().size(), "No files should be left"); - } - - @Test - void testCurrentForm() { - String[] newArgs = {"-t", "FOOBAR"}; - Main m = new Main(className, newArgs); - m.parseArguments(); - assertEquals(newArgs[1], m.getCurrentForm(), "Specified current form should stick"); - assertEquals(0, m.getFileArgs().size(), "No files should be left"); - } - - @Test - void testSetParams() { - String[] newArgs = {"-p", "FOO=BAR"}; - Main m = new Main(className, newArgs); - m.parseArguments(); - IBaseDataObject payload = DataObjectFactory.getInstance("aaa".getBytes(), "test", "UNKNOWN"); - payload.setParameters(m.getParameters()); - assertEquals("BAR", payload.getStringParameter("FOO"), "Specified parameter should stick"); - } - - @Test - void testLoggerLevels() { - // Careful, DevNull and root category will not be reset after this - String[] newArgs = {"-l", "FATAL", "-L", "INFO"}; - Main m = new Main(className, newArgs); - m.parseArguments(); - assertEquals(0, m.getFileArgs().size(), "No files should be left"); - } - - @Test - void testFilesRemaining() { - String[] newArgs = {"-s", "foo.txt", "bar.txt"}; - Main m = new Main(className, newArgs); - m.parseArguments(); - assertEquals(2, m.getFileArgs().size(), "Two files should be left"); - } - - @Test - void testPrintHooks() { - String[] args = {"-X", TMPDIR + "/testmain.dat"}; - MainWithHooks m = new MainWithHooks(className, args); - m.run(); - assertTrue(m.printhook[0], "PrePrintHook must be called"); - assertTrue(m.printhook[1], "PostPrintHook must be called"); - } - - @Test - void testProcHooks() { - String[] args = {"-X", TMPDIR + "/testmain.dat"}; - MainWithHooks m = new MainWithHooks(className, args); - m.run(); - assertTrue(m.processhook[0], "PreProcessHook must be called"); - assertTrue(m.processhook[1], "PostProcessHook must be called"); - } - - @Test - void testArgumentHooks() { - String[] args = {"-X", "foo.txt"}; - MainWithHooks m = new MainWithHooks(className, args); - m.parseArguments(); - assertTrue(m.arghook[0], "PreArgumentsHook must be called"); - assertTrue(m.arghook[1], "PostArgumentsHook must be called"); - } - - @Test - void testSplitHook() throws IOException { - String[] args = {"-s", "-d", TMPDIR, "-S", TMPDIR + "/testmain.dat"}; - MainWithHooks m = new MainWithHooks(className, args); - m.run(); - assertEquals(1, m.getFileArgs().size(), "Must process args and leave file"); - assertTrue(m.splithook[0], "PreSplitHook must be called"); - assertTrue(m.splithook[1], "PostSplitHook must be called"); - Path expectedSplitFile = Paths.get(TMPDIR, "testmain.dat." + Form.UNKNOWN); - assertTrue(Files.exists(expectedSplitFile), "File must exist after split"); - Files.deleteIfExists(expectedSplitFile); - } - - @Test - void testExceptionHandlingInPlaceProcessing() { - String[] args = {"-s", "-X", TMPDIR + "/testmain.dat"}; - Main m = new Main(OOMPlace.class.getName(), args); - m.parseArguments(); - assertDoesNotThrow(m::run); - } - - @Test - void testMetaDataFromChildrenAndExtractedRecordsIsCaptured() { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - Main m = new Main("emissary.place.MainTest$PlaceTest", new String[] {"-m", ".*"}); - m.setOutputStream(new PrintStream(baos)); - m.run(); - - IBaseDataObject payload = DataObjectFactory.getInstance("aaa".getBytes(), "test", "UNKNOWN"); - payload.putParameter("PARENT_KEY", "parent value"); - m.processPayload(payload); - - String s = baos.toString(); - assertTrue(s.contains("PARENT_KEY"), "Should have recorded parent metadata - " + s); - assertTrue(s.contains("CHILD_KEY"), "Should have recorded child metadata - " + s); - assertTrue(s.contains("RECORD_KEY"), "Should have recorded record metadata - " + s); - } - - @Test - void testHandleSplitOutput() throws IOException { - Path outputFolder = Files.createDirectories(testOutputFolder.resolve("testmain-split.dat")); - String[] args = {"-S", "-d", outputFolder.toRealPath().toString()}; - Main m = new Main(this.className, args); - m.parseArguments(); - try { - m.run(); - } catch (Throwable t) { - fail("Main runner allowed exception to escape", t); - } - IBaseDataObject payload = DataObjectFactory.getInstance("aaa".getBytes(), "test", "UNKNOWN"); - List atts = new ArrayList<>(); - IBaseDataObject att1 = DataObjectFactory.getInstance("bbb".getBytes(), "safe_attempt", "SAFE_ATTEMPT"); - atts.add(att1); - IBaseDataObject att2 = DataObjectFactory.getInstance("ccc".getBytes(), "escape_attempt", "./../../pwned/ESCAPE_ATTEMPT"); - atts.add(att2); - - IBaseDataObject att3 = DataObjectFactory.getInstance("ccc".getBytes(), "attempt", "./../../testmain.dat_sibling/ESCAPE_ATTEMPT"); - atts.add(att3); - - m.handleSplitOutput(payload, atts); - - // validate that output files "test.UNKNOWN" and "safe_attempt.SAFE_ATTEMPT" were created, but file - // "escape_attempt../../../pwned/ESCAPE_ATTEMPT" was not - String normalizedPath = outputFolder + "/" + payload.shortName() + "." + payload.currentForm(); - assertTrue(Files.exists(Paths.get(normalizedPath).normalize()), "File \"" + normalizedPath + "\" should have been created"); - - normalizedPath = outputFolder + "/" + att1.shortName() + "." + att1.currentForm(); - assertTrue(Files.exists(Paths.get(normalizedPath).normalize()), "File \"" + normalizedPath + "\" should have been created"); - - normalizedPath = outputFolder + "/" + att2.shortName() + "." + att2.currentForm(); - assertFalse(Files.exists(Paths.get(normalizedPath).normalize()), "File \"" + normalizedPath + "\" should have NOT been created"); - - normalizedPath = outputFolder + "/" + att3.shortName() + "." + att3.currentForm(); - assertFalse(Files.exists(Paths.get(normalizedPath).normalize()), "File \"" + normalizedPath + "\" should have NOT been created"); - } - - @Test - void testFilePathIsWithinBaseDirectory() { - - String basePath = testOutputFolder.resolve("foo").toString(); - assertEquals(basePath + "/somefile", filePathIsWithinBaseDirectory(basePath, basePath + "/somefile")); - assertEquals(basePath + "/otherfile", filePathIsWithinBaseDirectory(basePath, basePath + "//otherfile")); - assertEquals(basePath + "/foo/otherfile", filePathIsWithinBaseDirectory(basePath, basePath + "/./foo/otherfile")); - assertEquals(basePath + "/sub/otherfile", filePathIsWithinBaseDirectory(basePath, basePath + "/sub/././otherfile")); - - // Each of these should thrown an Exception - assertThrows(IllegalArgumentException.class, () -> filePathIsWithinBaseDirectory(basePath, "/var/log/somelog")); - - assertThrows(IllegalArgumentException.class, - () -> filePathIsWithinBaseDirectory(basePath, basePath + "/../foo2/otherfile"), - "Expected an IllegalArgumentException from input " + basePath + "/../foo2/otherfile"); - - assertThrows(IllegalArgumentException.class, - () -> filePathIsWithinBaseDirectory(basePath, basePath + "/../../somefile"), - "Expected an IllegalArgumentException from input " + basePath + "/../../somefile"); - - assertThrows(IllegalArgumentException.class, - () -> filePathIsWithinBaseDirectory(basePath, basePath + "/path/../../../otherpath"), - "Expected an IllegalArgumentException from input " + basePath + "/path/../../../otherpath"); - } - - // Override the hooks. The calls to super in these - // hooks are not normal, but help us mark the super class - // hooks as having been tested. - private static final class MainWithHooks extends Main { - public boolean[] printhook = {false, false}; - public boolean[] arghook = {false, false}; - public boolean[] processhook = {false, false}; - public boolean[] splithook = {false, false}; - - public MainWithHooks(String className, String[] args) { - super(className, args); - } - - @Override - protected boolean prePrintHook(IBaseDataObject payload, List att) { - super.prePrintHook(payload, att); - printhook[0] = true; - return false; - } - - @Override - protected void postPrintHook(IBaseDataObject payload, List att) { - super.postPrintHook(payload, att); - printhook[1] = true; - } - - @Override - protected boolean preArgumentsHook(CommandLine cmd) { - super.preArgumentsHook(cmd); - arghook[0] = true; - return true; - } - - @Override - protected void postArgumentsHook(CommandLine cmd) { - super.postArgumentsHook(cmd); - arghook[1] = true; - } - - @Override - protected boolean preProcessHook(IBaseDataObject payload, List attachments) { - super.preProcessHook(payload, attachments); - processhook[0] = true; - // Form checked in split hook test for file - if (payload.currentForm() == null) { - payload.setCurrentForm(Form.UNKNOWN); - } - return true; - } - - @Override - protected void postProcessHook(IBaseDataObject payload, List att) { - super.postProcessHook(payload, att); - processhook[1] = true; - } - - @Override - protected boolean preSplitHook(IBaseDataObject payload, List att) { - super.preSplitHook(payload, att); - splithook[0] = true; - return true; - } - - @Override - protected void postSplitHook(IBaseDataObject payload, List att) { - super.postSplitHook(payload, att); - splithook[1] = true; - } - } - - public static class PlaceTest extends ServiceProviderPlace { - public PlaceTest(String configInfo, String dir, String placeLoc) throws IOException { - super(); - } - - @Override - public List processHeavyDuty(IBaseDataObject payload) throws ResourceException { - IBaseDataObject extr = DataObjectFactory.getInstance("cde".getBytes(), "test-att-2", "UNKNOWN"); - extr.putParameter("RECORD_KEY", "record value"); - payload.addExtractedRecord(extr); - - IBaseDataObject child = DataObjectFactory.getInstance("cde".getBytes(), "test-att-1", "UNKNOWN"); - payload.putParameter("CHILD_KEY", "child value"); - - List children = new ArrayList<>(); - children.add(child); - - return children; - } - - @Override - public void process(IBaseDataObject payload) throws ResourceException {} - } -}