diff --git a/README.md b/README.md index ced4a0a80c..b99b98f8a4 100644 --- a/README.md +++ b/README.md @@ -79,10 +79,10 @@ of all the configuration subcommands and a brief description. Running `./emissary help` will give you the same output as running with no arguments. If you want to see more detailed information on a command, add the command name after help. For example, see all the -arguments with descriptions for the *what* command, run: +arguments with descriptions for the *server* command, run: ``` -./emissary help what +./emissary help server ``` #### Common parameters @@ -97,16 +97,6 @@ Logging is handled by logback. You can point to a custom file with the *--logbac See the *help -c * for each command to get more info. -#### What - -This command will use the configured engines to identify the file. Emissary currently only comes with the -SizeIdPlace, so the id will be TINY or SMALL etc. See that class for more info. The *-i or --input* argument -is required as well as *-b*. Here is how to run the command - -``` -./emissary what -i -``` - #### Server (Standalone) This command will start up an Emissary server and initialize all the places, diff --git a/src/main/java/emissary/Emissary.java b/src/main/java/emissary/Emissary.java index b458a80e88..15824eae15 100644 --- a/src/main/java/emissary/Emissary.java +++ b/src/main/java/emissary/Emissary.java @@ -14,7 +14,6 @@ import emissary.command.ServerCommand; import emissary.command.TopologyCommand; import emissary.command.VersionCommand; -import emissary.command.WhatCommand; import emissary.util.GitRepositoryState; import emissary.util.io.LoggingPrintStream; @@ -56,7 +55,7 @@ public class Emissary { static { List> cmds = - Arrays.asList(ServerCommand.class, HelpCommand.class, WhatCommand.class, TopologyCommand.class, FeedCommand.class, + Arrays.asList(ServerCommand.class, HelpCommand.class, TopologyCommand.class, FeedCommand.class, AgentsCommand.class, PoolCommand.class, VersionCommand.class, RunCommand.class, EnvCommand.class, PeersCommand.class, ConfigCommand.class, DirectoryCommand.class); Map staticCopy = new HashMap<>(); diff --git a/src/main/java/emissary/command/WhatCommand.java b/src/main/java/emissary/command/WhatCommand.java deleted file mode 100644 index cf1946320d..0000000000 --- a/src/main/java/emissary/command/WhatCommand.java +++ /dev/null @@ -1,310 +0,0 @@ -package emissary.command; - -import emissary.command.converter.WhatCommandPathExistsConverter; -import emissary.config.ConfigUtil; -import emissary.config.Configurator; -import emissary.core.DataObjectFactory; -import emissary.core.Factory; -import emissary.core.Form; -import emissary.core.IBaseDataObject; -import emissary.core.Namespace; -import emissary.core.ResourceException; -import emissary.directory.DirectoryPlace; -import emissary.directory.EmissaryNode; -import emissary.id.Identification; -import emissary.parser.DecomposedSession; -import emissary.parser.ParserEOFException; -import emissary.parser.ParserException; -import emissary.parser.ParserFactory; -import emissary.parser.SessionParser; -import emissary.parser.SessionProducer; -import emissary.place.IServiceProviderPlace; -import emissary.util.shell.Executrix; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import picocli.CommandLine; -import picocli.CommandLine.Command; -import picocli.CommandLine.Option; - -import java.io.IOException; -import java.lang.reflect.InvocationTargetException; -import java.nio.file.DirectoryStream; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.List; -import java.util.Set; -import javax.annotation.Nullable; - -import static emissary.core.Form.HTML; -import static emissary.core.Form.TEXT; - -@Deprecated -// @Parameters(commandDescription = "Run Identification places on a payload to determine the file type") -@Command(description = "Run Identification places on a payload to determine the file type", - subcommands = {HelpCommand.class}) -public class WhatCommand extends BaseCommand { - - static final Logger LOG = LoggerFactory.getLogger(WhatCommand.class); - - public static String COMMAND_NAME = "what"; - - public WhatCommand() - throws ClassNotFoundException, InstantiationException, IllegalAccessException, InvocationTargetException, NoSuchMethodException {} - - @Override - public String getCommandName() { - return COMMAND_NAME; - } - - @Option(names = {"-h", "--header"}, description = "use header identification, defaults to true\nDefault: ${DEFAULT-VALUE}", arity = "1") - private boolean headerIdentification = true; - - @Option(names = {"-i", "--input"}, description = "input file or directory", converter = WhatCommandPathExistsConverter.class, required = true) - private Path input; - - @Option(names = {"-r", "--recursive"}, description = "recurse on input directories, defaults to false\nDefault: ${DEFAULT-VALUE}") - private boolean recursive = false; - - protected List places; - protected List placeLabels = new ArrayList<>(); - protected Set textPlaces; - // protected DropOffUtil dropOffUtil; - private ParserFactory parserFactory; - - @Override - public void run(CommandLine c) { - // TODO This entire class should be refactored into a testable unit - setup(); - processPath(this.input); - } - - @Override - public void setupCommand() { - setupWhat(); - } - - public void setupWhat() { - setupConfig(); - // since we are not really running a server, setup a few things so the code doesn't know - System.setProperty(EmissaryNode.NODE_NAME_PROPERTY, "localhost"); - System.setProperty(EmissaryNode.NODE_PORT_PROPERTY, "8001"); - EmissaryNode node = new EmissaryNode(); - String directoryPlaceName = "http://" + node.getNodeName() + ":" + node.getNodePort() + "/DirectoryPlace"; - try { - Namespace.bind(directoryPlaceName, new DirectoryPlace(directoryPlaceName, node)); - parserFactory = new ParserFactory(); - final Configurator configG = ConfigUtil.getConfigInfo(this.getClass()); - final List placeNames = configG.findEntries("PLACE"); - this.places = new ArrayList<>(); - for (final String entry : placeNames) { - final String[] parts = entry.split("/"); - final String instanceName = parts[0]; - final String className = parts[1]; - final String configName = configG.findStringEntry("CONFIG_" + instanceName, className + ".cfg"); - try { - final IServiceProviderPlace place = - (IServiceProviderPlace) Factory.create(className, configName, "FAKE.DIR.DIR.http://localhost:8001/DirectoryPlace", - "http://localhost:8001/" + instanceName); - this.places.add(place); - this.placeLabels.add(instanceName); - } catch (Exception e) { - LOG.error(String.format("Could not create {} ({}): ", instanceName, className, e)); - } - } - LOG.debug("Created " + this.places.size() + " places"); - - this.textPlaces = configG.findEntriesAsSet("TEXT_PLACE"); - } catch (IOException e) { - LOG.error("Could not instantiate the WhatCommand: instanceName" + e.getMessage()); - } - } - - protected void processPath(final Path inputPath) { - if (Files.isDirectory(inputPath)) { - if (this.recursive) { - try (DirectoryStream stream = Files.newDirectoryStream(inputPath)) { - for (Path entry : stream) { - LOG.debug("Handling entry {} in directory {}", entry, inputPath); - this.processPath(entry); - } - } catch (IOException ex) { - LOG.error(inputPath + ": Cannot read, " + ex.getMessage()); - } - } else { - LOG.info(inputPath + ": DIRECTORY"); - } - } else if (Files.isRegularFile(inputPath)) { - try { - processFilePath(inputPath); - } catch (ParserException ex) { - LOG.error(inputPath + ": Cannot parse, " + ex.getMessage()); - } catch (IOException ex) { - LOG.error(inputPath + ": Cannot read, " + ex.getMessage()); - } - } else { - LOG.error(inputPath + ": NOT_A_FILE"); - } - } - - /** - * Process one regular file from the main routine - * - * @param path the string name of the file to process - */ - protected void processFilePath(final Path path) throws ParserException, IOException { - if (!Files.exists(path)) { - LOG.error(path + " - NONEXISTENT"); - return; - } - - if (!Files.isReadable(path)) { - LOG.error(path + " - UNREADABLE"); - return; - } - - if (Files.size(path) == 0) { - LOG.error(path + " - EMPTY"); - return; - } - - if (!this.headerIdentification) { - final IBaseDataObject payload = - DataObjectFactory.getInstance(Executrix.readDataFromFile(path.toString()), path.toAbsolutePath().toString(), - Form.UNKNOWN); - final Identification id = identify(payload, this.headerIdentification); - LOG.info(path + " - " + payload.getFilename() + ": " + id.getTypeString()); - return; - } - - // Using Header - final SessionParser parser = parserFactory.makeSessionParser(Files.newByteChannel(path)); - LOG.debug("Using parser " + parser.getClass().getName() + " for " + path); - final SessionProducer producer = new SessionProducer(parser, Form.UNKNOWN); - final String parent = path.getParent().toString(); - final String fileName = path.getFileName().toString(); - while (true) { - try { - final DecomposedSession sessionHash = parser.getNextSession(); - final IBaseDataObject payload = producer.createAndLoadDataObject(sessionHash, parent); - final Identification id = identify(payload, this.headerIdentification); - - LOG.info(payload.getFilename() + "/" + fileName + ": " + id.getTypeString() + " [" + payload.dataLength() + "]"); - } catch (ParserEOFException eof) { - LOG.debug("Parser reached end of file: ", eof); - // Expected EOF - break; - } - } - } - - protected Identification identify(@Nullable final IBaseDataObject b, final boolean useHeaderArg) { - Identification ident = null; - if (b == null || b.data() == null) { - ident = new Identification("BAD_SESSION"); - } else if (b != null && b.dataLength() == 0) { - ident = new Identification(Form.EMPTY); - } else { - ident = runEngines(b); - } - if (ident.getTypeCount() == 0) { - ident.addType(Form.UNKNOWN); - } - return ident; - } - - private Identification runEngines(@Nullable final IBaseDataObject b) { - - final Identification ident = new Identification(); - final List typesFound = new ArrayList<>(); - - if (b != null && b.dataLength() == 0) { - ident.addType(Form.EMPTY); - return ident; - } - - if (!b.currentForm().equals(Form.UNKNOWN)) { - typesFound.add(b.currentForm()); - } - - final byte[] data = b.data(); - - if (LOG.isDebugEnabled()) { - final String ds = (data.length > 10 ? new String(data, 0, 9) : new String(data)); - LOG.debug("Running engines with data=" + ds + "..."); - } - - for (int i = 0; i < this.places.size(); i++) { - b.setCurrentForm(Form.UNKNOWN); - final IServiceProviderPlace place = this.places.get(i); - final String placeLabel = this.placeLabels.get(i); - - if (unknown(ident) || (isText(ident) && this.textPlaces.contains(placeLabel))) { - try { - place.process(b); - ident.setType(b.currentForm()); - } catch (ResourceException rex) { - // Ignore. - LOG.debug("Error processing object", rex); - } - LOG.info(placeLabel + "\tsaid " + ident); - if (this.textPlaces.contains(placeLabel)) { - accumulateText(typesFound, ident); - } else { - accumulate(typesFound, ident); - } - } - } - // Nb. set not add... - ident.setTypes(typesFound); - return ident; - } - - /** - * Accumulate newly found types in our list with no duplication - */ - private void accumulate(final List l, final Identification i) { - for (final String type : i.getTypes()) { - if (!l.contains(type)) { - l.add(type); - } - } - l.remove(Form.UNKNOWN); - } - - private void accumulateText(final List l, final Identification i) { - for (final String s : i.getTypes()) { - if (!l.contains(s)) { - if ("QUOTED_PRINTABLE".equals(s)) { - l.add(s); - } else { - l.add("/" + s + "/"); - } - } - } - l.remove(Form.UNKNOWN); - } - - private static boolean unknown(final Identification i) { - if (i.getTypeCount() == 0) { - return true; - } - - if (i.getTypeCount() > 1) { - return false; - } - - final String t = i.getFirstType(); - return t.equals(Form.UNKNOWN); - } - - private static boolean isText(final Identification i) { - if (i.getTypeCount() == 0) { - return false; - } - - final String t = i.getFirstType(); - return HTML.equals(t) || t.endsWith(TEXT); - } -} diff --git a/src/main/java/emissary/command/converter/WhatCommandPathExistsConverter.java b/src/main/java/emissary/command/converter/WhatCommandPathExistsConverter.java deleted file mode 100644 index d5afaf6502..0000000000 --- a/src/main/java/emissary/command/converter/WhatCommandPathExistsConverter.java +++ /dev/null @@ -1,22 +0,0 @@ -package emissary.command.converter; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.nio.file.Files; -import java.nio.file.Path; - -public class WhatCommandPathExistsConverter extends PathExistsReadableConverter { - private static final Logger LOG = LoggerFactory.getLogger(WhatCommandPathExistsConverter.class); - - @Override - public Path convert(String value) { - Path p = super.convert("-i", value); - if (!Files.isReadable(p)) { - String msg = String.format("The option '-i' was configured with path '%s' which is not readable", p); - LOG.error(msg); - throw new IllegalArgumentException(msg); - } - return p; - } -} diff --git a/src/main/resources/emissary/command/WhatCommand.cfg b/src/main/resources/emissary/command/WhatCommand.cfg deleted file mode 100644 index b127c2ad95..0000000000 --- a/src/main/resources/emissary/command/WhatCommand.cfg +++ /dev/null @@ -1,7 +0,0 @@ -#TYPE_ENGINE_FILE = "mime.types" - -# Ordered list of places -#PLACE = "SIZE/emissary.id.SizeIdPlace" -PLACE = "UNIXFILE/emissary.id.UnixFilePlace" - -#CONFIG_SIZE = "emissary.id.SizeIdPlace.cfg" diff --git a/src/test/java/emissary/command/WhatCommandIT.java b/src/test/java/emissary/command/WhatCommandIT.java deleted file mode 100644 index c6a1cf6ff3..0000000000 --- a/src/test/java/emissary/command/WhatCommandIT.java +++ /dev/null @@ -1,150 +0,0 @@ -package emissary.command; - -import emissary.config.ConfigUtil; -import emissary.test.core.junit5.UnitTest; -import emissary.util.io.UnitTestFileUtils; - -import com.google.common.collect.Lists; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtensionContext; -import org.junit.jupiter.api.io.TempDir; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.params.provider.ArgumentsProvider; -import org.junit.jupiter.params.provider.ArgumentsSource; - -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.attribute.PosixFilePermission; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.stream.Stream; - -import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; - -class WhatCommandIT extends UnitTest { - - public static final String[] PROJECT_BASE_ARGS = {"-b", "--projectBase"}; - public static final String[] INPUT_ARGS = {"-i", "--input"}; - public static final String[] BOOLEAN_ARGS_WITHOUT_VALUE = {"-r", "--recursive"}; - public static final String[] BOOLEAN_ARGS_WITH_VALUE = {"-h", "--header"}; - public static final String[] STRING_ARGS = {"--logbackConfig"}; - - private WhatCommand command; - private Path baseDir; - private Path inputDir; - private final List arguments = new ArrayList<>(); - - @TempDir - public Path tmpDir; - - @BeforeEach - @Override - public void setUp() throws Exception { - command = null; - baseDir = Paths.get(System.getenv(ConfigUtil.PROJECT_BASE_ENV)); - inputDir = Files.createTempDirectory(tmpDir, "input"); - arguments.clear(); - } - - @AfterEach - @Override - public void tearDown() throws Exception { - UnitTestFileUtils.cleanupDirectoryRecursively(inputDir); - } - - static class CartesianArgumentsProvider implements ArgumentsProvider { - List> cartesian = Lists.cartesianProduct( - Arrays.asList(PROJECT_BASE_ARGS), - Arrays.asList(INPUT_ARGS), - Arrays.asList(STRING_ARGS), - Arrays.asList(BOOLEAN_ARGS_WITHOUT_VALUE), - Arrays.asList(BOOLEAN_ARGS_WITH_VALUE)); - - @Override - public Stream provideArguments(ExtensionContext context) { - return cartesian.stream().map(list -> Arguments.of((Object[]) list.toArray(new String[0]))); - } - } - - @ParameterizedTest - @ArgumentsSource(CartesianArgumentsProvider.class) - void verifyExpectedOptions(String baseDirArg, - String inputDirArg, - String stringArg, - String booleanArgWithoutValue, - String booleanArgWithValue) { - // setup - arguments.add(baseDirArg); - arguments.add(baseDir.toString()); - arguments.add(inputDirArg); - arguments.add(inputDir.toString()); - arguments.add(stringArg); - arguments.add("validateStringArg"); - arguments.add(booleanArgWithoutValue); - arguments.add(booleanArgWithValue); - arguments.add(Boolean.FALSE.toString()); - - // test (no exceptions thrown) - assertDoesNotThrow(() -> WhatCommand.parse(WhatCommand.class, arguments)); - } - - @Test - void missingInputDirectory() { - // setup - arguments.add(PROJECT_BASE_ARGS[0]); - arguments.add(baseDir.toString()); - arguments.add(INPUT_ARGS[0]); - arguments.add("missingInputDirectory"); - - // test - RuntimeException thrown = assertThrows(RuntimeException.class, () -> WhatCommand.parse(WhatCommand.class, arguments)); - // verify - assertTrue(thrown.getMessage().contains("The option '-i' was configured with path 'missingInputDirectory' which does not exist")); - } - - @Test - void missingConfigDirectory() { - // setup - arguments.add(PROJECT_BASE_ARGS[0]); - arguments.add("missingConfigDirectory"); - arguments.add(INPUT_ARGS[0]); - arguments.add(inputDir.toAbsolutePath().toString()); - - // test - RuntimeException thrown = assertThrows(RuntimeException.class, () -> WhatCommand.parse(WhatCommand.class, arguments)); - // verify - assertTrue(thrown.getMessage().contains("The option '-b' was configured with path 'missingConfigDirectory' which does not exist")); - } - - @Test - void unreadableInput() throws Exception { - // setup - arguments.add(PROJECT_BASE_ARGS[0]); - arguments.add(baseDir.toString()); - arguments.add(INPUT_ARGS[0]); - Set perms = new HashSet<>(); - perms.add(PosixFilePermission.OWNER_WRITE); - Files.setPosixFilePermissions(inputDir, perms); - arguments.add(inputDir.toAbsolutePath().toString()); - - // test - RuntimeException thrown = assertThrows(RuntimeException.class, () -> WhatCommand.parse(WhatCommand.class, arguments)); - - // Reset perms for cleanup - perms.add(PosixFilePermission.OWNER_READ); - Files.setPosixFilePermissions(inputDir, perms); - - // verify - assertTrue(thrown.getMessage().contains("The option '-i' was configured with path '" + inputDir + "' which is not readable")); - } - -}