diff --git a/src/main/java/picard/cmdline/CommandLineProgram.java b/src/main/java/picard/cmdline/CommandLineProgram.java index 50ae238abd..7ad91dcc59 100644 --- a/src/main/java/picard/cmdline/CommandLineProgram.java +++ b/src/main/java/picard/cmdline/CommandLineProgram.java @@ -53,8 +53,10 @@ import org.broadinstitute.barclay.argparser.SpecialArgumentsCollection; import picard.cmdline.argumentcollections.OptionalReferenceArgumentCollection; import picard.cmdline.argumentcollections.ReferenceArgumentCollection; +import picard.cmdline.argumentcollections.RequesterPaysArgumentCollection; import picard.cmdline.argumentcollections.RequiredReferenceArgumentCollection; -import picard.nio.PathProvider; +import picard.nio.GoogleStorageUtils; +import picard.nio.HttpNioUtils; import picard.nio.PicardHtsPath; import picard.util.RExecutor; @@ -62,11 +64,9 @@ import java.net.InetAddress; import java.text.DecimalFormat; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.Date; import java.util.List; -import java.util.stream.Collectors; /** * Abstract class to facilitate writing command-line programs. @@ -85,8 +85,8 @@ * */ public abstract class CommandLineProgram { - private static String PROPERTY_USE_LEGACY_PARSER = "picard.useLegacyParser"; - private static String PROPERTY_CONVERT_LEGACY_COMMAND_LINE = "picard.convertCommandLine"; + private static final String PROPERTY_USE_LEGACY_PARSER = "picard.useLegacyParser"; + private static final String PROPERTY_CONVERT_LEGACY_COMMAND_LINE = "picard.convertCommandLine"; private static Boolean useLegacyParser; public static String SYNTAX_TRANSITION_URL = "https://github.com/broadinstitute/picard/wiki/Command-Line-Syntax-Transition-For-Users-(Pre-Transition)"; @@ -132,6 +132,9 @@ public abstract class CommandLineProgram { // after argument parsing using the value established by the user in the referenceSequence argument collection. protected File REFERENCE_SEQUENCE = Defaults.REFERENCE_FASTA; + @ArgumentCollection + public RequesterPaysArgumentCollection requesterPays = new RequesterPaysArgumentCollection(); + @ArgumentCollection(doc="Special Arguments that have meaning to the argument parsing system. " + "It is unlikely these will ever need to be accessed by the command line program") public Object specialArgumentsCollection = useLegacyParser() ? @@ -182,7 +185,7 @@ public void instanceMainWithExit(final String[] argv) { } public int instanceMain(final String[] argv) { - String actualArgs[] = argv; + String[] actualArgs = argv; if (System.getProperty(PROPERTY_CONVERT_LEGACY_COMMAND_LINE, "false").equals("true")) { actualArgs = CommandLineSyntaxTranslater.convertPicardStyleToPosixStyle(argv); @@ -249,16 +252,15 @@ public int instanceMain(final String[] argv) { // default reader factory. At least until https://github.com/samtools/htsjdk/issues/1666 is resolved SamReaderFactory.setDefaultValidationStringency(VALIDATION_STRINGENCY); + // Configure the various filesystem providers + GoogleStorageUtils.initialize(requesterPays.getProjectForRequesterPays()); + HttpNioUtils.initialize(); + if (!QUIET) { System.err.println("[" + new Date() + "] " + commandLine); - // Output a one liner about who/where and what software/os we're running on + // Output a one-liner about who/where and what software/os we're running on try { - final String pathProvidersMessage = - Arrays.stream(PathProvider.values()) - .map(provider -> String.format("Provider %s is%s available;", provider.name(), provider.isAvailable ? "" : " not")) - .collect(Collectors.joining(" ")); - final boolean usingIntelDeflater = (BlockCompressedOutputStream.getDefaultDeflaterFactory() instanceof IntelDeflaterFactory && ((IntelDeflaterFactory)BlockCompressedOutputStream.getDefaultDeflaterFactory()).usingIntelDeflater()); final boolean usingIntelInflater = (BlockGunzipper.getDefaultInflaterFactory() instanceof IntelInflaterFactory && @@ -269,7 +271,7 @@ public int instanceMain(final String[] argv) { System.getProperty("os.name"), System.getProperty("os.version"), System.getProperty("os.arch"), System.getProperty("java.vm.name"), System.getProperty("java.runtime.version"), usingIntelDeflater ? "Intel" : "Jdk", usingIntelInflater ? "Intel" : "Jdk", - pathProvidersMessage, + requesterPays.getDescription(), getCommandLineParser().getVersion()); System.err.println(msg); } @@ -409,7 +411,7 @@ public CommandLineParser getCommandLineParser() { commandLineParser = useLegacyParser() ? new LegacyCommandLineArgumentParser(this) : new CommandLineArgumentParser(this, - Collections.EMPTY_LIST, + Collections.emptyList(), Collections.singleton(CommandLineParserOptions.APPEND_TO_COLLECTIONS)); } return commandLineParser; @@ -427,9 +429,7 @@ public CommandLineParser getCommandLineParser() { public static boolean useLegacyParser() { if (useLegacyParser == null) { final String legacyPropertyValue = System.getProperty(PROPERTY_USE_LEGACY_PARSER); - useLegacyParser = legacyPropertyValue == null ? - false : - Boolean.parseBoolean(legacyPropertyValue); + useLegacyParser = Boolean.parseBoolean(legacyPropertyValue); } return useLegacyParser; } @@ -475,7 +475,7 @@ public List
getDefaultHeaders() { public static String getStandardUsagePreamble(final Class mainClass) { return "USAGE: " + mainClass.getSimpleName() +" [options]\n\n" + (hasWebDocumentation(mainClass) ? - "Documentation: http://broadinstitute.github.io/picard/command-line-overview.html" + + "Documentation: https://broadinstitute.github.io/picard/command-line-overview.html" + mainClass.getSimpleName() + "\n\n" : ""); } @@ -499,7 +499,7 @@ public static boolean hasWebDocumentation(final Class clazz){ * @return the link to a FAQ */ public static String getFaqLink() { - return "To get help, see http://broadinstitute.github.io/picard/index.html#GettingHelp"; + return "To get help, see https://broadinstitute.github.io/picard/index.html#GettingHelp"; } /** diff --git a/src/main/java/picard/cmdline/argumentcollections/RequesterPaysArgumentCollection.java b/src/main/java/picard/cmdline/argumentcollections/RequesterPaysArgumentCollection.java new file mode 100644 index 0000000000..6b2a4af8a9 --- /dev/null +++ b/src/main/java/picard/cmdline/argumentcollections/RequesterPaysArgumentCollection.java @@ -0,0 +1,42 @@ +package picard.cmdline.argumentcollections; + +import com.google.common.base.Strings; +import org.broadinstitute.barclay.argparser.Argument; + +/** + * Argument collection to encapsulate special behavior of the REQUESTER_PAYS_PROJECT argument + */ +public class RequesterPaysArgumentCollection { + /** The System property which acts as a default for REQUESTER_PAYS_PROJECT */ + public static final String PROPERTY_NAME = "picard.googleProjectForRequesterPays"; + public static final String REQUESTER_PAYS_PROJECT_FULL_NAME = "REQUESTER_PAYS_PROJECT"; + + @Argument(doc="Google project for access to 'requester pays' buckets and objects. " + + "If this is not specified then value of the system property " + PROPERTY_NAME + " acts as the default.", + common = true, optional = true, + fullName = REQUESTER_PAYS_PROJECT_FULL_NAME) + public String requesterPaysProject = null; + + public String getProjectForRequesterPays() { + final String value = ! Strings.isNullOrEmpty(requesterPaysProject) + ? requesterPaysProject + : getSystemProperty(); + return Strings.isNullOrEmpty(value) ? null : value; // "" -> null + } + + private String getSystemProperty() { + return System.getProperty(PROPERTY_NAME); + } + + public String getDescription(){ + final String value = getProjectForRequesterPays(); + if(!Strings.isNullOrEmpty(requesterPaysProject)){ + return "Requester Pays Project set by argument: " + value; + } else if( !Strings.isNullOrEmpty(getSystemProperty())){ + return "Requester Pays Project set by system property: " + value; + } else { + return "Requester Pays Project not set."; + } + } + +} diff --git a/src/main/java/picard/nio/GoogleStorageUtils.java b/src/main/java/picard/nio/GoogleStorageUtils.java index 1e5f7c2d39..8225522cfe 100644 --- a/src/main/java/picard/nio/GoogleStorageUtils.java +++ b/src/main/java/picard/nio/GoogleStorageUtils.java @@ -36,22 +36,26 @@ /** * This class serves as a connection to google's implementation of nio support for GCS housed files. * - * While the actual code required to connect isn't packaged with Picard (only compiled), the Readme.md file in the - * github repository describes how it can be used. Additionally, Picard is bundled in GATK4, and its tools exposed via - * the GATK engine, since the nio library _is_ included the GATK4 jar. NIO enabled tools in picard can connect to - * GCS when called through GATK4. - * * This class contains hard-coded setting that have been found to work for the access patterns that characterize genomics * work. In the future it would make sense to expose these parameters so that they can be controlled via the commandline. * However, as the list of Path-enabled tools in picard is small, there seems to be little impetus to do so right now. * * */ -class GoogleStorageUtils { +public final class GoogleStorageUtils { + + public static final int MAX_REOPENS = 20; + + private GoogleStorageUtils(){} - public static void initialize() { + /** + * Set appropriate configuration options for the GCS file system provider. + * + * @param requesterProject the project to pay with when accessing requester pays buckets + */ + public static void initialize(final String requesterProject) { // requester pays support is currently not configured - CloudStorageFileSystemProvider.setDefaultCloudStorageConfiguration(GoogleStorageUtils.getCloudStorageConfiguration(20, null)); + CloudStorageFileSystemProvider.setDefaultCloudStorageConfiguration(GoogleStorageUtils.getCloudStorageConfiguration(MAX_REOPENS, requesterProject)); CloudStorageFileSystemProvider.setStorageOptions(GoogleStorageUtils.setGenerousTimeouts(StorageOptions.newBuilder()).build()); } @@ -62,7 +66,8 @@ private static CloudStorageConfiguration getCloudStorageConfiguration(int maxReo .maxChannelReopens(maxReopens); if (!Strings.isNullOrEmpty(requesterProject)) { // enable requester pays and indicate who pays - builder = builder.autoDetectRequesterPays(true).userProject(requesterProject); + builder.autoDetectRequesterPays(true) + .userProject(requesterProject); } // this causes the gcs filesystem to treat files that end in a / as a directory diff --git a/src/main/java/picard/nio/HttpNioUtils.java b/src/main/java/picard/nio/HttpNioUtils.java new file mode 100644 index 0000000000..3cd25ffc79 --- /dev/null +++ b/src/main/java/picard/nio/HttpNioUtils.java @@ -0,0 +1,37 @@ +package picard.nio; + +import org.broadinstitute.http.nio.HttpFileSystemProvider; +import org.broadinstitute.http.nio.HttpFileSystemProviderSettings; +import org.broadinstitute.http.nio.RetryHandler; + +import java.time.Duration; + +/** + * This class provides a way to easily configure the HttpNioProvider + * + * This class contains hard-coded setting that have been found to work for the access patterns that characterize genomics + * work. + * + */ +public final class HttpNioUtils { + + private HttpNioUtils() {} + public static final Duration MAX_TIMEOUT = Duration.ofMillis(120_000); + public static final int MAX_RETRIES = 20; + + public static void initialize() { + final HttpFileSystemProviderSettings.RetrySettings retrySettings = new HttpFileSystemProviderSettings.RetrySettings( + MAX_RETRIES, + RetryHandler.DEFAULT_RETRYABLE_HTTP_CODES, + RetryHandler.DEFAULT_RETRYABLE_EXCEPTIONS, + RetryHandler.DEFALT_RETRYABLE_MESSAGES, + e -> false); + + final HttpFileSystemProviderSettings settings = new HttpFileSystemProviderSettings( + MAX_TIMEOUT, + HttpFileSystemProviderSettings.DEFAULT_SETTINGS.redirect(), + retrySettings); + + HttpFileSystemProvider.setSettings(settings); + } +} diff --git a/src/main/java/picard/nio/PathProvider.java b/src/main/java/picard/nio/PathProvider.java deleted file mode 100644 index 2951574ca1..0000000000 --- a/src/main/java/picard/nio/PathProvider.java +++ /dev/null @@ -1,30 +0,0 @@ -package picard.nio; - -/** - * A class whose purpose is to initialize the various plugins that provide Path support. - *

- * At the moment only google-GCS support is available, but it should be easy to follow the pattern and add support for - * other Path providers. - */ -public enum PathProvider { - GCS { - @Override - public boolean initialize() { - try { - GoogleStorageUtils.initialize(); - return true; - } catch (NoClassDefFoundError e) { - return false; - } - } - }; - - public final boolean isAvailable; - - protected abstract boolean initialize(); - - PathProvider() { - //noinspection AbstractMethodCallInConstructor - isAvailable = initialize(); - } -} diff --git a/src/test/java/picard/cmdline/argumentcollections/RequesterPaysArgumentCollectionTest.java b/src/test/java/picard/cmdline/argumentcollections/RequesterPaysArgumentCollectionTest.java new file mode 100644 index 0000000000..79ce89f66b --- /dev/null +++ b/src/test/java/picard/cmdline/argumentcollections/RequesterPaysArgumentCollectionTest.java @@ -0,0 +1,70 @@ +package picard.cmdline.argumentcollections; + +import org.testng.Assert; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +public class RequesterPaysArgumentCollectionTest { + + public static final String P1 = "project1"; + public static final String P2 = "project2"; + + @DataProvider + public static Object[][] settings() { + + return new Object[][]{ + {null, null, null}, + {"", null, null}, + {null, "", null}, + {"", "", null}, + {P1, null, P1}, + {null, P2, P2}, + {P1, P2, P1}, + {"", P2, P2}, + {P1, "", P1} + }; + } + + @Test(dataProvider = "settings") + public void testCorrectValues(String arg, String sys, String expected){ + runWithSystemProperty( + () -> { + final RequesterPaysArgumentCollection rpc = new RequesterPaysArgumentCollection(); + rpc.requesterPaysProject = arg; + Assert.assertEquals(rpc.getProjectForRequesterPays(), expected); + }, RequesterPaysArgumentCollection.PROPERTY_NAME, sys + ); + } + + @Test(dataProvider = "settings") + public void testCorrectDescription(String arg, String sys, String expected){ + runWithSystemProperty( + () -> { + final RequesterPaysArgumentCollection rpc = new RequesterPaysArgumentCollection(); + rpc.requesterPaysProject = arg; + final String description = rpc.getDescription(); + final String value = rpc.getProjectForRequesterPays(); + if(expected == null) {Assert.assertEquals(description, "Requester Pays Project not set."); } + else if(expected.equals(P1)) { Assert.assertEquals(description, "Requester Pays Project set by argument: " + value); } + else if(expected.equals(P2)) { Assert.assertEquals(description, "Requester Pays Project set by system property: " + value); } + else { Assert.fail("should have been one of the of the previous"); } + }, RequesterPaysArgumentCollection.PROPERTY_NAME, sys); + } + private static void runWithSystemProperty(Runnable toRun, String name, String value){ + String previousValue = null; + try { + if(value != null) { + previousValue = System.setProperty(name, value); + } + + toRun.run(); + + } finally { + if(previousValue == null){ + System.clearProperty(name); + } else { + System.setProperty(name, previousValue); + } + } + } +} \ No newline at end of file