-
Notifications
You must be signed in to change notification settings - Fork 371
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add new argument to allow setting the google requester pays project
* New argument REQUESTER_PAYS_PROJECT added to command line program. This allows configuring the project to use with requester pays buckets in GCS * This can also be configured by setting a new system property: picard.googleProjectForRequesterPays * Updated the initialization of GCS and HTTP filesystem providers
- Loading branch information
1 parent
63138fc
commit 7f30feb
Showing
6 changed files
with
182 additions
and
58 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
42 changes: 42 additions & 0 deletions
42
src/main/java/picard/cmdline/argumentcollections/RequesterPaysArgumentCollection.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,42 @@ | ||
| package picard.cmdline.argumentcollections; | ||
|
|
||
| import com.google.common.base.Strings; | ||
| import org.broadinstitute.barclay.argparser.Argument; | ||
|
|
||
| /** | ||
| * Argument collection to encapsulate special behavior of the REQUESTER_PAYS_PROJECT argument | ||
| */ | ||
| public class RequesterPaysArgumentCollection { | ||
| /** The System property which acts as a default for REQUESTER_PAYS_PROJECT */ | ||
| public static final String PROPERTY_NAME = "picard.googleProjectForRequesterPays"; | ||
| public static final String REQUESTER_PAYS_PROJECT_FULL_NAME = "REQUESTER_PAYS_PROJECT"; | ||
|
|
||
| @Argument(doc="Google project for access to 'requester pays' buckets and objects. " + | ||
| "If this is not specified then value of the system property " + PROPERTY_NAME + " acts as the default.", | ||
| common = true, optional = true, | ||
| fullName = REQUESTER_PAYS_PROJECT_FULL_NAME) | ||
| public String requesterPaysProject = null; | ||
|
|
||
| public String getProjectForRequesterPays() { | ||
| final String value = ! Strings.isNullOrEmpty(requesterPaysProject) | ||
| ? requesterPaysProject | ||
| : getSystemProperty(); | ||
| return Strings.isNullOrEmpty(value) ? null : value; // "" -> null | ||
| } | ||
|
|
||
| private String getSystemProperty() { | ||
| return System.getProperty(PROPERTY_NAME); | ||
| } | ||
|
|
||
| public String getDescription(){ | ||
| final String value = getProjectForRequesterPays(); | ||
| if(!Strings.isNullOrEmpty(requesterPaysProject)){ | ||
| return "Requester Pays Project set by argument: " + value; | ||
| } else if( !Strings.isNullOrEmpty(getSystemProperty())){ | ||
| return "Requester Pays Project set by system property: " + value; | ||
| } else { | ||
| return "Requester Pays Project not set."; | ||
| } | ||
| } | ||
|
|
||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,37 @@ | ||
| package picard.nio; | ||
|
|
||
| import org.broadinstitute.http.nio.HttpFileSystemProvider; | ||
| import org.broadinstitute.http.nio.HttpFileSystemProviderSettings; | ||
| import org.broadinstitute.http.nio.RetryHandler; | ||
|
|
||
| import java.time.Duration; | ||
|
|
||
| /** | ||
| * This class provides a way to easily configure the HttpNioProvider | ||
| * | ||
| * This class contains hard-coded setting that have been found to work for the access patterns that characterize genomics | ||
| * work. | ||
| * | ||
| */ | ||
| public final class HttpNioUtils { | ||
|
|
||
| private HttpNioUtils() {} | ||
| public static final Duration MAX_TIMEOUT = Duration.ofMillis(120_000); | ||
| public static final int MAX_RETRIES = 20; | ||
|
|
||
| public static void initialize() { | ||
| final HttpFileSystemProviderSettings.RetrySettings retrySettings = new HttpFileSystemProviderSettings.RetrySettings( | ||
| MAX_RETRIES, | ||
| RetryHandler.DEFAULT_RETRYABLE_HTTP_CODES, | ||
| RetryHandler.DEFAULT_RETRYABLE_EXCEPTIONS, | ||
| RetryHandler.DEFALT_RETRYABLE_MESSAGES, | ||
| e -> false); | ||
|
|
||
| final HttpFileSystemProviderSettings settings = new HttpFileSystemProviderSettings( | ||
| MAX_TIMEOUT, | ||
| HttpFileSystemProviderSettings.DEFAULT_SETTINGS.redirect(), | ||
| retrySettings); | ||
|
|
||
| HttpFileSystemProvider.setSettings(settings); | ||
| } | ||
| } |
This file was deleted.
Oops, something went wrong.
70 changes: 70 additions & 0 deletions
70
src/test/java/picard/cmdline/argumentcollections/RequesterPaysArgumentCollectionTest.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,70 @@ | ||
| package picard.cmdline.argumentcollections; | ||
|
|
||
| import org.testng.Assert; | ||
| import org.testng.annotations.DataProvider; | ||
| import org.testng.annotations.Test; | ||
|
|
||
| public class RequesterPaysArgumentCollectionTest { | ||
|
|
||
| public static final String P1 = "project1"; | ||
| public static final String P2 = "project2"; | ||
|
|
||
| @DataProvider | ||
| public static Object[][] settings() { | ||
|
|
||
| return new Object[][]{ | ||
| {null, null, null}, | ||
| {"", null, null}, | ||
| {null, "", null}, | ||
| {"", "", null}, | ||
| {P1, null, P1}, | ||
| {null, P2, P2}, | ||
| {P1, P2, P1}, | ||
| {"", P2, P2}, | ||
| {P1, "", P1} | ||
| }; | ||
| } | ||
|
|
||
| @Test(dataProvider = "settings") | ||
| public void testCorrectValues(String arg, String sys, String expected){ | ||
| runWithSystemProperty( | ||
| () -> { | ||
| final RequesterPaysArgumentCollection rpc = new RequesterPaysArgumentCollection(); | ||
| rpc.requesterPaysProject = arg; | ||
| Assert.assertEquals(rpc.getProjectForRequesterPays(), expected); | ||
| }, RequesterPaysArgumentCollection.PROPERTY_NAME, sys | ||
| ); | ||
| } | ||
|
|
||
| @Test(dataProvider = "settings") | ||
| public void testCorrectDescription(String arg, String sys, String expected){ | ||
| runWithSystemProperty( | ||
| () -> { | ||
| final RequesterPaysArgumentCollection rpc = new RequesterPaysArgumentCollection(); | ||
| rpc.requesterPaysProject = arg; | ||
| final String description = rpc.getDescription(); | ||
| final String value = rpc.getProjectForRequesterPays(); | ||
| if(expected == null) {Assert.assertEquals(description, "Requester Pays Project not set."); } | ||
| else if(expected.equals(P1)) { Assert.assertEquals(description, "Requester Pays Project set by argument: " + value); } | ||
| else if(expected.equals(P2)) { Assert.assertEquals(description, "Requester Pays Project set by system property: " + value); } | ||
| else { Assert.fail("should have been one of the of the previous"); } | ||
| }, RequesterPaysArgumentCollection.PROPERTY_NAME, sys); | ||
| } | ||
| private static void runWithSystemProperty(Runnable toRun, String name, String value){ | ||
| String previousValue = null; | ||
| try { | ||
| if(value != null) { | ||
| previousValue = System.setProperty(name, value); | ||
| } | ||
|
|
||
| toRun.run(); | ||
|
|
||
| } finally { | ||
| if(previousValue == null){ | ||
| System.clearProperty(name); | ||
| } else { | ||
| System.setProperty(name, previousValue); | ||
| } | ||
| } | ||
| } | ||
| } |