Fix: ./gradlew :plugins:repository-s3:yamlRestTest runs locally.

Signed-off-by: dblock <[email protected]>
raghuvanshraj · Jun 9, 2023 · 24b039d · 24b039d
1 parent 6b8dcae
commit 24b039d
Show file tree

Hide file tree

Showing 6 changed files with 38 additions and 3 deletions.
diff --git a/plugins/repository-s3/build.gradle b/plugins/repository-s3/build.gradle
@@ -132,12 +132,14 @@ String s3PermanentAccessKey = System.getenv("amazon_s3_access_key")
 String s3PermanentSecretKey = System.getenv("amazon_s3_secret_key")
 String s3PermanentBucket = System.getenv("amazon_s3_bucket")
 String s3PermanentBasePath = System.getenv("amazon_s3_base_path")
+String s3PermanentRegion = System.getenv("amazon_s3_region")
 
 String s3TemporaryAccessKey = System.getenv("amazon_s3_access_key_temporary")
 String s3TemporarySecretKey = System.getenv("amazon_s3_secret_key_temporary")
 String s3TemporarySessionToken = System.getenv("amazon_s3_session_token_temporary")
 String s3TemporaryBucket = System.getenv("amazon_s3_bucket_temporary")
 String s3TemporaryBasePath = System.getenv("amazon_s3_base_path_temporary")
+String s3TemporaryRegion = System.getenv("amazon_s3_region_temporary")
 
 String s3EC2Bucket = System.getenv("amazon_s3_bucket_ec2")
 String s3EC2BasePath = System.getenv("amazon_s3_base_path_ec2")
@@ -150,28 +152,40 @@ String s3EKSBasePath = System.getenv("amazon_s3_base_path_eks")
 
 boolean s3DisableChunkedEncoding = (new Random(Long.parseUnsignedLong(BuildParams.testSeed.tokenize(':').get(0), 16))).nextBoolean()
 
+// TODO: remove after https://github.com/opensearch-project/opensearch-build/issues/3615
+if (s3PermanentBucket && !s3PermanentRegion) {
+  s3PermanentRegion = "us-west-2"
+}
+
+if (s3TemporaryBucket && !s3TemporaryRegion) {
+  s3TemporaryRegion = "us-west-2"
+}
+// ----
+
 // If all these variables are missing then we are testing against the internal fixture instead, which has the following
 // credentials hard-coded in.
 
-if (!s3PermanentAccessKey && !s3PermanentSecretKey && !s3PermanentBucket && !s3PermanentBasePath) {
+if (!s3PermanentAccessKey && !s3PermanentSecretKey && !s3PermanentBucket && !s3PermanentBasePath && !s3PermanentRegion) {
   s3PermanentAccessKey = 'access_key'
   s3PermanentSecretKey = 'secret_key'
   s3PermanentBucket = 'bucket'
   s3PermanentBasePath = 'base_path'
+  s3PermanentRegion = 'region'
 
   apply plugin: 'opensearch.test.fixtures'
   useFixture = true
 
-} else if (!s3PermanentAccessKey || !s3PermanentSecretKey || !s3PermanentBucket || !s3PermanentBasePath) {
+} else if (!s3PermanentAccessKey || !s3PermanentSecretKey || !s3PermanentBucket || !s3PermanentBasePath || !s3PermanentRegion) {
   throw new IllegalArgumentException("not all options specified to run against external S3 service as permanent credentials are present")
 }
 
-if (!s3TemporaryAccessKey && !s3TemporarySecretKey && !s3TemporaryBucket && !s3TemporaryBasePath && !s3TemporarySessionToken) {
+if (!s3TemporaryAccessKey && !s3TemporarySecretKey && !s3TemporaryBucket && !s3TemporaryBasePath && !s3TemporarySessionToken && !s3TemporaryRegion) {
   s3TemporaryAccessKey = 'session_token_access_key'
   s3TemporarySecretKey = 'session_token_secret_key'
   s3TemporaryBucket = 'session_token_bucket'
   s3TemporaryBasePath = 'session_token_base_path'
   s3TemporarySessionToken = 'session_token'
+  s3TemporaryRegion = 'session_token_region'
 
 } else if (!s3TemporaryAccessKey || !s3TemporarySecretKey || !s3TemporaryBucket || !s3TemporaryBasePath || !s3TemporarySessionToken) {
   throw new IllegalArgumentException("not all options specified to run against external S3 service as temporary credentials are present")
@@ -192,8 +206,10 @@ processYamlRestTestResources {
   Map<String, Object> expansions = [
     'permanent_bucket': s3PermanentBucket,
     'permanent_base_path': s3PermanentBasePath + "_integration_tests_" + BuildParams.testSeed,
+    'permanent_region': s3PermanentRegion,
     'temporary_bucket': s3TemporaryBucket,
     'temporary_base_path': s3TemporaryBasePath + "_integration_tests_" + BuildParams.testSeed,
+    'temporary_region': s3TemporaryRegion,
     'ec2_bucket': s3EC2Bucket,
     'ec2_base_path': s3EC2BasePath,
     'ecs_bucket': s3ECSBucket,
@@ -368,6 +384,7 @@ TaskProvider s3ThirdPartyTest = tasks.register("s3ThirdPartyTest", Test) {
   systemProperty 'test.s3.account', s3PermanentAccessKey
   systemProperty 'test.s3.key', s3PermanentSecretKey
   systemProperty 'test.s3.bucket', s3PermanentBucket
+  systemProperty 'test.s3.region', s3PermanentRegion
   nonInputProperties.systemProperty 'test.s3.base', s3PermanentBasePath + "_third_party_tests_" + BuildParams.testSeed
   if (useFixture) {
     nonInputProperties.systemProperty 'test.s3.endpoint', "${-> fixtureAddress('minio-fixture', 'minio-fixture', '9000') }"

diff --git a/...stTest/resources/rest-api-spec/test/repository_s3/20_repository_permanent_credentials.yml b/...stTest/resources/rest-api-spec/test/repository_s3/20_repository_permanent_credentials.yml
@@ -11,6 +11,7 @@ setup:
           type: s3
           settings:
             bucket: ${permanent_bucket}
+            region: ${permanent_region}
             client: integration_test_permanent
             base_path: "${permanent_base_path}"
             canned_acl: private
@@ -42,6 +43,7 @@ setup:
           type: s3
           settings:
             bucket: ${permanent_bucket}
+            region: ${permanent_region}
             client: integration_test_permanent
             base_path: "${permanent_base_path}"
             endpoint: 127.0.0.1:5
@@ -57,6 +59,7 @@ setup:
           type: s3
           settings:
             bucket: ${permanent_bucket}
+            region: ${permanent_region}
             client: integration_test_permanent
             base_path: "${permanent_base_path}"
             endpoint: 127.0.0.1:5
@@ -275,6 +278,7 @@ setup:
           type: s3
           settings:
             bucket: zHHkfSqlbnBsbpSgvCYtxrEfFLqghXtyPvvvKPNBnRCicNHQLE
+            region: ${permanent_region}
             client: integration_test_permanent
 
 ---
@@ -288,6 +292,7 @@ setup:
           type: s3
           settings:
             bucket: repository_permanent
+            region: ${permanent_region}
             client: unknown
 
 ---
@@ -302,6 +307,7 @@ setup:
         settings:
           readonly: true
           bucket: zHHkfSqlbnBsbpSgvCYtxrEfFLqghXtyPvvvKPNBnRCicNHQLE
+          region: ${permanent_region}
           client: integration_test_permanent
 
 ---
@@ -316,6 +322,7 @@ setup:
         settings:
           readonly: true
           bucket: repository_permanent
+          region: ${permanent_region}
           client: unknown
 
 ---

diff --git a/...stTest/resources/rest-api-spec/test/repository_s3/30_repository_temporary_credentials.yml b/...stTest/resources/rest-api-spec/test/repository_s3/30_repository_temporary_credentials.yml
@@ -11,6 +11,7 @@ setup:
           type: s3
           settings:
             bucket: ${temporary_bucket}
+            region: ${temporary_region}
             client: integration_test_temporary
             base_path: "${temporary_base_path}"
             canned_acl: private
@@ -26,6 +27,7 @@ setup:
         repository: repository_temporary
 
   - match: { repository_temporary.settings.bucket : ${temporary_bucket} }
+  - match: { repository_temporary.settings.region : ${temporary_region} }
   - match: { repository_temporary.settings.client : "integration_test_temporary" }
   - match: { repository_temporary.settings.base_path : "${temporary_base_path}" }
   - match: { repository_temporary.settings.canned_acl : "private" }
@@ -185,6 +187,7 @@ setup:
           type: s3
           settings:
             bucket: zHHkfSqlbnBsbpSgvCYtxrEfFLqghXtyPvvvKPNBnRCicNHQLE
+            region: ${temporary_region}
             client: integration_test_temporary
 
 ---
@@ -198,6 +201,7 @@ setup:
           type: s3
           settings:
             bucket: repository_temporary
+            region: ${temporary_region}
             client: unknown
 
 ---
@@ -212,6 +216,7 @@ setup:
         settings:
           readonly: true
           bucket: zHHkfSqlbnBsbpSgvCYtxrEfFLqghXtyPvvvKPNBnRCicNHQLE
+          region: ${temporary_region}
           client: integration_test_temporary
 
 ---
@@ -226,6 +231,7 @@ setup:
         settings:
           readonly: true
           bucket: repository_temporary
+          region: ${temporary_region}
           client: unknown
 
 ---

diff --git a/...yamlRestTest/resources/rest-api-spec/test/repository_s3/40_repository_ec2_credentials.yml b/...yamlRestTest/resources/rest-api-spec/test/repository_s3/40_repository_ec2_credentials.yml
@@ -11,6 +11,7 @@ setup:
           type: s3
           settings:
             bucket: ${ec2_bucket}
+            region: region
             client: integration_test_ec2
             base_path: "${ec2_base_path}"
             canned_acl: private
@@ -30,6 +31,7 @@ setup:
   - match: { repository_ec2.settings.base_path : "${ec2_base_path}" }
   - match: { repository_ec2.settings.canned_acl : "private" }
   - match: { repository_ec2.settings.storage_class : "standard" }
+  - is_false: repository_ec2.settings.region
   - is_false: repository_ec2.settings.access_key
   - is_false: repository_ec2.settings.secret_key
   - is_false: repository_ec2.settings.session_token

diff --git a/...yamlRestTest/resources/rest-api-spec/test/repository_s3/50_repository_ecs_credentials.yml b/...yamlRestTest/resources/rest-api-spec/test/repository_s3/50_repository_ecs_credentials.yml
@@ -11,6 +11,7 @@ setup:
           type: s3
           settings:
             bucket: ${ecs_bucket}
+            region: region
             client: integration_test_ecs
             base_path: "${ecs_base_path}"
             canned_acl: private
@@ -30,6 +31,7 @@ setup:
   - match: { repository_ecs.settings.base_path : "${ecs_base_path}" }
   - match: { repository_ecs.settings.canned_acl : "private" }
   - match: { repository_ecs.settings.storage_class : "standard" }
+  - is_false: repository_ecs.settings.region
   - is_false: repository_ecs.settings.access_key
   - is_false: repository_ecs.settings.secret_key
   - is_false: repository_ecs.settings.session_token

diff --git a/...yamlRestTest/resources/rest-api-spec/test/repository_s3/60_repository_eks_credentials.yml b/...yamlRestTest/resources/rest-api-spec/test/repository_s3/60_repository_eks_credentials.yml
@@ -11,6 +11,7 @@ setup:
           type: s3
           settings:
             bucket: ${eks_bucket}
+            region: region
             client: integration_test_eks
             base_path: "${eks_base_path}"
             canned_acl: private