From 05a39cf7459ebd214436dc2f0cbf4abd1b35447b Mon Sep 17 00:00:00 2001 From: Xavier Gourmandin Date: Wed, 2 Mar 2022 15:36:11 +0100 Subject: [PATCH 01/55] Support for Junit 5 Extension mechanism Fixes #288 --- pom.xml | 16 +- .../gaul/s3proxy/junit/S3ProxyExtension.java | 108 +++++++++++ .../gaul/s3proxy/junit/S3ProxyJunitCore.java | 177 ++++++++++++++++++ .../org/gaul/s3proxy/junit/S3ProxyRule.java | 125 ++----------- .../s3proxy/junit/S3ProxyExtensionTest.java | 96 ++++++++++ 5 files changed, 414 insertions(+), 108 deletions(-) create mode 100644 src/main/java/org/gaul/s3proxy/junit/S3ProxyExtension.java create mode 100644 src/main/java/org/gaul/s3proxy/junit/S3ProxyJunitCore.java create mode 100644 src/test/java/org/gaul/s3proxy/junit/S3ProxyExtensionTest.java diff --git a/pom.xml b/pom.xml index ffcbcc59..9d7bfdc5 100644 --- a/pom.xml +++ b/pom.xml @@ -233,7 +233,7 @@ - + org.apache.maven.plugins maven-assembly-plugin @@ -287,6 +287,11 @@ surefire-testng ${surefire.version} + + org.apache.maven.surefire + surefire-junit-platform + ${surefire.version} + all @@ -381,8 +386,6 @@ 1.7.28 ${project.groupId}.shaded 2.22.2 - - 2021-11-26T09:00:00Z @@ -430,6 +433,13 @@ provided + + org.junit.jupiter + junit-jupiter + 5.8.1 + + provided + com.fasterxml.jackson.dataformat jackson-dataformat-xml diff --git a/src/main/java/org/gaul/s3proxy/junit/S3ProxyExtension.java b/src/main/java/org/gaul/s3proxy/junit/S3ProxyExtension.java new file mode 100644 index 00000000..87c9cd0a --- /dev/null +++ b/src/main/java/org/gaul/s3proxy/junit/S3ProxyExtension.java @@ -0,0 +1,108 @@ +/* + * Copyright 2014-2021 Andrew Gaul + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gaul.s3proxy.junit; + +import java.net.URI; + +import org.gaul.s3proxy.AuthenticationType; +import org.junit.jupiter.api.extension.AfterEachCallback; +import org.junit.jupiter.api.extension.BeforeEachCallback; +import org.junit.jupiter.api.extension.ExtensionContext; + +/** + * A JUnit 5 Extension that manages an S3Proxy instance which tests + * can use as an S3 API endpoint. + */ +public final class S3ProxyExtension + implements AfterEachCallback, BeforeEachCallback { + + private final S3ProxyJunitCore core; + + public static final class Builder { + + private final S3ProxyJunitCore.Builder builder; + + private Builder() { + builder = new S3ProxyJunitCore.Builder(); + } + + public Builder withCredentials(AuthenticationType authType, + String accessKey, String secretKey) { + builder.withCredentials(authType, accessKey, secretKey); + return this; + } + + public Builder withCredentials(String accessKey, String secretKey) { + builder.withCredentials(accessKey, secretKey); + return this; + } + + public Builder withSecretStore(String path, String password) { + builder.withSecretStore(path, password); + return this; + } + + public Builder withPort(int port) { + builder.withPort(port); + return this; + } + + public Builder withBlobStoreProvider(String blobStoreProvider) { + builder.withBlobStoreProvider(blobStoreProvider); + return this; + } + + public Builder ignoreUnknownHeaders() { + builder.ignoreUnknownHeaders(); + return this; + } + + public S3ProxyExtension build() { + return new S3ProxyExtension(this); + } + } + + private S3ProxyExtension(Builder builder) { + core = new S3ProxyJunitCore(builder.builder); + } + + public static Builder builder() { + return new Builder(); + } + + @Override + public void beforeEach(ExtensionContext extensionContext) throws Exception { + core.beforeEach(); + } + + @Override + public void afterEach(ExtensionContext extensionContext) { + core.afterEach(); + } + + public URI getUri() { + return core.getUri(); + } + + public String getAccessKey() { + return core.getAccessKey(); + } + + public String getSecretKey() { + return core.getSecretKey(); + } +} diff --git a/src/main/java/org/gaul/s3proxy/junit/S3ProxyJunitCore.java b/src/main/java/org/gaul/s3proxy/junit/S3ProxyJunitCore.java new file mode 100644 index 00000000..2003da8a --- /dev/null +++ b/src/main/java/org/gaul/s3proxy/junit/S3ProxyJunitCore.java @@ -0,0 +1,177 @@ +/* + * Copyright 2014-2021 Andrew Gaul + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gaul.s3proxy.junit; + +import java.io.File; +import java.io.IOException; +import java.net.URI; +import java.nio.file.Files; +import java.util.Properties; + +import org.apache.commons.io.FileUtils; +import org.eclipse.jetty.util.component.AbstractLifeCycle; +import org.gaul.s3proxy.AuthenticationType; +import org.gaul.s3proxy.S3Proxy; +import org.jclouds.ContextBuilder; +import org.jclouds.blobstore.BlobStore; +import org.jclouds.blobstore.BlobStoreContext; +import org.jclouds.blobstore.domain.StorageMetadata; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class S3ProxyJunitCore { + + private static final Logger logger = LoggerFactory.getLogger( + S3ProxyJunitCore.class); + + private static final String LOCALHOST = "127.0.0.1"; + + private final String accessKey; + private final String secretKey; + private final String endpointFormat; + private final S3Proxy s3Proxy; + + private final BlobStoreContext blobStoreContext; + private URI endpointUri; + private final File blobStoreLocation; + + public static final class Builder { + private AuthenticationType authType = AuthenticationType.NONE; + private String accessKey; + private String secretKey; + private String secretStorePath; + private String secretStorePassword; + private int port = -1; + private boolean ignoreUnknownHeaders; + private String blobStoreProvider = "filesystem"; + + public Builder withCredentials(AuthenticationType authType, + String accessKey, String secretKey) { + this.authType = authType; + this.accessKey = accessKey; + this.secretKey = secretKey; + return this; + } + + public Builder withCredentials(String accessKey, String secretKey) { + return withCredentials(AuthenticationType.AWS_V2_OR_V4, accessKey, + secretKey); + } + + public Builder withSecretStore(String path, String password) { + secretStorePath = path; + secretStorePassword = password; + return this; + } + + public Builder withPort(int port) { + this.port = port; + return this; + } + + public Builder withBlobStoreProvider(String blobStoreProvider) { + this.blobStoreProvider = blobStoreProvider; + return this; + } + + public Builder ignoreUnknownHeaders() { + ignoreUnknownHeaders = true; + return this; + } + + public S3ProxyJunitCore build() { + return new S3ProxyJunitCore(this); + } + } + + S3ProxyJunitCore(Builder builder) { + accessKey = builder.accessKey; + secretKey = builder.secretKey; + + Properties properties = new Properties(); + try { + blobStoreLocation = Files.createTempDirectory("S3Proxy") + .toFile(); + properties.setProperty("jclouds.filesystem.basedir", + blobStoreLocation.getCanonicalPath()); + } catch (IOException e) { + throw new RuntimeException("Unable to initialize Blob Store", e); + } + + blobStoreContext = ContextBuilder.newBuilder( + builder.blobStoreProvider) + .credentials(accessKey, secretKey) + .overrides(properties).build(BlobStoreContext.class); + + S3Proxy.Builder s3ProxyBuilder = S3Proxy.builder() + .blobStore(blobStoreContext.getBlobStore()) + .awsAuthentication(builder.authType, accessKey, secretKey) + .ignoreUnknownHeaders(builder.ignoreUnknownHeaders); + + if (builder.secretStorePath != null || + builder.secretStorePassword != null) { + s3ProxyBuilder.keyStore(builder.secretStorePath, + builder.secretStorePassword); + } + + int port = Math.max(builder.port, 0); + endpointFormat = "http://%s:%d"; + String endpoint = String.format(endpointFormat, LOCALHOST, port); + s3ProxyBuilder.endpoint(URI.create(endpoint)); + + s3Proxy = s3ProxyBuilder.build(); + } + + public final void beforeEach() throws Exception { + logger.debug("S3 proxy is starting"); + s3Proxy.start(); + while (!s3Proxy.getState().equals(AbstractLifeCycle.STARTED)) { + Thread.sleep(10); + } + endpointUri = URI.create(String.format(endpointFormat, LOCALHOST, + s3Proxy.getPort())); + logger.debug("S3 proxy is running"); + } + + public final void afterEach() { + logger.debug("S3 proxy is stopping"); + try { + s3Proxy.stop(); + BlobStore blobStore = blobStoreContext.getBlobStore(); + for (StorageMetadata metadata : blobStore.list()) { + blobStore.deleteContainer(metadata.getName()); + } + blobStoreContext.close(); + } catch (Exception e) { + throw new RuntimeException("Unable to stop S3 proxy", e); + } + FileUtils.deleteQuietly(blobStoreLocation); + logger.debug("S3 proxy has stopped"); + } + + public final URI getUri() { + return endpointUri; + } + + public final String getAccessKey() { + return accessKey; + } + + public final String getSecretKey() { + return secretKey; + } +} diff --git a/src/main/java/org/gaul/s3proxy/junit/S3ProxyRule.java b/src/main/java/org/gaul/s3proxy/junit/S3ProxyRule.java index 506ceb04..b21fbd19 100644 --- a/src/main/java/org/gaul/s3proxy/junit/S3ProxyRule.java +++ b/src/main/java/org/gaul/s3proxy/junit/S3ProxyRule.java @@ -16,25 +16,12 @@ package org.gaul.s3proxy.junit; -import java.io.File; -import java.io.IOException; import java.net.URI; -import java.nio.file.Files; -import java.util.Properties; import com.google.common.annotations.Beta; -import org.apache.commons.io.FileUtils; -import org.eclipse.jetty.util.component.AbstractLifeCycle; import org.gaul.s3proxy.AuthenticationType; -import org.gaul.s3proxy.S3Proxy; -import org.jclouds.ContextBuilder; -import org.jclouds.blobstore.BlobStore; -import org.jclouds.blobstore.BlobStoreContext; -import org.jclouds.blobstore.domain.StorageMetadata; import org.junit.rules.ExternalResource; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * A JUnit Rule that manages an S3Proxy instance which tests can use as an S3 @@ -42,63 +29,45 @@ */ @Beta public final class S3ProxyRule extends ExternalResource { - private static final Logger logger = LoggerFactory.getLogger( - S3ProxyRule.class); - private static final String LOCALHOST = "127.0.0.1"; - - private final String accessKey; - private final String secretKey; - private final String endpointFormat; - private final S3Proxy s3Proxy; - - private final BlobStoreContext blobStoreContext; - private URI endpointUri; - private final File blobStoreLocation; + private final S3ProxyJunitCore core; public static final class Builder { - private AuthenticationType authType = AuthenticationType.NONE; - private String accessKey; - private String secretKey; - private String secretStorePath; - private String secretStorePassword; - private int port = -1; - private boolean ignoreUnknownHeaders; - private String blobStoreProvider = "filesystem"; - private Builder() { } + private final S3ProxyJunitCore.Builder builder; + + private Builder() { + builder = new S3ProxyJunitCore.Builder(); + } public Builder withCredentials(AuthenticationType authType, - String accessKey, String secretKey) { - this.authType = authType; - this.accessKey = accessKey; - this.secretKey = secretKey; + String accessKey, String secretKey) { + builder.withCredentials(authType, accessKey, secretKey); return this; } public Builder withCredentials(String accessKey, String secretKey) { - return withCredentials(AuthenticationType.AWS_V2_OR_V4, accessKey, - secretKey); + builder.withCredentials(accessKey, secretKey); + return this; } public Builder withSecretStore(String path, String password) { - secretStorePath = path; - secretStorePassword = password; + builder.withSecretStore(path, password); return this; } public Builder withPort(int port) { - this.port = port; + builder.withPort(port); return this; } public Builder withBlobStoreProvider(String blobStoreProvider) { - this.blobStoreProvider = blobStoreProvider; + builder.withBlobStoreProvider(blobStoreProvider); return this; } public Builder ignoreUnknownHeaders() { - ignoreUnknownHeaders = true; + builder.ignoreUnknownHeaders(); return this; } @@ -108,41 +77,7 @@ public S3ProxyRule build() { } private S3ProxyRule(Builder builder) { - accessKey = builder.accessKey; - secretKey = builder.secretKey; - - Properties properties = new Properties(); - try { - blobStoreLocation = Files.createTempDirectory("S3ProxyRule") - .toFile(); - properties.setProperty("jclouds.filesystem.basedir", - blobStoreLocation.getCanonicalPath()); - } catch (IOException e) { - throw new RuntimeException("Unable to initialize Blob Store", e); - } - - blobStoreContext = ContextBuilder.newBuilder( - builder.blobStoreProvider) - .credentials(accessKey, secretKey) - .overrides(properties).build(BlobStoreContext.class); - - S3Proxy.Builder s3ProxyBuilder = S3Proxy.builder() - .blobStore(blobStoreContext.getBlobStore()) - .awsAuthentication(builder.authType, accessKey, secretKey) - .ignoreUnknownHeaders(builder.ignoreUnknownHeaders); - - if (builder.secretStorePath != null || - builder.secretStorePassword != null) { - s3ProxyBuilder.keyStore(builder.secretStorePath, - builder.secretStorePassword); - } - - int port = builder.port < 0 ? 0 : builder.port; - endpointFormat = "http://%s:%d"; - String endpoint = String.format(endpointFormat, LOCALHOST, port); - s3ProxyBuilder.endpoint(URI.create(endpoint)); - - s3Proxy = s3ProxyBuilder.build(); + core = new S3ProxyJunitCore(builder.builder); } public static Builder builder() { @@ -151,43 +86,23 @@ public static Builder builder() { @Override protected void before() throws Throwable { - logger.debug("S3 proxy is starting"); - s3Proxy.start(); - while (!s3Proxy.getState().equals(AbstractLifeCycle.STARTED)) { - Thread.sleep(10); - } - endpointUri = URI.create(String.format(endpointFormat, LOCALHOST, - s3Proxy.getPort())); - logger.debug("S3 proxy is running"); + core.beforeEach(); } @Override protected void after() { - logger.debug("S3 proxy is stopping"); - try { - s3Proxy.stop(); - BlobStore blobStore = blobStoreContext.getBlobStore(); - for (StorageMetadata metadata : blobStore.list()) { - blobStore.deleteContainer(metadata.getName()); - } - blobStoreContext.close(); - } catch (Exception e) { - throw new RuntimeException("Unable to stop S3 proxy", e); - } - FileUtils.deleteQuietly(blobStoreLocation); - logger.debug("S3 proxy has stopped"); + core.afterEach(); } public URI getUri() { - return endpointUri; + return core.getUri(); } public String getAccessKey() { - return accessKey; + return core.getAccessKey(); } public String getSecretKey() { - return secretKey; + return core.getSecretKey(); } - } diff --git a/src/test/java/org/gaul/s3proxy/junit/S3ProxyExtensionTest.java b/src/test/java/org/gaul/s3proxy/junit/S3ProxyExtensionTest.java new file mode 100644 index 00000000..cc118876 --- /dev/null +++ b/src/test/java/org/gaul/s3proxy/junit/S3ProxyExtensionTest.java @@ -0,0 +1,96 @@ +/* + * Copyright 2014-2021 Andrew Gaul + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gaul.s3proxy.junit; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.List; + +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.client.builder.AwsClientBuilder; +import com.amazonaws.regions.Regions; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3ClientBuilder; +import com.amazonaws.services.s3.model.Bucket; +import com.amazonaws.services.s3.model.S3ObjectSummary; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +/** + * This is an example of how one would use the S3Proxy JUnit extension in a unit + * test as opposed to a proper test of the S3ProxyExtension class. + */ +public class S3ProxyExtensionTest { + + @RegisterExtension + static final S3ProxyExtension EXTENSION = S3ProxyExtension + .builder() + .withCredentials("access", "secret") + .build(); + + private static final String MY_TEST_BUCKET = "my-test-bucket"; + + private AmazonS3 s3Client; + + @BeforeEach + public final void setUp() throws Exception { + s3Client = AmazonS3ClientBuilder + .standard() + .withCredentials( + new AWSStaticCredentialsProvider( + new BasicAWSCredentials( + EXTENSION.getAccessKey(), EXTENSION.getSecretKey()))) + .withEndpointConfiguration( + new AwsClientBuilder.EndpointConfiguration( + EXTENSION.getUri().toString(), Regions.US_EAST_1.getName())) + .build(); + + s3Client.createBucket(MY_TEST_BUCKET); + } + + @Test + public final void listBucket() { + List buckets = s3Client.listBuckets(); + assertThat(buckets).hasSize(1); + assertThat(buckets.get(0).getName()) + .isEqualTo(MY_TEST_BUCKET); + } + + @Test + public final void uploadFile() throws Exception { + String testInput = "content"; + s3Client.putObject(MY_TEST_BUCKET, "file.txt", testInput); + + List summaries = s3Client + .listObjects(MY_TEST_BUCKET) + .getObjectSummaries(); + assertThat(summaries).hasSize(1); + assertThat(summaries.get(0).getKey()).isEqualTo("file.txt"); + assertThat(summaries.get(0).getSize()).isEqualTo(testInput.length()); + } + + @Test + public final void doesBucketExistV2() { + assertThat(s3Client.doesBucketExistV2(MY_TEST_BUCKET)).isTrue(); + + // Issue #299 + assertThat(s3Client.doesBucketExistV2("nonexistingbucket")).isFalse(); + } +} From f536835aa85cf4f88d6232391fb5f9a7f599bbd1 Mon Sep 17 00:00:00 2001 From: Andrew Gaul Date: Sat, 26 Mar 2022 19:42:12 +0900 Subject: [PATCH 02/55] pgrade to jclouds 2.5.0 Release notes: https://jclouds.apache.org/releasenotes/2.5.0/ Fixes #396. --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 9d7bfdc5..7e648487 100644 --- a/pom.xml +++ b/pom.xml @@ -382,7 +382,7 @@ UTF-8 1.8 - 2.4.0 + 2.5.0 1.7.28 ${project.groupId}.shaded 2.22.2 From 217308abd735c5f6e31ef4cf8351d5b287a5dc03 Mon Sep 17 00:00:00 2001 From: Florin Peter Date: Sat, 26 Mar 2022 14:05:08 +0100 Subject: [PATCH 03/55] Support for transparent encryption Co-authored-by: Florin Peter --- Dockerfile | 3 + docs/Encryption.md | 76 ++ pom.xml | 5 + .../org/gaul/s3proxy/EncryptedBlobStore.java | 773 ++++++++++++++++ src/main/java/org/gaul/s3proxy/Main.java | 8 + src/main/java/org/gaul/s3proxy/S3Proxy.java | 3 +- .../org/gaul/s3proxy/S3ProxyConstants.java | 7 + .../java/org/gaul/s3proxy/S3ProxyHandler.java | 28 +- .../org/gaul/s3proxy/crypto/Constants.java | 48 + .../org/gaul/s3proxy/crypto/Decryption.java | 319 +++++++ .../s3proxy/crypto/DecryptionInputStream.java | 382 ++++++++ .../org/gaul/s3proxy/crypto/Encryption.java | 56 ++ .../s3proxy/crypto/EncryptionInputStream.java | 126 +++ .../org/gaul/s3proxy/crypto/PartPadding.java | 88 ++ src/main/resources/run-docker-container.sh | 3 + .../s3proxy/EncryptedBlobStoreLiveTest.java | 282 ++++++ .../gaul/s3proxy/EncryptedBlobStoreTest.java | 835 ++++++++++++++++++ src/test/java/org/gaul/s3proxy/TestUtils.java | 8 + src/test/resources/s3proxy-encryption.conf | 20 + 19 files changed, 3066 insertions(+), 4 deletions(-) create mode 100644 docs/Encryption.md create mode 100644 src/main/java/org/gaul/s3proxy/EncryptedBlobStore.java create mode 100644 src/main/java/org/gaul/s3proxy/crypto/Constants.java create mode 100644 src/main/java/org/gaul/s3proxy/crypto/Decryption.java create mode 100644 src/main/java/org/gaul/s3proxy/crypto/DecryptionInputStream.java create mode 100644 src/main/java/org/gaul/s3proxy/crypto/Encryption.java create mode 100644 src/main/java/org/gaul/s3proxy/crypto/EncryptionInputStream.java create mode 100644 src/main/java/org/gaul/s3proxy/crypto/PartPadding.java create mode 100644 src/test/java/org/gaul/s3proxy/EncryptedBlobStoreLiveTest.java create mode 100644 src/test/java/org/gaul/s3proxy/EncryptedBlobStoreTest.java create mode 100644 src/test/resources/s3proxy-encryption.conf diff --git a/Dockerfile b/Dockerfile index 9bff009c..fffae63c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -31,6 +31,9 @@ ENV \ S3PROXY_CORS_ALLOW_METHODS="" \ S3PROXY_CORS_ALLOW_HEADERS="" \ S3PROXY_IGNORE_UNKNOWN_HEADERS="false" \ + S3PROXY_ENCRYPTED_BLOBSTORE="" \ + S3PROXY_ENCRYPTED_BLOBSTORE_PASSWORD="" \ + S3PROXY_ENCRYPTED_BLOBSTORE_SALT="" \ JCLOUDS_PROVIDER="filesystem" \ JCLOUDS_ENDPOINT="" \ JCLOUDS_REGION="" \ diff --git a/docs/Encryption.md b/docs/Encryption.md new file mode 100644 index 00000000..92110862 --- /dev/null +++ b/docs/Encryption.md @@ -0,0 +1,76 @@ +S3Proxy + +# Encryption + +## Motivation +The motivation behind this implementation is to provide a fully transparent and secure encryption to the s3 client while having the ability to write into different clouds. + +## Cipher mode +The chosen cipher is ```AES/CFB/NoPadding``` because it provides the ability to read from an offset like in the middle of a ```Blob```. +While reading from an offset the decryption process needs to consider the previous 16 bytes of the AES block. + +### Key generation +The encryption uses a 128-bit key that will be derived from a given password and salt in combination with random initialization vector that will be stored in each part padding. + +## How a blob is encrypted +Every uploaded part get a padding of 64 bytes that includes the necessary information for decryption. The input stream from a s3 client is passed through ```CipherInputStream``` and piped to append the 64 byte part padding at the end the encrypted stream. The encrypted input stream is then processed by the ```BlobStore``` to save the ```Blob```. + +| Name | Byte size | Description | +|-----------|-----------|----------------------------------------------------------------| +| Delimiter | 8 byte | The delimiter is used to detect if the ```Blob``` is encrypted | +| IV | 16 byte | AES initialization vector | +| Part | 4 byte | The part number | +| Size | 8 byte | The unencrypted size of the ```Blob``` | +| Version | 2 byte | Version can be used in the future if changes are necessary | +| Reserved | 26 byte | Reserved for future use | + +### Multipart handling +A single ```Blob``` can be uploaded by the client into multiple parts. After the completion all parts are concatenated into a single ```Blob```. +This procedure will result in multiple parts and paddings being held by a single ```Blob```. + +### Single blob example +``` +------------------------------------- +| ENCRYPTED BYTES | PADDING | +------------------------------------- +``` + +### Multipart blob example +``` +------------------------------------------------------------------------------------- +| ENCRYPTED BYTES | PADDING | ENCRYPTED BYTES | PADDING | ENCRYPTED BYTES | PADDING | +------------------------------------------------------------------------------------- +``` + +## How a blob is decrypted +The decryption is way more complex than the encryption. Decryption process needs to take care of the following circumstances: +- decryption of the entire ```Blob``` +- decryption from a specific offset by skipping initial bytes +- decryption of bytes by reading from the end (tail) +- decryption of a specific byte range like middle of the ```Blob``` +- decryption of all previous situation by considering a underlying multipart ```Blob``` + +### Single blob decryption +First the ```BlobMetadata``` is requested to get the encrypted ```Blob``` size. The last 64 bytes of ```PartPadding``` are fetched and inspected to detect if a decryption is necessary. +The cipher is than initialized with the IV and the key. + +### Multipart blob decryption +The process is similar to the single ```Blob``` decryption but with the difference that a list of parts is computed by fetching all ```PartPadding``` from end to the beginning. + +## Blob suffix +Each stored ```Blob``` will get a suffix named ```.s3enc``` this helps to determine if a ```Blob``` is encrypted. For the s3 client the ```.s3enc``` suffix is not visible and the ```Blob``` size will always show the unencrypted size. + +## Tested jClouds provider +- S3 + - Minio + - OBS from OpenTelekomCloud +- AWS S3 +- Azure +- GCP +- Local + +## Limitation +- All blobs are encrypted with the same key that is derived from a given password +- No support for re-encryption +- Returned eTag always differs therefore clients should not verify it +- Decryption of a ```Blob``` will always result in multiple calls against the backend for instance a GET will result in a HEAD + GET because the size of the blob needs to be determined diff --git a/pom.xml b/pom.xml index 7e648487..b0bb74d7 100644 --- a/pom.xml +++ b/pom.xml @@ -461,6 +461,11 @@ commons-fileupload 1.4 + + commons-codec + commons-codec + 1.15 + org.apache.jclouds jclouds-allblobstore diff --git a/src/main/java/org/gaul/s3proxy/EncryptedBlobStore.java b/src/main/java/org/gaul/s3proxy/EncryptedBlobStore.java new file mode 100644 index 00000000..9616c6b7 --- /dev/null +++ b/src/main/java/org/gaul/s3proxy/EncryptedBlobStore.java @@ -0,0 +1,773 @@ +/* + * Copyright 2014-2021 Andrew Gaul + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gaul.s3proxy; + +import static com.google.common.base.Preconditions.checkArgument; + +import java.io.IOException; +import java.io.InputStream; +import java.security.spec.KeySpec; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.regex.Matcher; + +import javax.crypto.SecretKey; +import javax.crypto.SecretKeyFactory; +import javax.crypto.spec.PBEKeySpec; +import javax.crypto.spec.SecretKeySpec; + +import com.google.common.base.Strings; +import com.google.common.collect.ImmutableSet; +import com.google.common.hash.HashCode; + +import org.apache.commons.codec.digest.DigestUtils; +import org.gaul.s3proxy.crypto.Constants; +import org.gaul.s3proxy.crypto.Decryption; +import org.gaul.s3proxy.crypto.Encryption; +import org.jclouds.blobstore.BlobStore; +import org.jclouds.blobstore.domain.Blob; +import org.jclouds.blobstore.domain.BlobAccess; +import org.jclouds.blobstore.domain.BlobBuilder; +import org.jclouds.blobstore.domain.BlobMetadata; +import org.jclouds.blobstore.domain.MultipartPart; +import org.jclouds.blobstore.domain.MultipartUpload; +import org.jclouds.blobstore.domain.MutableBlobMetadata; +import org.jclouds.blobstore.domain.PageSet; +import org.jclouds.blobstore.domain.StorageMetadata; +import org.jclouds.blobstore.domain.internal.MutableBlobMetadataImpl; +import org.jclouds.blobstore.domain.internal.PageSetImpl; +import org.jclouds.blobstore.options.CopyOptions; +import org.jclouds.blobstore.options.GetOptions; +import org.jclouds.blobstore.options.ListContainerOptions; +import org.jclouds.blobstore.options.PutOptions; +import org.jclouds.blobstore.util.ForwardingBlobStore; +import org.jclouds.io.ContentMetadata; +import org.jclouds.io.MutableContentMetadata; +import org.jclouds.io.Payload; +import org.jclouds.io.Payloads; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@SuppressWarnings("UnstableApiUsage") +public final class EncryptedBlobStore extends ForwardingBlobStore { + private final Logger logger = + LoggerFactory.getLogger(EncryptedBlobStore.class); + private SecretKeySpec secretKey; + + private EncryptedBlobStore(BlobStore blobStore, Properties properties) + throws IllegalArgumentException { + super(blobStore); + + String password = properties.getProperty( + S3ProxyConstants.PROPERTY_ENCRYPTED_BLOBSTORE_PASSWORD); + checkArgument(!Strings.isNullOrEmpty(password), + "Password for encrypted blobstore is not set"); + + String salt = properties.getProperty( + S3ProxyConstants.PROPERTY_ENCRYPTED_BLOBSTORE_SALT); + checkArgument(!Strings.isNullOrEmpty(salt), + "Salt for encrypted blobstore is not set"); + initStore(password, salt); + } + + static BlobStore newEncryptedBlobStore(BlobStore blobStore, + Properties properties) throws IOException { + return new EncryptedBlobStore(blobStore, properties); + } + + private void initStore(String password, String salt) + throws IllegalArgumentException { + try { + SecretKeyFactory factory = + SecretKeyFactory.getInstance("PBKDF2WithHmacSHA256"); + KeySpec spec = + new PBEKeySpec(password.toCharArray(), salt.getBytes(), 65536, + 128); + SecretKey tmp = factory.generateSecret(spec); + secretKey = new SecretKeySpec(tmp.getEncoded(), "AES"); + } catch (Exception e) { + throw new IllegalArgumentException(e); + } + } + + private Blob cipheredBlob(String container, Blob blob, InputStream payload, + long contentLength, + boolean addEncryptedMetadata) { + + // make a copy of the blob with the new payload stream + BlobMetadata blobMeta = blob.getMetadata(); + ContentMetadata contentMeta = blob.getMetadata().getContentMetadata(); + Map userMetadata = blobMeta.getUserMetadata(); + String contentType = contentMeta.getContentType(); + + // suffix the content type with -s3enc if we need to encrypt + if (addEncryptedMetadata) { + blobMeta = setEncryptedSuffix(blobMeta); + } else { + // remove the -s3enc suffix while decrypting + // but not if it contains a multipart meta + if (!blobMeta.getUserMetadata() + .containsKey(Constants.METADATA_IS_ENCRYPTED_MULTIPART)) { + blobMeta = removeEncryptedSuffix(blobMeta); + } + } + + // we do not set contentMD5 as it will not match due to the encryption + Blob cipheredBlob = blobBuilder(container) + .name(blobMeta.getName()) + .type(blobMeta.getType()) + .tier(blobMeta.getTier()) + .userMetadata(userMetadata) + .payload(payload) + .cacheControl(contentMeta.getCacheControl()) + .contentDisposition(contentMeta.getContentDisposition()) + .contentEncoding(contentMeta.getContentEncoding()) + .contentLanguage(contentMeta.getContentLanguage()) + .contentLength(contentLength) + .contentType(contentType) + .build(); + + cipheredBlob.getMetadata().setUri(blobMeta.getUri()); + cipheredBlob.getMetadata().setETag(blobMeta.getETag()); + cipheredBlob.getMetadata().setLastModified(blobMeta.getLastModified()); + cipheredBlob.getMetadata().setSize(blobMeta.getSize()); + cipheredBlob.getMetadata().setPublicUri(blobMeta.getPublicUri()); + cipheredBlob.getMetadata().setContainer(blobMeta.getContainer()); + + return cipheredBlob; + } + + private Blob encryptBlob(String container, Blob blob) { + + try { + // open the streams and pass them through the encryption + InputStream isRaw = blob.getPayload().openStream(); + Encryption encryption = + new Encryption(secretKey, isRaw, 1); + InputStream is = encryption.openStream(); + + // adjust the encrypted content length by + // adding the padding block size + long contentLength = + blob.getMetadata().getContentMetadata().getContentLength() + + Constants.PADDING_BLOCK_SIZE; + + return cipheredBlob(container, blob, is, contentLength, true); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + private Payload encryptPayload(Payload payload, int partNumber) { + + try { + // open the streams and pass them through the encryption + InputStream isRaw = payload.openStream(); + Encryption encryption = + new Encryption(secretKey, isRaw, partNumber); + InputStream is = encryption.openStream(); + + Payload cipheredPayload = Payloads.newInputStreamPayload(is); + MutableContentMetadata contentMetadata = + payload.getContentMetadata(); + HashCode md5 = null; + contentMetadata.setContentMD5(md5); + cipheredPayload.setContentMetadata(payload.getContentMetadata()); + cipheredPayload.setSensitive(payload.isSensitive()); + + // adjust the encrypted content length by + // adding the padding block size + long contentLength = + payload.getContentMetadata().getContentLength() + + Constants.PADDING_BLOCK_SIZE; + cipheredPayload.getContentMetadata() + .setContentLength(contentLength); + + return cipheredPayload; + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + private Blob decryptBlob(Decryption decryption, String container, + Blob blob) { + try { + // handle blob does not exist + if (blob == null) { + return null; + } + + // open the streams and pass them through the decryption + InputStream isRaw = blob.getPayload().openStream(); + InputStream is = decryption.openStream(isRaw); + + // adjust the content length if the blob is encrypted + long contentLength = + blob.getMetadata().getContentMetadata().getContentLength(); + if (decryption.isEncrypted()) { + contentLength = decryption.getContentLength(); + } + + return cipheredBlob(container, blob, is, contentLength, false); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + // filter the list by showing the unencrypted blob size + private PageSet filteredList( + PageSet pageSet) { + ImmutableSet.Builder builder = ImmutableSet.builder(); + for (StorageMetadata sm : pageSet) { + if (sm instanceof BlobMetadata) { + MutableBlobMetadata mbm = + new MutableBlobMetadataImpl((BlobMetadata) sm); + + // if blob is encrypted remove the -s3enc suffix + // from content type + if (isEncrypted(mbm)) { + mbm = removeEncryptedSuffix((BlobMetadata) sm); + mbm = calculateBlobSize(mbm); + } + + builder.add(mbm); + } else { + builder.add(sm); + } + } + + // make sure the marker do not show blob with .s3enc suffix + String marker = pageSet.getNextMarker(); + if (marker != null && isEncrypted(marker)) { + marker = removeEncryptedSuffix(marker); + } + return new PageSetImpl<>(builder.build(), marker); + } + + private boolean isEncrypted(BlobMetadata blobMeta) { + return isEncrypted(blobMeta.getName()); + } + + private boolean isEncrypted(String blobName) { + return blobName.endsWith(Constants.S3_ENC_SUFFIX); + } + + private MutableBlobMetadata setEncryptedSuffix(BlobMetadata blobMeta) { + MutableBlobMetadata bm = new MutableBlobMetadataImpl(blobMeta); + if (blobMeta.getName() != null && !isEncrypted(blobMeta.getName())) { + bm.setName(blobNameWithSuffix(blobMeta.getName())); + } + + return bm; + } + + private String removeEncryptedSuffix(String blobName) { + return blobName.substring(0, + blobName.length() - Constants.S3_ENC_SUFFIX.length()); + } + + private MutableBlobMetadata removeEncryptedSuffix(BlobMetadata blobMeta) { + MutableBlobMetadata bm = new MutableBlobMetadataImpl(blobMeta); + if (isEncrypted(bm.getName())) { + String blobName = bm.getName(); + bm.setName(removeEncryptedSuffix(blobName)); + } + + return bm; + } + + private MutableBlobMetadata calculateBlobSize(BlobMetadata blobMeta) { + MutableBlobMetadata mbm = removeEncryptedSuffix(blobMeta); + + // we are using on non-s3 backends like azure or gcp a metadata key to + // calculate the part padding sizes that needs to be removed + if (mbm.getUserMetadata() + .containsKey(Constants.METADATA_ENCRYPTION_PARTS)) { + int parts = Integer.parseInt( + mbm.getUserMetadata().get(Constants.METADATA_ENCRYPTION_PARTS)); + int partPaddingSizes = Constants.PADDING_BLOCK_SIZE * parts; + long size = blobMeta.getSize() - partPaddingSizes; + mbm.setSize(size); + mbm.getContentMetadata().setContentLength(size); + } else { + // on s3 backends like aws or minio we rely on the eTag suffix + Matcher matcher = + Constants.MPU_ETAG_SUFFIX_PATTERN.matcher(blobMeta.getETag()); + if (matcher.find()) { + int parts = Integer.parseInt(matcher.group(1)); + int partPaddingSizes = Constants.PADDING_BLOCK_SIZE * parts; + long size = blobMeta.getSize() - partPaddingSizes; + mbm.setSize(size); + mbm.getContentMetadata().setContentLength(size); + } else { + long size = blobMeta.getSize() - Constants.PADDING_BLOCK_SIZE; + mbm.setSize(size); + mbm.getContentMetadata().setContentLength(size); + } + } + + return mbm; + } + + private boolean multipartRequiresStub() { + String blobStoreType = getBlobStoreType(); + return Quirks.MULTIPART_REQUIRES_STUB.contains(blobStoreType); + } + + private String blobNameWithSuffix(String container, String name) { + String nameWithSuffix = blobNameWithSuffix(name); + if (delegate().blobExists(container, nameWithSuffix)) { + name = nameWithSuffix; + } + return name; + } + + private String blobNameWithSuffix(String name) { + return name + Constants.S3_ENC_SUFFIX; + } + + private String getBlobStoreType() { + return delegate().getContext().unwrap().getProviderMetadata().getId(); + } + + private String generateUploadId(String container, String blobName) { + String path = container + "/" + blobName; + return DigestUtils.sha256Hex(path); + } + + @Override + public Blob getBlob(String containerName, String blobName) { + return getBlob(containerName, blobName, new GetOptions()); + } + + @Override + public Blob getBlob(String containerName, String blobName, + GetOptions getOptions) { + + // adjust the blob name + blobName = blobNameWithSuffix(blobName); + + // get the metadata to determine the blob size + BlobMetadata meta = delegate().blobMetadata(containerName, blobName); + + try { + // we have a blob that ends with .s3enc + if (meta != null) { + // init defaults + long offset = 0; + long end = 0; + long length = -1; + + if (getOptions.getRanges().size() > 0) { + // S3 doesn't allow multiple ranges + String range = getOptions.getRanges().get(0); + String[] ranges = range.split("-", 2); + + if (ranges[0].isEmpty()) { + // handle to read from the end + end = Long.parseLong(ranges[1]); + length = end; + } else if (ranges[1].isEmpty()) { + // handle to read from an offset till the end + offset = Long.parseLong(ranges[0]); + } else { + // handle to read from an offset + offset = Long.parseLong(ranges[0]); + end = Long.parseLong(ranges[1]); + length = end - offset + 1; + } + } + + // init decryption + Decryption decryption = + new Decryption(secretKey, delegate(), meta, offset, length); + + if (decryption.isEncrypted() && + getOptions.getRanges().size() > 0) { + // clear current ranges to avoid multiple ranges + getOptions.getRanges().clear(); + + long startAt = decryption.getStartAt(); + long endAt = decryption.getEncryptedSize(); + + if (offset == 0 && end > 0 && length == end) { + // handle to read from the end + startAt = decryption.calculateTail(); + } else if (offset > 0 && end > 0) { + // handle to read from an offset + endAt = decryption.calculateEndAt(end); + } + + getOptions.range(startAt, endAt); + } + + Blob blob = + delegate().getBlob(containerName, blobName, getOptions); + return decryptBlob(decryption, containerName, blob); + } else { + // we suppose to return a unencrypted blob + // since no metadata was found + blobName = removeEncryptedSuffix(blobName); + return delegate().getBlob(containerName, blobName, getOptions); + } + + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Override + public String putBlob(String containerName, Blob blob) { + return delegate().putBlob(containerName, + encryptBlob(containerName, blob)); + } + + @Override + public String putBlob(String containerName, Blob blob, + PutOptions putOptions) { + return delegate().putBlob(containerName, + encryptBlob(containerName, blob), putOptions); + } + + @Override + public String copyBlob(String fromContainer, String fromName, + String toContainer, String toName, CopyOptions options) { + + // if we copy an encrypted blob + // make sure to add suffix to the destination blob name + String blobName = blobNameWithSuffix(fromName); + if (delegate().blobExists(fromContainer, blobName)) { + fromName = blobName; + toName = blobNameWithSuffix(toName); + } + + return delegate().copyBlob(fromContainer, fromName, toContainer, toName, + options); + } + + @Override + public void removeBlob(String container, String name) { + name = blobNameWithSuffix(container, name); + delegate().removeBlob(container, name); + } + + @Override + public void removeBlobs(String container, Iterable names) { + List filteredNames = new ArrayList<>(); + + // filter the list of blobs to determine + // if we need to delete encrypted blobs + for (String name : names) { + name = blobNameWithSuffix(container, name); + filteredNames.add(name); + } + + delegate().removeBlobs(container, filteredNames); + } + + @Override + public BlobAccess getBlobAccess(String container, String name) { + name = blobNameWithSuffix(container, name); + return delegate().getBlobAccess(container, name); + } + + @Override + public boolean blobExists(String container, String name) { + name = blobNameWithSuffix(container, name); + return delegate().blobExists(container, name); + } + + @Override + public void setBlobAccess(String container, String name, + BlobAccess access) { + name = blobNameWithSuffix(container, name); + delegate().setBlobAccess(container, name, access); + } + + @Override + public PageSet list() { + PageSet pageSet = delegate().list(); + return filteredList(pageSet); + } + + @Override + public PageSet list(String container) { + PageSet pageSet = delegate().list(container); + return filteredList(pageSet); + } + + @Override + public PageSet list(String container, + ListContainerOptions options) { + PageSet pageSet = + delegate().list(container, options); + return filteredList(pageSet); + } + + @Override + public MultipartUpload initiateMultipartUpload(String container, + BlobMetadata blobMetadata, PutOptions options) { + MutableBlobMetadata mbm = new MutableBlobMetadataImpl(blobMetadata); + mbm = setEncryptedSuffix(mbm); + + MultipartUpload mpu = + delegate().initiateMultipartUpload(container, mbm, options); + + // handle non-s3 backends + // by setting a metadata key for multipart stubs + if (multipartRequiresStub()) { + mbm.getUserMetadata() + .put(Constants.METADATA_IS_ENCRYPTED_MULTIPART, "true"); + + if (getBlobStoreType().equals("azureblob")) { + // use part 0 as a placeholder + delegate().uploadMultipartPart(mpu, 0, + Payloads.newStringPayload("dummy")); + + // since azure does not have a uploadId + // we use the sha256 of the path + String uploadId = generateUploadId(container, mbm.getName()); + + mpu = MultipartUpload.create(mpu.containerName(), + mpu.blobName(), uploadId, mpu.blobMetadata(), options); + } else if (getBlobStoreType().equals("google-cloud-storage")) { + mbm.getUserMetadata() + .put(Constants.METADATA_MULTIPART_KEY, mbm.getName()); + + // since gcp does not have a uploadId + // we use the sha256 of the path + String uploadId = generateUploadId(container, mbm.getName()); + + // to emulate later the list of multipart uploads + // we create a placeholer + BlobBuilder builder = + blobBuilder(Constants.MPU_FOLDER + uploadId) + .payload("") + .userMetadata(mbm.getUserMetadata()); + delegate().putBlob(container, builder.build(), options); + + // final mpu on gcp + mpu = MultipartUpload.create(mpu.containerName(), + mpu.blobName(), uploadId, mpu.blobMetadata(), options); + } + } + + return mpu; + } + + @Override + public List listMultipartUploads(String container) { + List mpus = new ArrayList<>(); + + // emulate list of multipart uploads on gcp + if (getBlobStoreType().equals("google-cloud-storage")) { + ListContainerOptions options = new ListContainerOptions(); + PageSet mpuList = + delegate().list(container, + options.prefix(Constants.MPU_FOLDER)); + + // find all blobs in .mpu folder and build the list + for (StorageMetadata blob : mpuList) { + Map meta = blob.getUserMetadata(); + if (meta.containsKey(Constants.METADATA_MULTIPART_KEY)) { + String blobName = + meta.get(Constants.METADATA_MULTIPART_KEY); + String uploadId = + blob.getName() + .substring(blob.getName().lastIndexOf("/") + 1); + MultipartUpload mpu = + MultipartUpload.create(container, + blobName, uploadId, null, null); + mpus.add(mpu); + } + } + } else { + mpus = delegate().listMultipartUploads(container); + } + + List filtered = new ArrayList<>(); + // filter the list uploads by removing the .s3enc suffix + for (MultipartUpload mpu : mpus) { + String blobName = mpu.blobName(); + if (isEncrypted(blobName)) { + blobName = removeEncryptedSuffix(mpu.blobName()); + + String uploadId = mpu.id(); + + // since azure not have a uploadId + // we use the sha256 of the path + if (getBlobStoreType().equals("azureblob")) { + uploadId = generateUploadId(container, mpu.blobName()); + } + + MultipartUpload mpuWithoutSuffix = + MultipartUpload.create(mpu.containerName(), + blobName, uploadId, mpu.blobMetadata(), + mpu.putOptions()); + + filtered.add(mpuWithoutSuffix); + } else { + filtered.add(mpu); + } + } + return filtered; + } + + @Override + public List listMultipartUpload(MultipartUpload mpu) { + mpu = filterMultipartUpload(mpu); + List parts = delegate().listMultipartUpload(mpu); + List filteredParts = new ArrayList<>(); + + // fix wrong multipart size due to the part padding + for (MultipartPart part : parts) { + + // we use part 0 as a placeholder and hide it on azure + if (getBlobStoreType().equals("azureblob") && + part.partNumber() == 0) { + continue; + } + + MultipartPart newPart = MultipartPart.create( + part.partNumber(), + part.partSize() - Constants.PADDING_BLOCK_SIZE, + part.partETag(), + part.lastModified() + ); + filteredParts.add(newPart); + } + return filteredParts; + } + + @Override + public MultipartPart uploadMultipartPart(MultipartUpload mpu, + int partNumber, Payload payload) { + + mpu = filterMultipartUpload(mpu); + return delegate().uploadMultipartPart(mpu, partNumber, + encryptPayload(payload, partNumber)); + } + + private MultipartUpload filterMultipartUpload(MultipartUpload mpu) { + MutableBlobMetadata mbm = null; + if (mpu.blobMetadata() != null) { + mbm = new MutableBlobMetadataImpl(mpu.blobMetadata()); + mbm = setEncryptedSuffix(mbm); + } + + String blobName = mpu.blobName(); + if (!isEncrypted(blobName)) { + blobName = blobNameWithSuffix(blobName); + } + + return MultipartUpload.create(mpu.containerName(), blobName, mpu.id(), + mbm, mpu.putOptions()); + } + + @Override + public String completeMultipartUpload(MultipartUpload mpu, + List parts) { + + MutableBlobMetadata mbm = + new MutableBlobMetadataImpl(mpu.blobMetadata()); + String blobName = mpu.blobName(); + + // always set .s3enc suffix except on gcp + // and blob name starts with multipart upload id + if (getBlobStoreType().equals("google-cloud-storage") && + mpu.blobName().startsWith(mpu.id())) { + logger.debug("skip suffix on gcp"); + } else { + mbm = setEncryptedSuffix(mbm); + if (!isEncrypted(mpu.blobName())) { + blobName = blobNameWithSuffix(blobName); + } + } + + MultipartUpload mpuWithSuffix = + MultipartUpload.create(mpu.containerName(), + blobName, mpu.id(), mbm, mpu.putOptions()); + + // this will only work for non s3 backends like azure and gcp + if (multipartRequiresStub()) { + long partCount = parts.size(); + + // special handling for GCP to sum up all parts + if (getBlobStoreType().equals("google-cloud-storage")) { + partCount = 0; + for (MultipartPart part : parts) { + blobName = + String.format("%s_%08d", + mpu.id(), + part.partNumber()); + BlobMetadata metadata = + delegate().blobMetadata(mpu.containerName(), blobName); + if (metadata != null && metadata.getUserMetadata() + .containsKey(Constants.METADATA_ENCRYPTION_PARTS)) { + String partMetaCount = metadata.getUserMetadata() + .get(Constants.METADATA_ENCRYPTION_PARTS); + partCount = partCount + Long.parseLong(partMetaCount); + } else { + partCount++; + } + } + } + + mpuWithSuffix.blobMetadata().getUserMetadata() + .put(Constants.METADATA_ENCRYPTION_PARTS, + String.valueOf(partCount)); + mpuWithSuffix.blobMetadata().getUserMetadata() + .remove(Constants.METADATA_IS_ENCRYPTED_MULTIPART); + } + + String eTag = delegate().completeMultipartUpload(mpuWithSuffix, parts); + + // cleanup mpu placeholder on gcp + if (getBlobStoreType().equals("google-cloud-storage")) { + delegate().removeBlob(mpu.containerName(), + Constants.MPU_FOLDER + mpu.id()); + } + + return eTag; + } + + @Override + public BlobMetadata blobMetadata(String container, String name) { + + name = blobNameWithSuffix(container, name); + BlobMetadata blobMetadata = delegate().blobMetadata(container, name); + if (blobMetadata != null) { + // only remove the -s3enc suffix + // if the blob is encrypted and not a multipart stub + if (isEncrypted(blobMetadata) && + !blobMetadata.getUserMetadata() + .containsKey(Constants.METADATA_IS_ENCRYPTED_MULTIPART)) { + blobMetadata = removeEncryptedSuffix(blobMetadata); + blobMetadata = calculateBlobSize(blobMetadata); + } + } + return blobMetadata; + } + + @Override + public long getMaximumMultipartPartSize() { + long max = delegate().getMaximumMultipartPartSize(); + return max - Constants.PADDING_BLOCK_SIZE; + } +} diff --git a/src/main/java/org/gaul/s3proxy/Main.java b/src/main/java/org/gaul/s3proxy/Main.java index e0b6b28e..59eb357f 100644 --- a/src/main/java/org/gaul/s3proxy/Main.java +++ b/src/main/java/org/gaul/s3proxy/Main.java @@ -257,6 +257,14 @@ private static BlobStore parseMiddlewareProperties(BlobStore blobStore, shards, prefixes); } + String encryptedBlobStore = properties.getProperty( + S3ProxyConstants.PROPERTY_ENCRYPTED_BLOBSTORE); + if ("true".equalsIgnoreCase(encryptedBlobStore)) { + System.err.println("Using encrypted storage backend"); + blobStore = EncryptedBlobStore.newEncryptedBlobStore(blobStore, + properties); + } + return blobStore; } diff --git a/src/main/java/org/gaul/s3proxy/S3Proxy.java b/src/main/java/org/gaul/s3proxy/S3Proxy.java index f48984b7..6067f7cc 100644 --- a/src/main/java/org/gaul/s3proxy/S3Proxy.java +++ b/src/main/java/org/gaul/s3proxy/S3Proxy.java @@ -99,7 +99,8 @@ public final class S3Proxy { } if (builder.secureEndpoint != null) { - SslContextFactory sslContextFactory = new SslContextFactory(); + SslContextFactory sslContextFactory = + new SslContextFactory.Server(); sslContextFactory.setKeyStorePath(builder.keyStorePath); sslContextFactory.setKeyStorePassword(builder.keyStorePassword); connector = new ServerConnector(server, sslContextFactory, diff --git a/src/main/java/org/gaul/s3proxy/S3ProxyConstants.java b/src/main/java/org/gaul/s3proxy/S3ProxyConstants.java index eec35bdd..9936343a 100644 --- a/src/main/java/org/gaul/s3proxy/S3ProxyConstants.java +++ b/src/main/java/org/gaul/s3proxy/S3ProxyConstants.java @@ -107,6 +107,13 @@ public final class S3ProxyConstants { public static final String PROPERTY_MAXIMUM_TIME_SKEW = "s3proxy.maximum-timeskew"; + public static final String PROPERTY_ENCRYPTED_BLOBSTORE = + "s3proxy.encrypted-blobstore"; + public static final String PROPERTY_ENCRYPTED_BLOBSTORE_PASSWORD = + "s3proxy.encrypted-blobstore-password"; + public static final String PROPERTY_ENCRYPTED_BLOBSTORE_SALT = + "s3proxy.encrypted-blobstore-salt"; + static final String PROPERTY_ALT_JCLOUDS_PREFIX = "alt."; private S3ProxyConstants() { diff --git a/src/main/java/org/gaul/s3proxy/S3ProxyHandler.java b/src/main/java/org/gaul/s3proxy/S3ProxyHandler.java index a99ff612..235ee7b1 100644 --- a/src/main/java/org/gaul/s3proxy/S3ProxyHandler.java +++ b/src/main/java/org/gaul/s3proxy/S3ProxyHandler.java @@ -1176,13 +1176,15 @@ private void handleListMultipartUploads(HttpServletRequest request, HttpServletResponse response, BlobStore blobStore, String container) throws IOException, S3Exception { if (request.getParameter("delimiter") != null || - request.getParameter("prefix") != null || request.getParameter("max-uploads") != null || request.getParameter("key-marker") != null || request.getParameter("upload-id-marker") != null) { throw new UnsupportedOperationException(); } + String encodingType = request.getParameter("encoding-type"); + String prefix = request.getParameter("prefix"); + List uploads = blobStore.listMultipartUploads( container); @@ -1203,11 +1205,23 @@ private void handleListMultipartUploads(HttpServletRequest request, xml.writeEmptyElement("NextKeyMarker"); xml.writeEmptyElement("NextUploadIdMarker"); xml.writeEmptyElement("Delimiter"); - xml.writeEmptyElement("Prefix"); + + if (Strings.isNullOrEmpty(prefix)) { + xml.writeEmptyElement("Prefix"); + } else { + writeSimpleElement(xml, "Prefix", encodeBlob( + encodingType, prefix)); + } + writeSimpleElement(xml, "MaxUploads", "1000"); writeSimpleElement(xml, "IsTruncated", "false"); for (MultipartUpload upload : uploads) { + if (prefix != null && + !upload.blobName().startsWith(prefix)) { + continue; + } + xml.writeStartElement("Upload"); writeSimpleElement(xml, "Key", upload.blobName()); @@ -2578,6 +2592,15 @@ private void handleCopyPart(HttpServletRequest request, "ArgumentValue", partNumberString)); } + // GCS only supports 32 parts so partition MPU into 32-part chunks. + String blobStoreType = getBlobStoreType(blobStore); + if (blobStoreType.equals("google-cloud-storage")) { + // fix up 1-based part numbers + uploadId = String.format( + "%s_%08d", uploadId, ((partNumber - 1) / 32) + 1); + partNumber = ((partNumber - 1) % 32) + 1; + } + // TODO: how to reconstruct original mpu? MultipartUpload mpu = MultipartUpload.create(containerName, blobName, uploadId, createFakeBlobMetadata(blobStore), @@ -2629,7 +2652,6 @@ blobName, uploadId, createFakeBlobMetadata(blobStore), long contentLength = blobMetadata.getContentMetadata().getContentLength(); - String blobStoreType = getBlobStoreType(blobStore); try (InputStream is = blob.getPayload().openStream()) { if (blobStoreType.equals("azureblob")) { // Azure has a smaller maximum part size than S3. Split a diff --git a/src/main/java/org/gaul/s3proxy/crypto/Constants.java b/src/main/java/org/gaul/s3proxy/crypto/Constants.java new file mode 100644 index 00000000..4e809571 --- /dev/null +++ b/src/main/java/org/gaul/s3proxy/crypto/Constants.java @@ -0,0 +1,48 @@ +/* + * Copyright 2014-2021 Andrew Gaul + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gaul.s3proxy.crypto; + +import java.nio.charset.StandardCharsets; +import java.util.regex.Pattern; + +public final class Constants { + public static final short VERSION = 1; + public static final String AES_CIPHER = "AES/CFB/NoPadding"; + public static final String S3_ENC_SUFFIX = ".s3enc"; + public static final String MPU_FOLDER = ".mpu/"; + public static final Pattern MPU_ETAG_SUFFIX_PATTERN = + Pattern.compile(".*-([0-9]+)"); + public static final String METADATA_ENCRYPTION_PARTS = + "s3proxy_encryption_parts"; + public static final String METADATA_IS_ENCRYPTED_MULTIPART = + "s3proxy_encryption_multipart"; + public static final String METADATA_MULTIPART_KEY = + "s3proxy_mpu_key"; + public static final int AES_BLOCK_SIZE = 16; + public static final int PADDING_BLOCK_SIZE = 64; + public static final byte[] DELIMITER = + "-S3-ENC-".getBytes(StandardCharsets.UTF_8); + public static final int PADDING_DELIMITER_LENGTH = DELIMITER.length; + public static final int PADDING_IV_LENGTH = 16; + public static final int PADDING_PART_LENGTH = 4; + public static final int PADDING_SIZE_LENGTH = 8; + public static final int PADDING_VERSION_LENGTH = 2; + + private Constants() { + throw new AssertionError("Cannot instantiate utility constructor"); + } +} diff --git a/src/main/java/org/gaul/s3proxy/crypto/Decryption.java b/src/main/java/org/gaul/s3proxy/crypto/Decryption.java new file mode 100644 index 00000000..4d38b79e --- /dev/null +++ b/src/main/java/org/gaul/s3proxy/crypto/Decryption.java @@ -0,0 +1,319 @@ +/* + * Copyright 2014-2021 Andrew Gaul + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gaul.s3proxy.crypto; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Map; +import java.util.TreeMap; + +import javax.annotation.concurrent.ThreadSafe; +import javax.crypto.SecretKey; +import javax.crypto.spec.SecretKeySpec; + +import org.apache.commons.io.IOUtils; +import org.apache.commons.io.input.BoundedInputStream; +import org.jclouds.blobstore.BlobStore; +import org.jclouds.blobstore.domain.Blob; +import org.jclouds.blobstore.domain.BlobMetadata; +import org.jclouds.blobstore.options.GetOptions; + +@ThreadSafe +public class Decryption { + private final SecretKey encryptionKey; + private TreeMap partList; + private long outputOffset; + private long outputLength; + private boolean skipFirstBlock; + private long unencryptedSize; + private long encryptedSize; + private long startAt; + private int skipParts; + private long skipPartBytes; + private boolean isEncrypted; + + public Decryption(SecretKeySpec key, BlobStore blobStore, + BlobMetadata meta, + long offset, long length) throws IOException { + encryptionKey = key; + outputLength = length; + isEncrypted = true; + + // if blob does not exist or size is smaller than the part padding + // then the file is considered not encrypted + if (meta == null || meta.getSize() <= 64) { + blobIsNotEncrypted(offset); + return; + } + + // get the 64 byte of part padding from the end of the blob + GetOptions options = new GetOptions(); + options.range(meta.getSize() - Constants.PADDING_BLOCK_SIZE, + meta.getSize()); + Blob blob = + blobStore.getBlob(meta.getContainer(), meta.getName(), options); + + // read the padding structure + PartPadding lastPartPadding = PartPadding.readPartPaddingFromBlob(blob); + if (!Arrays.equals( + lastPartPadding.getDelimiter().getBytes(StandardCharsets.UTF_8), + Constants.DELIMITER)) { + blobIsNotEncrypted(offset); + return; + } + + partList = new TreeMap<>(); + + // detect multipart + if (lastPartPadding.getPart() > 1 && + meta.getSize() > + (lastPartPadding.getSize() + Constants.PADDING_BLOCK_SIZE)) { + unencryptedSize = lastPartPadding.getSize(); + encryptedSize = + lastPartPadding.getSize() + Constants.PADDING_BLOCK_SIZE; + + // note that parts are in reversed order + int part = 1; + + // add the last part to the list + partList.put(part, lastPartPadding); + + // loop part by part from end to the beginning + // to build a list of all blocks + while (encryptedSize < meta.getSize()) { + // get the next block + // rewind by the current encrypted block size + // minus the encryption padding + options = new GetOptions(); + long startAt = (meta.getSize() - encryptedSize) - + Constants.PADDING_BLOCK_SIZE; + long endAt = meta.getSize() - encryptedSize - 1; + options.range(startAt, endAt); + blob = blobStore.getBlob(meta.getContainer(), meta.getName(), + options); + + part++; + + // read the padding structure + PartPadding partPadding = + PartPadding.readPartPaddingFromBlob(blob); + + // add the part to the list + this.partList.put(part, partPadding); + + // update the encrypted size + encryptedSize = encryptedSize + + (partPadding.getSize() + Constants.PADDING_BLOCK_SIZE); + unencryptedSize = this.unencryptedSize + partPadding.getSize(); + } + + } else { + // add the single part to the list + partList.put(1, lastPartPadding); + + // update the unencrypted size + unencryptedSize = meta.getSize() - Constants.PADDING_BLOCK_SIZE; + + // update the encrypted size + encryptedSize = meta.getSize(); + } + + // calculate the offset + calculateOffset(offset); + + // if there is a offset and a length set the output length + if (offset > 0 && length == 0) { + outputLength = unencryptedSize - offset; + } + } + + private void blobIsNotEncrypted(long offset) { + isEncrypted = false; + startAt = offset; + } + + // calculate the tail bytes we need to read + // because we know the unencryptedSize we can return startAt offset + public final long calculateTail() { + long offset = unencryptedSize - outputLength; + calculateOffset(offset); + + return startAt; + } + + public final long getEncryptedSize() { + return encryptedSize; + } + + public final long calculateEndAt(long endAt) { + // need to have always one more + endAt++; + + // handle multipart + if (partList.size() > 1) { + long plaintextSize = 0; + + // always skip 1 part at the end + int partCounter = 1; + + // we need the map in reversed order + for (Map.Entry part : partList.descendingMap() + .entrySet()) { + + // check the parts that are between offset and end + plaintextSize = plaintextSize + part.getValue().getSize(); + if (endAt > plaintextSize) { + partCounter++; + } else { + break; + } + } + + // add the paddings of all parts + endAt = endAt + ((long) Constants.PADDING_BLOCK_SIZE * partCounter); + } else { + // we need to read one AES block more in AES CFB mode + long rest = endAt % Constants.AES_BLOCK_SIZE; + if (rest > 0) { + endAt = endAt + Constants.AES_BLOCK_SIZE; + } + } + + return endAt; + } + + // open the streams and pipes + public final InputStream openStream(InputStream is) throws IOException { + // if the blob is not encrypted return the unencrypted stream + if (!isEncrypted) { + return is; + } + + // pass input stream through decryption + InputStream dis = new DecryptionInputStream(is, encryptionKey, partList, + skipParts, skipPartBytes); + + // skip some bytes if necessary + long offset = outputOffset; + if (this.skipFirstBlock) { + offset = offset + Constants.AES_BLOCK_SIZE; + } + IOUtils.skipFully(dis, offset); + + // trim the stream to a specific length if needed + return new BoundedInputStream(dis, outputLength); + } + + private void calculateOffset(long offset) { + startAt = 0; + skipParts = 0; + + // handle multipart + if (partList.size() > 1) { + + // init counters + long plaintextSize = 0; + long encryptedSize = 0; + long partOffset; + long partStartAt = 0; + + // we need the map in reversed order + for (Map.Entry part : partList.descendingMap() + .entrySet()) { + + // compute the plaintext size of the current part + plaintextSize = plaintextSize + part.getValue().getSize(); + + // check if the offset is located in another part + if (offset > plaintextSize) { + // compute the encrypted size of the skipped part + encryptedSize = encryptedSize + part.getValue().getSize() + + Constants.PADDING_BLOCK_SIZE; + + // compute offset in this part + partOffset = offset - plaintextSize; + + // skip the first block in CFB mode + skipFirstBlock = partOffset >= 16; + + // compute the offset of the output + outputOffset = partOffset % Constants.AES_BLOCK_SIZE; + + // skip this part + skipParts++; + + // we always need to read one previous AES block in CFB mode + // if we read from offset + if (partOffset > Constants.AES_BLOCK_SIZE) { + long rest = partOffset % Constants.AES_BLOCK_SIZE; + partStartAt = + (partOffset - Constants.AES_BLOCK_SIZE) - rest; + } else { + partStartAt = 0; + } + } else { + // start at a specific byte position + // while respecting other parts + startAt = encryptedSize + partStartAt; + + // skip part bytes if we are not starting + // from the beginning of a part + skipPartBytes = partStartAt; + break; + } + } + } + + // handle single part + if (skipParts == 0) { + // skip the first block in CFB mode + skipFirstBlock = offset >= 16; + + // compute the offset of the output + outputOffset = offset % Constants.AES_BLOCK_SIZE; + + // we always need to read one previous AES block in CFB mode + // if we read from offset + if (offset > Constants.AES_BLOCK_SIZE) { + long rest = offset % Constants.AES_BLOCK_SIZE; + startAt = (offset - Constants.AES_BLOCK_SIZE) - rest; + } + + // skip part bytes if we are not starting + // from the beginning of a part + skipPartBytes = startAt; + } + } + + public final long getStartAt() { + return startAt; + } + + public final boolean isEncrypted() { + return isEncrypted; + } + + public final long getContentLength() { + if (outputLength > 0) { + return outputLength; + } else { + return unencryptedSize; + } + } +} diff --git a/src/main/java/org/gaul/s3proxy/crypto/DecryptionInputStream.java b/src/main/java/org/gaul/s3proxy/crypto/DecryptionInputStream.java new file mode 100644 index 00000000..c5b40efa --- /dev/null +++ b/src/main/java/org/gaul/s3proxy/crypto/DecryptionInputStream.java @@ -0,0 +1,382 @@ +/* + * Copyright 2014-2021 Andrew Gaul + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gaul.s3proxy.crypto; + +import java.io.FilterInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.TreeMap; + +import javax.annotation.concurrent.ThreadSafe; +import javax.crypto.Cipher; +import javax.crypto.SecretKey; +import javax.crypto.ShortBufferException; + +import org.apache.commons.io.IOUtils; + +@ThreadSafe +public class DecryptionInputStream extends FilterInputStream { + + // the cipher engine to use to process stream data + private final Cipher cipher; + + // the secret key + private final SecretKey key; + + // the list of parts we expect in the stream + private final TreeMap parts; + + /* the buffer holding data that have been read in from the + underlying stream, but have not been processed by the cipher + engine. */ + private final byte[] ibuffer = new byte[4096]; + + // having reached the end of the underlying input stream + private boolean done; + + /* the buffer holding data that have been processed by the cipher + engine, but have not been read out */ + private byte[] obuffer; + // the offset pointing to the next "new" byte + private int ostart; + // the offset pointing to the last "new" byte + private int ofinish; + // stream status + private boolean closed; + // the current part + private int part; + // the remaining bytes of the current part + private long partBytesRemain; + + /** + * Constructs a CipherInputStream from an InputStream and a + * Cipher. + *
Note: if the specified input stream or cipher is + * null, a NullPointerException may be thrown later when + * they are used. + * + * @param is the to-be-processed input stream + * @param key the decryption key + * @param parts the list of parts + * @param skipParts the amount of parts to skip + * @param skipPartBytes the amount of part bytes to skip + * @throws IOException if cipher fails + */ + public DecryptionInputStream(InputStream is, SecretKey key, + TreeMap parts, int skipParts, + long skipPartBytes) + throws IOException { + super(is); + in = is; + this.parts = parts; + this.key = key; + + PartPadding partPadding = parts.get(parts.size() - skipParts); + + try { + // init the cipher + cipher = Cipher.getInstance(Constants.AES_CIPHER); + cipher.init(Cipher.DECRYPT_MODE, key, partPadding.getIv()); + } catch (Exception e) { + throw new IOException(e); + } + + // set the part to begin with + part = parts.size() - skipParts; + + // adjust part size due to offset + partBytesRemain = parts.get(part).getSize() - skipPartBytes; + } + + /** + * Ensure obuffer is big enough for the next update or doFinal + * operation, given the input length inLen (in bytes) + * The ostart and ofinish indices are reset to 0. + * + * @param inLen the input length (in bytes) + */ + private void ensureCapacity(int inLen) { + int minLen = cipher.getOutputSize(inLen); + if (obuffer == null || obuffer.length < minLen) { + obuffer = new byte[minLen]; + } + ostart = 0; + ofinish = 0; + } + + /** + * Private convenience function, read in data from the underlying + * input stream and process them with cipher. This method is called + * when the processed bytes inside obuffer has been exhausted. + *

+ * Entry condition: ostart = ofinish + *

+ * Exit condition: ostart = 0 AND ostart <= ofinish + *

+ * return (ofinish-ostart) (we have this many bytes for you) + * return 0 (no data now, but could have more later) + * return -1 (absolutely no more data) + *

+ * Note: Exceptions are only thrown after the stream is completely read. + * For AEAD ciphers a read() of any length will internally cause the + * whole stream to be read fully and verify the authentication tag before + * returning decrypted data or exceptions. + */ + private int getMoreData() throws IOException { + if (done) { + return -1; + } + + int readLimit = ibuffer.length; + if (partBytesRemain < ibuffer.length) { + readLimit = (int) partBytesRemain; + } + + int readin; + if (partBytesRemain == 0) { + readin = -1; + } else { + readin = in.read(ibuffer, 0, readLimit); + } + + if (readin == -1) { + ensureCapacity(0); + try { + ofinish = cipher.doFinal(obuffer, 0); + } catch (Exception e) { + throw new IOException(e); + } + + int nextPart = part - 1; + if (parts.containsKey(nextPart)) { + // reset cipher + PartPadding partPadding = parts.get(nextPart); + try { + cipher.init(Cipher.DECRYPT_MODE, key, partPadding.getIv()); + } catch (Exception e) { + throw new IOException(e); + } + + // update to the next part + part = nextPart; + + // update the remaining bytes of the next part + partBytesRemain = parts.get(nextPart).getSize(); + + IOUtils.skip(in, Constants.PADDING_BLOCK_SIZE); + + return ofinish; + } else { + done = true; + if (ofinish == 0) { + return -1; + } else { + return ofinish; + } + } + } + ensureCapacity(readin); + try { + ofinish = cipher.update(ibuffer, 0, readin, obuffer, ostart); + } catch (ShortBufferException e) { + throw new IOException(e); + } + + partBytesRemain = partBytesRemain - readin; + return ofinish; + } + + /** + * Reads the next byte of data from this input stream. The value + * byte is returned as an int in the range + * 0 to 255. If no byte is available + * because the end of the stream has been reached, the value + * -1 is returned. This method blocks until input data + * is available, the end of the stream is detected, or an exception + * is thrown. + * + * @return the next byte of data, or -1 if the end of the + * stream is reached. + * @throws IOException if an I/O error occurs. + */ + @Override + public final int read() throws IOException { + if (ostart >= ofinish) { + // we loop for new data as the spec says we are blocking + int i = 0; + while (i == 0) { + i = getMoreData(); + } + if (i == -1) { + return -1; + } + } + return (int) obuffer[ostart++] & 0xff; + } + + /** + * Reads up to b.length bytes of data from this input + * stream into an array of bytes. + *

+ * The read method of InputStream calls + * the read method of three arguments with the arguments + * b, 0, and b.length. + * + * @param b the buffer into which the data is read. + * @return the total number of bytes read into the buffer, or + * -1 is there is no more data because the end of + * the stream has been reached. + * @throws IOException if an I/O error occurs. + * @see java.io.InputStream#read(byte[], int, int) + */ + @Override + public final int read(byte[] b) throws IOException { + return read(b, 0, b.length); + } + + /** + * Reads up to len bytes of data from this input stream + * into an array of bytes. This method blocks until some input is + * available. If the first argument is null, up to + * len bytes are read and discarded. + * + * @param b the buffer into which the data is read. + * @param off the start offset in the destination array + * buf + * @param len the maximum number of bytes read. + * @return the total number of bytes read into the buffer, or + * -1 if there is no more data because the end of + * the stream has been reached. + * @throws IOException if an I/O error occurs. + * @see java.io.InputStream#read() + */ + @Override + public final int read(byte[] b, int off, int len) throws IOException { + if (ostart >= ofinish) { + // we loop for new data as the spec says we are blocking + int i = 0; + while (i == 0) { + i = getMoreData(); + } + if (i == -1) { + return -1; + } + } + if (len <= 0) { + return 0; + } + int available = ofinish - ostart; + if (len < available) { + available = len; + } + if (b != null) { + System.arraycopy(obuffer, ostart, b, off, available); + } + ostart = ostart + available; + return available; + } + + /** + * Skips n bytes of input from the bytes that can be read + * from this input stream without blocking. + * + *

Fewer bytes than requested might be skipped. + * The actual number of bytes skipped is equal to n or + * the result of a call to + * {@link #available() available}, + * whichever is smaller. + * If n is less than zero, no bytes are skipped. + * + *

The actual number of bytes skipped is returned. + * + * @param n the number of bytes to be skipped. + * @return the actual number of bytes skipped. + * @throws IOException if an I/O error occurs. + */ + @Override + public final long skip(long n) throws IOException { + int available = ofinish - ostart; + if (n > available) { + n = available; + } + if (n < 0) { + return 0; + } + ostart += n; + return n; + } + + /** + * Returns the number of bytes that can be read from this input + * stream without blocking. The available method of + * InputStream returns 0. This method + * should be overridden by subclasses. + * + * @return the number of bytes that can be read from this input stream + * without blocking. + */ + @Override + public final int available() { + return ofinish - ostart; + } + + /** + * Closes this input stream and releases any system resources + * associated with the stream. + *

+ * The close method of CipherInputStream + * calls the close method of its underlying input + * stream. + * + * @throws IOException if an I/O error occurs. + */ + @Override + public final void close() throws IOException { + if (closed) { + return; + } + closed = true; + in.close(); + + // Throw away the unprocessed data and throw no crypto exceptions. + // AEAD ciphers are fully readed before closing. Any authentication + // exceptions would occur while reading. + if (!done) { + ensureCapacity(0); + try { + cipher.doFinal(obuffer, 0); + } catch (Exception e) { + // Catch exceptions as the rest of the stream is unused. + } + } + obuffer = null; + } + + /** + * Tests if this input stream supports the mark + * and reset methods, which it does not. + * + * @return false, since this class does not support the + * mark and reset methods. + * @see java.io.InputStream#mark(int) + * @see java.io.InputStream#reset() + */ + @Override + public final boolean markSupported() { + return false; + } +} diff --git a/src/main/java/org/gaul/s3proxy/crypto/Encryption.java b/src/main/java/org/gaul/s3proxy/crypto/Encryption.java new file mode 100644 index 00000000..3aafa3e3 --- /dev/null +++ b/src/main/java/org/gaul/s3proxy/crypto/Encryption.java @@ -0,0 +1,56 @@ +/* + * Copyright 2014-2021 Andrew Gaul + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gaul.s3proxy.crypto; + +import java.io.IOException; +import java.io.InputStream; +import java.security.SecureRandom; + +import javax.annotation.concurrent.ThreadSafe; +import javax.crypto.Cipher; +import javax.crypto.CipherInputStream; +import javax.crypto.spec.IvParameterSpec; +import javax.crypto.spec.SecretKeySpec; + +@ThreadSafe +public class Encryption { + private final InputStream cis; + private final IvParameterSpec iv; + private final int part; + + public Encryption(SecretKeySpec key, InputStream isRaw, int partNumber) + throws Exception { + iv = generateIV(); + + Cipher cipher = Cipher.getInstance(Constants.AES_CIPHER); + cipher.init(Cipher.ENCRYPT_MODE, key, iv); + cis = new CipherInputStream(isRaw, cipher); + part = partNumber; + } + + public final InputStream openStream() throws IOException { + return new EncryptionInputStream(cis, part, iv); + } + + private IvParameterSpec generateIV() { + byte[] iv = new byte[Constants.AES_BLOCK_SIZE]; + SecureRandom randomSecureRandom = new SecureRandom(); + randomSecureRandom.nextBytes(iv); + + return new IvParameterSpec(iv); + } +} diff --git a/src/main/java/org/gaul/s3proxy/crypto/EncryptionInputStream.java b/src/main/java/org/gaul/s3proxy/crypto/EncryptionInputStream.java new file mode 100644 index 00000000..7a67eecb --- /dev/null +++ b/src/main/java/org/gaul/s3proxy/crypto/EncryptionInputStream.java @@ -0,0 +1,126 @@ +/* + * Copyright 2014-2021 Andrew Gaul + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gaul.s3proxy.crypto; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; + +import javax.crypto.spec.IvParameterSpec; + +public class EncryptionInputStream extends InputStream { + + private final int part; + private final IvParameterSpec iv; + private boolean hasPadding; + private long size; + private InputStream in; + + public EncryptionInputStream(InputStream in, int part, + IvParameterSpec iv) { + this.part = part; + this.iv = iv; + this.in = in; + } + + // Padding (64 byte) + // Delimiter (8 byte) + // IV (16 byte) + // Part (4 byte) + // Size (8 byte) + // Version (2 byte) + // Reserved (26 byte) + final void padding() throws IOException { + if (in != null) { + in.close(); + } + + if (!hasPadding) { + ByteBuffer bb = ByteBuffer.allocate(Constants.PADDING_BLOCK_SIZE); + bb.put(Constants.DELIMITER); + bb.put(iv.getIV()); + bb.putInt(part); + bb.putLong(size); + bb.putShort(Constants.VERSION); + + in = new ByteArrayInputStream(bb.array()); + hasPadding = true; + } else { + in = null; + } + } + + public final int available() throws IOException { + if (in == null) { + return 0; // no way to signal EOF from available() + } + return in.available(); + } + + public final int read() throws IOException { + while (in != null) { + int c = in.read(); + if (c != -1) { + size++; + return c; + } + padding(); + } + return -1; + } + + public final int read(byte[] b, int off, int len) throws IOException { + if (in == null) { + return -1; + } else if (b == null) { + throw new NullPointerException(); + } else if (off < 0 || len < 0 || len > b.length - off) { + throw new IndexOutOfBoundsException(); + } else if (len == 0) { + return 0; + } + do { + int n = in.read(b, off, len); + if (n > 0) { + size = size + n; + return n; + } + padding(); + } while (in != null); + return -1; + } + + public final void close() throws IOException { + IOException ioe = null; + while (in != null) { + try { + in.close(); + } catch (IOException e) { + if (ioe == null) { + ioe = e; + } else { + ioe.addSuppressed(e); + } + } + padding(); + } + if (ioe != null) { + throw ioe; + } + } +} diff --git a/src/main/java/org/gaul/s3proxy/crypto/PartPadding.java b/src/main/java/org/gaul/s3proxy/crypto/PartPadding.java new file mode 100644 index 00000000..983ae272 --- /dev/null +++ b/src/main/java/org/gaul/s3proxy/crypto/PartPadding.java @@ -0,0 +1,88 @@ +/* + * Copyright 2014-2021 Andrew Gaul + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gaul.s3proxy.crypto; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; + +import javax.crypto.spec.IvParameterSpec; + +import org.apache.commons.io.IOUtils; +import org.jclouds.blobstore.domain.Blob; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class PartPadding { + private static final Logger logger = + LoggerFactory.getLogger(PartPadding.class); + + private String delimiter; + private IvParameterSpec iv; + private int part; + private long size; + private short version; + + public static PartPadding readPartPaddingFromBlob(Blob blob) + throws IOException { + PartPadding partPadding = new PartPadding(); + + InputStream is = blob.getPayload().openStream(); + + byte[] paddingBytes = IOUtils.toByteArray(is); + ByteBuffer bb = ByteBuffer.wrap(paddingBytes); + + byte[] delimiterBytes = new byte[Constants.PADDING_DELIMITER_LENGTH]; + bb.get(delimiterBytes); + partPadding.delimiter = + new String(delimiterBytes, StandardCharsets.UTF_8); + + byte[] ivBytes = new byte[Constants.PADDING_IV_LENGTH]; + bb.get(ivBytes); + partPadding.iv = new IvParameterSpec(ivBytes); + + partPadding.part = bb.getInt(); + partPadding.size = bb.getLong(); + partPadding.version = bb.getShort(); + + logger.debug("delimiter {}", partPadding.delimiter); + logger.debug("iv {}", Arrays.toString(ivBytes)); + logger.debug("part {}", partPadding.part); + logger.debug("size {}", partPadding.size); + logger.debug("version {}", partPadding.version); + + return partPadding; + } + + public final String getDelimiter() { + return delimiter; + } + + public final IvParameterSpec getIv() { + return iv; + } + + public final int getPart() { + return part; + } + + public final long getSize() { + return size; + } +} diff --git a/src/main/resources/run-docker-container.sh b/src/main/resources/run-docker-container.sh index a1623479..3b3cb718 100755 --- a/src/main/resources/run-docker-container.sh +++ b/src/main/resources/run-docker-container.sh @@ -12,6 +12,9 @@ exec java \ -Ds3proxy.cors-allow-methods="${S3PROXY_CORS_ALLOW_METHODS}" \ -Ds3proxy.cors-allow-headers="${S3PROXY_CORS_ALLOW_HEADERS}" \ -Ds3proxy.ignore-unknown-headers="${S3PROXY_IGNORE_UNKNOWN_HEADERS}" \ + -Ds3proxy.encrypted-blobstore="${S3PROXY_ENCRYPTED_BLOBSTORE}" \ + -Ds3proxy.encrypted-blobstore-password="${S3PROXY_ENCRYPTED_BLOBSTORE_PASSWORD}" \ + -Ds3proxy.encrypted-blobstore-salt="${S3PROXY_ENCRYPTED_BLOBSTORE_SALT}" \ -Djclouds.provider="${JCLOUDS_PROVIDER}" \ -Djclouds.identity="${JCLOUDS_IDENTITY}" \ -Djclouds.credential="${JCLOUDS_CREDENTIAL}" \ diff --git a/src/test/java/org/gaul/s3proxy/EncryptedBlobStoreLiveTest.java b/src/test/java/org/gaul/s3proxy/EncryptedBlobStoreLiveTest.java new file mode 100644 index 00000000..da328a9d --- /dev/null +++ b/src/test/java/org/gaul/s3proxy/EncryptedBlobStoreLiveTest.java @@ -0,0 +1,282 @@ +/* + * Copyright 2014-2021 Andrew Gaul + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gaul.s3proxy; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.Properties; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.stream.Collectors; + +import com.google.common.collect.ImmutableMap; +import com.google.common.io.ByteSource; +import com.google.common.util.concurrent.Uninterruptibles; + +import org.assertj.core.api.Fail; +import org.gaul.s3proxy.crypto.Constants; +import org.jclouds.aws.AWSResponseException; +import org.jclouds.blobstore.BlobStoreContext; +import org.jclouds.blobstore.domain.PageSet; +import org.jclouds.blobstore.domain.StorageMetadata; +import org.jclouds.blobstore.options.ListContainerOptions; +import org.jclouds.http.options.GetOptions; +import org.jclouds.io.Payload; +import org.jclouds.io.Payloads; +import org.jclouds.s3.S3ClientLiveTest; +import org.jclouds.s3.domain.ListMultipartUploadsResponse; +import org.jclouds.s3.domain.ObjectMetadataBuilder; +import org.jclouds.s3.domain.S3Object; +import org.jclouds.s3.reference.S3Constants; +import org.testng.SkipException; +import org.testng.annotations.AfterClass; +import org.testng.annotations.Test; + +@SuppressWarnings("UnstableApiUsage") +@Test(testName = "EncryptedBlobStoreLiveTest") +public final class EncryptedBlobStoreLiveTest extends S3ClientLiveTest { + private static final int AWAIT_CONSISTENCY_TIMEOUT_SECONDS = + Integer.parseInt( + System.getProperty( + "test.blobstore.await-consistency-timeout-seconds", + "0")); + private static final long MINIMUM_MULTIPART_SIZE = 5 * 1024 * 1024; + + private S3Proxy s3Proxy; + private BlobStoreContext context; + + @AfterClass + public void tearDown() throws Exception { + s3Proxy.stop(); + context.close(); + } + + @Override + protected void awaitConsistency() { + Uninterruptibles.sleepUninterruptibly( + AWAIT_CONSISTENCY_TIMEOUT_SECONDS, TimeUnit.SECONDS); + } + + @Override + protected Properties setupProperties() { + TestUtils.S3ProxyLaunchInfo info; + try { + info = TestUtils.startS3Proxy("s3proxy-encryption.conf"); + s3Proxy = info.getS3Proxy(); + context = info.getBlobStore().getContext(); + } catch (Exception e) { + throw new RuntimeException(e); + } + Properties props = super.setupProperties(); + props.setProperty(org.jclouds.Constants.PROPERTY_IDENTITY, + info.getS3Identity()); + props.setProperty(org.jclouds.Constants.PROPERTY_CREDENTIAL, + info.getS3Credential()); + props.setProperty(org.jclouds.Constants.PROPERTY_ENDPOINT, + info.getEndpoint().toString() + info.getServicePath()); + props.setProperty(org.jclouds.Constants.PROPERTY_STRIP_EXPECT_HEADER, + "true"); + props.setProperty(S3Constants.PROPERTY_S3_SERVICE_PATH, + info.getServicePath()); + endpoint = info.getEndpoint().toString() + info.getServicePath(); + return props; + } + + @Test + public void testOneCharAndCopy() throws InterruptedException { + String blobName = TestUtils.createRandomBlobName(); + String containerName = this.getContainerName(); + + S3Object object = this.getApi().newS3Object(); + object.getMetadata().setKey(blobName); + object.setPayload("1"); + this.getApi().putObject(containerName, object); + + object = this.getApi().getObject(containerName, blobName); + assertThat(object.getMetadata().getContentMetadata() + .getContentLength()).isEqualTo(1L); + + PageSet + list = view.getBlobStore().list(containerName); + assertThat(list).hasSize(1); + + StorageMetadata md = list.iterator().next(); + assertThat(md.getName()).isEqualTo(blobName); + assertThat(md.getSize()).isEqualTo(1L); + + this.getApi().copyObject(containerName, blobName, containerName, + blobName + "-copy"); + list = view.getBlobStore().list(containerName); + assertThat(list).hasSize(2); + + for (StorageMetadata sm : list) { + assertThat(sm.getSize()).isEqualTo(1L); + assertThat(sm.getName()).doesNotContain( + Constants.S3_ENC_SUFFIX); + } + + ListContainerOptions lco = new ListContainerOptions(); + lco.maxResults(1); + list = view.getBlobStore().list(containerName, lco); + assertThat(list).hasSize(1); + assertThat(list.getNextMarker()).doesNotContain( + Constants.S3_ENC_SUFFIX); + } + + @Test + public void testPartialContent() throws InterruptedException, IOException { + String blobName = TestUtils.createRandomBlobName(); + String containerName = this.getContainerName(); + String content = "123456789A123456789B123456"; + + S3Object object = this.getApi().newS3Object(); + object.getMetadata().setKey(blobName); + object.setPayload(content); + this.getApi().putObject(containerName, object); + + // get only 20 bytes + GetOptions options = new GetOptions(); + options.range(0, 19); + object = this.getApi().getObject(containerName, blobName, options); + + InputStreamReader r = + new InputStreamReader(object.getPayload().openStream()); + BufferedReader reader = new BufferedReader(r); + String partialContent = reader.lines().collect(Collectors.joining()); + assertThat(partialContent).isEqualTo(content.substring(0, 20)); + } + + @Test + public void testMultipart() throws InterruptedException, IOException { + String blobName = TestUtils.createRandomBlobName(); + String containerName = this.getContainerName(); + + // 15mb of data + ByteSource byteSource = TestUtils.randomByteSource().slice( + 0, MINIMUM_MULTIPART_SIZE * 3); + + // first 2 parts with 6mb and last part with 3mb + long partSize = 6 * 1024 * 1024; + long lastPartSize = 3 * 1024 * 1024; + ByteSource byteSource1 = byteSource.slice(0, partSize); + ByteSource byteSource2 = byteSource.slice(partSize, partSize); + ByteSource byteSource3 = byteSource.slice(partSize * 2, + lastPartSize); + + String uploadId = this.getApi().initiateMultipartUpload(containerName, + ObjectMetadataBuilder.create().key(blobName).build()); + assertThat(this.getApi().listMultipartPartsFull(containerName, + blobName, uploadId)).isEmpty(); + + ListMultipartUploadsResponse + response = this.getApi() + .listMultipartUploads(containerName, null, null, null, blobName, + null); + assertThat(response.uploads()).hasSize(1); + + Payload part1 = + Payloads.newInputStreamPayload(byteSource1.openStream()); + part1.getContentMetadata().setContentLength(byteSource1.size()); + Payload part2 = + Payloads.newInputStreamPayload(byteSource2.openStream()); + part2.getContentMetadata().setContentLength(byteSource2.size()); + Payload part3 = + Payloads.newInputStreamPayload(byteSource3.openStream()); + part3.getContentMetadata().setContentLength(byteSource3.size()); + + String eTagOf1 = this.getApi() + .uploadPart(containerName, blobName, 1, uploadId, part1); + String eTagOf2 = this.getApi() + .uploadPart(containerName, blobName, 2, uploadId, part2); + String eTagOf3 = this.getApi() + .uploadPart(containerName, blobName, 3, uploadId, part3); + + this.getApi().completeMultipartUpload(containerName, blobName, uploadId, + ImmutableMap.of(1, eTagOf1, 2, eTagOf2, 3, eTagOf3)); + S3Object object = this.getApi().getObject(containerName, blobName); + + try (InputStream actual = object.getPayload().openStream(); + InputStream expected = byteSource.openStream()) { + assertThat(actual).hasContentEqualTo(expected); + } + + // get a 5mb slice that overlap parts + long partialStart = 5 * 1024 * 1024; + ByteSource partialContent = + byteSource.slice(partialStart, partialStart); + + GetOptions options = new GetOptions(); + options.range(partialStart, (partialStart * 2) - 1); + object = this.getApi().getObject(containerName, blobName, options); + + try (InputStream actual = object.getPayload().openStream(); + InputStream expected = partialContent.openStream()) { + assertThat(actual).hasContentEqualTo(expected); + } + } + + @Override + public void testMultipartSynchronously() { + throw new SkipException("list multipart synchronously not supported"); + } + + @Override + @Test + public void testUpdateObjectACL() throws InterruptedException, + ExecutionException, TimeoutException, IOException { + try { + super.testUpdateObjectACL(); + Fail.failBecauseExceptionWasNotThrown(AWSResponseException.class); + } catch (AWSResponseException are) { + assertThat(are.getError().getCode()).isEqualTo("NotImplemented"); + throw new SkipException("XML ACLs not supported", are); + } + } + + @Override + @Test + public void testPublicWriteOnObject() throws InterruptedException, + ExecutionException, TimeoutException, IOException { + try { + super.testPublicWriteOnObject(); + Fail.failBecauseExceptionWasNotThrown(AWSResponseException.class); + } catch (AWSResponseException are) { + assertThat(are.getError().getCode()).isEqualTo("NotImplemented"); + throw new SkipException("public-read-write-acl not supported", are); + } + } + + @Override + public void testCopyCannedAccessPolicyPublic() { + throw new SkipException("blob access control not supported"); + } + + @Override + public void testPutCannedAccessPolicyPublic() { + throw new SkipException("blob access control not supported"); + } + + @Override + public void testUpdateObjectCannedACL() { + throw new SkipException("blob access control not supported"); + } +} diff --git a/src/test/java/org/gaul/s3proxy/EncryptedBlobStoreTest.java b/src/test/java/org/gaul/s3proxy/EncryptedBlobStoreTest.java new file mode 100644 index 00000000..f9fc3a6d --- /dev/null +++ b/src/test/java/org/gaul/s3proxy/EncryptedBlobStoreTest.java @@ -0,0 +1,835 @@ +/* + * Copyright 2014-2021 Andrew Gaul + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.gaul.s3proxy; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.io.BufferedReader; +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.Random; +import java.util.stream.Collectors; + +import com.google.common.collect.ImmutableList; +import com.google.inject.Module; + +import org.gaul.s3proxy.crypto.Constants; +import org.jclouds.ContextBuilder; +import org.jclouds.blobstore.BlobStore; +import org.jclouds.blobstore.BlobStoreContext; +import org.jclouds.blobstore.domain.Blob; +import org.jclouds.blobstore.domain.BlobAccess; +import org.jclouds.blobstore.domain.BlobMetadata; +import org.jclouds.blobstore.domain.MultipartPart; +import org.jclouds.blobstore.domain.MultipartUpload; +import org.jclouds.blobstore.domain.PageSet; +import org.jclouds.blobstore.domain.StorageMetadata; +import org.jclouds.blobstore.domain.StorageType; +import org.jclouds.blobstore.options.CopyOptions; +import org.jclouds.blobstore.options.GetOptions; +import org.jclouds.blobstore.options.ListContainerOptions; +import org.jclouds.blobstore.options.PutOptions; +import org.jclouds.io.Payload; +import org.jclouds.io.Payloads; +import org.jclouds.logging.slf4j.config.SLF4JLoggingModule; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@SuppressWarnings("UnstableApiUsage") +public final class EncryptedBlobStoreTest { + private static final Logger logger = + LoggerFactory.getLogger(EncryptedBlobStoreTest.class); + + private BlobStoreContext context; + private BlobStore blobStore; + private String containerName; + private BlobStore encryptedBlobStore; + + private static Blob makeBlob(BlobStore blobStore, String blobName, + InputStream is, long contentLength) { + + return blobStore.blobBuilder(blobName) + .payload(is) + .contentLength(contentLength) + .build(); + } + + private static Blob makeBlob(BlobStore blobStore, String blobName, + byte[] payload, long contentLength) { + + return blobStore.blobBuilder(blobName) + .payload(payload) + .contentLength(contentLength) + .build(); + } + + private static Blob makeBlobWithContentType(BlobStore blobStore, + String blobName, + long contentLength, + InputStream is, + String contentType) { + + return blobStore.blobBuilder(blobName) + .payload(is) + .contentLength(contentLength) + .contentType(contentType) + .build(); + } + + @Before + public void setUp() throws Exception { + String password = "Password1234567!"; + String salt = "12345678"; + + containerName = TestUtils.createRandomContainerName(); + + //noinspection UnstableApiUsage + context = ContextBuilder + .newBuilder("transient") + .credentials("identity", "credential") + .modules(ImmutableList.of(new SLF4JLoggingModule())) + .build(BlobStoreContext.class); + blobStore = context.getBlobStore(); + blobStore.createContainerInLocation(null, containerName); + + Properties properties = new Properties(); + properties.put(S3ProxyConstants.PROPERTY_ENCRYPTED_BLOBSTORE, "true"); + properties.put(S3ProxyConstants.PROPERTY_ENCRYPTED_BLOBSTORE_PASSWORD, + password); + properties.put(S3ProxyConstants.PROPERTY_ENCRYPTED_BLOBSTORE_SALT, + salt); + + encryptedBlobStore = + EncryptedBlobStore.newEncryptedBlobStore(blobStore, properties); + } + + @After + public void tearDown() throws Exception { + if (context != null) { + blobStore.deleteContainer(containerName); + context.close(); + } + } + + @Test + public void testBlobNotExists() { + + String blobName = TestUtils.createRandomBlobName(); + Blob blob = encryptedBlobStore.getBlob(containerName, blobName); + assertThat(blob).isNull(); + + blob = encryptedBlobStore.getBlob(containerName, blobName, + new GetOptions()); + assertThat(blob).isNull(); + } + + @Test + public void testBlobNotEncrypted() throws Exception { + + String[] tests = new String[] { + "1", // only 1 char + "123456789A12345", // lower then the AES block + "123456789A1234567", // one byte bigger then the AES block + "123456789A123456123456789B123456123456789C" + + "1234123456789A123456123456789B123456123456789C1234" + }; + + Map contentLengths = new HashMap<>(); + for (String content : tests) { + String blobName = TestUtils.createRandomBlobName(); + + InputStream is = new ByteArrayInputStream( + content.getBytes(StandardCharsets.UTF_8)); + contentLengths.put(blobName, (long) content.length()); + Blob blob = makeBlob(blobStore, blobName, is, content.length()); + blobStore.putBlob(containerName, blob); + blob = encryptedBlobStore.getBlob(containerName, blobName); + + InputStream blobIs = blob.getPayload().openStream(); + InputStreamReader r = new InputStreamReader(blobIs); + BufferedReader reader = new BufferedReader(r); + String plaintext = reader.lines().collect(Collectors.joining()); + logger.debug("plaintext {}", plaintext); + + assertThat(content).isEqualTo(plaintext); + + GetOptions options = new GetOptions(); + blob = encryptedBlobStore.getBlob(containerName, blobName, options); + + blobIs = blob.getPayload().openStream(); + r = new InputStreamReader(blobIs); + reader = new BufferedReader(r); + plaintext = reader.lines().collect(Collectors.joining()); + logger.debug("plaintext {} with empty options ", plaintext); + + assertThat(content).isEqualTo(plaintext); + } + + PageSet blobs = + encryptedBlobStore.list(containerName, new ListContainerOptions()); + for (StorageMetadata blob : blobs) { + assertThat(blob.getSize()).isEqualTo( + contentLengths.get(blob.getName())); + } + + blobs = encryptedBlobStore.list(); + StorageMetadata metadata = blobs.iterator().next(); + assertThat(StorageType.CONTAINER).isEqualTo(metadata.getType()); + } + + @Test + public void testListEncrypted() { + String[] contents = new String[] { + "1", // only 1 char + "123456789A12345", // lower then the AES block + "123456789A1234567", // one byte bigger then the AES block + "123456789A123456123456789B123456123456789C1234" + }; + + Map contentLengths = new HashMap<>(); + for (String content : contents) { + String blobName = TestUtils.createRandomBlobName(); + + InputStream is = new ByteArrayInputStream( + content.getBytes(StandardCharsets.UTF_8)); + contentLengths.put(blobName, (long) content.length()); + Blob blob = + makeBlob(encryptedBlobStore, blobName, is, content.length()); + encryptedBlobStore.putBlob(containerName, blob); + } + + PageSet blobs = + encryptedBlobStore.list(containerName); + for (StorageMetadata blob : blobs) { + assertThat(blob.getSize()).isEqualTo( + contentLengths.get(blob.getName())); + } + + blobs = + encryptedBlobStore.list(containerName, new ListContainerOptions()); + for (StorageMetadata blob : blobs) { + assertThat(blob.getSize()).isEqualTo( + contentLengths.get(blob.getName())); + encryptedBlobStore.removeBlob(containerName, blob.getName()); + } + + blobs = + encryptedBlobStore.list(containerName, new ListContainerOptions()); + assertThat(blobs.size()).isEqualTo(0); + } + + @Test + public void testListEncryptedMultipart() { + + String blobName = TestUtils.createRandomBlobName(); + + String[] contentParts = new String[] { + "123456789A123456123456789B123456123456789C1234", + "123456789D123456123456789E123456123456789F123456", + "123456789G123456123456789H123456123456789I123" + }; + + String content = contentParts[0] + contentParts[1] + contentParts[2]; + BlobMetadata blobMetadata = makeBlob(encryptedBlobStore, blobName, + content.getBytes(StandardCharsets.UTF_8), + content.length()).getMetadata(); + + MultipartUpload mpu = + encryptedBlobStore.initiateMultipartUpload(containerName, + blobMetadata, new PutOptions()); + + Payload payload1 = Payloads.newByteArrayPayload( + contentParts[0].getBytes(StandardCharsets.UTF_8)); + Payload payload2 = Payloads.newByteArrayPayload( + contentParts[1].getBytes(StandardCharsets.UTF_8)); + Payload payload3 = Payloads.newByteArrayPayload( + contentParts[2].getBytes(StandardCharsets.UTF_8)); + + encryptedBlobStore.uploadMultipartPart(mpu, 1, payload1); + encryptedBlobStore.uploadMultipartPart(mpu, 2, payload2); + encryptedBlobStore.uploadMultipartPart(mpu, 3, payload3); + + List parts = encryptedBlobStore.listMultipartUpload(mpu); + + int index = 0; + for (MultipartPart part : parts) { + assertThat((long) contentParts[index].length()).isEqualTo( + part.partSize()); + index++; + } + + encryptedBlobStore.completeMultipartUpload(mpu, parts); + + PageSet blobs = + encryptedBlobStore.list(containerName); + StorageMetadata metadata = blobs.iterator().next(); + assertThat((long) content.length()).isEqualTo(metadata.getSize()); + + ListContainerOptions options = new ListContainerOptions(); + blobs = encryptedBlobStore.list(containerName, options.withDetails()); + metadata = blobs.iterator().next(); + assertThat((long) content.length()).isEqualTo(metadata.getSize()); + + blobs = encryptedBlobStore.list(); + metadata = blobs.iterator().next(); + assertThat(StorageType.CONTAINER).isEqualTo(metadata.getType()); + + List singleList = new ArrayList<>(); + singleList.add(blobName); + encryptedBlobStore.removeBlobs(containerName, singleList); + blobs = encryptedBlobStore.list(containerName); + assertThat(blobs.size()).isEqualTo(0); + } + + @Test + public void testBlobNotEncryptedRanges() throws Exception { + + for (int run = 0; run < 100; run++) { + String[] tests = new String[] { + "123456789A12345", // lower then the AES block + "123456789A1234567", // one byte bigger then the AES block + "123456789A123456123456789B123456123456789C" + + "1234123456789A123456123456789B123456123456789C1234" + }; + + for (String content : tests) { + String blobName = TestUtils.createRandomBlobName(); + Random rand = new Random(); + + InputStream is = new ByteArrayInputStream( + content.getBytes(StandardCharsets.UTF_8)); + Blob blob = makeBlob(blobStore, blobName, is, content.length()); + blobStore.putBlob(containerName, blob); + + GetOptions options = new GetOptions(); + int offset = rand.nextInt(content.length() - 1); + logger.debug("content {} with offset {}", content, offset); + + options.startAt(offset); + blob = encryptedBlobStore.getBlob(containerName, blobName, + options); + + InputStream blobIs = blob.getPayload().openStream(); + InputStreamReader r = new InputStreamReader(blobIs); + BufferedReader reader = new BufferedReader(r); + String plaintext = reader.lines().collect(Collectors.joining()); + logger.debug("plaintext {} with offset {}", plaintext, offset); + + assertThat(plaintext).isEqualTo(content.substring(offset)); + + options = new GetOptions(); + int tail = rand.nextInt(content.length()); + if (tail == 0) { + tail++; + } + logger.debug("content {} with tail {}", content, tail); + + options.tail(tail); + blob = encryptedBlobStore.getBlob(containerName, blobName, + options); + + blobIs = blob.getPayload().openStream(); + r = new InputStreamReader(blobIs); + reader = new BufferedReader(r); + plaintext = reader.lines().collect(Collectors.joining()); + logger.debug("plaintext {} with tail {}", plaintext, tail); + + assertThat(plaintext).isEqualTo( + content.substring(content.length() - tail)); + + options = new GetOptions(); + offset = 1; + int end = content.length() - 2; + logger.debug("content {} with range {}-{}", content, offset, + end); + + options.range(offset, end); + blob = encryptedBlobStore.getBlob(containerName, blobName, + options); + + blobIs = blob.getPayload().openStream(); + r = new InputStreamReader(blobIs); + reader = new BufferedReader(r); + plaintext = reader.lines().collect(Collectors.joining()); + logger.debug("plaintext {} with range {}-{}", plaintext, offset, + end); + + assertThat(plaintext).isEqualTo( + content.substring(offset, end + 1)); + } + } + } + + @Test + public void testEncryptContent() throws Exception { + String[] tests = new String[] { + "1", // only 1 char + "123456789A12345", // lower then the AES block + "123456789A1234567", // one byte bigger then the AES block + "123456789A123456123456789B123456123456789C1234" + }; + + for (String content : tests) { + String blobName = TestUtils.createRandomBlobName(); + String contentType = "plain/text"; + + InputStream is = new ByteArrayInputStream( + content.getBytes(StandardCharsets.UTF_8)); + Blob blob = makeBlobWithContentType(encryptedBlobStore, blobName, + content.length(), is, contentType); + encryptedBlobStore.putBlob(containerName, blob); + + blob = encryptedBlobStore.getBlob(containerName, blobName); + + InputStream blobIs = blob.getPayload().openStream(); + InputStreamReader r = new InputStreamReader(blobIs); + BufferedReader reader = new BufferedReader(r); + String plaintext = reader.lines().collect(Collectors.joining()); + logger.debug("plaintext {}", plaintext); + + assertThat(plaintext).isEqualTo(content); + + blob = blobStore.getBlob(containerName, + blobName + Constants.S3_ENC_SUFFIX); + blobIs = blob.getPayload().openStream(); + r = new InputStreamReader(blobIs); + reader = new BufferedReader(r); + String encrypted = reader.lines().collect(Collectors.joining()); + logger.debug("encrypted {}", encrypted); + + assertThat(content).isNotEqualTo(encrypted); + + assertThat(encryptedBlobStore.blobExists(containerName, + blobName)).isTrue(); + + BlobAccess access = + encryptedBlobStore.getBlobAccess(containerName, blobName); + assertThat(access).isEqualTo(BlobAccess.PRIVATE); + + encryptedBlobStore.setBlobAccess(containerName, blobName, + BlobAccess.PUBLIC_READ); + access = encryptedBlobStore.getBlobAccess(containerName, blobName); + assertThat(access).isEqualTo(BlobAccess.PUBLIC_READ); + } + } + + @Test + public void testEncryptContentWithOptions() throws Exception { + String[] tests = new String[] { + "1", // only 1 char + "123456789A12345", // lower then the AES block + "123456789A1234567", // one byte bigger then the AES block + "123456789A123456123456789B123456123456789C1234" + }; + + for (String content : tests) { + String blobName = TestUtils.createRandomBlobName(); + String contentType = "plain/text; charset=utf-8"; + + InputStream is = new ByteArrayInputStream( + content.getBytes(StandardCharsets.UTF_8)); + Blob blob = makeBlobWithContentType(encryptedBlobStore, blobName, + content.length(), is, contentType); + PutOptions options = new PutOptions(); + encryptedBlobStore.putBlob(containerName, blob, options); + + blob = encryptedBlobStore.getBlob(containerName, blobName); + + InputStream blobIs = blob.getPayload().openStream(); + InputStreamReader r = new InputStreamReader(blobIs); + BufferedReader reader = new BufferedReader(r); + String plaintext = reader.lines().collect(Collectors.joining()); + logger.debug("plaintext {}", plaintext); + + assertThat(content).isEqualTo(plaintext); + + blob = blobStore.getBlob(containerName, + blobName + Constants.S3_ENC_SUFFIX); + blobIs = blob.getPayload().openStream(); + r = new InputStreamReader(blobIs); + reader = new BufferedReader(r); + String encrypted = reader.lines().collect(Collectors.joining()); + logger.debug("encrypted {}", encrypted); + + assertThat(content).isNotEqualTo(encrypted); + + BlobMetadata metadata = + encryptedBlobStore.blobMetadata(containerName, + blobName + Constants.S3_ENC_SUFFIX); + assertThat(contentType).isEqualTo( + metadata.getContentMetadata().getContentType()); + + encryptedBlobStore.copyBlob(containerName, blobName, + containerName, blobName + "-copy", CopyOptions.NONE); + + blob = blobStore.getBlob(containerName, + blobName + Constants.S3_ENC_SUFFIX); + blobIs = blob.getPayload().openStream(); + r = new InputStreamReader(blobIs); + reader = new BufferedReader(r); + encrypted = reader.lines().collect(Collectors.joining()); + logger.debug("encrypted {}", encrypted); + + assertThat(content).isNotEqualTo(encrypted); + + blob = + encryptedBlobStore.getBlob(containerName, blobName + "-copy"); + blobIs = blob.getPayload().openStream(); + r = new InputStreamReader(blobIs); + reader = new BufferedReader(r); + plaintext = reader.lines().collect(Collectors.joining()); + logger.debug("plaintext {}", plaintext); + + assertThat(content).isEqualTo(plaintext); + } + } + + @Test + public void testEncryptMultipartContent() throws Exception { + String blobName = TestUtils.createRandomBlobName(); + + String content1 = "123456789A123456123456789B123456123456789C1234"; + String content2 = "123456789D123456123456789E123456123456789F123456"; + String content3 = "123456789G123456123456789H123456123456789I123"; + + String content = content1 + content2 + content3; + BlobMetadata blobMetadata = makeBlob(encryptedBlobStore, blobName, + content.getBytes(StandardCharsets.UTF_8), + content.length()).getMetadata(); + MultipartUpload mpu = + encryptedBlobStore.initiateMultipartUpload(containerName, + blobMetadata, new PutOptions()); + + Payload payload1 = Payloads.newByteArrayPayload( + content1.getBytes(StandardCharsets.UTF_8)); + Payload payload2 = Payloads.newByteArrayPayload( + content2.getBytes(StandardCharsets.UTF_8)); + Payload payload3 = Payloads.newByteArrayPayload( + content3.getBytes(StandardCharsets.UTF_8)); + + encryptedBlobStore.uploadMultipartPart(mpu, 1, payload1); + encryptedBlobStore.uploadMultipartPart(mpu, 2, payload2); + encryptedBlobStore.uploadMultipartPart(mpu, 3, payload3); + + List mpus = + encryptedBlobStore.listMultipartUploads(containerName); + assertThat(mpus.size()).isEqualTo(1); + + List parts = encryptedBlobStore.listMultipartUpload(mpu); + assertThat(mpus.get(0).id()).isEqualTo(mpu.id()); + + encryptedBlobStore.completeMultipartUpload(mpu, parts); + Blob blob = encryptedBlobStore.getBlob(containerName, blobName); + + InputStream blobIs = blob.getPayload().openStream(); + InputStreamReader r = new InputStreamReader(blobIs); + BufferedReader reader = new BufferedReader(r); + String plaintext = reader.lines().collect(Collectors.joining()); + logger.debug("plaintext {}", plaintext); + assertThat(plaintext).isEqualTo(content); + + blob = blobStore.getBlob(containerName, + blobName + Constants.S3_ENC_SUFFIX); + blobIs = blob.getPayload().openStream(); + r = new InputStreamReader(blobIs); + reader = new BufferedReader(r); + String encrypted = reader.lines().collect(Collectors.joining()); + logger.debug("encrypted {}", encrypted); + + assertThat(content).isNotEqualTo(encrypted); + } + + @Test + public void testReadPartial() throws Exception { + + for (int offset = 0; offset < 60; offset++) { + logger.debug("Test with offset {}", offset); + + String blobName = TestUtils.createRandomBlobName(); + String content = + "123456789A123456123456789B123456123456789" + + "C123456789D123456789E12345"; + InputStream is = new ByteArrayInputStream( + content.getBytes(StandardCharsets.UTF_8)); + + Blob blob = + makeBlob(encryptedBlobStore, blobName, is, content.length()); + encryptedBlobStore.putBlob(containerName, blob); + + GetOptions options = new GetOptions(); + options.startAt(offset); + blob = encryptedBlobStore.getBlob(containerName, blobName, options); + + InputStream blobIs = blob.getPayload().openStream(); + InputStreamReader r = new InputStreamReader(blobIs); + BufferedReader reader = new BufferedReader(r); + String plaintext = reader.lines().collect(Collectors.joining()); + logger.debug("plaintext {}", plaintext); + + assertThat(plaintext).isEqualTo(content.substring(offset)); + } + } + + @Test + public void testReadTail() throws Exception { + + for (int length = 1; length < 60; length++) { + logger.debug("Test with length {}", length); + + String blobName = TestUtils.createRandomBlobName(); + + String content = + "123456789A123456123456789B123456123456789C" + + "123456789D123456789E12345"; + InputStream is = new ByteArrayInputStream( + content.getBytes(StandardCharsets.UTF_8)); + + Blob blob = + makeBlob(encryptedBlobStore, blobName, is, content.length()); + encryptedBlobStore.putBlob(containerName, blob); + + GetOptions options = new GetOptions(); + options.tail(length); + blob = encryptedBlobStore.getBlob(containerName, blobName, options); + + InputStream blobIs = blob.getPayload().openStream(); + InputStreamReader r = new InputStreamReader(blobIs); + BufferedReader reader = new BufferedReader(r); + String plaintext = reader.lines().collect(Collectors.joining()); + logger.debug("plaintext {}", plaintext); + + assertThat(plaintext).isEqualTo( + content.substring(content.length() - length)); + } + } + + @Test + public void testReadPartialWithRandomEnd() throws Exception { + + for (int run = 0; run < 100; run++) { + for (int offset = 0; offset < 50; offset++) { + Random rand = new Random(); + int end = offset + rand.nextInt(20) + 2; + int size = end - offset + 1; + + logger.debug("Test with offset {} and end {} size {}", + offset, end, size); + + String blobName = TestUtils.createRandomBlobName(); + + String content = + "123456789A123456-123456789B123456-123456789C123456-" + + "123456789D123456-123456789E123456"; + InputStream is = new ByteArrayInputStream( + content.getBytes(StandardCharsets.UTF_8)); + + Blob blob = makeBlob(encryptedBlobStore, blobName, is, + content.length()); + encryptedBlobStore.putBlob(containerName, blob); + + GetOptions options = new GetOptions(); + options.range(offset, end); + blob = encryptedBlobStore.getBlob(containerName, blobName, + options); + + InputStream blobIs = blob.getPayload().openStream(); + InputStreamReader r = new InputStreamReader(blobIs); + BufferedReader reader = new BufferedReader(r); + String plaintext = reader.lines().collect(Collectors.joining()); + logger.debug("plaintext {}", plaintext); + + assertThat(plaintext).hasSize(size); + assertThat(plaintext).isEqualTo( + content.substring(offset, end + 1)); + } + } + } + + @Test + public void testMultipartReadPartial() throws Exception { + + for (int offset = 0; offset < 130; offset++) { + logger.debug("Test with offset {}", offset); + + String blobName = TestUtils.createRandomBlobName(); + + String content1 = "PART1-789A123456123456789B123456123456789C1234"; + String content2 = + "PART2-789D123456123456789E123456123456789F123456"; + String content3 = "PART3-789G123456123456789H123456123456789I123"; + String content = content1 + content2 + content3; + + BlobMetadata blobMetadata = makeBlob(encryptedBlobStore, blobName, + content.getBytes(StandardCharsets.UTF_8), + content.length()).getMetadata(); + MultipartUpload mpu = + encryptedBlobStore.initiateMultipartUpload(containerName, + blobMetadata, new PutOptions()); + + Payload payload1 = Payloads.newByteArrayPayload( + content1.getBytes(StandardCharsets.UTF_8)); + Payload payload2 = Payloads.newByteArrayPayload( + content2.getBytes(StandardCharsets.UTF_8)); + Payload payload3 = Payloads.newByteArrayPayload( + content3.getBytes(StandardCharsets.UTF_8)); + + encryptedBlobStore.uploadMultipartPart(mpu, 1, payload1); + encryptedBlobStore.uploadMultipartPart(mpu, 2, payload2); + encryptedBlobStore.uploadMultipartPart(mpu, 3, payload3); + + List parts = + encryptedBlobStore.listMultipartUpload(mpu); + encryptedBlobStore.completeMultipartUpload(mpu, parts); + + GetOptions options = new GetOptions(); + options.startAt(offset); + Blob blob = + encryptedBlobStore.getBlob(containerName, blobName, options); + + InputStream blobIs = blob.getPayload().openStream(); + InputStreamReader r = new InputStreamReader(blobIs); + BufferedReader reader = new BufferedReader(r); + String plaintext = reader.lines().collect(Collectors.joining()); + logger.debug("plaintext {}", plaintext); + + assertThat(plaintext).isEqualTo(content.substring(offset)); + } + } + + @Test + public void testMultipartReadTail() throws Exception { + + for (int length = 1; length < 130; length++) { + logger.debug("Test with length {}", length); + + String blobName = TestUtils.createRandomBlobName(); + + String content1 = "PART1-789A123456123456789B123456123456789C1234"; + String content2 = + "PART2-789D123456123456789E123456123456789F123456"; + String content3 = "PART3-789G123456123456789H123456123456789I123"; + String content = content1 + content2 + content3; + BlobMetadata blobMetadata = makeBlob(encryptedBlobStore, blobName, + content.getBytes(StandardCharsets.UTF_8), + content.length()).getMetadata(); + MultipartUpload mpu = + encryptedBlobStore.initiateMultipartUpload(containerName, + blobMetadata, new PutOptions()); + + Payload payload1 = Payloads.newByteArrayPayload( + content1.getBytes(StandardCharsets.UTF_8)); + Payload payload2 = Payloads.newByteArrayPayload( + content2.getBytes(StandardCharsets.UTF_8)); + Payload payload3 = Payloads.newByteArrayPayload( + content3.getBytes(StandardCharsets.UTF_8)); + + encryptedBlobStore.uploadMultipartPart(mpu, 1, payload1); + encryptedBlobStore.uploadMultipartPart(mpu, 2, payload2); + encryptedBlobStore.uploadMultipartPart(mpu, 3, payload3); + + List parts = + encryptedBlobStore.listMultipartUpload(mpu); + encryptedBlobStore.completeMultipartUpload(mpu, parts); + + GetOptions options = new GetOptions(); + options.tail(length); + Blob blob = + encryptedBlobStore.getBlob(containerName, blobName, options); + + InputStream blobIs = blob.getPayload().openStream(); + InputStreamReader r = new InputStreamReader(blobIs); + BufferedReader reader = new BufferedReader(r); + String plaintext = reader.lines().collect(Collectors.joining()); + logger.debug("plaintext {}", plaintext); + + assertThat(plaintext).isEqualTo( + content.substring(content.length() - length)); + } + } + + @Test + public void testMultipartReadPartialWithRandomEnd() throws Exception { + + for (int run = 0; run < 100; run++) { + // total len = 139 + for (int offset = 0; offset < 70; offset++) { + Random rand = new Random(); + int end = offset + rand.nextInt(60) + 2; + int size = end - offset + 1; + logger.debug("Test with offset {} and end {} size {}", + offset, end, size); + + String blobName = TestUtils.createRandomBlobName(); + + String content1 = + "PART1-789A123456123456789B123456123456789C1234"; + String content2 = + "PART2-789D123456123456789E123456123456789F123456"; + String content3 = + "PART3-789G123456123456789H123456123456789I123"; + + String content = content1 + content2 + content3; + BlobMetadata blobMetadata = + makeBlob(encryptedBlobStore, blobName, + content.getBytes(StandardCharsets.UTF_8), + content.length()).getMetadata(); + MultipartUpload mpu = + encryptedBlobStore.initiateMultipartUpload(containerName, + blobMetadata, new PutOptions()); + + Payload payload1 = Payloads.newByteArrayPayload( + content1.getBytes(StandardCharsets.UTF_8)); + Payload payload2 = Payloads.newByteArrayPayload( + content2.getBytes(StandardCharsets.UTF_8)); + Payload payload3 = Payloads.newByteArrayPayload( + content3.getBytes(StandardCharsets.UTF_8)); + + encryptedBlobStore.uploadMultipartPart(mpu, 1, payload1); + encryptedBlobStore.uploadMultipartPart(mpu, 2, payload2); + encryptedBlobStore.uploadMultipartPart(mpu, 3, payload3); + + List parts = + encryptedBlobStore.listMultipartUpload(mpu); + encryptedBlobStore.completeMultipartUpload(mpu, parts); + + GetOptions options = new GetOptions(); + options.range(offset, end); + Blob blob = encryptedBlobStore.getBlob(containerName, blobName, + options); + + InputStream blobIs = blob.getPayload().openStream(); + InputStreamReader r = new InputStreamReader(blobIs); + BufferedReader reader = new BufferedReader(r); + String plaintext = reader.lines().collect(Collectors.joining()); + logger.debug("plaintext {}", plaintext); + + assertThat(plaintext).isEqualTo( + content.substring(offset, end + 1)); + } + } + } +} diff --git a/src/test/java/org/gaul/s3proxy/TestUtils.java b/src/test/java/org/gaul/s3proxy/TestUtils.java index e76e9bb7..e64d0387 100644 --- a/src/test/java/org/gaul/s3proxy/TestUtils.java +++ b/src/test/java/org/gaul/s3proxy/TestUtils.java @@ -188,6 +188,14 @@ static S3ProxyLaunchInfo startS3Proxy(String configFile) throws Exception { BlobStoreContext context = builder.build(BlobStoreContext.class); info.blobStore = context.getBlobStore(); + String encrypted = info.getProperties().getProperty( + S3ProxyConstants.PROPERTY_ENCRYPTED_BLOBSTORE); + if (encrypted != null && encrypted.equals("true")) { + info.blobStore = + EncryptedBlobStore.newEncryptedBlobStore(info.blobStore, + info.getProperties()); + } + S3Proxy.Builder s3ProxyBuilder = S3Proxy.Builder.fromProperties( info.getProperties()); s3ProxyBuilder.blobStore(info.blobStore); diff --git a/src/test/resources/s3proxy-encryption.conf b/src/test/resources/s3proxy-encryption.conf new file mode 100644 index 00000000..7d4b83f1 --- /dev/null +++ b/src/test/resources/s3proxy-encryption.conf @@ -0,0 +1,20 @@ +s3proxy.endpoint=http://127.0.0.1:0 +s3proxy.secure-endpoint=https://127.0.0.1:0 +#s3proxy.service-path=s3proxy +# authorization must be aws-v2, aws-v4, aws-v2-or-v4, or none +s3proxy.authorization=aws-v2-or-v4 +s3proxy.identity=local-identity +s3proxy.credential=local-credential +s3proxy.keystore-path=keystore.jks +s3proxy.keystore-password=password + +jclouds.provider=transient +jclouds.identity=remote-identity +jclouds.credential=remote-credential +# endpoint is optional for some providers +#jclouds.endpoint=http://127.0.0.1:8081 +jclouds.filesystem.basedir=/tmp/blobstore + +s3proxy.encrypted-blobstore=true +s3proxy.encrypted-blobstore-password=1234567890123456 +s3proxy.encrypted-blobstore-salt=12345678 From 2e17734dc12fc1fd80ff9e2f703ba2fd57407b63 Mon Sep 17 00:00:00 2001 From: "xavier.gourmandin1" Date: Thu, 31 Mar 2022 16:58:29 +0200 Subject: [PATCH 04/55] fix: #412 fix NPE when creating S3ProxyExtention or S3ProxyRule with Auth type = NONE --- src/main/java/org/gaul/s3proxy/S3Proxy.java | 6 ++++-- .../java/org/gaul/s3proxy/junit/S3ProxyJunitCore.java | 9 ++++++--- .../org/gaul/s3proxy/junit/S3ProxyExtensionTest.java | 10 ++++++++++ .../java/org/gaul/s3proxy/junit/S3ProxyRuleTest.java | 9 +++++++++ 4 files changed, 29 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/gaul/s3proxy/S3Proxy.java b/src/main/java/org/gaul/s3proxy/S3Proxy.java index 6067f7cc..de4533a8 100644 --- a/src/main/java/org/gaul/s3proxy/S3Proxy.java +++ b/src/main/java/org/gaul/s3proxy/S3Proxy.java @@ -316,8 +316,10 @@ public Builder secureEndpoint(URI secureEndpoint) { public Builder awsAuthentication(AuthenticationType authenticationType, String identity, String credential) { this.authenticationType = authenticationType; - this.identity = requireNonNull(identity); - this.credential = requireNonNull(credential); + if (!AuthenticationType.NONE.equals(authenticationType)) { + this.identity = requireNonNull(identity); + this.credential = requireNonNull(credential); + } return this; } diff --git a/src/main/java/org/gaul/s3proxy/junit/S3ProxyJunitCore.java b/src/main/java/org/gaul/s3proxy/junit/S3ProxyJunitCore.java index 2003da8a..90816735 100644 --- a/src/main/java/org/gaul/s3proxy/junit/S3ProxyJunitCore.java +++ b/src/main/java/org/gaul/s3proxy/junit/S3ProxyJunitCore.java @@ -112,10 +112,13 @@ public S3ProxyJunitCore build() { throw new RuntimeException("Unable to initialize Blob Store", e); } - blobStoreContext = ContextBuilder.newBuilder( + ContextBuilder blobStoreContextBuilder = ContextBuilder.newBuilder( builder.blobStoreProvider) - .credentials(accessKey, secretKey) - .overrides(properties).build(BlobStoreContext.class); + .overrides(properties); + if (!AuthenticationType.NONE.equals(builder.authType)) { + blobStoreContextBuilder = blobStoreContextBuilder.credentials(accessKey, secretKey); + } + blobStoreContext = blobStoreContextBuilder.build(BlobStoreContext.class); S3Proxy.Builder s3ProxyBuilder = S3Proxy.builder() .blobStore(blobStoreContext.getBlobStore()) diff --git a/src/test/java/org/gaul/s3proxy/junit/S3ProxyExtensionTest.java b/src/test/java/org/gaul/s3proxy/junit/S3ProxyExtensionTest.java index cc118876..d9f78517 100644 --- a/src/test/java/org/gaul/s3proxy/junit/S3ProxyExtensionTest.java +++ b/src/test/java/org/gaul/s3proxy/junit/S3ProxyExtensionTest.java @@ -93,4 +93,14 @@ public final void doesBucketExistV2() { // Issue #299 assertThat(s3Client.doesBucketExistV2("nonexistingbucket")).isFalse(); } + + @Test + public final void createExtentionWithoutCredentials() { + S3ProxyExtension extension = S3ProxyExtension + .builder() + .build(); + assertThat(extension.getAccessKey()).isNull(); + assertThat(extension.getSecretKey()).isNull(); + assertThat(extension.getUri()).isNull(); + } } diff --git a/src/test/java/org/gaul/s3proxy/junit/S3ProxyRuleTest.java b/src/test/java/org/gaul/s3proxy/junit/S3ProxyRuleTest.java index 664738e1..03795d38 100644 --- a/src/test/java/org/gaul/s3proxy/junit/S3ProxyRuleTest.java +++ b/src/test/java/org/gaul/s3proxy/junit/S3ProxyRuleTest.java @@ -95,4 +95,13 @@ public final void doesBucketExistV2() { assertThat(s3Client.doesBucketExistV2("nonexistingbucket")).isFalse(); } + @Test + public final void createExtentionWithoutCredentials() { + S3ProxyRule extension = S3ProxyRule + .builder() + .build(); + assertThat(extension.getAccessKey()).isNull(); + assertThat(extension.getSecretKey()).isNull(); + assertThat(extension.getUri()).isNull(); + } } From f1ba56a1a1679fa869c3d4a824020f8b0240a9ea Mon Sep 17 00:00:00 2001 From: Andrew Gaul Date: Fri, 1 Apr 2022 21:33:47 +0900 Subject: [PATCH 05/55] Upgrade to logback 1.2.11 Release notes: https://logback.qos.ch/news.html --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index b0bb74d7..4cbef07d 100644 --- a/pom.xml +++ b/pom.xml @@ -419,7 +419,7 @@ ch.qos.logback logback-classic - 1.2.3 + 1.2.11 javax.xml.bind From 04e35b1671ad9355f81788de6cba9f06f8dd517f Mon Sep 17 00:00:00 2001 From: Andrew Gaul Date: Fri, 1 Apr 2022 21:34:38 +0900 Subject: [PATCH 06/55] Upgrade to jackson-dataformat-xml 2.13.2 Release notes: https://github.com/FasterXML/jackson-dataformat-xml/blob/2.13/release-notes/VERSION-2.x --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 4cbef07d..360a418f 100644 --- a/pom.xml +++ b/pom.xml @@ -443,7 +443,7 @@ com.fasterxml.jackson.dataformat jackson-dataformat-xml - 2.12.3 + 2.13.2 com.github.spotbugs From 058a55d931a86aa9e717fa8ef5b4aa43eb2be1d1 Mon Sep 17 00:00:00 2001 From: Andrew Gaul Date: Fri, 1 Apr 2022 21:36:29 +0900 Subject: [PATCH 07/55] Upgrade to Jetty 9.4.45.v20220203 Release notes: https://www.eclipse.org/lists/jetty-announce/msg00164.html --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 360a418f..15877639 100644 --- a/pom.xml +++ b/pom.xml @@ -491,7 +491,7 @@ org.eclipse.jetty jetty-servlet - 9.4.41.v20210516 + 9.4.45.v20220203 org.slf4j From 508a43e1109803a9eb7b59526448c1990b95c0c7 Mon Sep 17 00:00:00 2001 From: Andrew Gaul Date: Fri, 1 Apr 2022 21:45:23 +0900 Subject: [PATCH 08/55] Upgrade slf4j 1.7.36 Release notes: https://www.slf4j.org/news.html --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 15877639..739afa91 100644 --- a/pom.xml +++ b/pom.xml @@ -383,7 +383,7 @@ UTF-8 1.8 2.5.0 - 1.7.28 + 1.7.36 ${project.groupId}.shaded 2.22.2 From 11dc9d3121480c390cb0ef36f31efc3770e8b1e1 Mon Sep 17 00:00:00 2001 From: Andrew Gaul Date: Fri, 1 Apr 2022 21:46:42 +0900 Subject: [PATCH 09/55] Upgrade to Modernizer Maven Plugin 2.4.0 Release notes: https://github.com/gaul/modernizer-maven-plugin/releases/tag/modernizer-maven-plugin-2.4.0 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 739afa91..dac1ef4d 100644 --- a/pom.xml +++ b/pom.xml @@ -351,7 +351,7 @@ org.gaul modernizer-maven-plugin - 2.3.0 + 2.4.0 modernizer From b5d090d9f8c077e9171d7f1f97392f904064c4d1 Mon Sep 17 00:00:00 2001 From: Andrew Gaul Date: Sun, 3 Apr 2022 19:54:43 +0900 Subject: [PATCH 10/55] S3Proxy 2.0.0 release Fixes #410. --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index dac1ef4d..42ab1120 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ org.gaul s3proxy - 1.9.1-SNAPSHOT + 2.0.0 jar S3Proxy From 71541ac16708f0755a917b29cc1ebcd56e044871 Mon Sep 17 00:00:00 2001 From: Andrew Gaul Date: Sun, 3 Apr 2022 20:05:29 +0900 Subject: [PATCH 11/55] Bump to 2.0.0-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 42ab1120..ea80da8e 100644 --- a/pom.xml +++ b/pom.xml @@ -10,7 +10,7 @@ org.gaul s3proxy - 2.0.0 + 2.1.0-SNAPSHOT jar S3Proxy From 7344b0c60e47f4805ecb107ff2a12735a3dc3b70 Mon Sep 17 00:00:00 2001 From: Adrian Woodhead Date: Wed, 25 May 2022 23:32:19 +0100 Subject: [PATCH 12/55] Upgrade Jackson to 2.13.3 The following vulnerabilities are fixed with an upgrade: - https://snyk.io/vuln/SNYK-JAVA-COMFASTERXMLJACKSONCORE-2421244 Co-authored-by: snyk-bot --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index ea80da8e..9e662789 100644 --- a/pom.xml +++ b/pom.xml @@ -443,7 +443,7 @@ com.fasterxml.jackson.dataformat jackson-dataformat-xml - 2.13.2 + 2.13.3 com.github.spotbugs From 29723040b5e2d34b00f70686a0d7d2f5c58fd517 Mon Sep 17 00:00:00 2001 From: Andrew Gaul Date: Mon, 20 Jun 2022 22:21:00 +0900 Subject: [PATCH 13/55] Minio removed its cloud gateway --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d06a1f2e..389d4496 100644 --- a/README.md +++ b/README.md @@ -154,12 +154,12 @@ for specific storage backends. * [Apache jclouds](https://jclouds.apache.org/) provides storage backend support for S3Proxy * [Ceph s3-tests](https://github.com/ceph/s3-tests) help maintain and improve compatibility with the S3 API -* [fake-s3](https://github.com/jubos/fake-s3), [gofakes3](https://github.com/johannesboyne/gofakes3), [S3 ninja](https://github.com/scireum/s3ninja), and [s3rver](https://github.com/jamhall/s3rver) provide functionality similar to S3Proxy when using the filesystem backend +* [fake-s3](https://github.com/jubos/fake-s3), [gofakes3](https://github.com/johannesboyne/gofakes3), [minio](https://github.com/minio/minio), [S3 ninja](https://github.com/scireum/s3ninja), and [s3rver](https://github.com/jamhall/s3rver) provide functionality similar to S3Proxy when using the filesystem backend * [GlacierProxy](https://github.com/bouncestorage/glacier-proxy) and [SwiftProxy](https://github.com/bouncestorage/swiftproxy) provide similar functionality for the Amazon Glacier and OpenStack Swift APIs -* [minio](https://github.com/minio/minio) and [Zenko](https://www.zenko.io/) provide similar multi-cloud functionality * [s3mock](https://github.com/findify/s3mock) mocks the S3 API for Java/Scala projects * [sbt-s3](https://github.com/localytics/sbt-s3) runs S3Proxy via the Scala Build Tool * [swift3](https://github.com/openstack/swift3) provides an S3 middleware for OpenStack Swift +* [Zenko](https://www.zenko.io/) provide similar multi-cloud functionality ## License From 1d450fa221b12e161020a79df2020d86090dfc4f Mon Sep 17 00:00:00 2001 From: Christoph Kreutzer Date: Mon, 20 Jun 2022 18:01:15 +0200 Subject: [PATCH 14/55] Docker entrypoint: authentication for Azure Blob When using the s3proxy as a proxy for S3 -> Azure Blob Storage, one needs to set the authentication parameters. When running in a container, these can be exposed via environment variables, but are currently not passed through to the jclouds config. --- src/main/resources/run-docker-container.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/resources/run-docker-container.sh b/src/main/resources/run-docker-container.sh index 3b3cb718..093bc467 100755 --- a/src/main/resources/run-docker-container.sh +++ b/src/main/resources/run-docker-container.sh @@ -25,5 +25,8 @@ exec java \ -Djclouds.keystone.scope="${JCLOUDS_KEYSTONE_SCOPE}" \ -Djclouds.keystone.project-domain-name="${JCLOUDS_KEYSTONE_PROJECT_DOMAIN_NAME}" \ -Djclouds.filesystem.basedir="${JCLOUDS_FILESYSTEM_BASEDIR}" \ + -Djclouds.azureblob.tenantId="${JCLOUDS_AZUREBLOB_TENANTID}" \ + -Djclouds.azureblob.auth="${JCLOUDS_AZUREBLOB_AUTH}" \ + -Djclouds.azureblob.account="${JCLOUDS_AZUREBLOB_ACCOUNT}" \ -jar /opt/s3proxy/s3proxy \ --properties /dev/null From ba0fd6dad7beec69ed822f5981ec0a554bdc6eac Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 15 Jul 2022 21:07:28 +0000 Subject: [PATCH 15/55] Bump aws-java-sdk-s3 from 1.12.63 to 1.12.261 Bumps [aws-java-sdk-s3](https://github.com/aws/aws-sdk-java) from 1.12.63 to 1.12.261. - [Release notes](https://github.com/aws/aws-sdk-java/releases) - [Changelog](https://github.com/aws/aws-sdk-java/blob/master/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-java/compare/1.12.63...1.12.261) --- updated-dependencies: - dependency-name: com.amazonaws:aws-java-sdk-s3 dependency-type: direct:development ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 9e662789..5a6c72a0 100644 --- a/pom.xml +++ b/pom.xml @@ -402,7 +402,7 @@ com.amazonaws aws-java-sdk-s3 - 1.12.63 + 1.12.261 test From d71e1e812ae0213be7eb0aba41dba447732961ca Mon Sep 17 00:00:00 2001 From: Ryan Faircloth Date: Wed, 29 Jun 2022 14:10:04 -0400 Subject: [PATCH 16/55] ci: Enhancements to CI and release fixes #408 fixes #423 This PR makes several changes to the CI* Adds PR lint to ensure conventional commit syntax is used for all PRs Uses semantic-release tool to review commit log on branch and generate version numbers Publishes containers (non pr) to ghcr.io Publishes release containers to hub.docker.com Completes common tags (versions, sha-1 and ref) on all branches Notes Unable to test actually publishing containers to dockerhub however this was taken from a similar working project. --- .github/dependabot.yml | 11 ++ .github/workflows/ci-main.yml | 206 ++++++++++++++++++++++++++++++++ .github/workflows/pr-lint.yml | 17 +++ .github/workflows/run-tests.yml | 54 --------- .releaserc | 46 +++++++ Dockerfile | 15 +-- 6 files changed, 282 insertions(+), 67 deletions(-) create mode 100644 .github/dependabot.yml create mode 100644 .github/workflows/ci-main.yml create mode 100644 .github/workflows/pr-lint.yml delete mode 100644 .github/workflows/run-tests.yml create mode 100644 .releaserc diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..5b063201 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,11 @@ +# To get started with Dependabot version updates, you'll need to specify which +# package ecosystems to update and where the package manifests are located. +# Please see the documentation for all configuration options: +# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates + +version: 2 +updates: + - package-ecosystem: "maven" # See documentation for possible values + directory: "/" # Location of package manifests + schedule: + interval: "daily" diff --git a/.github/workflows/ci-main.yml b/.github/workflows/ci-main.yml new file mode 100644 index 00000000..1925c6eb --- /dev/null +++ b/.github/workflows/ci-main.yml @@ -0,0 +1,206 @@ +name: Main CI +on: + push: + branches: + - "master" + - "develop" + pull_request: + branches: + - "*" +jobs: + meta: + runs-on: ubuntu-latest + outputs: + dockerhub-publish: ${{ steps.dockerhub-publish.outputs.defined }} + registry: ghcr.io/${{ github.repository }}/container:${{ fromJSON(steps.docker_action_meta.outputs.json).labels['org.opencontainers.image.version'] }} + container_tags: ${{ steps.docker_action_meta.outputs.tags }} + container_labels: ${{ steps.docker_action_meta.outputs.labels }} + container_buildtime: ${{ fromJSON(steps.docker_action_meta.outputs.json).labels['org.opencontainers.image.created'] }} + container_version: ${{ fromJSON(steps.docker_action_meta.outputs.json).labels['org.opencontainers.image.version'] }} + container_revision: ${{ fromJSON(steps.docker_action_meta.outputs.json).labels['org.opencontainers.image.revision'] }} + container_base: ${{ fromJSON(steps.docker_action_meta.outputs.json).tags[0] }} + new_release_version: ${{ steps.version.outputs.new_release_version }} + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + submodules: false + persist-credentials: false + - id: dockerhub-publish + if: "${{ env.MY_KEY != '' }}" + run: echo "::set-output name=defined::true" + env: + MY_KEY: ${{ secrets.DOCKER_PASS }} + - uses: actions/setup-node@v2 + with: + node-version: "14" + - name: Semantic Release + id: version + uses: cycjimmy/semantic-release-action@v3.0.0 + with: + semantic_version: 17 + extra_plugins: | + @semantic-release/exec + @semantic-release/git + @google/semantic-release-replace-plugin + conventional-changelog-conventionalcommits + dry_run: true + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Docker meta + id: docker_action_meta + uses: docker/metadata-action@v4.0.1 + with: + images: ghcr.io/${{ github.repository }}/container + flavor: | + latest=false + tags: | + type=sha,format=long + type=sha + type=semver,pattern={{version}},value=${{ steps.version.outputs.new_release_version }} + type=semver,pattern={{major}},value=${{ steps.version.outputs.new_release_version }} + type=semver,pattern={{major}}.{{minor}},value=${{ steps.version.outputs.new_release_version }} + type=ref,event=branch + type=ref,event=pr + type=ref,event=tag + labels: | + org.opencontainers.image.licenses=Apache-2.0 + runTests: + runs-on: ubuntu-latest + needs: [meta] + steps: + - uses: actions/checkout@v2 + with: + submodules: "recursive" + + # These steps are quick and will work or if fail only because of external issues + - uses: actions/setup-java@v2 + with: + distribution: "temurin" + java-version: "11" + cache: "maven" + - uses: actions/setup-python@v4 + with: + python-version: "3.8" + cache: "pip" + + #Run tests + - name: Maven Set version + run: | + mvn versions:set -DnewVersion=${{ needs.meta.outputs.new_release_version }} + - name: Maven Package + run: | + mvn package -DskipTests + - name: Maven Test + run: | + mvn test + - name: Other Test + run: | + ./src/test/resources/run-s3-tests.sh + + #Store the target + - uses: actions/upload-artifact@v2 + with: + name: s3proxy + path: target/s3proxy + - uses: actions/upload-artifact@v2 + with: + name: pom + path: pom.xml + + Containerize: + runs-on: ubuntu-latest + needs: [runTests, meta] + steps: + #Yes we need code + - uses: actions/checkout@v2 + - uses: actions/download-artifact@v2 + with: + name: s3proxy + path: target + - uses: actions/download-artifact@v2 + with: + name: pom + path: . + # These steps are quick and will work or if fail only because of external issues + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Login to DockerHub + uses: docker/login-action@v2.0.0 + if: github.event_name != 'pull_request' && needs.meta.outputs.dockerhub-publish == 'true' + with: + username: ${{ secrets.DOCKER_USER }} + password: ${{ secrets.DOCKER_PASS }} + + - name: Login to DockerHub + uses: docker/login-action@v2.0.0 + if: github.event_name != 'pull_request' + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + #Generate Meta + - name: Build and push + uses: docker/build-push-action@v3 + with: + context: . + platforms: linux/amd64,linux/arm64 + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ needs.meta.outputs.container_base }} + labels: ${{ needs.meta.outputs.container_labels }} + build-args: | + BUILDTIME=${{ needs.meta.outputs.container_buildtime }} + VERSION=${{ needs.meta.outputs.container_version }} + REVISION=${{ needs.meta.outputs.container_revision }} + cache-from: type=registry,ref=${{ needs.meta.outputs.container_base }} + cache-to: type=inline + - uses: actions/setup-node@v2 + with: + node-version: "14" + - name: Semantic Release + if: github.event_name != 'pull_request' && needs.meta.outputs.dockerhub-publish == 'true' + id: version + uses: cycjimmy/semantic-release-action@v3.0.0 + with: + semantic_version: 17 + extra_plugins: | + @semantic-release/exec + @semantic-release/git + conventional-changelog-conventionalcommits + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Setup regctl + if: github.event_name != 'pull_request' && needs.meta.outputs.dockerhub-publish == 'true' + run: | + curl -L https://github.com/regclient/regclient/releases/download/v0.3.5/regctl-linux-amd64 >/tmp/regctl + chmod 755 /tmp/regctl + + - name: Docker meta + if: github.event_name != 'pull_request' && needs.meta.outputs.dockerhub-publish == 'true' + id: docker_action_meta + uses: docker/metadata-action@v4.0.1 + with: + images: andrewgaul/s3proxy + flavor: | + latest=false + tags: | + type=sha + type=semver,pattern={{version}},value=${{ needs.meta.outputs.new_release_version }} + type=semver,pattern={{major}},value=${{ needs.meta.outputs.new_release_version }} + type=semver,pattern={{major}}.{{minor}},value=${{ needs.meta.outputs.new_release_version }} + type=ref,event=branch + type=ref,event=pr + type=ref,event=tag + labels: | + org.opencontainers.image.licenses=Apache-2.0 + - name: Publish to Docker + if: github.event_name != 'pull_request' && needs.meta.outputs.dockerhub-publish == 'true' + run: | + for line in $CONTAINER_DEST_TAGS; do echo working on "$line"; /tmp/regctl image copy $SOURCE_CONTAINER $line; done + env: + SOURCE_CONTAINER: ${{ needs.meta.outputs.new_release_version }} + CONTAINER_DEST_TAGS: ${{ steps.docker_action_meta.outputs.tags }} diff --git a/.github/workflows/pr-lint.yml b/.github/workflows/pr-lint.yml new file mode 100644 index 00000000..ad6df65f --- /dev/null +++ b/.github/workflows/pr-lint.yml @@ -0,0 +1,17 @@ +name: "Lint PR" + +on: + pull_request_target: + types: + - opened + - edited + - synchronize + +jobs: + main: + name: Validate PR title + runs-on: ubuntu-latest + steps: + - uses: amannn/action-semantic-pull-request@v4 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml deleted file mode 100644 index 4ecd7a7c..00000000 --- a/.github/workflows/run-tests.yml +++ /dev/null @@ -1,54 +0,0 @@ -name: Run Tests -on: - push: - branches: - - 'master' - tags: - - '*' - pull_request: - branches: - - '*' -jobs: - runTests: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - with: - submodules: 'recursive' - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - uses: actions/setup-java@v2 - with: - distribution: 'temurin' - java-version: '11' - - name: Install dependencies - run: sudo apt-get install -y libevent-dev python3-pip python3-virtualenv - - name: Run Tests Tests - run: | - mvn package - ./src/test/resources/run-s3-tests.sh - - name: Login to DockerHub - uses: docker/login-action@v1 - if: github.event_name != 'pull_request' - with: - username: ${{ secrets.DOCKER_USER }} - password: ${{ secrets.DOCKER_PASS }} - - name: Docker meta - id: meta - uses: docker/metadata-action@v3 - with: - images: andrewgaul/s3proxy - tags: | - type=sha,event=branch - type=semver,pattern={{version}} - type=semver,pattern={{major}}.{{minor}} - type=semver,pattern={{major}} - - name: Build and push - uses: docker/build-push-action@v2 - with: - context: . - platforms: linux/amd64,linux/arm64 - push: ${{ github.event_name != 'pull_request' }} - tags: ${{ steps.meta.outputs.tags }} diff --git a/.releaserc b/.releaserc new file mode 100644 index 00000000..a05ac080 --- /dev/null +++ b/.releaserc @@ -0,0 +1,46 @@ +{ + "tagFormat": 's3proxy-${version}', + "branches": [ + { + "name": 'master', + prerelease: false + }, + { + "name": 'releases\/+([0-9])?(\.\d+)(\.\d+|z|$)', + prerelease: false + }, + { + "name": 'next', + prerelease: false + }, + { + name: 'next-major', + prerelease: true + }, + { + name: 'develop', + prerelease: true + }, + { + name: 'develop\/.*', + prerelease: true + } + ], + plugins: [ + [ + "@semantic-release/commit-analyzer", + { + "preset": "conventionalcommits", + "presetConfig": "conventional-changelog-conventionalcommits" + } + ], + "@semantic-release/release-notes-generator", + ["@semantic-release/git", { + "assets": [ + "pom.xml", + ], + "message": "chore(release): ${nextRelease.version}\n\n${nextRelease.notes}" + }], + ["@semantic-release/github"], + ] +} \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index fffae63c..4378aae1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,22 +1,11 @@ -# Multistage - Builder -FROM maven:3.6.3-jdk-11-slim as s3proxy-builder -LABEL maintainer="Andrew Gaul " - -WORKDIR /opt/s3proxy -COPY . /opt/s3proxy/ - -RUN mvn package -DskipTests - -# Multistage - Image FROM openjdk:11-jre-slim LABEL maintainer="Andrew Gaul " WORKDIR /opt/s3proxy COPY \ - --from=s3proxy-builder \ - /opt/s3proxy/target/s3proxy \ - /opt/s3proxy/src/main/resources/run-docker-container.sh \ + target/s3proxy \ + src/main/resources/run-docker-container.sh \ /opt/s3proxy/ ENV \ From c38508edf6af3ed85cdb55b2a1e040322707fd07 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 17 Jul 2022 13:37:50 +0000 Subject: [PATCH 17/55] Bump nexus-staging-maven-plugin from 1.6.8 to 1.6.13 Bumps nexus-staging-maven-plugin from 1.6.8 to 1.6.13. --- updated-dependencies: - dependency-name: org.sonatype.plugins:nexus-staging-maven-plugin dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 5a6c72a0..2417b2ab 100644 --- a/pom.xml +++ b/pom.xml @@ -368,7 +368,7 @@ org.sonatype.plugins nexus-staging-maven-plugin - 1.6.8 + 1.6.13 true ossrh From 7dcf0a5af7a22d0b3c407c92e2c8b846b35f2f53 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 17 Jul 2022 13:37:45 +0000 Subject: [PATCH 18/55] Bump junit from 4.13.1 to 4.13.2 Bumps [junit](https://github.com/junit-team/junit4) from 4.13.1 to 4.13.2. - [Release notes](https://github.com/junit-team/junit4/releases) - [Changelog](https://github.com/junit-team/junit4/blob/main/doc/ReleaseNotes4.13.1.md) - [Commits](https://github.com/junit-team/junit4/compare/r4.13.1...r4.13.2) --- updated-dependencies: - dependency-name: junit:junit dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 2417b2ab..c2eecd43 100644 --- a/pom.xml +++ b/pom.xml @@ -429,7 +429,7 @@ junit junit - 4.13.1 + 4.13.2 provided From ed2a046c0926c268059c52d177f0b61cb40fe671 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 17 Jul 2022 13:37:49 +0000 Subject: [PATCH 19/55] Bump maven-clean-plugin from 3.1.0 to 3.2.0 Bumps [maven-clean-plugin](https://github.com/apache/maven-clean-plugin) from 3.1.0 to 3.2.0. - [Release notes](https://github.com/apache/maven-clean-plugin/releases) - [Commits](https://github.com/apache/maven-clean-plugin/compare/maven-clean-plugin-3.1.0...maven-clean-plugin-3.2.0) --- updated-dependencies: - dependency-name: org.apache.maven.plugins:maven-clean-plugin dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index c2eecd43..dc73a560 100644 --- a/pom.xml +++ b/pom.xml @@ -95,7 +95,7 @@ org.apache.maven.plugins maven-clean-plugin - 3.1.0 + 3.2.0 org.apache.maven.plugins From 6fef2aa779808e46d84961386955575160116ed8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 17 Jul 2022 13:37:41 +0000 Subject: [PATCH 20/55] Bump maven-enforcer-plugin from 3.0.0 to 3.1.0 Bumps [maven-enforcer-plugin](https://github.com/apache/maven-enforcer) from 3.0.0 to 3.1.0. - [Release notes](https://github.com/apache/maven-enforcer/releases) - [Commits](https://github.com/apache/maven-enforcer/compare/enforcer-3.0.0...enforcer-3.1.0) --- updated-dependencies: - dependency-name: org.apache.maven.plugins:maven-enforcer-plugin dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index dc73a560..00597e42 100644 --- a/pom.xml +++ b/pom.xml @@ -75,7 +75,7 @@ org.apache.maven.plugins maven-enforcer-plugin - 3.0.0 + 3.1.0 enforce-maven From 687973ba783a23e42d00e62e27cde19676656a36 Mon Sep 17 00:00:00 2001 From: Andrew Gaul Date: Sun, 17 Jul 2022 23:13:39 +0900 Subject: [PATCH 21/55] Rip out useless semantic lint --- .github/workflows/ci-main.yml | 25 ------------------------- .github/workflows/pr-lint.yml | 17 ----------------- .releaserc | 21 ++------------------- 3 files changed, 2 insertions(+), 61 deletions(-) delete mode 100644 .github/workflows/pr-lint.yml diff --git a/.github/workflows/ci-main.yml b/.github/workflows/ci-main.yml index 1925c6eb..c46371de 100644 --- a/.github/workflows/ci-main.yml +++ b/.github/workflows/ci-main.yml @@ -34,19 +34,6 @@ jobs: - uses: actions/setup-node@v2 with: node-version: "14" - - name: Semantic Release - id: version - uses: cycjimmy/semantic-release-action@v3.0.0 - with: - semantic_version: 17 - extra_plugins: | - @semantic-release/exec - @semantic-release/git - @google/semantic-release-replace-plugin - conventional-changelog-conventionalcommits - dry_run: true - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Docker meta id: docker_action_meta uses: docker/metadata-action@v4.0.1 @@ -161,18 +148,6 @@ jobs: - uses: actions/setup-node@v2 with: node-version: "14" - - name: Semantic Release - if: github.event_name != 'pull_request' && needs.meta.outputs.dockerhub-publish == 'true' - id: version - uses: cycjimmy/semantic-release-action@v3.0.0 - with: - semantic_version: 17 - extra_plugins: | - @semantic-release/exec - @semantic-release/git - conventional-changelog-conventionalcommits - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Setup regctl if: github.event_name != 'pull_request' && needs.meta.outputs.dockerhub-publish == 'true' run: | diff --git a/.github/workflows/pr-lint.yml b/.github/workflows/pr-lint.yml deleted file mode 100644 index ad6df65f..00000000 --- a/.github/workflows/pr-lint.yml +++ /dev/null @@ -1,17 +0,0 @@ -name: "Lint PR" - -on: - pull_request_target: - types: - - opened - - edited - - synchronize - -jobs: - main: - name: Validate PR title - runs-on: ubuntu-latest - steps: - - uses: amannn/action-semantic-pull-request@v4 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.releaserc b/.releaserc index a05ac080..0c80928f 100644 --- a/.releaserc +++ b/.releaserc @@ -25,22 +25,5 @@ name: 'develop\/.*', prerelease: true } - ], - plugins: [ - [ - "@semantic-release/commit-analyzer", - { - "preset": "conventionalcommits", - "presetConfig": "conventional-changelog-conventionalcommits" - } - ], - "@semantic-release/release-notes-generator", - ["@semantic-release/git", { - "assets": [ - "pom.xml", - ], - "message": "chore(release): ${nextRelease.version}\n\n${nextRelease.notes}" - }], - ["@semantic-release/github"], - ] -} \ No newline at end of file + ] +} From d1779202280328a88e88e2b637322e692900aea9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Jul 2022 22:08:20 +0000 Subject: [PATCH 22/55] Bump spotbugs from 4.4.0 to 4.7.1 Bumps [spotbugs](https://github.com/spotbugs/spotbugs) from 4.4.0 to 4.7.1. - [Release notes](https://github.com/spotbugs/spotbugs/releases) - [Changelog](https://github.com/spotbugs/spotbugs/blob/master/CHANGELOG.md) - [Commits](https://github.com/spotbugs/spotbugs/compare/4.4.0...4.7.1) --- updated-dependencies: - dependency-name: com.github.spotbugs:spotbugs dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 00597e42..2ddf2fe5 100644 --- a/pom.xml +++ b/pom.xml @@ -317,7 +317,7 @@ com.github.spotbugs spotbugs - 4.4.0 + 4.7.1 From 460a852d66b71c3a1b69e9bbc77953830e77becd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Jul 2022 22:08:16 +0000 Subject: [PATCH 23/55] Bump maven-shade-plugin from 3.2.4 to 3.3.0 Bumps [maven-shade-plugin](https://github.com/apache/maven-shade-plugin) from 3.2.4 to 3.3.0. - [Release notes](https://github.com/apache/maven-shade-plugin/releases) - [Commits](https://github.com/apache/maven-shade-plugin/compare/maven-shade-plugin-3.2.4...maven-shade-plugin-3.3.0) --- updated-dependencies: - dependency-name: org.apache.maven.plugins:maven-shade-plugin dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 2ddf2fe5..9168ccea 100644 --- a/pom.xml +++ b/pom.xml @@ -199,7 +199,7 @@ org.apache.maven.plugins maven-shade-plugin - 3.2.4 + 3.3.0 package From 4fade1f4edd3d9fe1cd198eba73527b7e91ddbb7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Jul 2022 22:08:35 +0000 Subject: [PATCH 24/55] Bump junit-jupiter from 5.8.1 to 5.8.2 Bumps [junit-jupiter](https://github.com/junit-team/junit5) from 5.8.1 to 5.8.2. - [Release notes](https://github.com/junit-team/junit5/releases) - [Commits](https://github.com/junit-team/junit5/compare/r5.8.1...r5.8.2) --- updated-dependencies: - dependency-name: org.junit.jupiter:junit-jupiter dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 9168ccea..2fd02a8b 100644 --- a/pom.xml +++ b/pom.xml @@ -436,7 +436,7 @@ org.junit.jupiter junit-jupiter - 5.8.1 + 5.8.2 provided From 271b050a525870e5ae802dbf06da9b1eb33c523c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Jul 2022 22:08:39 +0000 Subject: [PATCH 25/55] Bump maven-assembly-plugin from 3.3.0 to 3.4.1 Bumps [maven-assembly-plugin](https://github.com/apache/maven-assembly-plugin) from 3.3.0 to 3.4.1. - [Release notes](https://github.com/apache/maven-assembly-plugin/releases) - [Commits](https://github.com/apache/maven-assembly-plugin/compare/maven-assembly-plugin-3.3.0...maven-assembly-plugin-3.4.1) --- updated-dependencies: - dependency-name: org.apache.maven.plugins:maven-assembly-plugin dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 2fd02a8b..e98f2899 100644 --- a/pom.xml +++ b/pom.xml @@ -237,7 +237,7 @@ org.apache.maven.plugins maven-assembly-plugin - 3.3.0 + 3.4.1 src/main/assembly/jar-with-dependencies.xml From d66c669f7c0729ae800481d98116ef6daedd6575 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 19 Jul 2022 22:06:20 +0000 Subject: [PATCH 26/55] Bump spotbugs-maven-plugin from 4.4.2 to 4.7.1.0 Bumps [spotbugs-maven-plugin](https://github.com/spotbugs/spotbugs-maven-plugin) from 4.4.2 to 4.7.1.0. - [Release notes](https://github.com/spotbugs/spotbugs-maven-plugin/releases) - [Commits](https://github.com/spotbugs/spotbugs-maven-plugin/compare/spotbugs-maven-plugin-4.4.2...spotbugs-maven-plugin-4.7.1.0) --- updated-dependencies: - dependency-name: com.github.spotbugs:spotbugs-maven-plugin dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index e98f2899..13360e5e 100644 --- a/pom.xml +++ b/pom.xml @@ -312,7 +312,7 @@ com.github.spotbugs spotbugs-maven-plugin - 4.4.2 + 4.7.1.0 com.github.spotbugs From 9c681e944cf0bfe3d4d72f6505cf1085d9e4d1da Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 19 Jul 2022 22:06:13 +0000 Subject: [PATCH 27/55] Bump spotbugs-annotations from 3.1.12 to 4.7.1 Bumps [spotbugs-annotations](https://github.com/spotbugs/spotbugs) from 3.1.12 to 4.7.1. - [Release notes](https://github.com/spotbugs/spotbugs/releases) - [Changelog](https://github.com/spotbugs/spotbugs/blob/master/CHANGELOG.md) - [Commits](https://github.com/spotbugs/spotbugs/compare/3.1.12...4.7.1) --- updated-dependencies: - dependency-name: com.github.spotbugs:spotbugs-annotations dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 13360e5e..65b8265d 100644 --- a/pom.xml +++ b/pom.xml @@ -448,7 +448,7 @@ com.github.spotbugs spotbugs-annotations - 3.1.12 + 4.7.1 provided From 3528bf3ae14c83280ecc1fb85109933e043cc4ba Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 19 Jul 2022 22:06:08 +0000 Subject: [PATCH 28/55] Bump maven-compiler-plugin from 3.8.1 to 3.10.1 Bumps [maven-compiler-plugin](https://github.com/apache/maven-compiler-plugin) from 3.8.1 to 3.10.1. - [Release notes](https://github.com/apache/maven-compiler-plugin/releases) - [Commits](https://github.com/apache/maven-compiler-plugin/compare/maven-compiler-plugin-3.8.1...maven-compiler-plugin-3.10.1) --- updated-dependencies: - dependency-name: org.apache.maven.plugins:maven-compiler-plugin dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 65b8265d..926ca9b3 100644 --- a/pom.xml +++ b/pom.xml @@ -144,7 +144,7 @@ org.apache.maven.plugins maven-compiler-plugin - 3.8.1 + 3.10.1 ${java.version} ${java.version} From d0fec21e93312671563f6acf4eb235b80917d124 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 19 Jul 2022 22:06:25 +0000 Subject: [PATCH 29/55] Bump checkstyle from 9.1 to 10.3.1 Bumps [checkstyle](https://github.com/checkstyle/checkstyle) from 9.1 to 10.3.1. - [Release notes](https://github.com/checkstyle/checkstyle/releases) - [Commits](https://github.com/checkstyle/checkstyle/compare/checkstyle-9.1...checkstyle-10.3.1) --- updated-dependencies: - dependency-name: com.puppycrawl.tools:checkstyle dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 926ca9b3..70df9568 100644 --- a/pom.xml +++ b/pom.xml @@ -132,7 +132,7 @@ com.puppycrawl.tools checkstyle - 9.1 + 10.3.1 From 00894152cdc62f13341730c511d497e7085e6523 Mon Sep 17 00:00:00 2001 From: Thiago da Silva Date: Tue, 19 Jul 2022 10:48:16 -0700 Subject: [PATCH 30/55] Allow setting of v4-max-non-chunked-request-size in Docker container --- src/main/resources/run-docker-container.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/resources/run-docker-container.sh b/src/main/resources/run-docker-container.sh index 093bc467..6b4f4032 100755 --- a/src/main/resources/run-docker-container.sh +++ b/src/main/resources/run-docker-container.sh @@ -15,6 +15,7 @@ exec java \ -Ds3proxy.encrypted-blobstore="${S3PROXY_ENCRYPTED_BLOBSTORE}" \ -Ds3proxy.encrypted-blobstore-password="${S3PROXY_ENCRYPTED_BLOBSTORE_PASSWORD}" \ -Ds3proxy.encrypted-blobstore-salt="${S3PROXY_ENCRYPTED_BLOBSTORE_SALT}" \ + -Ds3proxy.v4-max-non-chunked-request-size="${S3PROXY_V4_MAX_NON_CHUNKED_REQ_SIZE}" \ -Djclouds.provider="${JCLOUDS_PROVIDER}" \ -Djclouds.identity="${JCLOUDS_IDENTITY}" \ -Djclouds.credential="${JCLOUDS_CREDENTIAL}" \ From bbb3bc4aa99824e131a5b81f754cb8d650b64ff1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 20 Jul 2022 22:13:26 +0000 Subject: [PATCH 31/55] Bump maven-javadoc-plugin from 3.3.1 to 3.4.0 Bumps [maven-javadoc-plugin](https://github.com/apache/maven-javadoc-plugin) from 3.3.1 to 3.4.0. - [Release notes](https://github.com/apache/maven-javadoc-plugin/releases) - [Commits](https://github.com/apache/maven-javadoc-plugin/compare/maven-javadoc-plugin-3.3.1...maven-javadoc-plugin-3.4.0) --- updated-dependencies: - dependency-name: org.apache.maven.plugins:maven-javadoc-plugin dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 70df9568..0a7a5ae1 100644 --- a/pom.xml +++ b/pom.xml @@ -186,7 +186,7 @@ org.apache.maven.plugins maven-javadoc-plugin - 3.3.1 + 3.4.0 attach-javadocs From 974139e14c4d8f17576a72b525c7fbd6637f3522 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 20 Jul 2022 22:13:21 +0000 Subject: [PATCH 32/55] Bump maven-jar-plugin from 3.2.0 to 3.2.2 Bumps [maven-jar-plugin](https://github.com/apache/maven-jar-plugin) from 3.2.0 to 3.2.2. - [Release notes](https://github.com/apache/maven-jar-plugin/releases) - [Commits](https://github.com/apache/maven-jar-plugin/compare/maven-jar-plugin-3.2.0...maven-jar-plugin-3.2.2) --- updated-dependencies: - dependency-name: org.apache.maven.plugins:maven-jar-plugin dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 0a7a5ae1..ba35e158 100644 --- a/pom.xml +++ b/pom.xml @@ -173,7 +173,7 @@ org.apache.maven.plugins maven-jar-plugin - 3.2.0 + 3.2.2 From e32ca2ca50b82fbdcfdd98ed1a6e982868ed3ed7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 20 Jul 2022 22:13:33 +0000 Subject: [PATCH 33/55] Bump maven-deploy-plugin from 2.8.2 to 3.0.0 Bumps [maven-deploy-plugin](https://github.com/apache/maven-deploy-plugin) from 2.8.2 to 3.0.0. - [Release notes](https://github.com/apache/maven-deploy-plugin/releases) - [Commits](https://github.com/apache/maven-deploy-plugin/compare/maven-deploy-plugin-2.8.2...maven-deploy-plugin-3.0.0) --- updated-dependencies: - dependency-name: org.apache.maven.plugins:maven-deploy-plugin dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index ba35e158..9d45bb37 100644 --- a/pom.xml +++ b/pom.xml @@ -105,7 +105,7 @@ org.apache.maven.plugins maven-deploy-plugin - 2.8.2 + 3.0.0 org.apache.maven.plugins From e4666932bb3e0405de74ee750deae111cdffc98d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 25 Jul 2022 22:06:06 +0000 Subject: [PATCH 34/55] Bump maven-install-plugin from 2.5.2 to 3.0.1 Bumps [maven-install-plugin](https://github.com/apache/maven-install-plugin) from 2.5.2 to 3.0.1. - [Release notes](https://github.com/apache/maven-install-plugin/releases) - [Commits](https://github.com/apache/maven-install-plugin/compare/maven-install-plugin-2.5.2...maven-install-plugin-3.0.1) --- updated-dependencies: - dependency-name: org.apache.maven.plugins:maven-install-plugin dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 9d45bb37..011821cc 100644 --- a/pom.xml +++ b/pom.xml @@ -100,7 +100,7 @@ org.apache.maven.plugins maven-install-plugin - 2.5.2 + 3.0.1 org.apache.maven.plugins From 277ef3c9d7f1222de91ec9929545d2f7df68a702 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 25 Jul 2022 22:05:35 +0000 Subject: [PATCH 35/55] Bump spotbugs-maven-plugin from 4.7.1.0 to 4.7.1.1 Bumps [spotbugs-maven-plugin](https://github.com/spotbugs/spotbugs-maven-plugin) from 4.7.1.0 to 4.7.1.1. - [Release notes](https://github.com/spotbugs/spotbugs-maven-plugin/releases) - [Commits](https://github.com/spotbugs/spotbugs-maven-plugin/compare/spotbugs-maven-plugin-4.7.1.0...spotbugs-maven-plugin-4.7.1.1) --- updated-dependencies: - dependency-name: com.github.spotbugs:spotbugs-maven-plugin dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 011821cc..bc3ab75f 100644 --- a/pom.xml +++ b/pom.xml @@ -312,7 +312,7 @@ com.github.spotbugs spotbugs-maven-plugin - 4.7.1.0 + 4.7.1.1 com.github.spotbugs From d375011388436e588fc646f035a9f626eef68ee8 Mon Sep 17 00:00:00 2001 From: Andrew Gaul Date: Sun, 7 Aug 2022 21:22:01 +0900 Subject: [PATCH 36/55] Upgrade to Jetty 11.0.11 Jetty 9 is EOL. This requires Java 11 and updating some Java EE imports. Fixes #422. --- README.md | 3 ++- pom.xml | 2 +- src/main/java/org/gaul/s3proxy/AwsSignature.java | 3 ++- src/main/java/org/gaul/s3proxy/S3ErrorCode.java | 4 ++-- src/main/java/org/gaul/s3proxy/S3Proxy.java | 11 ++++++++--- src/main/java/org/gaul/s3proxy/S3ProxyHandler.java | 5 +++-- .../java/org/gaul/s3proxy/S3ProxyHandlerJetty.java | 5 +++-- src/test/java/org/gaul/s3proxy/AwsSdkTest.java | 2 +- 8 files changed, 22 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 389d4496..a1053336 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,8 @@ and has instructions on how to run it. Users can [download releases](https://github.com/gaul/s3proxy/releases) from GitHub. Developers can build the project by running `mvn package` which -produces a binary at `target/s3proxy`. S3Proxy requires Java 8 or newer to run. +produces a binary at `target/s3proxy`. S3Proxy requires Java 11 or newer to +run. Configure S3Proxy via a properties file. An example using the local file system as the storage backend with anonymous access: diff --git a/pom.xml b/pom.xml index bc3ab75f..455a5288 100644 --- a/pom.xml +++ b/pom.xml @@ -491,7 +491,7 @@ org.eclipse.jetty jetty-servlet - 9.4.45.v20220203 + 11.0.11 org.slf4j diff --git a/src/main/java/org/gaul/s3proxy/AwsSignature.java b/src/main/java/org/gaul/s3proxy/AwsSignature.java index a16b40e4..0e62007c 100644 --- a/src/main/java/org/gaul/s3proxy/AwsSignature.java +++ b/src/main/java/org/gaul/s3proxy/AwsSignature.java @@ -34,7 +34,6 @@ import javax.annotation.Nullable; import javax.crypto.Mac; import javax.crypto.spec.SecretKeySpec; -import javax.servlet.http.HttpServletRequest; import com.google.common.base.Joiner; import com.google.common.base.Splitter; @@ -46,6 +45,8 @@ import com.google.common.net.HttpHeaders; import com.google.common.net.PercentEscaper; +import jakarta.servlet.http.HttpServletRequest; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/src/main/java/org/gaul/s3proxy/S3ErrorCode.java b/src/main/java/org/gaul/s3proxy/S3ErrorCode.java index d4158b22..e1399604 100644 --- a/src/main/java/org/gaul/s3proxy/S3ErrorCode.java +++ b/src/main/java/org/gaul/s3proxy/S3ErrorCode.java @@ -18,10 +18,10 @@ import static java.util.Objects.requireNonNull; -import javax.servlet.http.HttpServletResponse; - import com.google.common.base.CaseFormat; +import jakarta.servlet.http.HttpServletResponse; + /** * List of S3 error codes. Reference: * http://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html diff --git a/src/main/java/org/gaul/s3proxy/S3Proxy.java b/src/main/java/org/gaul/s3proxy/S3Proxy.java index de4533a8..1dd4494f 100644 --- a/src/main/java/org/gaul/s3proxy/S3Proxy.java +++ b/src/main/java/org/gaul/s3proxy/S3Proxy.java @@ -34,6 +34,7 @@ import org.eclipse.jetty.http.HttpCompliance; import org.eclipse.jetty.server.HttpConfiguration; import org.eclipse.jetty.server.HttpConnectionFactory; +import org.eclipse.jetty.server.SecureRequestCustomizer; import org.eclipse.jetty.server.Server; import org.eclipse.jetty.server.ServerConnector; import org.eclipse.jetty.server.handler.ContextHandler; @@ -84,9 +85,13 @@ public final class S3Proxy { context.setContextPath(builder.servicePath); } + HttpConfiguration httpConfiguration = new HttpConfiguration(); + httpConfiguration.setHttpCompliance(HttpCompliance.LEGACY); + SecureRequestCustomizer src = new SecureRequestCustomizer(); + src.setSniHostCheck(false); + httpConfiguration.addCustomizer(src); HttpConnectionFactory httpConnectionFactory = - new HttpConnectionFactory( - new HttpConfiguration(), HttpCompliance.LEGACY); + new HttpConnectionFactory(httpConfiguration); ServerConnector connector; if (builder.endpoint != null) { connector = new ServerConnector(server, httpConnectionFactory); @@ -99,7 +104,7 @@ public final class S3Proxy { } if (builder.secureEndpoint != null) { - SslContextFactory sslContextFactory = + SslContextFactory.Server sslContextFactory = new SslContextFactory.Server(); sslContextFactory.setKeyStorePath(builder.keyStorePath); sslContextFactory.setKeyStorePassword(builder.keyStorePassword); diff --git a/src/main/java/org/gaul/s3proxy/S3ProxyHandler.java b/src/main/java/org/gaul/s3proxy/S3ProxyHandler.java index 235ee7b1..064886b6 100644 --- a/src/main/java/org/gaul/s3proxy/S3ProxyHandler.java +++ b/src/main/java/org/gaul/s3proxy/S3ProxyHandler.java @@ -53,8 +53,6 @@ import javax.annotation.Nullable; import javax.crypto.Mac; import javax.crypto.spec.SecretKeySpec; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; import javax.xml.stream.XMLOutputFactory; import javax.xml.stream.XMLStreamException; import javax.xml.stream.XMLStreamWriter; @@ -82,6 +80,9 @@ import com.google.common.net.HttpHeaders; import com.google.common.net.PercentEscaper; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; + import org.apache.commons.fileupload.MultipartStream; import org.jclouds.blobstore.BlobStore; import org.jclouds.blobstore.KeyNotFoundException; diff --git a/src/main/java/org/gaul/s3proxy/S3ProxyHandlerJetty.java b/src/main/java/org/gaul/s3proxy/S3ProxyHandlerJetty.java index c2fb331a..b0f74cec 100644 --- a/src/main/java/org/gaul/s3proxy/S3ProxyHandlerJetty.java +++ b/src/main/java/org/gaul/s3proxy/S3ProxyHandlerJetty.java @@ -21,11 +21,12 @@ import java.util.concurrent.TimeoutException; import javax.annotation.Nullable; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; import com.google.common.collect.ImmutableMap; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.servlet.http.HttpServletResponse; + import org.eclipse.jetty.server.Request; import org.eclipse.jetty.server.handler.AbstractHandler; import org.jclouds.blobstore.BlobStore; diff --git a/src/test/java/org/gaul/s3proxy/AwsSdkTest.java b/src/test/java/org/gaul/s3proxy/AwsSdkTest.java index e178f241..c5e9b80b 100644 --- a/src/test/java/org/gaul/s3proxy/AwsSdkTest.java +++ b/src/test/java/org/gaul/s3proxy/AwsSdkTest.java @@ -1006,7 +1006,7 @@ public void testBlobRemove() throws Exception { @Test public void testSinglepartUploadJettyCachedHeader() throws Exception { String blobName = "singlepart-upload-jetty-cached"; - String contentType = "text/plain;charset=utf-8"; + String contentType = "text/plain"; ObjectMetadata metadata = new ObjectMetadata(); metadata.setContentLength(BYTE_SOURCE.size()); metadata.setContentType(contentType); From bbbacaa44237f6b9e8b428d104ecbfb58d167883 Mon Sep 17 00:00:00 2001 From: Andrew Gaul Date: Mon, 8 Aug 2022 21:14:03 +0900 Subject: [PATCH 37/55] Require Java 11 Jetty 11 requires this. Also address some Modernizer issues. References #422. --- pom.xml | 2 +- .../java/org/gaul/s3proxy/NullBlobStore.java | 5 +++-- .../java/org/gaul/s3proxy/S3ProxyHandler.java | 10 ++++++---- src/test/java/org/gaul/s3proxy/AwsSdkTest.java | 6 +++--- .../org/gaul/s3proxy/NullBlobStoreTest.java | 18 +++++++++--------- 5 files changed, 22 insertions(+), 19 deletions(-) diff --git a/pom.xml b/pom.xml index 455a5288..8c90f020 100644 --- a/pom.xml +++ b/pom.xml @@ -381,7 +381,7 @@ UTF-8 - 1.8 + 11 2.5.0 1.7.36 ${project.groupId}.shaded diff --git a/src/main/java/org/gaul/s3proxy/NullBlobStore.java b/src/main/java/org/gaul/s3proxy/NullBlobStore.java index 860bada0..14f23fd3 100644 --- a/src/main/java/org/gaul/s3proxy/NullBlobStore.java +++ b/src/main/java/org/gaul/s3proxy/NullBlobStore.java @@ -18,6 +18,7 @@ import java.io.IOException; import java.io.InputStream; +import java.io.OutputStream; import java.util.Arrays; import java.util.List; @@ -119,7 +120,7 @@ public String putBlob(String containerName, Blob blob, PutOptions options) { long length; try (InputStream is = blob.getPayload().openStream()) { - length = ByteStreams.copy(is, ByteStreams.nullOutputStream()); + length = is.transferTo(OutputStream.nullOutputStream()); } catch (IOException ioe) { throw new RuntimeException(ioe); } @@ -175,7 +176,7 @@ public MultipartPart uploadMultipartPart(MultipartUpload mpu, int partNumber, Payload payload) { long length; try (InputStream is = payload.openStream()) { - length = ByteStreams.copy(is, ByteStreams.nullOutputStream()); + length = is.transferTo(OutputStream.nullOutputStream()); } catch (IOException ioe) { throw new RuntimeException(ioe); } diff --git a/src/main/java/org/gaul/s3proxy/S3ProxyHandler.java b/src/main/java/org/gaul/s3proxy/S3ProxyHandler.java index 064886b6..6636cd8a 100644 --- a/src/main/java/org/gaul/s3proxy/S3ProxyHandler.java +++ b/src/main/java/org/gaul/s3proxy/S3ProxyHandler.java @@ -471,7 +471,7 @@ public final void doHandle(HttpServletRequest baseRequest, String[] path = uri.split("/", 3); for (int i = 0; i < path.length; i++) { - path[i] = URLDecoder.decode(path[i], UTF_8); + path[i] = URLDecoder.decode(path[i], StandardCharsets.UTF_8); } Map.Entry provider = @@ -1762,7 +1762,7 @@ private void handleGetBlob(HttpServletRequest request, try (InputStream is = blob.getPayload().openStream(); OutputStream os = response.getOutputStream()) { - ByteStreams.copy(is, os); + is.transferTo(os); os.flush(); } } @@ -1772,7 +1772,8 @@ private void handleCopyBlob(HttpServletRequest request, String destContainerName, String destBlobName) throws IOException, S3Exception { String copySourceHeader = request.getHeader(AwsHttpHeaders.COPY_SOURCE); - copySourceHeader = URLDecoder.decode(copySourceHeader, UTF_8); + copySourceHeader = URLDecoder.decode( + copySourceHeader, StandardCharsets.UTF_8); if (copySourceHeader.startsWith("/")) { // Some clients like boto do not include the leading slash copySourceHeader = copySourceHeader.substring(1); @@ -2520,7 +2521,8 @@ private void handleCopyPart(HttpServletRequest request, throws IOException, S3Exception { // TODO: duplicated from handlePutBlob String copySourceHeader = request.getHeader(AwsHttpHeaders.COPY_SOURCE); - copySourceHeader = URLDecoder.decode(copySourceHeader, UTF_8); + copySourceHeader = URLDecoder.decode( + copySourceHeader, StandardCharsets.UTF_8); if (copySourceHeader.startsWith("/")) { // Some clients like boto do not include the leading slash copySourceHeader = copySourceHeader.substring(1); diff --git a/src/test/java/org/gaul/s3proxy/AwsSdkTest.java b/src/test/java/org/gaul/s3proxy/AwsSdkTest.java index c5e9b80b..3ecf7532 100644 --- a/src/test/java/org/gaul/s3proxy/AwsSdkTest.java +++ b/src/test/java/org/gaul/s3proxy/AwsSdkTest.java @@ -20,6 +20,7 @@ import static org.junit.Assume.assumeTrue; import java.io.InputStream; +import java.io.OutputStream; import java.net.URI; import java.net.URL; import java.net.URLConnection; @@ -93,7 +94,6 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; import com.google.common.io.ByteSource; -import com.google.common.io.ByteStreams; import org.assertj.core.api.Fail; @@ -653,7 +653,7 @@ public void testOverrideResponseHeader() throws Exception { S3Object object = client.getObject(getObjectRequest); try (InputStream is = object.getObjectContent()) { assertThat(is).isNotNull(); - ByteStreams.copy(is, ByteStreams.nullOutputStream()); + is.transferTo(OutputStream.nullOutputStream()); } ObjectMetadata reponseMetadata = object.getObjectMetadata(); @@ -1446,7 +1446,7 @@ public void testConditionalGet() throws Exception { .withMatchingETagConstraint(result.getETag())); try (InputStream is = object.getObjectContent()) { assertThat(is).isNotNull(); - ByteStreams.copy(is, ByteStreams.nullOutputStream()); + is.transferTo(OutputStream.nullOutputStream()); } object = client.getObject( diff --git a/src/test/java/org/gaul/s3proxy/NullBlobStoreTest.java b/src/test/java/org/gaul/s3proxy/NullBlobStoreTest.java index d8e223a3..21687626 100644 --- a/src/test/java/org/gaul/s3proxy/NullBlobStoreTest.java +++ b/src/test/java/org/gaul/s3proxy/NullBlobStoreTest.java @@ -20,13 +20,13 @@ import java.io.IOException; import java.io.InputStream; +import java.io.OutputStream; import java.util.List; import java.util.Random; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.io.ByteSource; -import com.google.common.io.ByteStreams; import com.google.common.net.MediaType; import com.google.inject.Module; @@ -91,10 +91,10 @@ public void testCreateBlobGetBlob() throws Exception { // content differs, only compare length try (InputStream actual = blob.getPayload().openStream(); InputStream expected = BYTE_SOURCE.openStream()) { - long actualLength = ByteStreams.copy(actual, - ByteStreams.nullOutputStream()); - long expectedLength = ByteStreams.copy(expected, - ByteStreams.nullOutputStream()); + long actualLength = actual.transferTo( + OutputStream.nullOutputStream()); + long expectedLength = expected.transferTo( + OutputStream.nullOutputStream()); assertThat(actualLength).isEqualTo(expectedLength); } @@ -157,10 +157,10 @@ public void testCreateMultipartBlobGetBlob() throws Exception { // content differs, only compare length try (InputStream actual = newBlob.getPayload().openStream(); InputStream expected = byteSource.openStream()) { - long actualLength = ByteStreams.copy(actual, - ByteStreams.nullOutputStream()); - long expectedLength = ByteStreams.copy(expected, - ByteStreams.nullOutputStream()); + long actualLength = actual.transferTo( + OutputStream.nullOutputStream()); + long expectedLength = expected.transferTo( + OutputStream.nullOutputStream()); assertThat(actualLength).isEqualTo(expectedLength); } From 0245de405c275534c9a08348f33c9489743549bf Mon Sep 17 00:00:00 2001 From: Andrew Gaul Date: Sat, 20 Aug 2022 19:29:41 +0900 Subject: [PATCH 38/55] Address error-prone warnings --- .../java/org/gaul/s3proxy/crypto/DecryptionInputStream.java | 2 +- .../java/org/gaul/s3proxy/crypto/EncryptionInputStream.java | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/gaul/s3proxy/crypto/DecryptionInputStream.java b/src/main/java/org/gaul/s3proxy/crypto/DecryptionInputStream.java index c5b40efa..02c87cc8 100644 --- a/src/main/java/org/gaul/s3proxy/crypto/DecryptionInputStream.java +++ b/src/main/java/org/gaul/s3proxy/crypto/DecryptionInputStream.java @@ -316,7 +316,7 @@ public final long skip(long n) throws IOException { if (n < 0) { return 0; } - ostart += n; + ostart += (int) n; return n; } diff --git a/src/main/java/org/gaul/s3proxy/crypto/EncryptionInputStream.java b/src/main/java/org/gaul/s3proxy/crypto/EncryptionInputStream.java index 7a67eecb..0d4dfe24 100644 --- a/src/main/java/org/gaul/s3proxy/crypto/EncryptionInputStream.java +++ b/src/main/java/org/gaul/s3proxy/crypto/EncryptionInputStream.java @@ -65,6 +65,7 @@ final void padding() throws IOException { } } + @Override public final int available() throws IOException { if (in == null) { return 0; // no way to signal EOF from available() @@ -72,6 +73,7 @@ public final int available() throws IOException { return in.available(); } + @Override public final int read() throws IOException { while (in != null) { int c = in.read(); @@ -84,6 +86,7 @@ public final int read() throws IOException { return -1; } + @Override public final int read(byte[] b, int off, int len) throws IOException { if (in == null) { return -1; @@ -105,6 +108,7 @@ public final int read(byte[] b, int off, int len) throws IOException { return -1; } + @Override public final void close() throws IOException { IOException ioe = null; while (in != null) { From df57f5453b88b5d7ea3d551206f922612148179e Mon Sep 17 00:00:00 2001 From: Andrew Gaul Date: Sat, 20 Aug 2022 19:32:59 +0900 Subject: [PATCH 39/55] Upgrade to error-prone 2.15.0 Release notes: https://github.com/google/error-prone/releases --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 8c90f020..01636234 100644 --- a/pom.xml +++ b/pom.xml @@ -154,10 +154,10 @@ -Xlint -XDcompilePolicy=simple -Xplugin:ErrorProne + -Xep:CanIgnoreReturnValueSuggester:OFF -Xep:DefaultCharset:OFF -Xep:HidingField:OFF -Xep:JavaUtilDate:OFF - -Xep:MutableConstantField:OFF -Xep:ProtectedMembersInFinalClass:OFF @@ -165,7 +165,7 @@ com.google.errorprone error_prone_core - 2.10.0 + 2.15.0 From a856cc76817ab7ff9109815c743adae5015e4878 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 Aug 2022 12:27:17 +0000 Subject: [PATCH 40/55] Bump junit-jupiter from 5.8.2 to 5.9.0 Bumps [junit-jupiter](https://github.com/junit-team/junit5) from 5.8.2 to 5.9.0. - [Release notes](https://github.com/junit-team/junit5/releases) - [Commits](https://github.com/junit-team/junit5/compare/r5.8.2...r5.9.0) --- updated-dependencies: - dependency-name: org.junit.jupiter:junit-jupiter dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 01636234..f216ce96 100644 --- a/pom.xml +++ b/pom.xml @@ -436,7 +436,7 @@ org.junit.jupiter junit-jupiter - 5.8.2 + 5.9.0 provided From 68237a101253b72b9e4cf2f7815e8c8ae9f829dd Mon Sep 17 00:00:00 2001 From: Andrew Gaul Date: Fri, 23 Dec 2022 15:03:34 +0900 Subject: [PATCH 41/55] Upgrade spotbugs to 4.7.3 Release notes: https://github.com/spotbugs/spotbugs/blob/master/CHANGELOG.md#473---2022-10-15 --- pom.xml | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/pom.xml b/pom.xml index f216ce96..d6911c97 100644 --- a/pom.xml +++ b/pom.xml @@ -312,14 +312,7 @@ com.github.spotbugs spotbugs-maven-plugin - 4.7.1.1 - - - com.github.spotbugs - spotbugs - 4.7.1 - - + 4.7.3.0 Max CrossSiteScripting,DefaultEncodingDetector,FindNullDeref From ddd32686f54d3365124e8c7dbecf8347ae506dbe Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 23 Dec 2022 05:48:09 +0000 Subject: [PATCH 42/55] Bump maven-jar-plugin from 3.2.2 to 3.3.0 Bumps [maven-jar-plugin](https://github.com/apache/maven-jar-plugin) from 3.2.2 to 3.3.0. - [Release notes](https://github.com/apache/maven-jar-plugin/releases) - [Commits](https://github.com/apache/maven-jar-plugin/compare/maven-jar-plugin-3.2.2...maven-jar-plugin-3.3.0) --- updated-dependencies: - dependency-name: org.apache.maven.plugins:maven-jar-plugin dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index d6911c97..07731d66 100644 --- a/pom.xml +++ b/pom.xml @@ -173,7 +173,7 @@ org.apache.maven.plugins maven-jar-plugin - 3.2.2 + 3.3.0 From 9408f53cf5dcaeced122b1f20f29d2a268dc4d0f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 23 Dec 2022 06:05:24 +0000 Subject: [PATCH 43/55] Bump maven-resources-plugin from 3.2.0 to 3.3.0 Bumps [maven-resources-plugin](https://github.com/apache/maven-resources-plugin) from 3.2.0 to 3.3.0. - [Release notes](https://github.com/apache/maven-resources-plugin/releases) - [Commits](https://github.com/apache/maven-resources-plugin/compare/maven-resources-plugin-3.2.0...maven-resources-plugin-3.3.0) --- updated-dependencies: - dependency-name: org.apache.maven.plugins:maven-resources-plugin dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 07731d66..ba3880b0 100644 --- a/pom.xml +++ b/pom.xml @@ -139,7 +139,7 @@ org.apache.maven.plugins maven-resources-plugin - 3.2.0 + 3.3.0 org.apache.maven.plugins From 92c6171bfb38fb5f206d786a0ecec2adf6fbdd1e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 23 Dec 2022 06:05:20 +0000 Subject: [PATCH 44/55] Bump maven-assembly-plugin from 3.4.1 to 3.4.2 Bumps [maven-assembly-plugin](https://github.com/apache/maven-assembly-plugin) from 3.4.1 to 3.4.2. - [Release notes](https://github.com/apache/maven-assembly-plugin/releases) - [Commits](https://github.com/apache/maven-assembly-plugin/compare/maven-assembly-plugin-3.4.1...maven-assembly-plugin-3.4.2) --- updated-dependencies: - dependency-name: org.apache.maven.plugins:maven-assembly-plugin dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index ba3880b0..c909a22f 100644 --- a/pom.xml +++ b/pom.xml @@ -237,7 +237,7 @@ org.apache.maven.plugins maven-assembly-plugin - 3.4.1 + 3.4.2 src/main/assembly/jar-with-dependencies.xml From 81e885422fa48adfe5b8f0445188e8efcc902fc1 Mon Sep 17 00:00:00 2001 From: Andrew Gaul Date: Mon, 26 Dec 2022 12:20:23 +0900 Subject: [PATCH 45/55] Suppress spurious md5 deprecation warnings --- src/test/java/org/gaul/s3proxy/AliasBlobStoreTest.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/test/java/org/gaul/s3proxy/AliasBlobStoreTest.java b/src/test/java/org/gaul/s3proxy/AliasBlobStoreTest.java index 7e817ad8..1208456d 100644 --- a/src/test/java/org/gaul/s3proxy/AliasBlobStoreTest.java +++ b/src/test/java/org/gaul/s3proxy/AliasBlobStoreTest.java @@ -123,6 +123,7 @@ public void testAliasBlob() throws IOException { createContainer(aliasContainerName); String blobName = TestUtils.createRandomBlobName(); ByteSource content = TestUtils.randomByteSource().slice(0, 1024); + @SuppressWarnings("deprecation") String contentMD5 = Hashing.md5().hashBytes(content.read()).toString(); Blob blob = aliasBlobStore.blobBuilder(blobName).payload(content) .build(); @@ -143,6 +144,7 @@ public void testAliasMultipartUpload() throws IOException { createContainer(aliasContainerName); String blobName = TestUtils.createRandomBlobName(); ByteSource content = TestUtils.randomByteSource().slice(0, 1024); + @SuppressWarnings("deprecation") HashCode contentHash = Hashing.md5().hashBytes(content.read()); Blob blob = aliasBlobStore.blobBuilder(blobName).build(); MultipartUpload mpu = aliasBlobStore.initiateMultipartUpload( @@ -156,9 +158,10 @@ public void testAliasMultipartUpload() throws IOException { parts.add(part); String mpuETag = aliasBlobStore.completeMultipartUpload(mpu, parts.build()); + @SuppressWarnings("deprecation") + HashCode contentHash2 = Hashing.md5().hashBytes(contentHash.asBytes()); assertThat(mpuETag).isEqualTo( - String.format("\"%s-1\"", - Hashing.md5().hashBytes(contentHash.asBytes()))); + String.format("\"%s-1\"", contentHash2)); blob = aliasBlobStore.getBlob(aliasContainerName, blobName); try (InputStream actual = blob.getPayload().openStream(); InputStream expected = content.openStream()) { From b72a6fea9ee667cb785b2b784e796d0fca747d2b Mon Sep 17 00:00:00 2001 From: Andrew Gaul Date: Mon, 26 Dec 2022 12:24:08 +0900 Subject: [PATCH 46/55] Remove unneeded AliasBlobStore directory methods These are not needed and generate deprecation warnings. --- .../java/org/gaul/s3proxy/AliasBlobStore.java | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/src/main/java/org/gaul/s3proxy/AliasBlobStore.java b/src/main/java/org/gaul/s3proxy/AliasBlobStore.java index b1d5797a..5ecb10c7 100644 --- a/src/main/java/org/gaul/s3proxy/AliasBlobStore.java +++ b/src/main/java/org/gaul/s3proxy/AliasBlobStore.java @@ -193,21 +193,6 @@ public boolean deleteContainerIfEmpty(String container) { return delegate().deleteContainerIfEmpty(getContainer(container)); } - @Override - public boolean directoryExists(String container, String directory) { - return delegate().directoryExists(getContainer(container), directory); - } - - @Override - public void createDirectory(String container, String directory) { - delegate().createDirectory(getContainer(container), directory); - } - - @Override - public void deleteDirectory(String container, String directory) { - delegate().deleteDirectory(getContainer(container), directory); - } - @Override public boolean blobExists(String container, String name) { return delegate().blobExists(getContainer(container), name); From 302e2050b1564cc4d44354ee064c1d51d4a99054 Mon Sep 17 00:00:00 2001 From: Andrew Gaul Date: Mon, 26 Dec 2022 12:26:59 +0900 Subject: [PATCH 47/55] Upgrade to error-prone 2.16 Release notes: https://github.com/google/error-prone/releases/tag/v2.16 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index c909a22f..f6a32354 100644 --- a/pom.xml +++ b/pom.xml @@ -165,7 +165,7 @@ com.google.errorprone error_prone_core - 2.15.0 + 2.16 From 9524d032e1725c54c116e9a14372eb52d4dbe677 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Dec 2022 22:01:54 +0000 Subject: [PATCH 48/55] Bump jetty-servlet from 11.0.11 to 11.0.13 Bumps [jetty-servlet](https://github.com/eclipse/jetty.project) from 11.0.11 to 11.0.13. - [Release notes](https://github.com/eclipse/jetty.project/releases) - [Commits](https://github.com/eclipse/jetty.project/compare/jetty-11.0.11...jetty-11.0.13) --- updated-dependencies: - dependency-name: org.eclipse.jetty:jetty-servlet dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index f6a32354..613833f2 100644 --- a/pom.xml +++ b/pom.xml @@ -484,7 +484,7 @@ org.eclipse.jetty jetty-servlet - 11.0.11 + 11.0.13 org.slf4j From 69eadc5c5d45d3ac44771ff015e2c44f2d359c42 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Dec 2022 22:01:57 +0000 Subject: [PATCH 49/55] Bump maven-shade-plugin from 3.3.0 to 3.4.1 Bumps [maven-shade-plugin](https://github.com/apache/maven-shade-plugin) from 3.3.0 to 3.4.1. - [Release notes](https://github.com/apache/maven-shade-plugin/releases) - [Commits](https://github.com/apache/maven-shade-plugin/compare/maven-shade-plugin-3.3.0...maven-shade-plugin-3.4.1) --- updated-dependencies: - dependency-name: org.apache.maven.plugins:maven-shade-plugin dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 613833f2..ea91ba82 100644 --- a/pom.xml +++ b/pom.xml @@ -199,7 +199,7 @@ org.apache.maven.plugins maven-shade-plugin - 3.3.0 + 3.4.1 package From 56a63116e896091796b0658116f9900cd2f6084a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Dec 2022 22:01:59 +0000 Subject: [PATCH 50/55] Bump maven-checkstyle-plugin from 3.1.2 to 3.2.0 Bumps [maven-checkstyle-plugin](https://github.com/apache/maven-checkstyle-plugin) from 3.1.2 to 3.2.0. - [Release notes](https://github.com/apache/maven-checkstyle-plugin/releases) - [Commits](https://github.com/apache/maven-checkstyle-plugin/compare/maven-checkstyle-plugin-3.1.2...maven-checkstyle-plugin-3.2.0) --- updated-dependencies: - dependency-name: org.apache.maven.plugins:maven-checkstyle-plugin dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index ea91ba82..ac29e1ba 100644 --- a/pom.xml +++ b/pom.xml @@ -110,7 +110,7 @@ org.apache.maven.plugins maven-checkstyle-plugin - 3.1.2 + 3.2.0 check From d94c0d81ff3804ac5a892fdd40796bcd30f8f2df Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Dec 2022 22:01:53 +0000 Subject: [PATCH 51/55] Bump slf4j.version from 1.7.36 to 2.0.6 Bumps `slf4j.version` from 1.7.36 to 2.0.6. Updates `slf4j-api` from 1.7.36 to 2.0.6 - [Release notes](https://github.com/qos-ch/slf4j/releases) - [Commits](https://github.com/qos-ch/slf4j/compare/v_1.7.36...v_2.0.6) Updates `jcl-over-slf4j` from 1.7.36 to 2.0.6 - [Release notes](https://github.com/qos-ch/slf4j/releases) - [Commits](https://github.com/qos-ch/slf4j/compare/v_1.7.36...v_2.0.6) --- updated-dependencies: - dependency-name: org.slf4j:slf4j-api dependency-type: direct:production update-type: version-update:semver-major - dependency-name: org.slf4j:jcl-over-slf4j dependency-type: direct:development update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index ac29e1ba..9808a8c1 100644 --- a/pom.xml +++ b/pom.xml @@ -376,7 +376,7 @@ UTF-8 11 2.5.0 - 1.7.36 + 2.0.6 ${project.groupId}.shaded 2.22.2 From 47605c98641be10468de3866058c3fcaa3c7e006 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 27 Dec 2022 22:01:42 +0000 Subject: [PATCH 52/55] Bump logback-classic from 1.2.11 to 1.4.5 Bumps [logback-classic](https://github.com/qos-ch/logback) from 1.2.11 to 1.4.5. - [Release notes](https://github.com/qos-ch/logback/releases) - [Commits](https://github.com/qos-ch/logback/compare/v_1.2.11...v_1.4.5) --- updated-dependencies: - dependency-name: ch.qos.logback:logback-classic dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 9808a8c1..711bbdaf 100644 --- a/pom.xml +++ b/pom.xml @@ -412,7 +412,7 @@ ch.qos.logback logback-classic - 1.2.11 + 1.4.5 javax.xml.bind From ad6a9947b35fe943f27181295732d46936b1cce1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 27 Dec 2022 22:01:38 +0000 Subject: [PATCH 53/55] Bump maven-javadoc-plugin from 3.4.0 to 3.4.1 Bumps [maven-javadoc-plugin](https://github.com/apache/maven-javadoc-plugin) from 3.4.0 to 3.4.1. - [Release notes](https://github.com/apache/maven-javadoc-plugin/releases) - [Commits](https://github.com/apache/maven-javadoc-plugin/compare/maven-javadoc-plugin-3.4.0...maven-javadoc-plugin-3.4.1) --- updated-dependencies: - dependency-name: org.apache.maven.plugins:maven-javadoc-plugin dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 711bbdaf..dadf0a4b 100644 --- a/pom.xml +++ b/pom.xml @@ -186,7 +186,7 @@ org.apache.maven.plugins maven-javadoc-plugin - 3.4.0 + 3.4.1 attach-javadocs From 55f779480a9763fb3dacc39b589f1a7b3dd8cf4c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 27 Dec 2022 22:01:18 +0000 Subject: [PATCH 54/55] Bump maven-install-plugin from 3.0.1 to 3.1.0 Bumps [maven-install-plugin](https://github.com/apache/maven-install-plugin) from 3.0.1 to 3.1.0. - [Release notes](https://github.com/apache/maven-install-plugin/releases) - [Commits](https://github.com/apache/maven-install-plugin/compare/maven-install-plugin-3.0.1...maven-install-plugin-3.1.0) --- updated-dependencies: - dependency-name: org.apache.maven.plugins:maven-install-plugin dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index dadf0a4b..7bb51424 100644 --- a/pom.xml +++ b/pom.xml @@ -100,7 +100,7 @@ org.apache.maven.plugins maven-install-plugin - 3.0.1 + 3.1.0 org.apache.maven.plugins From 65725c4c1e17bc5d11f88f04296720f8ccff5f6f Mon Sep 17 00:00:00 2001 From: Andrew Gaul Date: Sat, 14 Jan 2023 17:25:06 +0900 Subject: [PATCH 55/55] Configure Dependabot for monthly notifications --- .github/dependabot.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 5b063201..23a3bbe5 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -8,4 +8,4 @@ updates: - package-ecosystem: "maven" # See documentation for possible values directory: "/" # Location of package manifests schedule: - interval: "daily" + interval: "monthly"