diff --git a/README.md b/README.md index a32e7ad..8cc3431 100644 --- a/README.md +++ b/README.md @@ -217,4 +217,20 @@ as starting the project using the shortcut in the `Makefile`: There will be more documentation evolving inside the `docs/` folder. +### Minio as Storage + +If you want to use Min.io as Storage, you can easily start a Min.io Server with +the following docker command: + +``` +docker run -p 9000:9000 --name minio1 -e "MINIO_ACCESS_KEY=admin" -e +"MINIO_SECRET_KEY=password" -v /home/youruser/.minio/data +-v /mnt/config:/home/youruser/.minio minio/minio server /data +``` +If this ran smoothly you can reach Min.io Browser under localhost:9000 +with credentials "admin:password" + +Currently min.io is the default store, if you want to chance this, +edit application.yml to your needs + Happy hacking! diff --git a/pom.xml b/pom.xml index d2a9cef..71d5b13 100644 --- a/pom.xml +++ b/pom.xml @@ -21,6 +21,8 @@ UTF-8 UTF-8 11 + 1.8 + 1.8 diff --git a/src/main/java/com/studiomediatech/contessa/store/ContessaException.java b/src/main/java/com/studiomediatech/contessa/store/ContessaException.java new file mode 100644 index 0000000..6efacb1 --- /dev/null +++ b/src/main/java/com/studiomediatech/contessa/store/ContessaException.java @@ -0,0 +1,38 @@ +/* + * J.Arraszu + */ +package com.studiomediatech.contessa.store; + +/** + * Wrapper for the concrete Exceptions of the underlying Storeimpls. + * + * @author Joachim Arrasz + */ +public class ContessaException extends Exception { + + private Exception cause; + + /** + * Needs a cause exception. + * + * @param ex the originator + */ + public ContessaException(Exception ex) { + this.cause = ex; + } + + /** + * @return the cause + */ + public Exception getCause() { + return cause; + } + + /** + * @param cause the cause to set + */ + public void setCause(Exception cause) { + this.cause = cause; + } + +} diff --git a/src/main/java/com/studiomediatech/contessa/store/Storage.java b/src/main/java/com/studiomediatech/contessa/store/Storage.java index 7364b96..f4e2a48 100644 --- a/src/main/java/com/studiomediatech/contessa/store/Storage.java +++ b/src/main/java/com/studiomediatech/contessa/store/Storage.java @@ -11,12 +11,46 @@ */ public interface Storage extends Loggable { + /** + * Stores an entry under a default path. + * + * @param entry the entry to store under default path. + */ void store(Entry entry); - - + + /** + * Check if an entry already exists under gioven path. + * + * @param entry the entry to check + * @param path the pasth to the entry, possibly a bucket, possibly a url/uri, can be null + * @return true if this entry is already saved under this path + */ + boolean exists(Entry entry, String path) throws ContessaException; + + + /** + * Gets the enty identified by identifier under default path. + * + * @param identifier the identifier of the entry + * @return the entry is exists + */ Optional retrieve(String identifier); - - + + /** + * Remove an object from a given bucket if exists. + * + * @param entry to remove + * @param path a bucket if not default , an url/uri , can be null + * @throws ContessaException + */ + void remove(Entry entry, String path) throws ContessaException; + + + /** + * Provides count of objects in all buckets. + * + * @return entry count + */ default long count() { return 0L; diff --git a/src/main/java/com/studiomediatech/contessa/store/cassandra/CassandraStorageImpl.java b/src/main/java/com/studiomediatech/contessa/store/cassandra/CassandraStorageImpl.java index 7d205e0..55d0871 100644 --- a/src/main/java/com/studiomediatech/contessa/store/cassandra/CassandraStorageImpl.java +++ b/src/main/java/com/studiomediatech/contessa/store/cassandra/CassandraStorageImpl.java @@ -3,6 +3,7 @@ import com.studiomediatech.contessa.domain.Entry; import com.studiomediatech.contessa.logging.Loggable; import com.studiomediatech.contessa.store.Storage; +import com.studiomediatech.contessa.store.ContessaException; import org.springframework.stereotype.Component; @@ -42,4 +43,14 @@ public long count() { return repo.count(); } + + @Override + public boolean exists(Entry entry, String path) { + throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates. + } + + @Override + public void remove(Entry entry, String hpat) throws ContessaException { + throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates. + } } diff --git a/src/main/java/com/studiomediatech/contessa/store/file/FileStorageImpl.java b/src/main/java/com/studiomediatech/contessa/store/file/FileStorageImpl.java index ea14fa1..17e30dc 100644 --- a/src/main/java/com/studiomediatech/contessa/store/file/FileStorageImpl.java +++ b/src/main/java/com/studiomediatech/contessa/store/file/FileStorageImpl.java @@ -6,6 +6,7 @@ import com.studiomediatech.contessa.domain.Entry; import com.studiomediatech.contessa.logging.Loggable; import com.studiomediatech.contessa.store.Storage; +import com.studiomediatech.contessa.store.ContessaException; import org.springframework.stereotype.Component; @@ -79,4 +80,14 @@ public long count() { return getStoragePath().toFile().list((dir, name) -> name.contains(".json")).length; } + + @Override + public boolean exists(Entry entry, String path) { + throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates. + } + + @Override + public void remove(Entry entry, String hpat) throws ContessaException { + throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates. + } } diff --git a/src/main/java/com/studiomediatech/contessa/store/minio/MinioStorageImpl.java b/src/main/java/com/studiomediatech/contessa/store/minio/MinioStorageImpl.java index b72474e..c2b9283 100644 --- a/src/main/java/com/studiomediatech/contessa/store/minio/MinioStorageImpl.java +++ b/src/main/java/com/studiomediatech/contessa/store/minio/MinioStorageImpl.java @@ -1,5 +1,6 @@ package com.studiomediatech.contessa.store.minio; +import com.studiomediatech.contessa.store.ContessaException; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.annotations.VisibleForTesting; @@ -8,8 +9,14 @@ import com.studiomediatech.contessa.store.Storage; import io.minio.MinioClient; +import io.minio.errors.ErrorResponseException; +import io.minio.errors.InsufficientDataException; +import io.minio.errors.InternalException; +import io.minio.errors.InvalidArgumentException; +import io.minio.errors.InvalidBucketNameException; import io.minio.errors.MinioException; +import io.minio.errors.NoResponseException; import org.springframework.boot.context.event.ApplicationReadyEvent; import org.springframework.boot.context.properties.EnableConfigurationProperties; @@ -30,7 +37,8 @@ import java.util.Optional; import java.util.concurrent.atomic.AtomicLong; import java.util.function.Supplier; - +import java.util.logging.Level; +import java.util.logging.Logger; @EnableConfigurationProperties(ContessaMinioProperties.class) @Component @@ -46,18 +54,24 @@ public class MinioStorageImpl implements Storage { @VisibleForTesting Supplier client; + /** + * Default Impl for Min.io Servers. + * + * @param props configuration of our minio client to connect minio server. + * @param objectMapper + */ public MinioStorageImpl(ContessaMinioProperties props, ObjectMapper objectMapper) { this.objectMapper = objectMapper; - this.client = - () -> { - try { - return new MinioClient(props.getEndpoint(), props.getAccessKey(), props.getSecretKey()); - } catch (MinioException e) { - throw new RuntimeException("Failed to initialize Minio client.", e); - } - }; + this.client + = () -> { + try { + return new MinioClient(props.getEndpoint(), props.getAccessKey(), props.getSecretKey()); + } catch (MinioException e) { + throw new RuntimeException("Failed to initialize Minio client.", e); + } + }; } @EventListener @@ -68,7 +82,6 @@ public void on(ApplicationReadyEvent _event) { updateObjectsCount(); } - private void ensureBucketIsCreated() { try { @@ -80,7 +93,6 @@ private void ensureBucketIsCreated() { } } - private void loadObjectsCount() { try { @@ -95,7 +107,6 @@ private void loadObjectsCount() { } } - private void updateObjectsCount() { try { @@ -112,10 +123,9 @@ private void updateObjectsCount() { } } - @Override public void store(Entry entry) { - + ensureBucketIsCreated(); try { String filename = String.format("%s.json", entry.getId()); MinioEntry m = MinioEntry.valueOf(entry); @@ -131,7 +141,6 @@ public void store(Entry entry) { } } - @Override public Optional retrieve(String identifier) { @@ -149,12 +158,43 @@ public Optional retrieve(String identifier) { } } - @Override public long count() { return this.count.get(); } + + @Override + public boolean exists(Entry entry, String path) throws ContessaException { + logger().debug("called with path :" + path + " for entry with id: " + entry.getId()); + InputStream stream = null; + String filename = String.format("%s.json", entry.getId()); + + try { + if (path != null) { + stream = this.client.get().getObject(path, filename); + } else { + stream = this.client.get().getObject(BUCKET, filename); + } + } catch (InvalidBucketNameException | NoSuchAlgorithmException | InsufficientDataException | IOException | InvalidKeyException | NoResponseException | XmlPullParserException | ErrorResponseException | InternalException | InvalidArgumentException ex) { + throw new ContessaException(ex); + } + + return stream != null; + } + + @Override + public void remove(Entry entry, String path) throws ContessaException { + logger().debug("entry.toString() is being deleted now. Path is given: " + path); + String filename = String.format("%s.json", entry.getId()); + + try { + this.client.get().removeObject(BUCKET, filename); + } catch (InvalidBucketNameException | NoSuchAlgorithmException | InsufficientDataException | IOException | InvalidKeyException | NoResponseException | XmlPullParserException | ErrorResponseException | InternalException | InvalidArgumentException ex) { + logger().error("Unable to remove object. Exception occured: ", ex); + throw new ContessaException(ex); + } + } } class MetaEntry { diff --git a/src/main/java/com/studiomediatech/contessa/store/mongo/MongoDbStorageImpl.java b/src/main/java/com/studiomediatech/contessa/store/mongo/MongoDbStorageImpl.java index eca6bc7..295bb9c 100644 --- a/src/main/java/com/studiomediatech/contessa/store/mongo/MongoDbStorageImpl.java +++ b/src/main/java/com/studiomediatech/contessa/store/mongo/MongoDbStorageImpl.java @@ -3,6 +3,7 @@ import com.studiomediatech.contessa.domain.Entry; import com.studiomediatech.contessa.logging.Loggable; import com.studiomediatech.contessa.store.Storage; +import com.studiomediatech.contessa.store.ContessaException; import org.springframework.stereotype.Component; @@ -42,4 +43,14 @@ public long count() { return repo.count(); } + + @Override + public boolean exists(Entry entry, String path) { + throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates. + } + + @Override + public void remove(Entry entry, String hpat) throws ContessaException { + throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates. + } } diff --git a/src/main/java/com/studiomediatech/contessa/store/none/NoneStorageImpl.java b/src/main/java/com/studiomediatech/contessa/store/none/NoneStorageImpl.java index 597e111..efe1cd9 100644 --- a/src/main/java/com/studiomediatech/contessa/store/none/NoneStorageImpl.java +++ b/src/main/java/com/studiomediatech/contessa/store/none/NoneStorageImpl.java @@ -3,6 +3,7 @@ import com.studiomediatech.contessa.domain.Entry; import com.studiomediatech.contessa.logging.Loggable; import com.studiomediatech.contessa.store.Storage; +import com.studiomediatech.contessa.store.ContessaException; import java.util.Optional; @@ -26,4 +27,14 @@ public Optional retrieve(String identifier) { return Optional.empty(); } + + @Override + public boolean exists(Entry entry, String path) { + throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates. + } + + @Override + public void remove(Entry entry, String hpat) throws ContessaException { + throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates. + } } diff --git a/src/main/java/com/studiomediatech/contessa/store/ram/RamStorageImpl.java b/src/main/java/com/studiomediatech/contessa/store/ram/RamStorageImpl.java index 23196a4..fb73fbd 100644 --- a/src/main/java/com/studiomediatech/contessa/store/ram/RamStorageImpl.java +++ b/src/main/java/com/studiomediatech/contessa/store/ram/RamStorageImpl.java @@ -3,6 +3,7 @@ import com.studiomediatech.contessa.domain.Entry; import com.studiomediatech.contessa.logging.Loggable; import com.studiomediatech.contessa.store.Storage; +import com.studiomediatech.contessa.store.ContessaException; import org.springframework.stereotype.Component; @@ -36,4 +37,14 @@ public long count() { return storage.size(); } + + @Override + public boolean exists(Entry entry, String path) { + throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates. + } + + @Override + public void remove(Entry entry, String hpat) throws ContessaException { + throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates. + } } diff --git a/src/main/java/com/studiomediatech/contessa/store/sql/SqlStorageImpl.java b/src/main/java/com/studiomediatech/contessa/store/sql/SqlStorageImpl.java index b3125af..07f082f 100644 --- a/src/main/java/com/studiomediatech/contessa/store/sql/SqlStorageImpl.java +++ b/src/main/java/com/studiomediatech/contessa/store/sql/SqlStorageImpl.java @@ -3,6 +3,7 @@ import com.studiomediatech.contessa.domain.Entry; import com.studiomediatech.contessa.logging.Loggable; import com.studiomediatech.contessa.store.Storage; +import com.studiomediatech.contessa.store.ContessaException; import org.springframework.stereotype.Component; @@ -48,4 +49,14 @@ public long count() { return repo.count(); } + + @Override + public boolean exists(Entry entry, String path) { + throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates. + } + + @Override + public void remove(Entry entry, String hpat) throws ContessaException { + throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates. + } } diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index a3eca3b..07b4f06 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -26,7 +26,7 @@ contessa: ## --[CURRENTLY NOT WORKING!]-- ### `REDIS` ...... Redis storage cluster ### - store: SQL + store: MINIO ### =========================================== ### ### Toggles the available UIs on/off, allowing @@ -84,11 +84,11 @@ contessa: ### --------------------------------------------- minio: ## URL to object storage service. - endpoint: + endpoint: http://172.17.0.2:9000 ## Access key is like user ID that uniquely identifies your account. - access-key: + access-key: admin ## Secret key is the password to your account. - secret-key: + secret-key: password ### --------------------------------------------- ### Customized Spring Boot configuration diff --git a/src/test/java/com/studiomediatech/contessa/store/files/MinioStorageImplTest.java b/src/test/java/com/studiomediatech/contessa/store/files/MinioStorageImplTest.java new file mode 100644 index 0000000..fd15be3 --- /dev/null +++ b/src/test/java/com/studiomediatech/contessa/store/files/MinioStorageImplTest.java @@ -0,0 +1,89 @@ +/* + * J.Arrasz + */ +package com.studiomediatech.contessa.store.files; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.studiomediatech.contessa.domain.Entry; +import com.studiomediatech.contessa.store.Storage; +import com.studiomediatech.contessa.store.ContessaException; +import com.studiomediatech.contessa.store.minio.ContessaMinioProperties; +import com.studiomediatech.contessa.store.minio.MinioStorageImpl; +import org.junit.Test; +import static org.junit.Assert.*; + +/** + * Test for all relevant interactions with MinioClient. + * + * @author Joachim Arrasz + */ +public class MinioStorageImplTest { + + private static final String TEST_BUCKET = "contessa"; + private static final String ENDPOINT_URL = "http://172.17.0.2:9000"; + private static final String SECRET = "password"; + private static final String ACCESS_KEY = "admin"; + + @Test + public void writeFileToStore() { + Entry reference = createReferenceData(); + Storage store = createMinioStoreImpl(); + + store.store(reference); + } + + @Test + public void readFileFromStore() { + Entry reference = createReferenceData(); + Storage store = createMinioStoreImpl(); + + assertNotNull(store.retrieve(reference.getId())); + + } + + @Test + public void fileExists() { + + Entry reference = createReferenceData(); + + Storage store = createMinioStoreImpl(); + + try { + + store.exists(reference, TEST_BUCKET); + } catch (ContessaException ex) { + assertNull(ex); + } + } + + private Storage createMinioStoreImpl() { + ContessaMinioProperties props = new ContessaMinioProperties(); + props.setAccessKey(ACCESS_KEY); + props.setSecretKey(SECRET); + props.setEndpoint(ENDPOINT_URL); + Storage store = new MinioStorageImpl(props, new ObjectMapper()); + return store; + } + + @Test + public void deleteFileFromStore() { + Entry reference = createReferenceData(); + Storage store = createMinioStoreImpl(); + + try { + store.remove(reference, null); + } catch (ContessaException ex) { + assertNull(ex); + } + } + + private Entry createReferenceData() { + //create reference data + Entry reference = new Entry(); + reference.setId("reference"); + reference.setSuffix("txt"); + reference.setType("text/txt"); + reference.setData("lorem Ipsum test data".getBytes()); + return reference; + } +}