Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Check this out , more implementation for min.io #4

Open
wants to merge 9 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 16 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -217,4 +217,20 @@ as starting the project using the shortcut in the `Makefile`:

There will be more documentation evolving inside the `docs/` folder.

### Minio as Storage

If you want to use Min.io as Storage, you can easily start a Min.io Server with
the following docker command:

```
docker run -p 9000:9000 --name minio1 -e "MINIO_ACCESS_KEY=admin" -e
"MINIO_SECRET_KEY=password" -v /home/youruser/.minio/data
-v /mnt/config:/home/youruser/.minio minio/minio server /data
```
If this ran smoothly you can reach Min.io Browser under localhost:9000
with credentials "admin:password"

Currently min.io is the default store, if you want to chance this,
edit application.yml to your needs

Happy hacking!
2 changes: 2 additions & 0 deletions pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<java.version>11</java.version>
<maven.compiler.source>1.8</maven.compiler.source>
<maven.compiler.target>1.8</maven.compiler.target>
</properties>

<dependencies>
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
/*
* J.Arraszu
*/
package com.studiomediatech.contessa.store;

/**
* Wrapper for the concrete Exceptions of the underlying Storeimpls.
*
* @author Joachim Arrasz <[email protected]>
*/
public class ContessaException extends Exception {

private Exception cause;

/**
* Needs a cause exception.
*
* @param ex the originator
*/
public ContessaException(Exception ex) {
this.cause = ex;
}

/**
* @return the cause
*/
public Exception getCause() {
return cause;
}

/**
* @param cause the cause to set
*/
public void setCause(Exception cause) {
this.cause = cause;
}

}
42 changes: 38 additions & 4 deletions src/main/java/com/studiomediatech/contessa/store/Storage.java
Original file line number Diff line number Diff line change
Expand Up @@ -11,12 +11,46 @@
*/
public interface Storage extends Loggable {

/**
* Stores an entry under a default path.
*
* @param entry the entry to store under default path.
*/
void store(Entry entry);



/**
* Check if an entry already exists under gioven path.
*
* @param entry the entry to check
* @param path the pasth to the entry, possibly a bucket, possibly a url/uri, can be null
* @return true if this entry is already saved under this path
*/
boolean exists(Entry entry, String path) throws ContessaException;


/**
* Gets the enty identified by identifier under default path.
*
* @param identifier the identifier of the entry
* @return the entry is exists
*/
Optional<Entry> retrieve(String identifier);



/**
* Remove an object from a given bucket if exists.
*
* @param entry to remove
* @param path a bucket if not default , an url/uri , can be null
* @throws ContessaException
*/
void remove(Entry entry, String path) throws ContessaException;


/**
* Provides count of objects in all buckets.
*
* @return entry count
*/
default long count() {

return 0L;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import com.studiomediatech.contessa.domain.Entry;
import com.studiomediatech.contessa.logging.Loggable;
import com.studiomediatech.contessa.store.Storage;
import com.studiomediatech.contessa.store.ContessaException;

import org.springframework.stereotype.Component;

Expand Down Expand Up @@ -42,4 +43,14 @@ public long count() {

return repo.count();
}

@Override
public boolean exists(Entry entry, String path) {
throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates.
}

@Override
public void remove(Entry entry, String hpat) throws ContessaException {
throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates.
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import com.studiomediatech.contessa.domain.Entry;
import com.studiomediatech.contessa.logging.Loggable;
import com.studiomediatech.contessa.store.Storage;
import com.studiomediatech.contessa.store.ContessaException;

import org.springframework.stereotype.Component;

Expand Down Expand Up @@ -79,4 +80,14 @@ public long count() {

return getStoragePath().toFile().list((dir, name) -> name.contains(".json")).length;
}

@Override
public boolean exists(Entry entry, String path) {
throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates.
}

@Override
public void remove(Entry entry, String hpat) throws ContessaException {
throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates.
}
}
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package com.studiomediatech.contessa.store.minio;

import com.studiomediatech.contessa.store.ContessaException;
import com.fasterxml.jackson.databind.ObjectMapper;

import com.google.common.annotations.VisibleForTesting;
Expand All @@ -8,8 +9,14 @@
import com.studiomediatech.contessa.store.Storage;

import io.minio.MinioClient;
import io.minio.errors.ErrorResponseException;
import io.minio.errors.InsufficientDataException;
import io.minio.errors.InternalException;
import io.minio.errors.InvalidArgumentException;
import io.minio.errors.InvalidBucketNameException;

import io.minio.errors.MinioException;
import io.minio.errors.NoResponseException;

import org.springframework.boot.context.event.ApplicationReadyEvent;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
Expand All @@ -30,7 +37,8 @@
import java.util.Optional;
import java.util.concurrent.atomic.AtomicLong;
import java.util.function.Supplier;

import java.util.logging.Level;
import java.util.logging.Logger;

@EnableConfigurationProperties(ContessaMinioProperties.class)
@Component
Expand All @@ -46,18 +54,24 @@ public class MinioStorageImpl implements Storage {
@VisibleForTesting
Supplier<MinioClient> client;

/**
* Default Impl for Min.io Servers.
*
* @param props configuration of our minio client to connect minio server.
* @param objectMapper
*/
public MinioStorageImpl(ContessaMinioProperties props, ObjectMapper objectMapper) {

this.objectMapper = objectMapper;

this.client =
() -> {
try {
return new MinioClient(props.getEndpoint(), props.getAccessKey(), props.getSecretKey());
} catch (MinioException e) {
throw new RuntimeException("Failed to initialize Minio client.", e);
}
};
this.client
= () -> {
try {
return new MinioClient(props.getEndpoint(), props.getAccessKey(), props.getSecretKey());
} catch (MinioException e) {
throw new RuntimeException("Failed to initialize Minio client.", e);
}
};
}

@EventListener
Expand All @@ -68,7 +82,6 @@ public void on(ApplicationReadyEvent _event) {
updateObjectsCount();
}


private void ensureBucketIsCreated() {

try {
Expand All @@ -80,7 +93,6 @@ private void ensureBucketIsCreated() {
}
}


private void loadObjectsCount() {

try {
Expand All @@ -95,7 +107,6 @@ private void loadObjectsCount() {
}
}


private void updateObjectsCount() {

try {
Expand All @@ -112,10 +123,9 @@ private void updateObjectsCount() {
}
}


@Override
public void store(Entry entry) {

ensureBucketIsCreated();
try {
String filename = String.format("%s.json", entry.getId());
MinioEntry m = MinioEntry.valueOf(entry);
Expand All @@ -131,7 +141,6 @@ public void store(Entry entry) {
}
}


@Override
public Optional<Entry> retrieve(String identifier) {

Expand All @@ -149,12 +158,43 @@ public Optional<Entry> retrieve(String identifier) {
}
}


@Override
public long count() {

return this.count.get();
}

@Override
public boolean exists(Entry entry, String path) throws ContessaException {
logger().debug("called with path :" + path + " for entry with id: " + entry.getId());
InputStream stream = null;
String filename = String.format("%s.json", entry.getId());

try {
if (path != null) {
stream = this.client.get().getObject(path, filename);
} else {
stream = this.client.get().getObject(BUCKET, filename);
}
} catch (InvalidBucketNameException | NoSuchAlgorithmException | InsufficientDataException | IOException | InvalidKeyException | NoResponseException | XmlPullParserException | ErrorResponseException | InternalException | InvalidArgumentException ex) {
throw new ContessaException(ex);
}

return stream != null;
}

@Override
public void remove(Entry entry, String path) throws ContessaException {
logger().debug("entry.toString() is being deleted now. Path is given: " + path);
String filename = String.format("%s.json", entry.getId());

try {
this.client.get().removeObject(BUCKET, filename);
} catch (InvalidBucketNameException | NoSuchAlgorithmException | InsufficientDataException | IOException | InvalidKeyException | NoResponseException | XmlPullParserException | ErrorResponseException | InternalException | InvalidArgumentException ex) {
logger().error("Unable to remove object. Exception occured: ", ex);
throw new ContessaException(ex);
}
}
}

class MetaEntry {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import com.studiomediatech.contessa.domain.Entry;
import com.studiomediatech.contessa.logging.Loggable;
import com.studiomediatech.contessa.store.Storage;
import com.studiomediatech.contessa.store.ContessaException;

import org.springframework.stereotype.Component;

Expand Down Expand Up @@ -42,4 +43,14 @@ public long count() {

return repo.count();
}

@Override
public boolean exists(Entry entry, String path) {
throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates.
}

@Override
public void remove(Entry entry, String hpat) throws ContessaException {
throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates.
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import com.studiomediatech.contessa.domain.Entry;
import com.studiomediatech.contessa.logging.Loggable;
import com.studiomediatech.contessa.store.Storage;
import com.studiomediatech.contessa.store.ContessaException;

import java.util.Optional;

Expand All @@ -26,4 +27,14 @@ public Optional<Entry> retrieve(String identifier) {

return Optional.empty();
}

@Override
public boolean exists(Entry entry, String path) {
throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates.
}

@Override
public void remove(Entry entry, String hpat) throws ContessaException {
throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates.
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import com.studiomediatech.contessa.domain.Entry;
import com.studiomediatech.contessa.logging.Loggable;
import com.studiomediatech.contessa.store.Storage;
import com.studiomediatech.contessa.store.ContessaException;

import org.springframework.stereotype.Component;

Expand Down Expand Up @@ -36,4 +37,14 @@ public long count() {

return storage.size();
}

@Override
public boolean exists(Entry entry, String path) {
throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates.
}

@Override
public void remove(Entry entry, String hpat) throws ContessaException {
throw new UnsupportedOperationException("Not supported yet."); //To change body of generated methods, choose Tools | Templates.
}
}
Loading