From fe114ac2304315db8ec6ad99043da5e6ae3bef65 Mon Sep 17 00:00:00 2001 From: David Gault Date: Sun, 10 Mar 2024 20:03:47 +0000 Subject: [PATCH 01/25] Reintroduce S3FileSystemStore --- pom.xml | 8 +- src/loci/formats/S3FileSystemStore.java | 306 ++++++++++++++++++ src/loci/formats/in/ZarrReader.java | 17 +- .../formats/services/JZarrServiceImpl.java | 48 +-- 4 files changed, 357 insertions(+), 22 deletions(-) create mode 100644 src/loci/formats/S3FileSystemStore.java diff --git a/pom.xml b/pom.xml index ca08d16..dadfe03 100644 --- a/pom.xml +++ b/pom.xml @@ -91,9 +91,11 @@ 2.7.2 runtime - - - + + software.amazon.awssdk + s3 + 2.13.8 + diff --git a/src/loci/formats/S3FileSystemStore.java b/src/loci/formats/S3FileSystemStore.java new file mode 100644 index 0000000..48cc817 --- /dev/null +++ b/src/loci/formats/S3FileSystemStore.java @@ -0,0 +1,306 @@ +package loci.formats; + +/*- + * #%L + * Implementation of Bio-Formats readers for the next-generation file formats + * %% + * Copyright (C) 2020 - 2022 Open Microscopy Environment + * %% + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * #L% + */ + +import com.bc.zarr.ZarrConstants; +import com.bc.zarr.ZarrUtils; +import com.bc.zarr.storage.Store; + +import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider; +import software.amazon.awssdk.auth.credentials.AwsCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.core.ResponseInputStream; +import software.amazon.awssdk.core.sync.ResponseTransformer; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.S3Configuration; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.model.ListObjectsRequest; +import software.amazon.awssdk.services.s3.model.ListObjectsResponse; +import software.amazon.awssdk.services.s3.model.S3Object; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.file.FileSystem; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.ListIterator; +import java.util.TreeSet; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class S3FileSystemStore implements Store { + + private Path root; + S3Client client; + protected static final Logger LOGGER = + LoggerFactory.getLogger(S3FileSystemStore.class); + + public S3FileSystemStore(String path, FileSystem fileSystem) { + if (fileSystem == null) { + root = Paths.get(path); + } else { + root = fileSystem.getPath(path); + } + setupClient(); + } + + public void updateRoot(String path) { + root = Paths.get(path); + } + + public String getRoot() { + return root.toString(); + } + + private void setupClient() { + String[] pathSplit = root.toString().split(File.separator); + String endpoint = "https://" + pathSplit[1] + File.separator; + URI endpoint_uri; + try { + endpoint_uri = new URI(endpoint); + final S3Configuration config = S3Configuration.builder() + .pathStyleAccessEnabled(true) + .build(); + AwsCredentials credentials = AnonymousCredentialsProvider.create().resolveCredentials(); + client = S3Client.builder() + .endpointOverride(endpoint_uri) + .serviceConfiguration(config) + .region(Region.EU_WEST_1) // Ignored but required by the client + .credentialsProvider(StaticCredentialsProvider.create(credentials)).build(); + + } catch (URISyntaxException e) { + LOGGER.info( "Syntax error generating URI from endpoint: " + endpoint); + e.printStackTrace(); + } catch (Exception e) { + LOGGER.info("Exception caught while constructing S3 client"); + e.printStackTrace(); + } + + } + + public void close() { + if (client != null) { + client.close(); + } + } + + public S3FileSystemStore(Path rootPath) { + root = rootPath; + setupClient(); + } + + @Override + public InputStream getInputStream(String key) throws IOException { + String[] pathSplit = root.toString().split(File.separator); + String bucketName = pathSplit[2]; + String key2 = root.toString().substring(root.toString().indexOf(pathSplit[3]), root.toString().length()) + File.separator + key; + + try { + GetObjectRequest getRequest = GetObjectRequest.builder().bucket(bucketName).key(key2).build(); + ResponseInputStream responseStream = client.getObject(getRequest, ResponseTransformer.toInputStream()); + return responseStream; + } catch (Exception e) { + LOGGER.info( "Unable to locate or access key: " + key2); + e.printStackTrace(); + } + + return null; + } + + @Override + public OutputStream getOutputStream(String key) throws IOException { + final Path filePath = root.resolve(key); + final Path dir = filePath.getParent(); + Files.createDirectories(dir); + return Files.newOutputStream(filePath); + } + + @Override + public void delete(String key) throws IOException { + final Path toBeDeleted = root.resolve(key); + if (Files.isDirectory(toBeDeleted)) { + ZarrUtils.deleteDirectoryTreeRecursively(toBeDeleted); + } + if (Files.exists(toBeDeleted)){ + Files.delete(toBeDeleted); + } + if (Files.exists(toBeDeleted)|| Files.isDirectory(toBeDeleted)) { + throw new IOException("Unable to initialize " + toBeDeleted.toAbsolutePath().toString()); + } + } + + @Override + public TreeSet getArrayKeys() throws IOException { + return getKeysFor(ZarrConstants.FILENAME_DOT_ZARRAY); + } + + @Override + public TreeSet getGroupKeys() throws IOException { + return getKeysFor(ZarrConstants.FILENAME_DOT_ZGROUP); + } + + /** + * Copied from {@com.bc.zarr.storage.FileSystemStorage#getKeysEndingWith(String). + * + * @param suffix + * @return + * @throws IOException + */ + public TreeSet getKeysEndingWith(String suffix) throws IOException { + return (TreeSet)Files.walk(this.root).filter((path) -> { + return path.toString().endsWith(suffix); + }).map((path) -> { + return this.root.relativize(path).toString(); + }).collect(Collectors.toCollection(TreeSet::new)); + } + + /** + * Copied from {@com.bc.zarr.storage.FileSystemStorage#getRelativeLeafKeys(String). + * + * @param key + * @return + * @throws IOException + */ + public Stream getRelativeLeafKeys(String key) throws IOException { + Path walkingRoot = this.root.resolve(key); + return Files.walk(walkingRoot).filter((path) -> { + return !Files.isDirectory(path, new LinkOption[0]); + }).map((path) -> { + return walkingRoot.relativize(path).toString(); + }).map(ZarrUtils::normalizeStoragePath).filter((s) -> { + return s.trim().length() > 0; + }); + } + + private TreeSet getKeysFor(String suffix) throws IOException { + TreeSet keys = new TreeSet(); + + String[] pathSplit = root.toString().split(File.separator); + + String bucketName = pathSplit[2]; + String key2 = root.toString().substring(root.toString().indexOf(pathSplit[3]), root.toString().length()); + + ListObjectsRequest listObjectsRequest = ListObjectsRequest + .builder() + .bucket(bucketName) + .prefix(key2) + .build() + ; + + ListObjectsResponse listObjectsResponse = null; + String lastKey = null; + + do { + if ( listObjectsResponse != null ) { + listObjectsRequest = listObjectsRequest.toBuilder() + .marker(lastKey) + .build() + ; + } + + listObjectsResponse = client.listObjects(listObjectsRequest); + List objects = listObjectsResponse.contents(); + + // Iterate over results + ListIterator iterVals = objects.listIterator(); + while (iterVals.hasNext()) { + S3Object object = (S3Object) iterVals.next(); + String k = object.key(); + if (k.contains(suffix)) { + String key = k.substring(k.indexOf(key2) + key2.length() + 1, k.indexOf(suffix)); + if (!key.isEmpty()) { + keys.add(key.substring(0, key.length()-1)); + } + } + lastKey = k; + } + } while ( listObjectsResponse.isTruncated() ); + + return keys; + } + + public ArrayList getFiles() throws IOException { + ArrayList keys = new ArrayList(); + + String[] pathSplit = root.toString().split(File.separator); + String bucketName = pathSplit[2]; + String key2 = root.toString().substring(root.toString().indexOf(pathSplit[3]), root.toString().length()); + + ListObjectsRequest listObjectsRequest = ListObjectsRequest + .builder() + .bucket(bucketName) + .prefix(key2) + .build() + ; + + ListObjectsResponse listObjectsResponse = null; + String lastKey = null; + + do { + if ( listObjectsResponse != null ) { + listObjectsRequest = listObjectsRequest.toBuilder() + .marker(lastKey) + .build() + ; + } + + listObjectsResponse = client.listObjects(listObjectsRequest); + List objects = listObjectsResponse.contents(); + + // Iterate over results + ListIterator iterVals = objects.listIterator(); + while (iterVals.hasNext()) { + S3Object object = (S3Object) iterVals.next(); + String k = object.key(); + String key = k.substring(k.indexOf(key2) + key2.length() + 1, k.length()); + if (!key.isEmpty()) { + keys.add(key.substring(0, key.length()-1)); + } + lastKey = k; + } + } while ( listObjectsResponse.isTruncated() ); + return keys; + } + + +} \ No newline at end of file diff --git a/src/loci/formats/in/ZarrReader.java b/src/loci/formats/in/ZarrReader.java index 5bfc736..19c153c 100644 --- a/src/loci/formats/in/ZarrReader.java +++ b/src/loci/formats/in/ZarrReader.java @@ -97,6 +97,8 @@ public class ZarrReader extends FormatReader { public static final String LIST_PIXELS_ENV_KEY = "OME_ZARR_LIST_PIXELS"; public static final String INCLUDE_LABELS_KEY = "omezarr.include_labels"; public static final boolean INCLUDE_LABELS_DEFAULT = false; + public static final String ALT_STORE_KEY = "omezarr.alt_store"; + public static final String ALT_STORE_DEFAULT = null; protected transient ZarrService zarrService; private ArrayList arrayPaths = new ArrayList(); @@ -468,7 +470,7 @@ public void reopenFile() throws IOException { } protected void initializeZarrService(String rootPath) throws IOException, FormatException { - zarrService = new JZarrServiceImpl(rootPath); + zarrService = new JZarrServiceImpl(altStore()); openZarr(); } @@ -1200,6 +1202,19 @@ public boolean includeLabels() { } return INCLUDE_LABELS_DEFAULT; } + + /** + * Used to provide the location of an alternative file store where the data is located + * @return String representing the root path of the alternative file store or null if no alternative location exist + */ + public String altStore() { + MetadataOptions options = getMetadataOptions(); + if (options instanceof DynamicMetadataOptions) { + return ((DynamicMetadataOptions) options).get( + ALT_STORE_KEY, ALT_STORE_DEFAULT); + } + return ALT_STORE_DEFAULT; + } private boolean systemEnvListPixels() { String value = System.getenv(LIST_PIXELS_ENV_KEY); diff --git a/src/loci/formats/services/JZarrServiceImpl.java b/src/loci/formats/services/JZarrServiceImpl.java index dce0186..482315f 100644 --- a/src/loci/formats/services/JZarrServiceImpl.java +++ b/src/loci/formats/services/JZarrServiceImpl.java @@ -54,6 +54,7 @@ import loci.common.services.AbstractService; import loci.formats.FormatException; import loci.formats.FormatTools; +import loci.formats.S3FileSystemStore; import loci.formats.meta.IPyramidStore; import loci.formats.meta.MetadataRetrieve; import ucar.ma2.InvalidRangeException; @@ -65,6 +66,7 @@ public class JZarrServiceImpl extends AbstractService public static final String NO_ZARR_MSG = "JZARR is required to read Zarr files."; // -- Fields -- + S3FileSystemStore s3fs; ZarrArray zarrArray; String currentId; Compressor zlibComp = CompressorFactory.create("zlib", "level", 8); // 8 = compression level .. valid values 0 .. 9 @@ -76,20 +78,20 @@ public class JZarrServiceImpl extends AbstractService */ public JZarrServiceImpl(String root) { checkClassDependency(com.bc.zarr.ZarrArray.class); - if (root != null && root.toLowerCase().contains("s3:")) { - LOGGER.warn("S3 access currently not supported"); + if (root != null && (root.toLowerCase().contains("s3:") || root.toLowerCase().contains("s3."))) { + s3fs = new S3FileSystemStore(Paths.get(root)); } } @Override public void open(String file) throws IOException, FormatException { currentId = file; - // TODO: Update s3 location identification - if (!file.toLowerCase().contains("s3:")) { + if (s3fs == null) { zarrArray = ZarrArray.open(file); } - else { - LOGGER.warn("S3 access currently not supported"); + else { + s3fs.updateRoot(getZarrRoot(s3fs.getRoot()) + stripZarrRoot(file)); + zarrArray = ZarrArray.open(s3fs); } } @@ -100,48 +102,48 @@ public void open(String id, ZarrArray array) { public Map getGroupAttr(String path) throws IOException, FormatException { ZarrGroup group = null; - if (!path.toLowerCase().contains("s3:")) { + if (s3fs == null) { group = ZarrGroup.open(path); } else { - LOGGER.warn("S3 access currently not supported"); - return null; + s3fs.updateRoot(getZarrRoot(s3fs.getRoot()) + stripZarrRoot(path)); + group = ZarrGroup.open(s3fs); } return group.getAttributes(); } public Map getArrayAttr(String path) throws IOException, FormatException { ZarrArray array = null; - if (!path.toLowerCase().contains("s3:")) { + if (s3fs == null) { array = ZarrArray.open(path); } else { - LOGGER.warn("S3 access currently not supported"); - return null; + s3fs.updateRoot(getZarrRoot(s3fs.getRoot()) + stripZarrRoot(path)); + array = ZarrArray.open(s3fs); } return array.getAttributes(); } public Set getGroupKeys(String path) throws IOException, FormatException { ZarrGroup group = null; - if (!path.toLowerCase().contains("s3:")) { + if (s3fs == null) { group = ZarrGroup.open(path); } else { - LOGGER.warn("S3 access currently not supported"); - return null; + s3fs.updateRoot(getZarrRoot(s3fs.getRoot()) + stripZarrRoot(path)); + group = ZarrGroup.open(s3fs); } return group.getGroupKeys(); } public Set getArrayKeys(String path) throws IOException, FormatException { ZarrGroup group = null; - if (!path.toLowerCase().contains("s3:")) { + if (s3fs == null) { group = ZarrGroup.open(path); } else { - LOGGER.warn("S3 access currently not supported"); - return null; + s3fs.updateRoot(getZarrRoot(s3fs.getRoot()) + stripZarrRoot(path)); + group = ZarrGroup.open(s3fs); } return group.getArrayKeys(); } @@ -247,6 +249,9 @@ public boolean isLittleEndian() { public void close() throws IOException { zarrArray = null; currentId = null; + if (s3fs != null) { + s3fs.close(); + } } @Override @@ -358,5 +363,12 @@ public void create(String id, MetadataRetrieve meta, int[] chunks) throws IOExce create(id, meta, chunks, Compression.NONE); } + private String stripZarrRoot(String path) { + return path.substring(path.indexOf(".zarr")+5); + } + + private String getZarrRoot(String path) { + return path.substring(0, path.indexOf(".zarr")+5); + } } From 35a2fef7b3c1f4def220806c286ad6582c600a3e Mon Sep 17 00:00:00 2001 From: David Gault Date: Mon, 11 Mar 2024 15:12:18 +0000 Subject: [PATCH 02/25] Add new openPlane function to read pixel data without initializing --- src/loci/formats/in/ZarrReader.java | 103 +++++++++++++++++++++++++++- 1 file changed, 102 insertions(+), 1 deletion(-) diff --git a/src/loci/formats/in/ZarrReader.java b/src/loci/formats/in/ZarrReader.java index 19c153c..122118f 100644 --- a/src/loci/formats/in/ZarrReader.java +++ b/src/loci/formats/in/ZarrReader.java @@ -447,7 +447,7 @@ private int[] get5DShape(int [] originalShape) { * @param size of the shape required by jzarr * @return a 5D shape to be used within the reader */ - private int[] getOriginalShape(int [] shape5D, int size) { + private static int[] getOriginalShape(int [] shape5D, int size) { int [] shape = new int[size]; int shape5DIndex = 4; for (int s = shape.length - 1; s >= 0; s--) { @@ -1147,6 +1147,7 @@ protected ArrayList getAvailableOptions() { optionsList.add(LIST_PIXELS_KEY); optionsList.add(QUICK_READ_KEY); optionsList.add(INCLUDE_LABELS_KEY); + optionsList.add(ALT_STORE_KEY); return optionsList; } @@ -1241,4 +1242,104 @@ private void reloadOptionsFile(String id) { } } } + + public static byte[] openPlane(String path, byte[] buf) throws FormatException, IOException { + String rootPath = path.substring(0, path.indexOf(".zarr") + 5); + String altPath = loadAltStoreOption(rootPath); + if (altPath == null) altPath = rootPath; + ZarrService service = new JZarrServiceImpl(altPath); + openZarr(path, service); + int [] shape = {1, 1, 1, 1, 1}; + int [] zarrShape = service.getShape(); + shape[4] = zarrShape[zarrShape.length - 1]; + shape[3] = zarrShape[zarrShape.length - 2]; + int zarrArrayShapeSize = zarrShape.length; + if (zarrArrayShapeSize < 5) { + shape = getOriginalShape(shape, zarrArrayShapeSize); + } + int [] offsets = {0, 0, 0, 0, 0}; + Object image = service.readBytes(shape, offsets); + int w = shape[4]; + int h = shape[3]; + System.out.println("Success: Read image"); + boolean little = service.isLittleEndian(); + int bpp = FormatTools.getBytesPerPixel(service.getPixelType()); + if (image instanceof byte[]) { + byte [] data = (byte []) image; + for (int i = 0; i < data.length; i++) { + DataTools.unpackBytes(data[i], buf, i, 1, little); + } + } + else if (image instanceof short[]) { + short[] data = (short[]) image; + for (int row = 0; row < h; row++) { + int base = row * w * bpp; + for (int i = 0; i < w; i++) { + DataTools.unpackBytes(data[(row * w) + i], buf, base + 2 * i, 2, little); + } + } + } + else if (image instanceof int[]) { + int[] data = (int[]) image; + for (int row = 0; row < h; row++) { + int base = row * w * bpp; + for (int i = 0; i < w; i++) { + DataTools.unpackBytes(data[(row * w) + i], buf, base + 4 * i, 4, little); + } + } + } + else if (image instanceof float[]) { + float[] data = (float[]) image; + for (int row = 0; row < h; row++) { + int base = row * w * bpp; + for (int i = 0; i < w; i++) { + int value = Float.floatToIntBits(data[(row * w) + i]); + DataTools.unpackBytes(value, buf, base + 4 * i, 4, little); + } + } + } + else if (image instanceof double[]) { + double[] data = (double[]) image; + for (int row = 0; row < h; row++) { + int base = row * w * bpp; + for (int i = 0; i < w; i++) { + long value = Double.doubleToLongBits(data[(row * w) + i]); + DataTools.unpackBytes(value, buf, base + 8 * i, 8, little); + } + } + } + return buf; + } + + private static void openZarr(String path, ZarrService service) { + try { + String canonicalPath = new Location(path).getCanonicalPath(); + LOGGER.debug("Opening zarr statically at path: {}", canonicalPath); + service.open(canonicalPath); + } catch (IOException | FormatException e) { + e.printStackTrace(); + } + } + + /** + * Reloads the bfoptions file statically, only reading the value for the alterntiave file store + */ + private static String loadAltStoreOption(String id) { + String optionsFile = DynamicMetadataOptions.getMetadataOptionsFile(id); + if (optionsFile != null) { + MetadataOptions options = new DynamicMetadataOptions(); + if (options != null && options instanceof DynamicMetadataOptions) { + try { + ArrayList optionsList = new ArrayList(); + optionsList.add(ALT_STORE_KEY); + ((DynamicMetadataOptions) options).loadOptions(optionsFile, optionsList); + return ((DynamicMetadataOptions) options).get( + ALT_STORE_KEY, ALT_STORE_DEFAULT); + } catch (Exception e) { + LOGGER.warn("Exception while attempting to read metadata options file", e); + } + } + } + return ALT_STORE_DEFAULT; + } } From fcec4c874d082699567eac075ae477c088223ba0 Mon Sep 17 00:00:00 2001 From: David Gault Date: Wed, 13 Mar 2024 12:25:41 +0000 Subject: [PATCH 03/25] Bump awssdk version --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index dadfe03..102c89b 100644 --- a/pom.xml +++ b/pom.xml @@ -94,7 +94,7 @@ software.amazon.awssdk s3 - 2.13.8 + 2.25.7 From 07b60e22f53e1e0430b8651349bcce20b103334a Mon Sep 17 00:00:00 2001 From: David Gault Date: Thu, 14 Mar 2024 13:30:02 +0000 Subject: [PATCH 04/25] Remove extra debug output --- src/loci/formats/in/ZarrReader.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/loci/formats/in/ZarrReader.java b/src/loci/formats/in/ZarrReader.java index 122118f..32c777c 100644 --- a/src/loci/formats/in/ZarrReader.java +++ b/src/loci/formats/in/ZarrReader.java @@ -1261,7 +1261,6 @@ public static byte[] openPlane(String path, byte[] buf) throws FormatException, Object image = service.readBytes(shape, offsets); int w = shape[4]; int h = shape[3]; - System.out.println("Success: Read image"); boolean little = service.isLittleEndian(); int bpp = FormatTools.getBytesPerPixel(service.getPixelType()); if (image instanceof byte[]) { From f0683dcb1b0a0d8d1e62409841990e6d5ee07b6c Mon Sep 17 00:00:00 2001 From: David Gault Date: Fri, 15 Mar 2024 14:25:53 +0000 Subject: [PATCH 05/25] Update dependencies --- pom.xml | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 102c89b..2a935d4 100644 --- a/pom.xml +++ b/pom.xml @@ -60,6 +60,20 @@ dev.zarr jzarr 0.4.2 + + + com.fasterxml.jackson.core + jackson-databind + + + com.fasterxml.jackson.core + jackson-core + + + com.fasterxml.jackson.core + jackson-annotations + + @@ -94,7 +108,7 @@ software.amazon.awssdk s3 - 2.25.7 + 2.17.100 From 5c4cc8b8829bff9882153331cb9ff2715e18286a Mon Sep 17 00:00:00 2001 From: David Gault Date: Wed, 3 Apr 2024 13:11:23 +0100 Subject: [PATCH 06/25] S3FileStore: Use V1 AWS SDK to match microservices --- pom.xml | 6 +- src/loci/formats/S3FileSystemStore.java | 103 +++++++++++------------- 2 files changed, 48 insertions(+), 61 deletions(-) diff --git a/pom.xml b/pom.xml index 2a935d4..eacc60f 100644 --- a/pom.xml +++ b/pom.xml @@ -106,9 +106,9 @@ runtime - software.amazon.awssdk - s3 - 2.17.100 + com.amazonaws + aws-java-sdk-s3 + 1.12.659 diff --git a/src/loci/formats/S3FileSystemStore.java b/src/loci/formats/S3FileSystemStore.java index 48cc817..7ad06af 100644 --- a/src/loci/formats/S3FileSystemStore.java +++ b/src/loci/formats/S3FileSystemStore.java @@ -1,5 +1,21 @@ package loci.formats; +import com.amazonaws.auth.AWSCredentials; +import com.amazonaws.internal.StaticCredentialsProvider; +import com.amazonaws.regions.Region; +import com.amazonaws.regions.Regions; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3ClientBuilder; +import com.amazonaws.services.s3.model.GetObjectRequest; +import com.amazonaws.services.s3.model.ListObjectsRequest; +import com.amazonaws.services.s3.model.ObjectListing; +import com.amazonaws.services.s3.model.S3Object; +import com.amazonaws.services.s3.model.S3ObjectInputStream; +import com.amazonaws.services.s3.model.S3ObjectSummary; +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.auth.AnonymousAWSCredentials; +import com.amazonaws.client.builder.AwsClientBuilder; + /*- * #%L * Implementation of Bio-Formats readers for the next-generation file formats @@ -33,19 +49,6 @@ import com.bc.zarr.ZarrUtils; import com.bc.zarr.storage.Store; -import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider; -import software.amazon.awssdk.auth.credentials.AwsCredentials; -import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; -import software.amazon.awssdk.core.ResponseInputStream; -import software.amazon.awssdk.core.sync.ResponseTransformer; -import software.amazon.awssdk.regions.Region; -import software.amazon.awssdk.services.s3.S3Client; -import software.amazon.awssdk.services.s3.S3Configuration; -import software.amazon.awssdk.services.s3.model.GetObjectRequest; -import software.amazon.awssdk.services.s3.model.GetObjectResponse; -import software.amazon.awssdk.services.s3.model.ListObjectsRequest; -import software.amazon.awssdk.services.s3.model.ListObjectsResponse; -import software.amazon.awssdk.services.s3.model.S3Object; import java.io.File; import java.io.IOException; @@ -71,7 +74,7 @@ public class S3FileSystemStore implements Store { private Path root; - S3Client client; + AmazonS3 client; protected static final Logger LOGGER = LoggerFactory.getLogger(S3FileSystemStore.class); @@ -95,22 +98,12 @@ public String getRoot() { private void setupClient() { String[] pathSplit = root.toString().split(File.separator); String endpoint = "https://" + pathSplit[1] + File.separator; - URI endpoint_uri; try { - endpoint_uri = new URI(endpoint); - final S3Configuration config = S3Configuration.builder() - .pathStyleAccessEnabled(true) - .build(); - AwsCredentials credentials = AnonymousCredentialsProvider.create().resolveCredentials(); - client = S3Client.builder() - .endpointOverride(endpoint_uri) - .serviceConfiguration(config) - .region(Region.EU_WEST_1) // Ignored but required by the client - .credentialsProvider(StaticCredentialsProvider.create(credentials)).build(); - - } catch (URISyntaxException e) { - LOGGER.info( "Syntax error generating URI from endpoint: " + endpoint); - e.printStackTrace(); + client = AmazonS3ClientBuilder.standard() + .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(endpoint, "auto")) + .withPathStyleAccessEnabled(true) + .withRegion(Regions.DEFAULT_REGION) // Ignored but required by the client + .withCredentials(new AWSStaticCredentialsProvider(new AnonymousAWSCredentials())).build(); } catch (Exception e) { LOGGER.info("Exception caught while constructing S3 client"); e.printStackTrace(); @@ -120,7 +113,7 @@ private void setupClient() { public void close() { if (client != null) { - client.close(); + client.shutdown(); } } @@ -136,8 +129,8 @@ public InputStream getInputStream(String key) throws IOException { String key2 = root.toString().substring(root.toString().indexOf(pathSplit[3]), root.toString().length()) + File.separator + key; try { - GetObjectRequest getRequest = GetObjectRequest.builder().bucket(bucketName).key(key2).build(); - ResponseInputStream responseStream = client.getObject(getRequest, ResponseTransformer.toInputStream()); + S3Object o = client.getObject(bucketName, key2); + S3ObjectInputStream responseStream = o.getObjectContent(); return responseStream; } catch (Exception e) { LOGGER.info( "Unable to locate or access key: " + key2); @@ -220,32 +213,29 @@ private TreeSet getKeysFor(String suffix) throws IOException { String bucketName = pathSplit[2]; String key2 = root.toString().substring(root.toString().indexOf(pathSplit[3]), root.toString().length()); - ListObjectsRequest listObjectsRequest = ListObjectsRequest - .builder() - .bucket(bucketName) - .prefix(key2) - .build() + ListObjectsRequest listObjectsRequest = new ListObjectsRequest() + .withBucketName(bucketName) + .withPrefix(key2) ; - ListObjectsResponse listObjectsResponse = null; + ObjectListing listObjectsResponse = null; String lastKey = null; do { if ( listObjectsResponse != null ) { - listObjectsRequest = listObjectsRequest.toBuilder() - .marker(lastKey) - .build() + listObjectsRequest = listObjectsRequest + .withMarker(lastKey) ; } listObjectsResponse = client.listObjects(listObjectsRequest); - List objects = listObjectsResponse.contents(); + List objects = listObjectsResponse.getObjectSummaries(); // Iterate over results - ListIterator iterVals = objects.listIterator(); + ListIterator iterVals = objects.listIterator(); while (iterVals.hasNext()) { - S3Object object = (S3Object) iterVals.next(); - String k = object.key(); + S3ObjectSummary object = (S3ObjectSummary) iterVals.next(); + String k = object.getKey(); if (k.contains(suffix)) { String key = k.substring(k.indexOf(key2) + key2.length() + 1, k.indexOf(suffix)); if (!key.isEmpty()) { @@ -266,32 +256,29 @@ public ArrayList getFiles() throws IOException { String bucketName = pathSplit[2]; String key2 = root.toString().substring(root.toString().indexOf(pathSplit[3]), root.toString().length()); - ListObjectsRequest listObjectsRequest = ListObjectsRequest - .builder() - .bucket(bucketName) - .prefix(key2) - .build() + ListObjectsRequest listObjectsRequest = new ListObjectsRequest() + .withBucketName(bucketName) + .withPrefix(key2) ; - ListObjectsResponse listObjectsResponse = null; + ObjectListing listObjectsResponse = null; String lastKey = null; do { if ( listObjectsResponse != null ) { - listObjectsRequest = listObjectsRequest.toBuilder() - .marker(lastKey) - .build() + listObjectsRequest = listObjectsRequest + .withMarker(lastKey) ; } listObjectsResponse = client.listObjects(listObjectsRequest); - List objects = listObjectsResponse.contents(); + List objects = listObjectsResponse.getObjectSummaries(); // Iterate over results - ListIterator iterVals = objects.listIterator(); + ListIterator iterVals = objects.listIterator(); while (iterVals.hasNext()) { - S3Object object = (S3Object) iterVals.next(); - String k = object.key(); + S3ObjectSummary object = (S3ObjectSummary) iterVals.next(); + String k = object.getKey(); String key = k.substring(k.indexOf(key2) + key2.length() + 1, k.length()); if (!key.isEmpty()) { keys.add(key.substring(0, key.length()-1)); From aa228665ca7e18520f264d369270c6303c17cf4a Mon Sep 17 00:00:00 2001 From: David Gault Date: Wed, 3 Apr 2024 13:30:29 +0100 Subject: [PATCH 07/25] Add commons-logging to excisions for AWS SDK --- pom.xml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pom.xml b/pom.xml index eacc60f..014fdfc 100644 --- a/pom.xml +++ b/pom.xml @@ -109,6 +109,12 @@ com.amazonaws aws-java-sdk-s3 1.12.659 + + + commons-logging + commons-logging + + From ce1eed0b244cd3b52bdaef495b89c806d0354361 Mon Sep 17 00:00:00 2001 From: David Gault Date: Wed, 3 Apr 2024 15:33:46 +0100 Subject: [PATCH 08/25] Declare commons-logging dependency at top level --- pom.xml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pom.xml b/pom.xml index 014fdfc..17a56bd 100644 --- a/pom.xml +++ b/pom.xml @@ -116,6 +116,11 @@ + + commons-logging + commons-logging + 1.2 + From d9c0bb62e7c060fbc3323fd78742fa3efe2bc575 Mon Sep 17 00:00:00 2001 From: David Gault Date: Wed, 3 Apr 2024 16:19:03 +0100 Subject: [PATCH 09/25] Add httpcore to JZarr exculsions --- pom.xml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pom.xml b/pom.xml index 17a56bd..63af9fa 100644 --- a/pom.xml +++ b/pom.xml @@ -73,6 +73,10 @@ com.fasterxml.jackson.core jackson-annotations + + org.apache.httpcomponents + httpcore + From 141f8bebc9dfdf3a1744213df1020c640c031a17 Mon Sep 17 00:00:00 2001 From: David Gault Date: Thu, 4 Apr 2024 16:07:29 +0100 Subject: [PATCH 10/25] AWS client: Don't set region when endpoint configured --- src/loci/formats/S3FileSystemStore.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/loci/formats/S3FileSystemStore.java b/src/loci/formats/S3FileSystemStore.java index 7ad06af..cad86d6 100644 --- a/src/loci/formats/S3FileSystemStore.java +++ b/src/loci/formats/S3FileSystemStore.java @@ -102,7 +102,6 @@ private void setupClient() { client = AmazonS3ClientBuilder.standard() .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(endpoint, "auto")) .withPathStyleAccessEnabled(true) - .withRegion(Regions.DEFAULT_REGION) // Ignored but required by the client .withCredentials(new AWSStaticCredentialsProvider(new AnonymousAWSCredentials())).build(); } catch (Exception e) { LOGGER.info("Exception caught while constructing S3 client"); From 44d5622f9621602f4250986289570edc2985ad07 Mon Sep 17 00:00:00 2001 From: David Gault Date: Fri, 26 Apr 2024 17:22:00 +0100 Subject: [PATCH 11/25] Only use S3FileSystemStore for https endpoint --- src/loci/formats/S3FileSystemStore.java | 5 +++-- src/loci/formats/services/JZarrServiceImpl.java | 12 ++++++++++-- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/src/loci/formats/S3FileSystemStore.java b/src/loci/formats/S3FileSystemStore.java index cad86d6..7d66928 100644 --- a/src/loci/formats/S3FileSystemStore.java +++ b/src/loci/formats/S3FileSystemStore.java @@ -75,6 +75,7 @@ public class S3FileSystemStore implements Store { private Path root; AmazonS3 client; + public static final String ENDPPOINT_PROTOCOL= "https://"; protected static final Logger LOGGER = LoggerFactory.getLogger(S3FileSystemStore.class); @@ -97,7 +98,7 @@ public String getRoot() { private void setupClient() { String[] pathSplit = root.toString().split(File.separator); - String endpoint = "https://" + pathSplit[1] + File.separator; + String endpoint = "ENDPPOINT_PROTOCOL" + pathSplit[1] + File.separator; try { client = AmazonS3ClientBuilder.standard() .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(endpoint, "auto")) @@ -179,7 +180,7 @@ public TreeSet getGroupKeys() throws IOException { * @throws IOException */ public TreeSet getKeysEndingWith(String suffix) throws IOException { - return (TreeSet)Files.walk(this.root).filter((path) -> { + return (TreeSet)Files.walk(this.root).filter((path) -> { return path.toString().endsWith(suffix); }).map((path) -> { return this.root.relativize(path).toString(); diff --git a/src/loci/formats/services/JZarrServiceImpl.java b/src/loci/formats/services/JZarrServiceImpl.java index 482315f..ff6b8f9 100644 --- a/src/loci/formats/services/JZarrServiceImpl.java +++ b/src/loci/formats/services/JZarrServiceImpl.java @@ -1,5 +1,7 @@ package loci.formats.services; +import java.io.File; + /*- * #%L * Implementation of Bio-Formats readers for the next-generation file formats @@ -79,7 +81,13 @@ public class JZarrServiceImpl extends AbstractService public JZarrServiceImpl(String root) { checkClassDependency(com.bc.zarr.ZarrArray.class); if (root != null && (root.toLowerCase().contains("s3:") || root.toLowerCase().contains("s3."))) { - s3fs = new S3FileSystemStore(Paths.get(root)); + String[] pathSplit = root.toString().split(File.separator); + if (!S3FileSystemStore.ENDPPOINT_PROTOCOL.contains(pathSplit[0].toLowerCase())) { + s3fs = new S3FileSystemStore(Paths.get(root)); + } + else { + LOGGER.warn("Zarr Reader is not using S3FileSystemStore as this is currently for use with S3 configured with a https endpoint"); + } } } @@ -89,7 +97,7 @@ public void open(String file) throws IOException, FormatException { if (s3fs == null) { zarrArray = ZarrArray.open(file); } - else { + else { s3fs.updateRoot(getZarrRoot(s3fs.getRoot()) + stripZarrRoot(file)); zarrArray = ZarrArray.open(s3fs); } From 6944b02cb0b3ad86ea102aebfe866fe9141be3ca Mon Sep 17 00:00:00 2001 From: David Gault Date: Wed, 1 May 2024 15:52:26 +0100 Subject: [PATCH 12/25] Update src/loci/formats/S3FileSystemStore.java MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Sébastien Besson --- src/loci/formats/S3FileSystemStore.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/loci/formats/S3FileSystemStore.java b/src/loci/formats/S3FileSystemStore.java index 7d66928..ed631f4 100644 --- a/src/loci/formats/S3FileSystemStore.java +++ b/src/loci/formats/S3FileSystemStore.java @@ -98,7 +98,7 @@ public String getRoot() { private void setupClient() { String[] pathSplit = root.toString().split(File.separator); - String endpoint = "ENDPPOINT_PROTOCOL" + pathSplit[1] + File.separator; + String endpoint = ENDPPOINT_PROTOCOL + pathSplit[1] + File.separator; try { client = AmazonS3ClientBuilder.standard() .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(endpoint, "auto")) From f28dab4e4c2374c52db00bc4cb9928179f2f0c48 Mon Sep 17 00:00:00 2001 From: David Gault Date: Wed, 1 May 2024 17:16:10 +0100 Subject: [PATCH 13/25] Correct spelling --- src/loci/formats/S3FileSystemStore.java | 4 ++-- src/loci/formats/services/JZarrServiceImpl.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/loci/formats/S3FileSystemStore.java b/src/loci/formats/S3FileSystemStore.java index ed631f4..c61d2ca 100644 --- a/src/loci/formats/S3FileSystemStore.java +++ b/src/loci/formats/S3FileSystemStore.java @@ -75,7 +75,7 @@ public class S3FileSystemStore implements Store { private Path root; AmazonS3 client; - public static final String ENDPPOINT_PROTOCOL= "https://"; + public static final String ENDPOINT_PROTOCOL= "https://"; protected static final Logger LOGGER = LoggerFactory.getLogger(S3FileSystemStore.class); @@ -98,7 +98,7 @@ public String getRoot() { private void setupClient() { String[] pathSplit = root.toString().split(File.separator); - String endpoint = ENDPPOINT_PROTOCOL + pathSplit[1] + File.separator; + String endpoint = ENDPOINT_PROTOCOL + pathSplit[1] + File.separator; try { client = AmazonS3ClientBuilder.standard() .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(endpoint, "auto")) diff --git a/src/loci/formats/services/JZarrServiceImpl.java b/src/loci/formats/services/JZarrServiceImpl.java index ff6b8f9..b59daf5 100644 --- a/src/loci/formats/services/JZarrServiceImpl.java +++ b/src/loci/formats/services/JZarrServiceImpl.java @@ -82,7 +82,7 @@ public JZarrServiceImpl(String root) { checkClassDependency(com.bc.zarr.ZarrArray.class); if (root != null && (root.toLowerCase().contains("s3:") || root.toLowerCase().contains("s3."))) { String[] pathSplit = root.toString().split(File.separator); - if (!S3FileSystemStore.ENDPPOINT_PROTOCOL.contains(pathSplit[0].toLowerCase())) { + if (!S3FileSystemStore.ENDPOINT_PROTOCOL.contains(pathSplit[0].toLowerCase())) { s3fs = new S3FileSystemStore(Paths.get(root)); } else { From b1583c48659b7ff8d6886d00bbbe35fcf181b399 Mon Sep 17 00:00:00 2001 From: David Gault Date: Wed, 1 May 2024 17:17:46 +0100 Subject: [PATCH 14/25] Removed redundant parameter --- src/loci/formats/in/ZarrReader.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/loci/formats/in/ZarrReader.java b/src/loci/formats/in/ZarrReader.java index 32c777c..7ff44a2 100644 --- a/src/loci/formats/in/ZarrReader.java +++ b/src/loci/formats/in/ZarrReader.java @@ -191,7 +191,7 @@ protected void initFile(String id) throws FormatException, IOException { Location omeMetaFile = new Location( zarrRootPath + File.separator + "OME", "METADATA.ome.xml" ); String canonicalPath = new Location(zarrRootPath).getCanonicalPath(); - initializeZarrService(canonicalPath); + initializeZarrService(); reloadOptionsFile(zarrRootPath); ArrayList omeSeriesOrder = new ArrayList(); @@ -462,14 +462,14 @@ private static int[] getOriginalShape(int [] shape5D, int size) { public void reopenFile() throws IOException { try { String canonicalPath = new Location(currentId).getCanonicalPath(); - initializeZarrService(canonicalPath); + initializeZarrService(); } catch (FormatException e) { throw new IOException(e); } } - protected void initializeZarrService(String rootPath) throws IOException, FormatException { + protected void initializeZarrService() throws IOException, FormatException { zarrService = new JZarrServiceImpl(altStore()); openZarr(); } From c0a835242373680c8131a7edb6687cceb0a2d077 Mon Sep 17 00:00:00 2001 From: David Gault Date: Wed, 1 May 2024 17:18:40 +0100 Subject: [PATCH 15/25] Remove unused openPlane and openZarr --- src/loci/formats/in/ZarrReader.java | 77 ----------------------------- 1 file changed, 77 deletions(-) diff --git a/src/loci/formats/in/ZarrReader.java b/src/loci/formats/in/ZarrReader.java index 7ff44a2..7814144 100644 --- a/src/loci/formats/in/ZarrReader.java +++ b/src/loci/formats/in/ZarrReader.java @@ -1242,83 +1242,6 @@ private void reloadOptionsFile(String id) { } } } - - public static byte[] openPlane(String path, byte[] buf) throws FormatException, IOException { - String rootPath = path.substring(0, path.indexOf(".zarr") + 5); - String altPath = loadAltStoreOption(rootPath); - if (altPath == null) altPath = rootPath; - ZarrService service = new JZarrServiceImpl(altPath); - openZarr(path, service); - int [] shape = {1, 1, 1, 1, 1}; - int [] zarrShape = service.getShape(); - shape[4] = zarrShape[zarrShape.length - 1]; - shape[3] = zarrShape[zarrShape.length - 2]; - int zarrArrayShapeSize = zarrShape.length; - if (zarrArrayShapeSize < 5) { - shape = getOriginalShape(shape, zarrArrayShapeSize); - } - int [] offsets = {0, 0, 0, 0, 0}; - Object image = service.readBytes(shape, offsets); - int w = shape[4]; - int h = shape[3]; - boolean little = service.isLittleEndian(); - int bpp = FormatTools.getBytesPerPixel(service.getPixelType()); - if (image instanceof byte[]) { - byte [] data = (byte []) image; - for (int i = 0; i < data.length; i++) { - DataTools.unpackBytes(data[i], buf, i, 1, little); - } - } - else if (image instanceof short[]) { - short[] data = (short[]) image; - for (int row = 0; row < h; row++) { - int base = row * w * bpp; - for (int i = 0; i < w; i++) { - DataTools.unpackBytes(data[(row * w) + i], buf, base + 2 * i, 2, little); - } - } - } - else if (image instanceof int[]) { - int[] data = (int[]) image; - for (int row = 0; row < h; row++) { - int base = row * w * bpp; - for (int i = 0; i < w; i++) { - DataTools.unpackBytes(data[(row * w) + i], buf, base + 4 * i, 4, little); - } - } - } - else if (image instanceof float[]) { - float[] data = (float[]) image; - for (int row = 0; row < h; row++) { - int base = row * w * bpp; - for (int i = 0; i < w; i++) { - int value = Float.floatToIntBits(data[(row * w) + i]); - DataTools.unpackBytes(value, buf, base + 4 * i, 4, little); - } - } - } - else if (image instanceof double[]) { - double[] data = (double[]) image; - for (int row = 0; row < h; row++) { - int base = row * w * bpp; - for (int i = 0; i < w; i++) { - long value = Double.doubleToLongBits(data[(row * w) + i]); - DataTools.unpackBytes(value, buf, base + 8 * i, 8, little); - } - } - } - return buf; - } - - private static void openZarr(String path, ZarrService service) { - try { - String canonicalPath = new Location(path).getCanonicalPath(); - LOGGER.debug("Opening zarr statically at path: {}", canonicalPath); - service.open(canonicalPath); - } catch (IOException | FormatException e) { - e.printStackTrace(); - } - } /** * Reloads the bfoptions file statically, only reading the value for the alterntiave file store From 50ffec180f02d156493a785890b6cb5f36e6dbf7 Mon Sep 17 00:00:00 2001 From: David Gault Date: Wed, 1 May 2024 17:36:02 +0100 Subject: [PATCH 16/25] Update ZrrReaderMock --- test/loci/formats/utests/ZarrReaderMock.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/loci/formats/utests/ZarrReaderMock.java b/test/loci/formats/utests/ZarrReaderMock.java index 8575e4a..bc0cd23 100644 --- a/test/loci/formats/utests/ZarrReaderMock.java +++ b/test/loci/formats/utests/ZarrReaderMock.java @@ -44,7 +44,7 @@ public ZarrReaderMock(ZarrService zarrService) { } @Override - protected void initializeZarrService(String rootPath) throws IOException, FormatException { + protected void initializeZarrService() throws IOException, FormatException { zarrService = mockService; } } From 45736c763a55a4c44917a9dcc532f60b462b8510 Mon Sep 17 00:00:00 2001 From: David Gault Date: Fri, 10 May 2024 16:09:17 +0100 Subject: [PATCH 17/25] JZarrServiceImpl: Correct protocol check --- src/loci/formats/services/JZarrServiceImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/loci/formats/services/JZarrServiceImpl.java b/src/loci/formats/services/JZarrServiceImpl.java index b59daf5..731a6b8 100644 --- a/src/loci/formats/services/JZarrServiceImpl.java +++ b/src/loci/formats/services/JZarrServiceImpl.java @@ -82,7 +82,7 @@ public JZarrServiceImpl(String root) { checkClassDependency(com.bc.zarr.ZarrArray.class); if (root != null && (root.toLowerCase().contains("s3:") || root.toLowerCase().contains("s3."))) { String[] pathSplit = root.toString().split(File.separator); - if (!S3FileSystemStore.ENDPOINT_PROTOCOL.contains(pathSplit[0].toLowerCase())) { + if (S3FileSystemStore.ENDPOINT_PROTOCOL.contains(pathSplit[0].toLowerCase())) { s3fs = new S3FileSystemStore(Paths.get(root)); } else { From ba5b955533c5bd6720ac662ad705fca9de9b9547 Mon Sep 17 00:00:00 2001 From: David Gault Date: Tue, 14 May 2024 10:25:28 +0100 Subject: [PATCH 18/25] S3FileSystemStore: Log stack trace rather than printing --- src/loci/formats/S3FileSystemStore.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/loci/formats/S3FileSystemStore.java b/src/loci/formats/S3FileSystemStore.java index c61d2ca..dde2339 100644 --- a/src/loci/formats/S3FileSystemStore.java +++ b/src/loci/formats/S3FileSystemStore.java @@ -105,8 +105,7 @@ private void setupClient() { .withPathStyleAccessEnabled(true) .withCredentials(new AWSStaticCredentialsProvider(new AnonymousAWSCredentials())).build(); } catch (Exception e) { - LOGGER.info("Exception caught while constructing S3 client"); - e.printStackTrace(); + LOGGER.info("Exception caught while constructing S3 client", e); } } @@ -133,8 +132,7 @@ public InputStream getInputStream(String key) throws IOException { S3ObjectInputStream responseStream = o.getObjectContent(); return responseStream; } catch (Exception e) { - LOGGER.info( "Unable to locate or access key: " + key2); - e.printStackTrace(); + LOGGER.info( "Unable to locate or access key: " + key2, e); } return null; From 7ab9c352f8da0a0334ca77917a4374ce44ef25f7 Mon Sep 17 00:00:00 2001 From: David Gault Date: Tue, 14 May 2024 10:26:21 +0100 Subject: [PATCH 19/25] Remove unused method --- src/loci/formats/in/ZarrReader.java | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/src/loci/formats/in/ZarrReader.java b/src/loci/formats/in/ZarrReader.java index 7814144..efd2a31 100644 --- a/src/loci/formats/in/ZarrReader.java +++ b/src/loci/formats/in/ZarrReader.java @@ -1242,26 +1242,4 @@ private void reloadOptionsFile(String id) { } } } - - /** - * Reloads the bfoptions file statically, only reading the value for the alterntiave file store - */ - private static String loadAltStoreOption(String id) { - String optionsFile = DynamicMetadataOptions.getMetadataOptionsFile(id); - if (optionsFile != null) { - MetadataOptions options = new DynamicMetadataOptions(); - if (options != null && options instanceof DynamicMetadataOptions) { - try { - ArrayList optionsList = new ArrayList(); - optionsList.add(ALT_STORE_KEY); - ((DynamicMetadataOptions) options).loadOptions(optionsFile, optionsList); - return ((DynamicMetadataOptions) options).get( - ALT_STORE_KEY, ALT_STORE_DEFAULT); - } catch (Exception e) { - LOGGER.warn("Exception while attempting to read metadata options file", e); - } - } - } - return ALT_STORE_DEFAULT; - } } From 93ee98d863274054decee812166b5168bc222e91 Mon Sep 17 00:00:00 2001 From: David Gault Date: Tue, 14 May 2024 11:18:03 +0100 Subject: [PATCH 20/25] JZarrServiceImpl: Refactor fetching of ZarrArray and ZarrGroup --- .../formats/services/JZarrServiceImpl.java | 63 +++++++++---------- 1 file changed, 28 insertions(+), 35 deletions(-) diff --git a/src/loci/formats/services/JZarrServiceImpl.java b/src/loci/formats/services/JZarrServiceImpl.java index 731a6b8..26a8540 100644 --- a/src/loci/formats/services/JZarrServiceImpl.java +++ b/src/loci/formats/services/JZarrServiceImpl.java @@ -94,13 +94,7 @@ public JZarrServiceImpl(String root) { @Override public void open(String file) throws IOException, FormatException { currentId = file; - if (s3fs == null) { - zarrArray = ZarrArray.open(file); - } - else { - s3fs.updateRoot(getZarrRoot(s3fs.getRoot()) + stripZarrRoot(file)); - zarrArray = ZarrArray.open(s3fs); - } + zarrArray = getArray(file); } public void open(String id, ZarrArray array) { @@ -109,15 +103,7 @@ public void open(String id, ZarrArray array) { } public Map getGroupAttr(String path) throws IOException, FormatException { - ZarrGroup group = null; - if (s3fs == null) { - group = ZarrGroup.open(path); - } - else { - s3fs.updateRoot(getZarrRoot(s3fs.getRoot()) + stripZarrRoot(path)); - group = ZarrGroup.open(s3fs); - } - return group.getAttributes(); + return getGroup(path).getAttributes(); } public Map getArrayAttr(String path) throws IOException, FormatException { @@ -129,31 +115,15 @@ public Map getArrayAttr(String path) throws IOException, FormatE s3fs.updateRoot(getZarrRoot(s3fs.getRoot()) + stripZarrRoot(path)); array = ZarrArray.open(s3fs); } - return array.getAttributes(); + return getArray(path).getAttributes(); } public Set getGroupKeys(String path) throws IOException, FormatException { - ZarrGroup group = null; - if (s3fs == null) { - group = ZarrGroup.open(path); - } - else { - s3fs.updateRoot(getZarrRoot(s3fs.getRoot()) + stripZarrRoot(path)); - group = ZarrGroup.open(s3fs); - } - return group.getGroupKeys(); + return getGroup(path).getGroupKeys(); } public Set getArrayKeys(String path) throws IOException, FormatException { - ZarrGroup group = null; - if (s3fs == null) { - group = ZarrGroup.open(path); - } - else { - s3fs.updateRoot(getZarrRoot(s3fs.getRoot()) + stripZarrRoot(path)); - group = ZarrGroup.open(s3fs); - } - return group.getArrayKeys(); + return getGroup(path).getArrayKeys(); } public DataType getZarrPixelType(int pixType) { @@ -379,4 +349,27 @@ private String getZarrRoot(String path) { return path.substring(0, path.indexOf(".zarr")+5); } + private ZarrGroup getGroup(String path) throws IOException { + ZarrGroup group = null; + if (s3fs == null) { + group = ZarrGroup.open(path); + } + else { + s3fs.updateRoot(getZarrRoot(s3fs.getRoot()) + stripZarrRoot(path)); + group = ZarrGroup.open(s3fs); + } + return group; + } + + private ZarrArray getArray(String path) throws IOException { + ZarrArray array = null; + if (s3fs == null) { + array = ZarrArray.open(path); + } + else { + s3fs.updateRoot(getZarrRoot(s3fs.getRoot()) + stripZarrRoot(path)); + array = ZarrArray.open(s3fs); + } + return array; + } } From 2bfe3db330252332a4907abae560f6cac154df56 Mon Sep 17 00:00:00 2001 From: David Gault Date: Tue, 14 May 2024 11:22:12 +0100 Subject: [PATCH 21/25] JZarrServiceImple: Add new usingS3FileSystemStore method --- src/loci/formats/services/JZarrServiceImpl.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/loci/formats/services/JZarrServiceImpl.java b/src/loci/formats/services/JZarrServiceImpl.java index 26a8540..3f32518 100644 --- a/src/loci/formats/services/JZarrServiceImpl.java +++ b/src/loci/formats/services/JZarrServiceImpl.java @@ -372,4 +372,8 @@ private ZarrArray getArray(String path) throws IOException { } return array; } + + public boolean usingS3FileSystemStore() { + return s3fs != null; + } } From a54a40ee8efd1d8ae34bf85c886a9956ff9a0631 Mon Sep 17 00:00:00 2001 From: David Gault Date: Tue, 14 May 2024 11:34:02 +0100 Subject: [PATCH 22/25] S3FileSystemStore: Add comments for path splitting --- src/loci/formats/S3FileSystemStore.java | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/loci/formats/S3FileSystemStore.java b/src/loci/formats/S3FileSystemStore.java index dde2339..5d908e7 100644 --- a/src/loci/formats/S3FileSystemStore.java +++ b/src/loci/formats/S3FileSystemStore.java @@ -123,8 +123,11 @@ public S3FileSystemStore(Path rootPath) { @Override public InputStream getInputStream(String key) throws IOException { + // Get the base bucket name from splitting the root path and removing the prefixed protocol and end-point String[] pathSplit = root.toString().split(File.separator); String bucketName = pathSplit[2]; + + // Append the desired key onto the remaining prefix String key2 = root.toString().substring(root.toString().indexOf(pathSplit[3]), root.toString().length()) + File.separator + key; try { @@ -205,10 +208,12 @@ public Stream getRelativeLeafKeys(String key) throws IOException { private TreeSet getKeysFor(String suffix) throws IOException { TreeSet keys = new TreeSet(); - + + // Get the base bucket name from splitting the root path and removing the prefixed protocol and end-point String[] pathSplit = root.toString().split(File.separator); - String bucketName = pathSplit[2]; + + // Append the desired key onto the remaining prefix String key2 = root.toString().substring(root.toString().indexOf(pathSplit[3]), root.toString().length()); ListObjectsRequest listObjectsRequest = new ListObjectsRequest() @@ -249,11 +254,13 @@ private TreeSet getKeysFor(String suffix) throws IOException { public ArrayList getFiles() throws IOException { ArrayList keys = new ArrayList(); - + + // Get the base bucket name from splitting the root path and removing the prefixed protocol and end-point String[] pathSplit = root.toString().split(File.separator); String bucketName = pathSplit[2]; + + // Append the desired key onto the remaining prefix String key2 = root.toString().substring(root.toString().indexOf(pathSplit[3]), root.toString().length()); - ListObjectsRequest listObjectsRequest = new ListObjectsRequest() .withBucketName(bucketName) .withPrefix(key2) From 51399b7e713e386142497fc51acaee3d6863a496 Mon Sep 17 00:00:00 2001 From: David Gault Date: Tue, 14 May 2024 11:39:46 +0100 Subject: [PATCH 23/25] S3FileSystemStore: Removed unused getFiles method --- src/loci/formats/S3FileSystemStore.java | 44 ------------------------- 1 file changed, 44 deletions(-) diff --git a/src/loci/formats/S3FileSystemStore.java b/src/loci/formats/S3FileSystemStore.java index 5d908e7..f2f3f62 100644 --- a/src/loci/formats/S3FileSystemStore.java +++ b/src/loci/formats/S3FileSystemStore.java @@ -251,48 +251,4 @@ private TreeSet getKeysFor(String suffix) throws IOException { return keys; } - - public ArrayList getFiles() throws IOException { - ArrayList keys = new ArrayList(); - - // Get the base bucket name from splitting the root path and removing the prefixed protocol and end-point - String[] pathSplit = root.toString().split(File.separator); - String bucketName = pathSplit[2]; - - // Append the desired key onto the remaining prefix - String key2 = root.toString().substring(root.toString().indexOf(pathSplit[3]), root.toString().length()); - ListObjectsRequest listObjectsRequest = new ListObjectsRequest() - .withBucketName(bucketName) - .withPrefix(key2) - ; - - ObjectListing listObjectsResponse = null; - String lastKey = null; - - do { - if ( listObjectsResponse != null ) { - listObjectsRequest = listObjectsRequest - .withMarker(lastKey) - ; - } - - listObjectsResponse = client.listObjects(listObjectsRequest); - List objects = listObjectsResponse.getObjectSummaries(); - - // Iterate over results - ListIterator iterVals = objects.listIterator(); - while (iterVals.hasNext()) { - S3ObjectSummary object = (S3ObjectSummary) iterVals.next(); - String k = object.getKey(); - String key = k.substring(k.indexOf(key2) + key2.length() + 1, k.length()); - if (!key.isEmpty()) { - keys.add(key.substring(0, key.length()-1)); - } - lastKey = k; - } - } while ( listObjectsResponse.isTruncated() ); - return keys; - } - - } \ No newline at end of file From 9a9bd3493628e132ce9cb0d014951ae08dfa1342 Mon Sep 17 00:00:00 2001 From: David Gault Date: Tue, 14 May 2024 11:41:22 +0100 Subject: [PATCH 24/25] S3FileSystemStore: Cleanup unused imports --- src/loci/formats/S3FileSystemStore.java | 31 +++++++++---------------- 1 file changed, 11 insertions(+), 20 deletions(-) diff --git a/src/loci/formats/S3FileSystemStore.java b/src/loci/formats/S3FileSystemStore.java index f2f3f62..2d5a40f 100644 --- a/src/loci/formats/S3FileSystemStore.java +++ b/src/loci/formats/S3FileSystemStore.java @@ -1,21 +1,5 @@ package loci.formats; -import com.amazonaws.auth.AWSCredentials; -import com.amazonaws.internal.StaticCredentialsProvider; -import com.amazonaws.regions.Region; -import com.amazonaws.regions.Regions; -import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.AmazonS3ClientBuilder; -import com.amazonaws.services.s3.model.GetObjectRequest; -import com.amazonaws.services.s3.model.ListObjectsRequest; -import com.amazonaws.services.s3.model.ObjectListing; -import com.amazonaws.services.s3.model.S3Object; -import com.amazonaws.services.s3.model.S3ObjectInputStream; -import com.amazonaws.services.s3.model.S3ObjectSummary; -import com.amazonaws.auth.AWSStaticCredentialsProvider; -import com.amazonaws.auth.AnonymousAWSCredentials; -import com.amazonaws.client.builder.AwsClientBuilder; - /*- * #%L * Implementation of Bio-Formats readers for the next-generation file formats @@ -49,19 +33,15 @@ import com.bc.zarr.ZarrUtils; import com.bc.zarr.storage.Store; - import java.io.File; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import java.net.URI; -import java.net.URISyntaxException; import java.nio.file.FileSystem; import java.nio.file.Files; import java.nio.file.LinkOption; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.ArrayList; import java.util.List; import java.util.ListIterator; import java.util.TreeSet; @@ -71,6 +51,17 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3ClientBuilder; +import com.amazonaws.services.s3.model.ListObjectsRequest; +import com.amazonaws.services.s3.model.ObjectListing; +import com.amazonaws.services.s3.model.S3Object; +import com.amazonaws.services.s3.model.S3ObjectInputStream; +import com.amazonaws.services.s3.model.S3ObjectSummary; +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.auth.AnonymousAWSCredentials; +import com.amazonaws.client.builder.AwsClientBuilder; + public class S3FileSystemStore implements Store { private Path root; From ea4ce08d1f569837239fa90c5fb7173f7a75c849 Mon Sep 17 00:00:00 2001 From: David Gault Date: Wed, 15 May 2024 16:01:53 +0100 Subject: [PATCH 25/25] JZarrServiceImpl: Remove unused code --- src/loci/formats/services/JZarrServiceImpl.java | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/loci/formats/services/JZarrServiceImpl.java b/src/loci/formats/services/JZarrServiceImpl.java index 3f32518..732b8c2 100644 --- a/src/loci/formats/services/JZarrServiceImpl.java +++ b/src/loci/formats/services/JZarrServiceImpl.java @@ -107,14 +107,6 @@ public Map getGroupAttr(String path) throws IOException, FormatE } public Map getArrayAttr(String path) throws IOException, FormatException { - ZarrArray array = null; - if (s3fs == null) { - array = ZarrArray.open(path); - } - else { - s3fs.updateRoot(getZarrRoot(s3fs.getRoot()) + stripZarrRoot(path)); - array = ZarrArray.open(s3fs); - } return getArray(path).getAttributes(); }