From 36d9f777f7e27655de06a38626d6bf26b75930b9 Mon Sep 17 00:00:00 2001 From: vikram-vikram_bveng Date: Tue, 23 Jul 2024 16:31:38 +0530 Subject: [PATCH 1/6] PD-249429: integrated emodb with datastorage-media-service for getting the metadata from s3. --- blob/pom.xml | 5 + .../emodb/blob/config/ApiClient.java | 143 ++++++++++++++++++ .../emodb/blob/core/DefaultBlobStore.java | 18 +-- .../resources/blob/BlobStoreResource1.java | 7 +- 4 files changed, 158 insertions(+), 15 deletions(-) create mode 100644 blob/src/main/java/com/bazaarvoice/emodb/blob/config/ApiClient.java diff --git a/blob/pom.xml b/blob/pom.xml index a1708a21d8..31f1d9dce8 100644 --- a/blob/pom.xml +++ b/blob/pom.xml @@ -234,5 +234,10 @@ jersey-client test + + org.glassfish + javax.json + 1.0.4 + diff --git a/blob/src/main/java/com/bazaarvoice/emodb/blob/config/ApiClient.java b/blob/src/main/java/com/bazaarvoice/emodb/blob/config/ApiClient.java new file mode 100644 index 0000000000..1d66d5bf9f --- /dev/null +++ b/blob/src/main/java/com/bazaarvoice/emodb/blob/config/ApiClient.java @@ -0,0 +1,143 @@ +package com.bazaarvoice.emodb.blob.config; + +import com.bazaarvoice.emodb.blob.api.BlobMetadata; +import com.bazaarvoice.emodb.blob.api.DefaultBlobMetadata; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.json.*; +import java.io.BufferedReader; +import java.io.InputStreamReader; +import java.io.StringReader; +import java.net.HttpURLConnection; +import java.net.URL; +import java.net.URLEncoder; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.*; + +public class ApiClient { + + private static final Logger LOGGER = LoggerFactory.getLogger(ApiClient.class); + private final String BASE_URL = "http://localhost:8082/blob"; + private final String TENANT_NAME = "datastorage"; + public Iterator getBlobMetadata(String tableName) { + try { + LOGGER.debug(" Constructing URL and consuming datastorage-media-service URL "); + String[] parts = tableName.split(":"); + String table = parts[0]; + String clientName = parts[1]; + + // Constructing URL with path variable and query parameters. + String urlString = String.format("%s/%s/%s/%s", + BASE_URL, + URLEncoder.encode(TENANT_NAME, "UTF-8"), + URLEncoder.encode(table, "UTF-8"), + URLEncoder.encode(clientName, "UTF-8")); + + URL url = new URL(urlString); + HttpURLConnection connection = (HttpURLConnection) url.openConnection(); + connection.setRequestMethod("GET"); + + // Setting headers + connection.setRequestProperty("Accept", "application/json"); + + int responseCode = connection.getResponseCode(); + + if (responseCode == HttpURLConnection.HTTP_OK) { + BufferedReader in = new BufferedReader(new InputStreamReader(connection.getInputStream())); + String inputLine; + StringBuilder response = new StringBuilder(); + + while ((inputLine = in.readLine()) != null) { + response.append(inputLine); + } + in.close(); + System.out.println(response); + LOGGER.info(" Before mapping of the response "); + return mapResponseToBlobMetaData(response.toString()).iterator(); + } else { + System.out.println("GET request not worked"); + } + } catch (Exception e) { + e.printStackTrace(); + } + + return null; + } + + public List mapResponseToBlobMetaData(String response) { + + // Parse JSON string to JsonArray + JsonReader jsonReader = Json.createReader(new StringReader(response)); + JsonArray jsonArray = jsonReader.readArray(); + jsonReader.close(); + + // Convert JsonArray to List + List blobMetadata = new ArrayList<>(); + for (JsonObject jsonObject : jsonArray.getValuesAs(JsonObject.class)) { + long length = Long.parseLong(String.valueOf(jsonObject.getInt("length"))); + + System.out.println(" Length " + length); + Map attributes = convertStringAttributesToMap((JsonObject) jsonObject.get("attributes")); + BlobMetadata blobMetadataObject = new DefaultBlobMetadata(jsonObject.getString("id"), + convertToDate(jsonObject.getString("timestamp")), + length, + jsonObject.getString("md5"), + jsonObject.getString("sha1"), + attributes); + blobMetadata.add(blobMetadataObject); + System.out.println(jsonObject); + } + LOGGER.info(" After mapping of the response "); + System.out.println(" BlobMetaData " + blobMetadata); + return blobMetadata; + } + + public Date convertToDate(String timestamp) { + LOGGER.info(" Date to be parsed {} ", timestamp); + SimpleDateFormat formatter = new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss z", Locale.ENGLISH); + try { + // Parse the string into a Date object + return formatter.parse(timestamp); + } catch (ParseException e) { + LOGGER.error(" Date could not be parsed {} ", timestamp); + } + return null; + } + + public Map convertStringAttributesToMap(JsonObject attributes) { + LOGGER.info(" Attributes to be parsed {} ", attributes); + // Convert JsonObject to Map + Map attributesMap = new HashMap<>(); + for (Map.Entry entry : attributes.entrySet()) { + String key = entry.getKey(); + JsonValue value = entry.getValue(); + String stringValue; + + // Determine the type of the value and convert accordingly + switch (value.getValueType()) { + case STRING: + stringValue = ((JsonString) value).getString(); + break; + // Handles integers and floats + case TRUE: + stringValue = "true"; + break; + case FALSE: + stringValue = "false"; + break; + case NULL: + stringValue = null; + break; + // Convert JSON object/array to string + default: + stringValue = value.toString(); // Fallback for any other types + break; + } + attributesMap.put(key, stringValue); + } + + return attributesMap; + } +} diff --git a/blob/src/main/java/com/bazaarvoice/emodb/blob/core/DefaultBlobStore.java b/blob/src/main/java/com/bazaarvoice/emodb/blob/core/DefaultBlobStore.java index 8eba374154..cc44d9edc4 100644 --- a/blob/src/main/java/com/bazaarvoice/emodb/blob/core/DefaultBlobStore.java +++ b/blob/src/main/java/com/bazaarvoice/emodb/blob/core/DefaultBlobStore.java @@ -11,6 +11,7 @@ import com.bazaarvoice.emodb.blob.api.Range; import com.bazaarvoice.emodb.blob.api.RangeSpecification; import com.bazaarvoice.emodb.blob.api.StreamSupplier; +import com.bazaarvoice.emodb.blob.config.ApiClient; import com.bazaarvoice.emodb.blob.db.MetadataProvider; import com.bazaarvoice.emodb.blob.db.StorageProvider; import com.bazaarvoice.emodb.blob.db.StorageSummary; @@ -227,20 +228,9 @@ public BlobMetadata getMetadata(String tableName, String blobId) throws BlobNotF @Override public Iterator scanMetadata(String tableName, @Nullable String fromBlobIdExclusive, long limit) { checkLegalTableName(tableName); - checkArgument(fromBlobIdExclusive == null || Names.isLegalBlobId(fromBlobIdExclusive), "fromBlobIdExclusive"); - checkArgument(limit > 0, "Limit must be >0"); - - final Table table = _tableDao.get(tableName); - - // Stream back results. Don't hold them all in memory at once. - LimitCounter remaining = new LimitCounter(limit); - return remaining.limit(Iterators.transform(_metadataProvider.scanMetadata(table, fromBlobIdExclusive, remaining), - new Function, BlobMetadata>() { - @Override - public BlobMetadata apply(Map.Entry entry) { - return newMetadata(table, entry.getKey(), entry.getValue()); - } - })); + ApiClient apiClient = new ApiClient(); + LOGGER.debug(" Before calling the endpoint "); + return apiClient.getBlobMetadata(tableName); } private static BlobMetadata newMetadata(Table table, String blobId, StorageSummary s) { diff --git a/web/src/main/java/com/bazaarvoice/emodb/web/resources/blob/BlobStoreResource1.java b/web/src/main/java/com/bazaarvoice/emodb/web/resources/blob/BlobStoreResource1.java index a3869e8214..aa5de872f0 100644 --- a/web/src/main/java/com/bazaarvoice/emodb/web/resources/blob/BlobStoreResource1.java +++ b/web/src/main/java/com/bazaarvoice/emodb/web/resources/blob/BlobStoreResource1.java @@ -148,6 +148,7 @@ public Meter load(String key) throws Exception { }); } + //change @GET @Path("_table") @Timed(name = "bv.emodb.blob.BlobStoreResource1.listTables", absolute = true) @@ -196,6 +197,7 @@ public SuccessResponse createTable(@PathParam("table") String table, return SuccessResponse.instance(); } + //change @DELETE @Path("_table/{table}") @RequiresPermissions("blob|drop_table|{table}") @@ -332,6 +334,7 @@ public Response head(@PathParam("table") String table, /** * Retrieves a list of content items in a particular table. */ + //change @GET @Path("{table}") @RequiresPermissions("blob|read|{table}") @@ -344,7 +347,7 @@ public Iterator scanMetadata(@PathParam("table") String table, @QueryParam("from") String blobId, @QueryParam("limit") @DefaultValue("10") LongParam limit, @Authenticated Subject subject) { - _scanMetadataRequestsByApiKey.getUnchecked(subject.getId()).mark(); +// _scanMetadataRequestsByApiKey.getUnchecked(subject.getId()).mark(); return streamingIterator(_blobStore.scanMetadata(table, Strings.emptyToNull(blobId), limit.get())); } @@ -364,6 +367,7 @@ public Collection getTablePlacements(@Authenticated Subject subject) { } + //change /** * Retrieves the current version of a piece of content from the data store. */ @@ -453,6 +457,7 @@ private String safeResponseContentType(String metadataContentType) { return MediaType.APPLICATION_OCTET_STREAM; } + //change @PUT @Path("{table}/{blobId}") @Consumes(MediaType.WILDCARD) From 8e506594f6ceda454b14d83f2ac168722b421d86 Mon Sep 17 00:00:00 2001 From: vikram-vikram_bveng Date: Wed, 24 Jul 2024 09:27:13 +0530 Subject: [PATCH 2/6] PD-249429: refactored code. --- .../emodb/web/resources/blob/BlobStoreResource1.java | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/web/src/main/java/com/bazaarvoice/emodb/web/resources/blob/BlobStoreResource1.java b/web/src/main/java/com/bazaarvoice/emodb/web/resources/blob/BlobStoreResource1.java index aa5de872f0..fb5fb7e2c5 100644 --- a/web/src/main/java/com/bazaarvoice/emodb/web/resources/blob/BlobStoreResource1.java +++ b/web/src/main/java/com/bazaarvoice/emodb/web/resources/blob/BlobStoreResource1.java @@ -148,7 +148,6 @@ public Meter load(String key) throws Exception { }); } - //change @GET @Path("_table") @Timed(name = "bv.emodb.blob.BlobStoreResource1.listTables", absolute = true) @@ -197,7 +196,6 @@ public SuccessResponse createTable(@PathParam("table") String table, return SuccessResponse.instance(); } - //change @DELETE @Path("_table/{table}") @RequiresPermissions("blob|drop_table|{table}") @@ -334,7 +332,6 @@ public Response head(@PathParam("table") String table, /** * Retrieves a list of content items in a particular table. */ - //change @GET @Path("{table}") @RequiresPermissions("blob|read|{table}") @@ -347,7 +344,7 @@ public Iterator scanMetadata(@PathParam("table") String table, @QueryParam("from") String blobId, @QueryParam("limit") @DefaultValue("10") LongParam limit, @Authenticated Subject subject) { -// _scanMetadataRequestsByApiKey.getUnchecked(subject.getId()).mark(); + _scanMetadataRequestsByApiKey.getUnchecked(subject.getId()).mark(); return streamingIterator(_blobStore.scanMetadata(table, Strings.emptyToNull(blobId), limit.get())); } @@ -366,8 +363,6 @@ public Collection getTablePlacements(@Authenticated Subject subject) { return _blobStore.getTablePlacements(); } - - //change /** * Retrieves the current version of a piece of content from the data store. */ @@ -457,7 +452,6 @@ private String safeResponseContentType(String metadataContentType) { return MediaType.APPLICATION_OCTET_STREAM; } - //change @PUT @Path("{table}/{blobId}") @Consumes(MediaType.WILDCARD) From 08fc399bb79bcf8e374bcf70129e0de9d8e37e2b Mon Sep 17 00:00:00 2001 From: vikram-vikram_bveng Date: Wed, 24 Jul 2024 10:51:30 +0530 Subject: [PATCH 3/6] PD-249428: DELETE blob with blob id. --- .../emodb/blob/api/TenantRequest.java | 23 ++++++++ .../emodb/blob/config/ApiClient.java | 56 ++++++++++++++++++- .../emodb/blob/core/DefaultBlobStore.java | 9 ++- .../resources/blob/BlobStoreResource1.java | 3 +- 4 files changed, 84 insertions(+), 7 deletions(-) create mode 100644 blob/src/main/java/com/bazaarvoice/emodb/blob/api/TenantRequest.java diff --git a/blob/src/main/java/com/bazaarvoice/emodb/blob/api/TenantRequest.java b/blob/src/main/java/com/bazaarvoice/emodb/blob/api/TenantRequest.java new file mode 100644 index 0000000000..61dc7eb524 --- /dev/null +++ b/blob/src/main/java/com/bazaarvoice/emodb/blob/api/TenantRequest.java @@ -0,0 +1,23 @@ +package com.bazaarvoice.emodb.blob.api; + +public class TenantRequest { + + private String tenantName; + + public TenantRequest(String tenantName) { + this.tenantName = tenantName; + } + + public String getTenantName() { + return tenantName; + } + + public void setTenantName(String tenantName) { + this.tenantName = tenantName; + } + + @Override + public String toString() { + return "{\"tenantName\":\"" + tenantName + "\"}"; + } +} diff --git a/blob/src/main/java/com/bazaarvoice/emodb/blob/config/ApiClient.java b/blob/src/main/java/com/bazaarvoice/emodb/blob/config/ApiClient.java index 1d66d5bf9f..736a11851b 100644 --- a/blob/src/main/java/com/bazaarvoice/emodb/blob/config/ApiClient.java +++ b/blob/src/main/java/com/bazaarvoice/emodb/blob/config/ApiClient.java @@ -2,16 +2,19 @@ import com.bazaarvoice.emodb.blob.api.BlobMetadata; import com.bazaarvoice.emodb.blob.api.DefaultBlobMetadata; +import com.bazaarvoice.emodb.blob.api.TenantRequest; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import javax.json.*; import java.io.BufferedReader; import java.io.InputStreamReader; +import java.io.OutputStream; import java.io.StringReader; import java.net.HttpURLConnection; import java.net.URL; import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.*; @@ -21,6 +24,8 @@ public class ApiClient { private static final Logger LOGGER = LoggerFactory.getLogger(ApiClient.class); private final String BASE_URL = "http://localhost:8082/blob"; private final String TENANT_NAME = "datastorage"; + public final String SUCCESS_MSG = "Successfully deleted blob."; + public Iterator getBlobMetadata(String tableName) { try { LOGGER.debug(" Constructing URL and consuming datastorage-media-service URL "); @@ -57,7 +62,56 @@ public Iterator getBlobMetadata(String tableName) { LOGGER.info(" Before mapping of the response "); return mapResponseToBlobMetaData(response.toString()).iterator(); } else { - System.out.println("GET request not worked"); + LOGGER.debug(" GET operation halted with error "); + } + } catch (Exception e) { + e.printStackTrace(); + } + + return null; + } + + public String deleteBlobFromTable(String tableName, String blobId) { + try { + LOGGER.debug(" Constructing URL and consuming datastorage-media-service delete blob URL "); + String[] parts = tableName.split(":"); + String table = parts[0]; + String clientName = parts[1]; + TenantRequest tenantRequest = new TenantRequest(TENANT_NAME); + System.out.println(" Tenant Request " + tenantRequest); + + // Constructing URL with path variable and query parameters. + String urlString = String.format("%s/%s:%s/%s", + BASE_URL + "/delete", + URLEncoder.encode(table, "UTF-8"), + URLEncoder.encode(clientName, "UTF-8"), + URLEncoder.encode(blobId, "UTF-8")); + + LOGGER.info(" URL {} ", urlString); + URL url = new URL(urlString); + HttpURLConnection connection = (HttpURLConnection) url.openConnection(); + connection.setRequestMethod("DELETE"); + + // Setting headers + connection.setRequestProperty("Content-Type", "application/json; utf-8"); + connection.setRequestProperty("Accept", "application/json"); + + // Enable output for the request body + connection.setDoOutput(true); + + // Write the request body + try (OutputStream os = connection.getOutputStream()) { + byte[] input = tenantRequest.toString().getBytes(StandardCharsets.UTF_8); + os.write(input, 0, input.length); + } + + int responseCode = connection.getResponseCode(); + + if (responseCode == HttpURLConnection.HTTP_OK) { + LOGGER.debug(" Blob with id {} deleted successfully", blobId); + return SUCCESS_MSG; + } else { + LOGGER.debug(" Blob with id {} didn't get deleted ", blobId); } } catch (Exception e) { e.printStackTrace(); diff --git a/blob/src/main/java/com/bazaarvoice/emodb/blob/core/DefaultBlobStore.java b/blob/src/main/java/com/bazaarvoice/emodb/blob/core/DefaultBlobStore.java index cc44d9edc4..cf25d5cbcc 100644 --- a/blob/src/main/java/com/bazaarvoice/emodb/blob/core/DefaultBlobStore.java +++ b/blob/src/main/java/com/bazaarvoice/emodb/blob/core/DefaultBlobStore.java @@ -405,11 +405,10 @@ public void delete(String tableName, String blobId) { checkLegalTableName(tableName); checkLegalBlobId(blobId); - Table table = _tableDao.get(tableName); - - StorageSummary storageSummary = _metadataProvider.readMetadata(table, blobId); - - delete(table, blobId, storageSummary); + ApiClient apiClient = new ApiClient(); + String response = apiClient.deleteBlobFromTable(tableName, blobId); + if (response.equalsIgnoreCase(apiClient.SUCCESS_MSG)) + LOGGER.info(" {} ", apiClient.SUCCESS_MSG); } private void delete(Table table, String blobId, StorageSummary storageSummary) { diff --git a/web/src/main/java/com/bazaarvoice/emodb/web/resources/blob/BlobStoreResource1.java b/web/src/main/java/com/bazaarvoice/emodb/web/resources/blob/BlobStoreResource1.java index fb5fb7e2c5..962fec915b 100644 --- a/web/src/main/java/com/bazaarvoice/emodb/web/resources/blob/BlobStoreResource1.java +++ b/web/src/main/java/com/bazaarvoice/emodb/web/resources/blob/BlobStoreResource1.java @@ -497,6 +497,7 @@ public SuccessResponse put(@PathParam("table") String table, return SuccessResponse.instance(); } + //change @DELETE @Path("{table}/{blobId}") @RequiresPermissions("blob|update|{table}") @@ -508,7 +509,7 @@ public SuccessResponse put(@PathParam("table") String table, public SuccessResponse delete(@PathParam("table") String table, @PathParam("blobId") String blobId, @Authenticated Subject subject) { - _deleteObjectRequestsByApiKey.getUnchecked(subject.getId()).mark(); +// _deleteObjectRequestsByApiKey.getUnchecked(subject.getId()).mark(); _blobStore.delete(table, blobId); return SuccessResponse.instance(); } From 886b2a20fd3eeda8dcd44a242172360aa6fe041c Mon Sep 17 00:00:00 2001 From: vikram-vikram_bveng Date: Wed, 24 Jul 2024 10:53:51 +0530 Subject: [PATCH 4/6] PD-249428: Refactored code.. --- .../emodb/web/resources/blob/BlobStoreResource1.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/web/src/main/java/com/bazaarvoice/emodb/web/resources/blob/BlobStoreResource1.java b/web/src/main/java/com/bazaarvoice/emodb/web/resources/blob/BlobStoreResource1.java index 962fec915b..fb5fb7e2c5 100644 --- a/web/src/main/java/com/bazaarvoice/emodb/web/resources/blob/BlobStoreResource1.java +++ b/web/src/main/java/com/bazaarvoice/emodb/web/resources/blob/BlobStoreResource1.java @@ -497,7 +497,6 @@ public SuccessResponse put(@PathParam("table") String table, return SuccessResponse.instance(); } - //change @DELETE @Path("{table}/{blobId}") @RequiresPermissions("blob|update|{table}") @@ -509,7 +508,7 @@ public SuccessResponse put(@PathParam("table") String table, public SuccessResponse delete(@PathParam("table") String table, @PathParam("blobId") String blobId, @Authenticated Subject subject) { -// _deleteObjectRequestsByApiKey.getUnchecked(subject.getId()).mark(); + _deleteObjectRequestsByApiKey.getUnchecked(subject.getId()).mark(); _blobStore.delete(table, blobId); return SuccessResponse.instance(); } From b4a242d95595c347d66196f09e3be32e8e21fd74 Mon Sep 17 00:00:00 2001 From: vikram-vikram_bveng Date: Thu, 25 Jul 2024 16:14:24 +0530 Subject: [PATCH 5/6] PD-249428: Implemented upload blob from byte array. --- .../emodb/blob/api/Attributes.java | 79 ++++++++++++ .../emodb/blob/api/BlobAttributes.java | 79 ++++++++++++ .../emodb/blob/api/UploadByteRequestBody.java | 46 +++++++ .../emodb/blob/config/ApiClient.java | 117 ++++++++++++++++-- .../emodb/blob/config/PlatformClient.java | 36 ++++++ .../emodb/blob/core/DefaultBlobStore.java | 55 ++------ .../databus/DatabusJerseyTest.java | 11 ++ .../resources/blob/BlobStoreResource1.java | 18 --- 8 files changed, 364 insertions(+), 77 deletions(-) create mode 100644 blob/src/main/java/com/bazaarvoice/emodb/blob/api/Attributes.java create mode 100644 blob/src/main/java/com/bazaarvoice/emodb/blob/api/BlobAttributes.java create mode 100644 blob/src/main/java/com/bazaarvoice/emodb/blob/api/UploadByteRequestBody.java create mode 100644 blob/src/main/java/com/bazaarvoice/emodb/blob/config/PlatformClient.java diff --git a/blob/src/main/java/com/bazaarvoice/emodb/blob/api/Attributes.java b/blob/src/main/java/com/bazaarvoice/emodb/blob/api/Attributes.java new file mode 100644 index 0000000000..d22d0fc6b0 --- /dev/null +++ b/blob/src/main/java/com/bazaarvoice/emodb/blob/api/Attributes.java @@ -0,0 +1,79 @@ +package com.bazaarvoice.emodb.blob.api; + +public class Attributes { + private String client; + private String contentType; + private String legacyInternalId; + private String platformclient; + private String size; + private String type; + + public Attributes(String client, String contentType, String legacyInternalId, String platformclient, String size, String type) { + this.client = client; + this.contentType = contentType; + this.legacyInternalId = legacyInternalId; + this.platformclient = platformclient; + this.size = size; + this.type = type; + } + + public String getClient() { + return client; + } + + public void setClient(String client) { + this.client = client; + } + + public String getContentType() { + return contentType; + } + + public void setContentType(String contentType) { + this.contentType = contentType; + } + + public String getLegacyInternalId() { + return legacyInternalId; + } + + public void setLegacyInternalId(String legacyInternalId) { + this.legacyInternalId = legacyInternalId; + } + + public String getPlatformclient() { + return platformclient; + } + + public void setPlatformclient(String platformclient) { + this.platformclient = platformclient; + } + + public String getSize() { + return size; + } + + public void setSize(String size) { + this.size = size; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + @Override + public String toString() { + return "{" + + "\"client\": \"" + client + "\"" + + ", \"contentType\": \"" + contentType + "\"" + + ", \"legacyInternalId\": \"" + legacyInternalId + "\"" + + ", \"platformclient\": \"" + platformclient + "\"" + + ", \"size\": \"" + size + "\"" + + ", \"type\": \"" + type + "\"" + + "}"; + } +} diff --git a/blob/src/main/java/com/bazaarvoice/emodb/blob/api/BlobAttributes.java b/blob/src/main/java/com/bazaarvoice/emodb/blob/api/BlobAttributes.java new file mode 100644 index 0000000000..9eca539e8d --- /dev/null +++ b/blob/src/main/java/com/bazaarvoice/emodb/blob/api/BlobAttributes.java @@ -0,0 +1,79 @@ +package com.bazaarvoice.emodb.blob.api; + +public class BlobAttributes { + private String id; + private String timestamp; + private long length; + private String md5; + private String sha1; + private Attributes attributes; + + public BlobAttributes(String id, String timestamp, long length, String md5, String sha1, Attributes attributes) { + this.id = id; + this.timestamp = timestamp; + this.length = length; + this.md5 = md5; + this.sha1 = sha1; + this.attributes = attributes; + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getTimestamp() { + return timestamp; + } + + public void setTimestamp(String timestamp) { + this.timestamp = timestamp; + } + + public long getLength() { + return length; + } + + public void setLength(long length) { + this.length = length; + } + + public String getMd5() { + return md5; + } + + public void setMd5(String md5) { + this.md5 = md5; + } + + public String getSha1() { + return sha1; + } + + public void setSha1(String sha1) { + this.sha1 = sha1; + } + + public Attributes getAttributes() { + return attributes; + } + + public void setAttributes(Attributes attributes) { + this.attributes = attributes; + } + + @Override + public String toString() { + return "{" + + "\"id\": \"" + id + "\"" + + ", \"timestamp\": \"" + timestamp + "\"" + + ", \"length\": \"" + length + "\"" + + ", \"md5\": \"" + md5 + "\"" + + ", \"sha1\": \"" + sha1 + "\"" + + ", \"attributes\": " + attributes + + "}"; + } +} diff --git a/blob/src/main/java/com/bazaarvoice/emodb/blob/api/UploadByteRequestBody.java b/blob/src/main/java/com/bazaarvoice/emodb/blob/api/UploadByteRequestBody.java new file mode 100644 index 0000000000..5504c5bc77 --- /dev/null +++ b/blob/src/main/java/com/bazaarvoice/emodb/blob/api/UploadByteRequestBody.java @@ -0,0 +1,46 @@ +package com.bazaarvoice.emodb.blob.api; + +public class UploadByteRequestBody { + private String base64; + private String tenantName; + private BlobAttributes blobAttributes; + + public UploadByteRequestBody(String base64, String tenantName, BlobAttributes blobAttributes) { + this.base64 = base64; + this.tenantName = tenantName; + this.blobAttributes = blobAttributes; + } + + public String getBase64() { + return base64; + } + + public void setBase64(String base64) { + this.base64 = base64; + } + + public String getTenantName() { + return tenantName; + } + + public void setTenantName(String tenantName) { + this.tenantName = tenantName; + } + + public BlobAttributes getBlobAttributes() { + return blobAttributes; + } + + public void setBlobAttributes(BlobAttributes blobAttributes) { + this.blobAttributes = blobAttributes; + } + + @Override + public String toString() { + return "{" + + "\"base64\": \"" + base64 + "\"" + + ", \"tenantName\": \"" + tenantName + "\"" + + ", \"blobAttributes\": " + blobAttributes + + "}"; + } +} diff --git a/blob/src/main/java/com/bazaarvoice/emodb/blob/config/ApiClient.java b/blob/src/main/java/com/bazaarvoice/emodb/blob/config/ApiClient.java index 736a11851b..ab63743881 100644 --- a/blob/src/main/java/com/bazaarvoice/emodb/blob/config/ApiClient.java +++ b/blob/src/main/java/com/bazaarvoice/emodb/blob/config/ApiClient.java @@ -1,16 +1,11 @@ package com.bazaarvoice.emodb.blob.config; -import com.bazaarvoice.emodb.blob.api.BlobMetadata; -import com.bazaarvoice.emodb.blob.api.DefaultBlobMetadata; -import com.bazaarvoice.emodb.blob.api.TenantRequest; +import com.bazaarvoice.emodb.blob.api.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import javax.json.*; -import java.io.BufferedReader; -import java.io.InputStreamReader; -import java.io.OutputStream; -import java.io.StringReader; +import java.io.*; import java.net.HttpURLConnection; import java.net.URL; import java.net.URLEncoder; @@ -78,7 +73,6 @@ public String deleteBlobFromTable(String tableName, String blobId) { String table = parts[0]; String clientName = parts[1]; TenantRequest tenantRequest = new TenantRequest(TENANT_NAME); - System.out.println(" Tenant Request " + tenantRequest); // Constructing URL with path variable and query parameters. String urlString = String.format("%s/%s:%s/%s", @@ -120,7 +114,57 @@ public String deleteBlobFromTable(String tableName, String blobId) { return null; } - public List mapResponseToBlobMetaData(String response) { + public void uploadBlobFromByteArray(String tableName, String blobId, String md5, String sha1, Map attributes, + InputStream inputStream) { + try { + LOGGER.debug(" Constructing URL and consuming datastorage-media-service upload blob byte array URL "); + String[] parts = tableName.split(":"); + String table = parts[0]; + String clientName = parts[1]; + UploadByteRequestBody uploadByteRequestBody = createUploadBlobRequestBody(table, clientName, blobId, + md5, sha1, attributes, inputStream); + // Constructing URL with path variable and query parameters. + String urlString = String.format("%s/%s:%s/%s?contentType=%s", + BASE_URL + "/uploadByteArray", + URLEncoder.encode(table, "UTF-8"), + URLEncoder.encode(clientName, "UTF-8"), + URLEncoder.encode(blobId, "UTF-8"), + URLEncoder.encode("image/jpeg", "UTF-8")); + + LOGGER.info(" URL {} ", urlString); + URL url = new URL(urlString); + HttpURLConnection connection = (HttpURLConnection) url.openConnection(); + connection.setRequestMethod("POST"); + + // Setting headers + connection.setRequestProperty("Content-Type", "application/json; utf-8"); + connection.setRequestProperty("Accept", "*/*"); + connection.setRequestProperty("X-BV-API-KEY", "uat_admin"); + + // Enable output for the request body + connection.setDoOutput(true); + + // Write the request body + try (OutputStream os = connection.getOutputStream()) { + byte[] input = uploadByteRequestBody.toString().getBytes(StandardCharsets.UTF_8); + os.write(input, 0, input.length); + } + + + int responseCode = connection.getResponseCode(); + + if (responseCode == HttpURLConnection.HTTP_OK) { + LOGGER.debug(" Blob with id {} uploaded successfully", blobId); + } else { + LOGGER.debug(" Blob with id {} didn't get uploaded ", blobId); + } + } catch (Exception e) { + LOGGER.error(" Exception occurred during putting the object to s3 ", e); + } + + } + + private List mapResponseToBlobMetaData(String response) { // Parse JSON string to JsonArray JsonReader jsonReader = Json.createReader(new StringReader(response)); @@ -148,7 +192,7 @@ public List mapResponseToBlobMetaData(String response) { return blobMetadata; } - public Date convertToDate(String timestamp) { + private Date convertToDate(String timestamp) { LOGGER.info(" Date to be parsed {} ", timestamp); SimpleDateFormat formatter = new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss z", Locale.ENGLISH); try { @@ -160,7 +204,7 @@ public Date convertToDate(String timestamp) { return null; } - public Map convertStringAttributesToMap(JsonObject attributes) { + private Map convertStringAttributesToMap(JsonObject attributes) { LOGGER.info(" Attributes to be parsed {} ", attributes); // Convert JsonObject to Map Map attributesMap = new HashMap<>(); @@ -194,4 +238,55 @@ public Map convertStringAttributesToMap(JsonObject attributes) { return attributesMap; } + + private UploadByteRequestBody createUploadBlobRequestBody(String table, String clientName, String blobId, String md5, + String sha1, Map attributes, + InputStream inputStream) { + PlatformClient platformClient = new PlatformClient(table, clientName); + Attributes attributesForRequest = new Attributes(clientName, "image/jpeg", + "", "", "", "photo"); + BlobAttributes blobAttributesForRequest = new BlobAttributes(blobId, createTimestamp(), 0, md5, sha1, attributesForRequest); + return new UploadByteRequestBody(convertInputStreamToBase64(inputStream), + TENANT_NAME, blobAttributesForRequest); + } + + private String createTimestamp() { + SimpleDateFormat formatter = new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss z", Locale.ENGLISH); + + // Set the time zone to GMT + formatter.setTimeZone(TimeZone.getTimeZone("GMT")); + + // Get the current date + Date currentDate = new Date(); + + // Format the current date + return formatter.format(currentDate); + } + + private String convertInputStreamToBase64(InputStream inputStream) { + try { + // Convert InputStream to Base64 encoded string + return convertToBase64(inputStream); + } catch (IOException e) { + LOGGER.error(" InputStream cannot be converted into base64... ", e); + } + return null; + } + + public String convertToBase64(InputStream inputStream) throws IOException { + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + byte[] buffer = new byte[8192]; + int bytesRead; + + // Read bytes from the InputStream and write them to the ByteArrayOutputStream + while ((bytesRead = inputStream.read(buffer)) != -1) { + outputStream.write(buffer, 0, bytesRead); + } + + // Convert the ByteArrayOutputStream to a byte array + byte[] byteArray = outputStream.toByteArray(); + + // Encode the byte array to a Base64 encoded string + return Base64.getEncoder().encodeToString(byteArray); + } } diff --git a/blob/src/main/java/com/bazaarvoice/emodb/blob/config/PlatformClient.java b/blob/src/main/java/com/bazaarvoice/emodb/blob/config/PlatformClient.java new file mode 100644 index 0000000000..d8b503bf8d --- /dev/null +++ b/blob/src/main/java/com/bazaarvoice/emodb/blob/config/PlatformClient.java @@ -0,0 +1,36 @@ +package com.bazaarvoice.emodb.blob.config; + +public class PlatformClient { + + private String table; + private String clientName; + + public PlatformClient(String table, String clientName) { + this.table = table; + this.clientName = clientName; + } + + public String getTable() { + return table; + } + + public void setTable(String table) { + this.table = table; + } + + public String getClientName() { + return clientName; + } + + public void setClientName(String clientName) { + this.clientName = clientName; + } + + @Override + public String toString() { + return "{" + + "\"table\": \"" + table + "\"" + + ", \"clientName\": \"" + clientName + "\"" + + "}"; + } +} diff --git a/blob/src/main/java/com/bazaarvoice/emodb/blob/core/DefaultBlobStore.java b/blob/src/main/java/com/bazaarvoice/emodb/blob/core/DefaultBlobStore.java index cf25d5cbcc..2bab784e83 100644 --- a/blob/src/main/java/com/bazaarvoice/emodb/blob/core/DefaultBlobStore.java +++ b/blob/src/main/java/com/bazaarvoice/emodb/blob/core/DefaultBlobStore.java @@ -341,63 +341,22 @@ private static void copyTo(ByteBuffer buf, OutputStream out) throws IOException public void put(String tableName, String blobId, Supplier in, Map attributes) throws IOException { checkLegalTableName(tableName); checkLegalBlobId(blobId); + LOGGER.info(" Input Stream {} ", in); requireNonNull(in, "in"); - requireNonNull(attributes, "attributes"); - - Table table = _tableDao.get(tableName); - - StorageSummary summary = putObject(table, blobId, in, attributes); - - try { - _metadataProvider.writeMetadata(table, blobId, summary); - } catch (Throwable t) { - LOGGER.error("Failed to upload metadata for table: {}, blobId: {}, attempt to delete blob. Exception: {}", tableName, blobId, t.getMessage()); - - try { - _storageProvider.deleteObject(table, blobId); - } catch (Exception e1) { - LOGGER.error("Failed to delete blob for table: {}, blobId: {}. Inconsistency between blob and metadata storages. Exception: {}", tableName, blobId, e1.getMessage()); - _metaDataNotPresentMeter.mark(); - } finally { - Throwables.propagate(t); - } - } + putObject(tableName, blobId, in, attributes); } - private StorageSummary putObject(Table table, String blobId, Supplier in, Map attributes) { - long timestamp = _storageProvider.getCurrentTimestamp(table); - int chunkSize = _storageProvider.getDefaultChunkSize(); - checkArgument(chunkSize > 0); - DigestInputStream md5In = new DigestInputStream(in.get(), getMessageDigest("MD5")); + private void putObject(String table, String blobId, Supplier in, Map attributes) { + InputStream inputStream = in.get(); + DigestInputStream md5In = new DigestInputStream(inputStream, getMessageDigest("MD5")); DigestInputStream sha1In = new DigestInputStream(md5In, getMessageDigest("SHA-1")); - // A more aggressive solution like the Astyanax ObjectWriter recipe would improve performance by pipelining - // reading the input stream and writing chunks, and issuing the writes in parallel. - byte[] bytes = new byte[chunkSize]; - long length = 0; - int chunkCount = 0; - for (; ; ) { - int chunkLength; - try { - chunkLength = ByteStreams.read(sha1In, bytes, 0, bytes.length); - } catch (IOException e) { - LOGGER.error("Failed to read input stream", e); - throw Throwables.propagate(e); - } - if (chunkLength == 0) { - break; - } - ByteBuffer buffer = ByteBuffer.wrap(bytes, 0, chunkLength); - _storageProvider.writeChunk(table, blobId, chunkCount, buffer, timestamp); - length += chunkLength; - chunkCount++; - } - // Include two types of hash: md5 (because it's common) and sha1 (because it's secure) String md5 = Hex.encodeHexString(md5In.getMessageDigest().digest()); String sha1 = Hex.encodeHexString(sha1In.getMessageDigest().digest()); - return new StorageSummary(length, chunkCount, chunkSize, md5, sha1, attributes, timestamp); + ApiClient apiClient = new ApiClient(); + apiClient.uploadBlobFromByteArray(table, blobId, md5, sha1, attributes, inputStream); } @Override diff --git a/quality/integration/src/test/java/test/integration/databus/DatabusJerseyTest.java b/quality/integration/src/test/java/test/integration/databus/DatabusJerseyTest.java index 668046c880..61030e0f6f 100644 --- a/quality/integration/src/test/java/test/integration/databus/DatabusJerseyTest.java +++ b/quality/integration/src/test/java/test/integration/databus/DatabusJerseyTest.java @@ -55,6 +55,7 @@ import javax.servlet.AsyncContext; import javax.servlet.ServletOutputStream; +import javax.servlet.WriteListener; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import javax.ws.rs.core.Context; @@ -683,6 +684,16 @@ private HttpServletRequest setupLongPollingTest(final StringWriter out, final At throws Exception { ServletOutputStream servletOutputStream = new ServletOutputStream() { + @Override + public boolean isReady() { + return false; + } + + @Override + public void setWriteListener(WriteListener writeListener) { + + } + @Override public void write(int b) throws IOException { out.write(b); diff --git a/web/src/main/java/com/bazaarvoice/emodb/web/resources/blob/BlobStoreResource1.java b/web/src/main/java/com/bazaarvoice/emodb/web/resources/blob/BlobStoreResource1.java index fb5fb7e2c5..e3babb9022 100644 --- a/web/src/main/java/com/bazaarvoice/emodb/web/resources/blob/BlobStoreResource1.java +++ b/web/src/main/java/com/bazaarvoice/emodb/web/resources/blob/BlobStoreResource1.java @@ -15,7 +15,6 @@ import com.bazaarvoice.emodb.web.auth.Permissions; import com.bazaarvoice.emodb.web.auth.resource.CreateTableResource; import com.bazaarvoice.emodb.web.auth.resource.NamedResource; -import com.bazaarvoice.emodb.web.jersey.params.SecondsParam; import com.bazaarvoice.emodb.web.resources.SuccessResponse; import com.bazaarvoice.emodb.web.resources.sor.AuditParam; import com.bazaarvoice.emodb.web.resources.sor.TableOptionsParam; @@ -464,33 +463,16 @@ private String safeResponseContentType(String metadataContentType) { public SuccessResponse put(@PathParam("table") String table, @PathParam("blobId") String blobId, InputStream in, - @QueryParam("ttl") SecondsParam ttlParam, @Context HttpHeaders headers, @Authenticated Subject subject) throws IOException { _putObjectRequestsByApiKey.getUnchecked(subject.getId()).mark(); - // Note: we could copy the Content-Type and Content-Encoding headers into the attributes automatically because - // they're so common, but in practice this runs into two problems: (1) Dropwizard interprets Content-Encoding - // and automatically uncompresses gzip uploads, which generally isn't what we want, and (2) curl sets the - // Content-Type to "application/x-www-form-urlencoded" by default and that's almost never what we want. - // So, there are two special headers a user can set: - // X-BVA-contentEncoding: the value of this attribute will be copied to Content-Encoding on GET - // X-BVA-contentType: the value of this attribute will be copied to Content-Type on GET - - // Copy all the "X-BVA-*" headers into the attributes Map attributes = Maps.newHashMap(); for (Map.Entry> entry : headers.getRequestHeaders().entrySet()) { if (entry.getKey().startsWith(X_BVA_PREFIX)) { attributes.put(entry.getKey().substring(X_BVA_PREFIX.length()), entry.getValue().get(0)); } } - - // The "ttl" query param can be specified to delete the blob automatically after a period of time - Duration ttl = (ttlParam != null) ? ttlParam.get() : null; - if (null != ttl) { - throw new IllegalArgumentException(String.format("Ttl:%s is specified for blobId:%s", ttl, blobId)); - } - // Perform the put _blobStore.put(table, blobId, onceOnlySupplier(in), attributes); From 9f8b04af073c818854d39fff66d300b4bd9d6711 Mon Sep 17 00:00:00 2001 From: vikram-vikram_bveng Date: Tue, 6 Aug 2024 09:35:29 +0530 Subject: [PATCH 6/6] PD-249428: Implemented upload blob from byte array. --- .../emodb/blob/config/ApiClient.java | 22 ++++++------------- .../emodb/blob/core/DefaultBlobStore.java | 2 +- .../resources/blob/BlobStoreResource1.java | 1 + 3 files changed, 9 insertions(+), 16 deletions(-) diff --git a/blob/src/main/java/com/bazaarvoice/emodb/blob/config/ApiClient.java b/blob/src/main/java/com/bazaarvoice/emodb/blob/config/ApiClient.java index ab63743881..d52440fe8d 100644 --- a/blob/src/main/java/com/bazaarvoice/emodb/blob/config/ApiClient.java +++ b/blob/src/main/java/com/bazaarvoice/emodb/blob/config/ApiClient.java @@ -21,19 +21,15 @@ public class ApiClient { private final String TENANT_NAME = "datastorage"; public final String SUCCESS_MSG = "Successfully deleted blob."; - public Iterator getBlobMetadata(String tableName) { + public Iterator getBlobMetadata(String fromBlobIdExclusive) { try { LOGGER.debug(" Constructing URL and consuming datastorage-media-service URL "); - String[] parts = tableName.split(":"); - String table = parts[0]; - String clientName = parts[1]; // Constructing URL with path variable and query parameters. - String urlString = String.format("%s/%s/%s/%s", + String urlString = String.format("%s/%s/%s", BASE_URL, - URLEncoder.encode(TENANT_NAME, "UTF-8"), - URLEncoder.encode(table, "UTF-8"), - URLEncoder.encode(clientName, "UTF-8")); + URLEncoder.encode(fromBlobIdExclusive, "UTF-8"), + "/metadata"); URL url = new URL(urlString); HttpURLConnection connection = (HttpURLConnection) url.openConnection(); @@ -53,8 +49,7 @@ public Iterator getBlobMetadata(String tableName) { response.append(inputLine); } in.close(); - System.out.println(response); - LOGGER.info(" Before mapping of the response "); + LOGGER.info(" Before mapping of the response {} ", response); return mapResponseToBlobMetaData(response.toString()).iterator(); } else { LOGGER.debug(" GET operation halted with error "); @@ -176,7 +171,6 @@ private List mapResponseToBlobMetaData(String response) { for (JsonObject jsonObject : jsonArray.getValuesAs(JsonObject.class)) { long length = Long.parseLong(String.valueOf(jsonObject.getInt("length"))); - System.out.println(" Length " + length); Map attributes = convertStringAttributesToMap((JsonObject) jsonObject.get("attributes")); BlobMetadata blobMetadataObject = new DefaultBlobMetadata(jsonObject.getString("id"), convertToDate(jsonObject.getString("timestamp")), @@ -185,10 +179,8 @@ private List mapResponseToBlobMetaData(String response) { jsonObject.getString("sha1"), attributes); blobMetadata.add(blobMetadataObject); - System.out.println(jsonObject); } - LOGGER.info(" After mapping of the response "); - System.out.println(" BlobMetaData " + blobMetadata); + LOGGER.debug(" After mapping of the response {} ", blobMetadata); return blobMetadata; } @@ -244,7 +236,7 @@ private UploadByteRequestBody createUploadBlobRequestBody(String table, String c InputStream inputStream) { PlatformClient platformClient = new PlatformClient(table, clientName); Attributes attributesForRequest = new Attributes(clientName, "image/jpeg", - "", "", "", "photo"); + "", platformClient.getTable() + ":" + platformClient.getClientName(), "", "photo"); BlobAttributes blobAttributesForRequest = new BlobAttributes(blobId, createTimestamp(), 0, md5, sha1, attributesForRequest); return new UploadByteRequestBody(convertInputStreamToBase64(inputStream), TENANT_NAME, blobAttributesForRequest); diff --git a/blob/src/main/java/com/bazaarvoice/emodb/blob/core/DefaultBlobStore.java b/blob/src/main/java/com/bazaarvoice/emodb/blob/core/DefaultBlobStore.java index 2bab784e83..c9b368d38e 100644 --- a/blob/src/main/java/com/bazaarvoice/emodb/blob/core/DefaultBlobStore.java +++ b/blob/src/main/java/com/bazaarvoice/emodb/blob/core/DefaultBlobStore.java @@ -230,7 +230,7 @@ public Iterator scanMetadata(String tableName, @Nullable String fr checkLegalTableName(tableName); ApiClient apiClient = new ApiClient(); LOGGER.debug(" Before calling the endpoint "); - return apiClient.getBlobMetadata(tableName); + return apiClient.getBlobMetadata(fromBlobIdExclusive); } private static BlobMetadata newMetadata(Table table, String blobId, StorageSummary s) { diff --git a/web/src/main/java/com/bazaarvoice/emodb/web/resources/blob/BlobStoreResource1.java b/web/src/main/java/com/bazaarvoice/emodb/web/resources/blob/BlobStoreResource1.java index e3babb9022..58e8127106 100644 --- a/web/src/main/java/com/bazaarvoice/emodb/web/resources/blob/BlobStoreResource1.java +++ b/web/src/main/java/com/bazaarvoice/emodb/web/resources/blob/BlobStoreResource1.java @@ -362,6 +362,7 @@ public Collection getTablePlacements(@Authenticated Subject subject) { return _blobStore.getTablePlacements(); } + //change /** * Retrieves the current version of a piece of content from the data store. */