From f4fd0c01eb441518f54e5791e3b6ce84a853188e Mon Sep 17 00:00:00 2001 From: Your Name Date: Thu, 14 Dec 2023 15:56:14 -0500 Subject: [PATCH 1/5] SFR-1828_APIS3ObjectLink --- CHANGELOG.md | 1 + api/app.py | 3 +- api/blueprints/__init__.py | 1 + api/blueprints/drbS3.py | 105 ++++++++++++++++++++++++++ api/utils.py | 2 +- scripts/nyplLoginFlags.py | 2 +- swagger.v4.json | 39 ++++++++++ tests/unit/test_api_s3_blueprint.py | 56 ++++++++++++++ tests/unit/test_api_work_blueprint.py | 2 +- 9 files changed, 207 insertions(+), 4 deletions(-) create mode 100644 api/blueprints/drbS3.py create mode 100644 tests/unit/test_api_s3_blueprint.py diff --git a/CHANGELOG.md b/CHANGELOG.md index d4f078c6e0..4f292172f8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ ## Added - New script to add nypl_login flag to Links objects - Added nypl_login flag to nypl mapping +- New API to return a link to an S3 object ## Fixed ## 2023-09-05 version -- v0.12.3 diff --git a/api/app.py b/api/app.py index a8a3ebbff3..5452a6b8fd 100644 --- a/api/app.py +++ b/api/app.py @@ -9,7 +9,7 @@ from logger import createLog from .blueprints import ( - search, work, info, edition, utils, link, opds, collection, citation + search, work, info, edition, utils, link, opds, collection, citation, s3 ) from .utils import APIUtils @@ -36,6 +36,7 @@ def __init__(self, dbEngine, redisClient): self.app.register_blueprint(opds) self.app.register_blueprint(collection) self.app.register_blueprint(citation) + self.app.register_blueprint(s3) def run(self): if 'local-compose' in os.environ['ENVIRONMENT'] or 'sample-compose' in os.environ['ENVIRONMENT']: diff --git a/api/blueprints/__init__.py b/api/blueprints/__init__.py index b7f4565309..a04613c105 100644 --- a/api/blueprints/__init__.py +++ b/api/blueprints/__init__.py @@ -7,3 +7,4 @@ from .drbSearch import search from .drbUtils import utils from .drbWork import work +from .drbS3 import s3 diff --git a/api/blueprints/drbS3.py b/api/blueprints/drbS3.py new file mode 100644 index 0000000000..9158d50848 --- /dev/null +++ b/api/blueprints/drbS3.py @@ -0,0 +1,105 @@ +from flask import Blueprint, request, current_app +from ..utils import APIUtils +from logger import createLog +from functools import wraps +from botocore.exceptions import ClientError +from urllib.parse import urlparse + +import requests +import argparse +import boto3 +import json + +logger = createLog(__name__) + +s3 = Blueprint('s3', __name__, url_prefix='/s3') + +@s3.route('/', methods=['GET']) +def s3ObjectLinkFetch(bucket): + + logger.info(f'Fetching AWS S3 Object Link') + + searchParams = APIUtils.normalizeQueryParams(request.args) + + key = searchParams.get('key')[0] + + url = f'https://{bucket}.s3.amazonaws.com/{key}' + + response = requests.get(url) + + if response.status_code == 200: + statusCode = 200 + responseBody = { + 'message': f'Object URL: {url}' + } + else: + statusCode = response.status_code + responseBody = { + 'message': f'URL does not exist {url}' + } + + return APIUtils.formatResponseObject( + statusCode, 's3ObjectLinkFetch', responseBody + ) + + # searchParams = APIUtils.normalizeQueryParams(request.args) + + # key = searchParams.get('key')[0] + # logger.info(key) + # logger.info(bucket) + + # parser = argparse.ArgumentParser() + + # logger.info(parser) + + # parser.add_argument(bucket, help=bucket) + # parser.add_argument(key,help=key) + # parser.add_argument("action", choices=("get"), help="get_object") + + # args = parser.parse_args() + # logger.info(args) + + # s3_client = boto3.client("s3") + # client_action = "get_object" #if args.action == "get" else None + # url = generate_presigned_url( + # s3_client, client_action, {"Bucket": bucket, "Key": key}, 1000 + # ) + # logger.info(url) + + # logger.info("Using the Requests package to send a request to the URL.") + # response = None + # response = requests.get(url) + # if args.action == "get": + # response = requests.get(url) + + # if response is not None: + # print("Got response:") + # print(f"Status: {response.status_code}") + # dictResponse = json.loads(response.text) + # linksResponse = dictResponse['links'] + # print(dictResponse) + # print('\n') + # print(linksResponse) + # return response.text + +# def generate_presigned_url(s3_client, client_method, method_parameters, expires_in): +# """ +# Generate a presigned Amazon S3 URL that can be used to perform an action. + +# :param s3_client: A Boto3 Amazon S3 client. +# :param client_method: The name of the client method that the URL performs. +# :param method_parameters: The parameters of the specified client method. +# :param expires_in: The number of seconds the presigned URL is valid for. +# :return: The presigned URL. +# """ +# try: +# url = s3_client.generate_presigned_url( +# ClientMethod=client_method, Params=method_parameters, ExpiresIn=expires_in +# ) +# logger.info("Got presigned URL: %s", url) +# except ClientError: +# logger.exception( +# "Couldn't get a presigned URL for client method '%s'.", client_method +# ) +# raise +# return url diff --git a/api/utils.py b/api/utils.py index 5f740e65b1..35f84150b6 100644 --- a/api/utils.py +++ b/api/utils.py @@ -10,7 +10,7 @@ class APIUtils(): QUERY_TERMS = [ 'keyword', 'title', 'author', 'subject', 'identifier', 'authority: identifier', 'viaf', 'lcnaf', - 'date', 'startYear', 'endYear', 'language', 'format', 'govDoc', 'showAll' + 'date', 'startYear', 'endYear', 'language', 'format', 'govDoc', 'showAll', 'key' ] FORMAT_CROSSWALK = { diff --git a/scripts/nyplLoginFlags.py b/scripts/nyplLoginFlags.py index 9911b8fb21..a356d991c8 100644 --- a/scripts/nyplLoginFlags.py +++ b/scripts/nyplLoginFlags.py @@ -23,7 +23,7 @@ def main(): for link in dbManager.session.query(Link) \ .filter(or_(Link.media_type == 'application/html+edd', Link.media_type == 'application/x.html+edd')).all(): - if link.flags['edd'] == True: + if link.flags and 'edd' in link.flags and link.flags['edd'] == True: #The link.flags doesn't update if the dict method isn't called on it newLinkFlag = dict(link.flags) newLinkFlag['nypl_login'] = True diff --git a/swagger.v4.json b/swagger.v4.json index 091cb89b5c..9cc72d8688 100644 --- a/swagger.v4.json +++ b/swagger.v4.json @@ -246,6 +246,45 @@ } } }, + "/s3/{bucket}": { + "get": { + "tags": ["digital-research-books"], + "summary": "v4 Get AWS S3 Object Link", + "description": "Return a single s3 object link from DRB buckets", + "parameters": [ + { + "name": "key", + "in": "path", + "description": "AWS key for S3 object", + "type": "string", + "required": true + } + ], + "responses": { + "200": { + "description": "A single DRB S3 Object URL" + }, + "404": { + "description": "Resource was not found", + "schema": { + "$ref": "#/definitions/ErrorResponse" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/ErrorResponse" + } + }, + "default": { + "description": "Unexpected Error", + "schema": { + "$ref": "#/definitions/ErrorResponse" + } + } + } + } + }, "/link/{id}": { "get": { "tags": ["digital-research-books"], diff --git a/tests/unit/test_api_s3_blueprint.py b/tests/unit/test_api_s3_blueprint.py new file mode 100644 index 0000000000..3157d8ffcd --- /dev/null +++ b/tests/unit/test_api_s3_blueprint.py @@ -0,0 +1,56 @@ +from flask import Flask +import pytest + +from api.blueprints.drbS3 import s3ObjectLinkFetch +from api.utils import APIUtils + +from werkzeug.datastructures import ImmutableMultiDict + +class TestS3Blueprint: + @pytest.fixture + def mockUtils(self, mocker): + return mocker.patch.multiple( + APIUtils, + normalizeQueryParams=mocker.DEFAULT, + formatResponseObject=mocker.DEFAULT + ) + + @pytest.fixture + def testApp(self): + flaskApp = Flask('test') + + return flaskApp + + def test_s3ObjectLinkFetch_success(self, mockUtils, testApp, mocker): + + mockUtils['normalizeQueryParams'].return_value = {'key': ['manifests/met/10935.json']} + mockUtils['formatResponseObject'].return_value = 'testS3Response' + + + with testApp.test_request_context('/?key=manifests/met/10935.json'): + testAPIResponse = s3ObjectLinkFetch('drb-files-qa') + + assert testAPIResponse == 'testS3Response' + + mockUtils['normalizeQueryParams'].assert_called_once_with(ImmutableMultiDict([('key', 'manifests/met/10935.json')])) + + mockUtils['formatResponseObject'].assert_called_once_with( + 200, 's3ObjectLinkFetch', {'message': f'Object URL: https://drb-files-qa.s3.amazonaws.com/manifests/met/10935.json'} + ) + + def test_s3ObjectLinkFetch_fail(self, mockUtils, testApp, mocker): + + mockUtils['normalizeQueryParams'].return_value = {'key': ['manifests/met/10935.json']} + mockUtils['formatResponseObject'].return_value = 'testS3Response' + + + with testApp.test_request_context('/?key=manifests/met/10935.json'): + testAPIResponse = s3ObjectLinkFetch('drb-files-qaFail') + + assert testAPIResponse == 'testS3Response' + + mockUtils['normalizeQueryParams'].assert_called_once_with(ImmutableMultiDict([('key', 'manifests/met/10935.json')])) + + mockUtils['formatResponseObject'].assert_called_once_with( + 404, 's3ObjectLinkFetch', {'message': f'URL does not exist https://drb-files-qaFail.s3.amazonaws.com/manifests/met/10935.json'} + ) diff --git a/tests/unit/test_api_work_blueprint.py b/tests/unit/test_api_work_blueprint.py index bec039bcc8..386f9df317 100644 --- a/tests/unit/test_api_work_blueprint.py +++ b/tests/unit/test_api_work_blueprint.py @@ -5,7 +5,7 @@ from api.utils import APIUtils -class TestSearchBlueprint: +class TestWorkBlueprint: @pytest.fixture def mockUtils(self, mocker): return mocker.patch.multiple( From 3589735bae9c591e38aa8d766bb6da13e7f28e2d Mon Sep 17 00:00:00 2001 From: Your Name Date: Mon, 18 Dec 2023 15:32:15 -0500 Subject: [PATCH 2/5] Returns unique link --- api/blueprints/drbS3.py | 112 +++++++++++++--------------------------- 1 file changed, 36 insertions(+), 76 deletions(-) diff --git a/api/blueprints/drbS3.py b/api/blueprints/drbS3.py index 9158d50848..f2e9a9215c 100644 --- a/api/blueprints/drbS3.py +++ b/api/blueprints/drbS3.py @@ -5,10 +5,8 @@ from botocore.exceptions import ClientError from urllib.parse import urlparse -import requests import argparse import boto3 -import json logger = createLog(__name__) @@ -23,83 +21,45 @@ def s3ObjectLinkFetch(bucket): key = searchParams.get('key')[0] - url = f'https://{bucket}.s3.amazonaws.com/{key}' + logger.info(f'Key: {key}') + logger.info(f'Bucket: {bucket}') - response = requests.get(url) + parser = argparse.ArgumentParser() - if response.status_code == 200: - statusCode = 200 - responseBody = { - 'message': f'Object URL: {url}' - } - else: - statusCode = response.status_code - responseBody = { - 'message': f'URL does not exist {url}' - } + parser.add_argument(bucket, help=bucket) + parser.add_argument(key, help=key) - return APIUtils.formatResponseObject( - statusCode, 's3ObjectLinkFetch', responseBody - ) - - # searchParams = APIUtils.normalizeQueryParams(request.args) - - # key = searchParams.get('key')[0] - # logger.info(key) - # logger.info(bucket) - - # parser = argparse.ArgumentParser() - - # logger.info(parser) + logger.info(parser) - # parser.add_argument(bucket, help=bucket) - # parser.add_argument(key,help=key) - # parser.add_argument("action", choices=("get"), help="get_object") - - # args = parser.parse_args() - # logger.info(args) + s3_client = boto3.client("s3") + client_action = "get_object" + url = generate_presigned_url( + s3_client, client_action, {"Bucket": bucket, "Key": key}, 1000 + ) + logger.info(url) - # s3_client = boto3.client("s3") - # client_action = "get_object" #if args.action == "get" else None - # url = generate_presigned_url( - # s3_client, client_action, {"Bucket": bucket, "Key": key}, 1000 - # ) - # logger.info(url) - - # logger.info("Using the Requests package to send a request to the URL.") - # response = None - # response = requests.get(url) - # if args.action == "get": - # response = requests.get(url) - - # if response is not None: - # print("Got response:") - # print(f"Status: {response.status_code}") - # dictResponse = json.loads(response.text) - # linksResponse = dictResponse['links'] - # print(dictResponse) - # print('\n') - # print(linksResponse) - # return response.text - -# def generate_presigned_url(s3_client, client_method, method_parameters, expires_in): -# """ -# Generate a presigned Amazon S3 URL that can be used to perform an action. + return APIUtils.formatResponseObject( + 200, 's3ObjectLinkFetch', {'message': f'{url}'} + ) -# :param s3_client: A Boto3 Amazon S3 client. -# :param client_method: The name of the client method that the URL performs. -# :param method_parameters: The parameters of the specified client method. -# :param expires_in: The number of seconds the presigned URL is valid for. -# :return: The presigned URL. -# """ -# try: -# url = s3_client.generate_presigned_url( -# ClientMethod=client_method, Params=method_parameters, ExpiresIn=expires_in -# ) -# logger.info("Got presigned URL: %s", url) -# except ClientError: -# logger.exception( -# "Couldn't get a presigned URL for client method '%s'.", client_method -# ) -# raise -# return url +def generate_presigned_url(s3_client, client_method, method_parameters, expires_in): + """ + Generate a presigned Amazon S3 URL that can be used to perform an action. + + :param s3_client: A Boto3 Amazon S3 client. + :param client_method: The name of the client method that the URL performs. + :param method_parameters: The parameters of the specified client method. + :param expires_in: The number of seconds the presigned URL is valid for. + :return: The presigned URL. + """ + try: + url = s3_client.generate_presigned_url( + ClientMethod=client_method, Params=method_parameters, ExpiresIn=expires_in + ) + logger.info("Got presigned URL: %s", url) + except ClientError: + logger.exception( + "Couldn't get a presigned URL for client method '%s'.", client_method + ) + raise + return url From 98e28aa69c6ae471541d75b5086f0b8f151c8fa9 Mon Sep 17 00:00:00 2001 From: Your Name Date: Tue, 19 Dec 2023 22:41:35 -0500 Subject: [PATCH 3/5] Returns unique link --- CHANGELOG.md | 2 +- api/blueprints/drbS3.py | 34 +++++++++-------------------- api/utils.py | 28 ++++++++++++++++++++++++ tests/unit/test_api_s3_blueprint.py | 12 +++++----- 4 files changed, 45 insertions(+), 31 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4f292172f8..c1e4cff5d0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ ## Added - New script to add nypl_login flag to Links objects - Added nypl_login flag to nypl mapping -- New API to return a link to an S3 object +- New API to return a unique, presigned link to an S3 object ## Fixed ## 2023-09-05 version -- v0.12.3 diff --git a/api/blueprints/drbS3.py b/api/blueprints/drbS3.py index f2e9a9215c..b81b99a29d 100644 --- a/api/blueprints/drbS3.py +++ b/api/blueprints/drbS3.py @@ -5,6 +5,7 @@ from botocore.exceptions import ClientError from urllib.parse import urlparse +import requests import argparse import boto3 @@ -33,33 +34,18 @@ def s3ObjectLinkFetch(bucket): s3_client = boto3.client("s3") client_action = "get_object" - url = generate_presigned_url( + url = APIUtils.generate_presigned_url( s3_client, client_action, {"Bucket": bucket, "Key": key}, 1000 ) logger.info(url) - return APIUtils.formatResponseObject( + response = requests.get(url) + + if response.status_code == 200: + return APIUtils.formatResponseObject( 200, 's3ObjectLinkFetch', {'message': f'{url}'} ) - -def generate_presigned_url(s3_client, client_method, method_parameters, expires_in): - """ - Generate a presigned Amazon S3 URL that can be used to perform an action. - - :param s3_client: A Boto3 Amazon S3 client. - :param client_method: The name of the client method that the URL performs. - :param method_parameters: The parameters of the specified client method. - :param expires_in: The number of seconds the presigned URL is valid for. - :return: The presigned URL. - """ - try: - url = s3_client.generate_presigned_url( - ClientMethod=client_method, Params=method_parameters, ExpiresIn=expires_in - ) - logger.info("Got presigned URL: %s", url) - except ClientError: - logger.exception( - "Couldn't get a presigned URL for client method '%s'.", client_method - ) - raise - return url + else: + return APIUtils.formatResponseObject( + response.status_code, 's3ObjectLinkFetch', {'Bucket/Key does not exist': f'{url}'} + ) diff --git a/api/utils.py b/api/utils.py index 35f84150b6..5af6829fbf 100644 --- a/api/utils.py +++ b/api/utils.py @@ -6,6 +6,11 @@ from model import Collection, Edition import re from model.postgres.collection import COLLECTION_EDITIONS +from logger import createLog +from botocore.exceptions import ClientError + + +logger = createLog(__name__) class APIUtils(): QUERY_TERMS = [ @@ -508,3 +513,26 @@ def validatePassword(password, hash, salt): hashedPassword = scrypt(password, salt=salt, n=2**14, r=8, p=1) return hashedPassword == hash + + @staticmethod + def generate_presigned_url(s3_client, client_method, method_parameters, expires_in): + """ + Generate a presigned Amazon S3 URL that can be used to perform an action. + + :param s3_client: A Boto3 Amazon S3 client. + :param client_method: The name of the client method that the URL performs. + :param method_parameters: The parameters of the specified client method. + :param expires_in: The number of seconds the presigned URL is valid for. + :return: The presigned URL. + """ + try: + url = s3_client.generate_presigned_url( + ClientMethod=client_method, Params=method_parameters, ExpiresIn=expires_in + ) + logger.info("Got presigned URL: %s", url) + except ClientError: + logger.exception( + "Couldn't get a presigned URL for client method '%s'.", client_method + ) + raise + return url \ No newline at end of file diff --git a/tests/unit/test_api_s3_blueprint.py b/tests/unit/test_api_s3_blueprint.py index 3157d8ffcd..c076a20b83 100644 --- a/tests/unit/test_api_s3_blueprint.py +++ b/tests/unit/test_api_s3_blueprint.py @@ -12,7 +12,8 @@ def mockUtils(self, mocker): return mocker.patch.multiple( APIUtils, normalizeQueryParams=mocker.DEFAULT, - formatResponseObject=mocker.DEFAULT + formatResponseObject=mocker.DEFAULT, + generate_presigned_url=mocker.DEFAULT ) @pytest.fixture @@ -25,6 +26,7 @@ def test_s3ObjectLinkFetch_success(self, mockUtils, testApp, mocker): mockUtils['normalizeQueryParams'].return_value = {'key': ['manifests/met/10935.json']} mockUtils['formatResponseObject'].return_value = 'testS3Response' + mockUtils['generate_presigned_url'].return_value = 'https://digital-research-books-beta.nypl.org/work/2263dbf4-51c2-4f6e-b9eb-e1def96b48d4?featured=9401263' with testApp.test_request_context('/?key=manifests/met/10935.json'): @@ -35,14 +37,14 @@ def test_s3ObjectLinkFetch_success(self, mockUtils, testApp, mocker): mockUtils['normalizeQueryParams'].assert_called_once_with(ImmutableMultiDict([('key', 'manifests/met/10935.json')])) mockUtils['formatResponseObject'].assert_called_once_with( - 200, 's3ObjectLinkFetch', {'message': f'Object URL: https://drb-files-qa.s3.amazonaws.com/manifests/met/10935.json'} + 200, 's3ObjectLinkFetch', {'message': 'https://digital-research-books-beta.nypl.org/work/2263dbf4-51c2-4f6e-b9eb-e1def96b48d4?featured=9401263'} ) def test_s3ObjectLinkFetch_fail(self, mockUtils, testApp, mocker): mockUtils['normalizeQueryParams'].return_value = {'key': ['manifests/met/10935.json']} mockUtils['formatResponseObject'].return_value = 'testS3Response' - + mockUtils['generate_presigned_url'].return_value = 'https://drb-files-qaFail.s3.amazonaws.com/manifests/met/10935.json?' with testApp.test_request_context('/?key=manifests/met/10935.json'): testAPIResponse = s3ObjectLinkFetch('drb-files-qaFail') @@ -51,6 +53,4 @@ def test_s3ObjectLinkFetch_fail(self, mockUtils, testApp, mocker): mockUtils['normalizeQueryParams'].assert_called_once_with(ImmutableMultiDict([('key', 'manifests/met/10935.json')])) - mockUtils['formatResponseObject'].assert_called_once_with( - 404, 's3ObjectLinkFetch', {'message': f'URL does not exist https://drb-files-qaFail.s3.amazonaws.com/manifests/met/10935.json'} - ) + mockUtils['formatResponseObject'].assert_called_once_with(404, 's3ObjectLinkFetch', {'Bucket/Key does not exist': 'https://drb-files-qaFail.s3.amazonaws.com/manifests/met/10935.json?'}) \ No newline at end of file From eaf125a85630c63fed2df77f31c5fd4bbce81abc Mon Sep 17 00:00:00 2001 From: Your Name Date: Thu, 21 Dec 2023 15:03:53 -0500 Subject: [PATCH 4/5] Removed API and blueprint for this branch --- CHANGELOG.md | 2 +- api/app.py | 3 +- api/blueprints/__init__.py | 1 - api/blueprints/drbS3.py | 51 -------------------------- swagger.v4.json | 39 -------------------- tests/unit/test_api_s3_blueprint.py | 56 ----------------------------- 6 files changed, 2 insertions(+), 150 deletions(-) delete mode 100644 api/blueprints/drbS3.py delete mode 100644 tests/unit/test_api_s3_blueprint.py diff --git a/CHANGELOG.md b/CHANGELOG.md index c1e4cff5d0..391fecfb24 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ ## Added - New script to add nypl_login flag to Links objects - Added nypl_login flag to nypl mapping -- New API to return a unique, presigned link to an S3 object +- New APIUtils method to generate a presigned url for S3 actions ## Fixed ## 2023-09-05 version -- v0.12.3 diff --git a/api/app.py b/api/app.py index 5452a6b8fd..a8a3ebbff3 100644 --- a/api/app.py +++ b/api/app.py @@ -9,7 +9,7 @@ from logger import createLog from .blueprints import ( - search, work, info, edition, utils, link, opds, collection, citation, s3 + search, work, info, edition, utils, link, opds, collection, citation ) from .utils import APIUtils @@ -36,7 +36,6 @@ def __init__(self, dbEngine, redisClient): self.app.register_blueprint(opds) self.app.register_blueprint(collection) self.app.register_blueprint(citation) - self.app.register_blueprint(s3) def run(self): if 'local-compose' in os.environ['ENVIRONMENT'] or 'sample-compose' in os.environ['ENVIRONMENT']: diff --git a/api/blueprints/__init__.py b/api/blueprints/__init__.py index a04613c105..b7f4565309 100644 --- a/api/blueprints/__init__.py +++ b/api/blueprints/__init__.py @@ -7,4 +7,3 @@ from .drbSearch import search from .drbUtils import utils from .drbWork import work -from .drbS3 import s3 diff --git a/api/blueprints/drbS3.py b/api/blueprints/drbS3.py deleted file mode 100644 index b81b99a29d..0000000000 --- a/api/blueprints/drbS3.py +++ /dev/null @@ -1,51 +0,0 @@ -from flask import Blueprint, request, current_app -from ..utils import APIUtils -from logger import createLog -from functools import wraps -from botocore.exceptions import ClientError -from urllib.parse import urlparse - -import requests -import argparse -import boto3 - -logger = createLog(__name__) - -s3 = Blueprint('s3', __name__, url_prefix='/s3') - -@s3.route('/', methods=['GET']) -def s3ObjectLinkFetch(bucket): - - logger.info(f'Fetching AWS S3 Object Link') - - searchParams = APIUtils.normalizeQueryParams(request.args) - - key = searchParams.get('key')[0] - - logger.info(f'Key: {key}') - logger.info(f'Bucket: {bucket}') - - parser = argparse.ArgumentParser() - - parser.add_argument(bucket, help=bucket) - parser.add_argument(key, help=key) - - logger.info(parser) - - s3_client = boto3.client("s3") - client_action = "get_object" - url = APIUtils.generate_presigned_url( - s3_client, client_action, {"Bucket": bucket, "Key": key}, 1000 - ) - logger.info(url) - - response = requests.get(url) - - if response.status_code == 200: - return APIUtils.formatResponseObject( - 200, 's3ObjectLinkFetch', {'message': f'{url}'} - ) - else: - return APIUtils.formatResponseObject( - response.status_code, 's3ObjectLinkFetch', {'Bucket/Key does not exist': f'{url}'} - ) diff --git a/swagger.v4.json b/swagger.v4.json index 9cc72d8688..c87184bc8d 100644 --- a/swagger.v4.json +++ b/swagger.v4.json @@ -245,45 +245,6 @@ } } } - }, - "/s3/{bucket}": { - "get": { - "tags": ["digital-research-books"], - "summary": "v4 Get AWS S3 Object Link", - "description": "Return a single s3 object link from DRB buckets", - "parameters": [ - { - "name": "key", - "in": "path", - "description": "AWS key for S3 object", - "type": "string", - "required": true - } - ], - "responses": { - "200": { - "description": "A single DRB S3 Object URL" - }, - "404": { - "description": "Resource was not found", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "500": { - "description": "Internal Server Error", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - }, - "default": { - "description": "Unexpected Error", - "schema": { - "$ref": "#/definitions/ErrorResponse" - } - } - } - } }, "/link/{id}": { "get": { diff --git a/tests/unit/test_api_s3_blueprint.py b/tests/unit/test_api_s3_blueprint.py deleted file mode 100644 index c076a20b83..0000000000 --- a/tests/unit/test_api_s3_blueprint.py +++ /dev/null @@ -1,56 +0,0 @@ -from flask import Flask -import pytest - -from api.blueprints.drbS3 import s3ObjectLinkFetch -from api.utils import APIUtils - -from werkzeug.datastructures import ImmutableMultiDict - -class TestS3Blueprint: - @pytest.fixture - def mockUtils(self, mocker): - return mocker.patch.multiple( - APIUtils, - normalizeQueryParams=mocker.DEFAULT, - formatResponseObject=mocker.DEFAULT, - generate_presigned_url=mocker.DEFAULT - ) - - @pytest.fixture - def testApp(self): - flaskApp = Flask('test') - - return flaskApp - - def test_s3ObjectLinkFetch_success(self, mockUtils, testApp, mocker): - - mockUtils['normalizeQueryParams'].return_value = {'key': ['manifests/met/10935.json']} - mockUtils['formatResponseObject'].return_value = 'testS3Response' - mockUtils['generate_presigned_url'].return_value = 'https://digital-research-books-beta.nypl.org/work/2263dbf4-51c2-4f6e-b9eb-e1def96b48d4?featured=9401263' - - - with testApp.test_request_context('/?key=manifests/met/10935.json'): - testAPIResponse = s3ObjectLinkFetch('drb-files-qa') - - assert testAPIResponse == 'testS3Response' - - mockUtils['normalizeQueryParams'].assert_called_once_with(ImmutableMultiDict([('key', 'manifests/met/10935.json')])) - - mockUtils['formatResponseObject'].assert_called_once_with( - 200, 's3ObjectLinkFetch', {'message': 'https://digital-research-books-beta.nypl.org/work/2263dbf4-51c2-4f6e-b9eb-e1def96b48d4?featured=9401263'} - ) - - def test_s3ObjectLinkFetch_fail(self, mockUtils, testApp, mocker): - - mockUtils['normalizeQueryParams'].return_value = {'key': ['manifests/met/10935.json']} - mockUtils['formatResponseObject'].return_value = 'testS3Response' - mockUtils['generate_presigned_url'].return_value = 'https://drb-files-qaFail.s3.amazonaws.com/manifests/met/10935.json?' - - with testApp.test_request_context('/?key=manifests/met/10935.json'): - testAPIResponse = s3ObjectLinkFetch('drb-files-qaFail') - - assert testAPIResponse == 'testS3Response' - - mockUtils['normalizeQueryParams'].assert_called_once_with(ImmutableMultiDict([('key', 'manifests/met/10935.json')])) - - mockUtils['formatResponseObject'].assert_called_once_with(404, 's3ObjectLinkFetch', {'Bucket/Key does not exist': 'https://drb-files-qaFail.s3.amazonaws.com/manifests/met/10935.json?'}) \ No newline at end of file From 47ee6234fa4df26a2bfe46b8ad0b2000cbd32145 Mon Sep 17 00:00:00 2001 From: Dmitri Slory Date: Thu, 21 Dec 2023 16:43:13 -0500 Subject: [PATCH 5/5] Update swagger.v4.json Co-authored-by: Lyndsey M. <2042238+Apophenia@users.noreply.github.com> --- swagger.v4.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swagger.v4.json b/swagger.v4.json index c87184bc8d..091cb89b5c 100644 --- a/swagger.v4.json +++ b/swagger.v4.json @@ -245,7 +245,7 @@ } } } - }, + }, "/link/{id}": { "get": { "tags": ["digital-research-books"],