Skip to content

Commit

Permalink
PdfRasterizer returns image path, rather than PIL Image
Browse files Browse the repository at this point in the history
Avoids potential issues with consumer trying to use closed image
  • Loading branch information
donaldgray committed Jul 16, 2024
1 parent b76b0f5 commit 3b7ac0d
Show file tree
Hide file tree
Showing 3 changed files with 14 additions and 12 deletions.
2 changes: 1 addition & 1 deletion src/app/engine/rasterizers.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ def rasterize_pdf(self, subfolder_path):
pdf_source = os.path.join(subfolder_path, "source.pdf")
images = self.__rasterize(pdf_source, subfolder_path, dpi=self._dpi)
images = self.__validate_rasterized_images(images, pdf_source, subfolder_path)
return images
return [i.filename for i in images]

def __rasterize(
self, pdf_source, subfolder_path, start_page=None, last_page=None, dpi=None
Expand Down
14 changes: 8 additions & 6 deletions src/app/engine/s3.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,9 @@ def __build_bucket_base_url(self):
else:
return f"https://s3.amazonaws.com/{self._bucket_name}"

def put_images(self, images, submission_id, composite_id, customer_id, space_id):
def put_images(
self, image_paths, submission_id, composite_id, customer_id, space_id
):
s3_uris = []

key_prefix = self.__get_key_prefix(
Expand All @@ -36,14 +38,14 @@ def put_images(self, images, submission_id, composite_id, customer_id, space_id)
with tqdm.tqdm(
desc=f"[{submission_id}] Upload images to S3",
unit=" image",
total=len(images),
total=len(image_paths),
) as progress_bar:
with ThreadPoolExecutor(max_workers=self._upload_threads) as executor:
# It's critical that the list of S3 URI's returned by this method is in the
# same order as the list of images provided to it. '.map(...)' gives us that,
# whilst '.submit(...)' does not.
for s3_uri in executor.map(
self.__put_image, repeat(key_prefix), images
self.__put_image, repeat(key_prefix), image_paths
):
s3_uris.append(s3_uri)
progress_bar.update(1)
Expand All @@ -52,8 +54,8 @@ def put_images(self, images, submission_id, composite_id, customer_id, space_id)
def __get_key_prefix(self, submission_id, composite_id, customer, space):
return f"{self._object_key_prefix}/{customer}/{space}/{composite_id or submission_id}"

def __put_image(self, key_prefix, image):
object_key = f"{key_prefix}/{os.path.basename(image.filename)}"
with open(image.filename, "rb") as file:
def __put_image(self, key_prefix, image_path):
object_key = f"{key_prefix}/{os.path.basename(image_path)}"
with open(image_path, "rb") as file:
self._client.put_object(Bucket=self._bucket_name, Key=object_key, Body=file)
return f"{self._bucket_base_url}/{object_key}"
10 changes: 5 additions & 5 deletions src/app/engine/tasks.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,8 @@ def process_member(args):
folder_path = None
try:
folder_path = __fetch_origin(member, member.json_data["origin"])
images = __rasterize_composite(member, folder_path)
s3_urls = __push_images_to_dlcs(member, images)
image_paths = __rasterize_composite(member, folder_path)
s3_urls = __push_images_to_dlcs(member, image_paths)
dlcs_requests = __build_dlcs_requests(member, s3_urls)
dlcs_responses = __initiate_dlcs_ingest(member, dlcs_requests, args["auth"])
return __build_result(member, dlcs_responses)
Expand All @@ -49,12 +49,12 @@ def __rasterize_composite(member, pdf_path):
return pdf_rasterizer.rasterize_pdf(pdf_path)


def __push_images_to_dlcs(member, images):
__update_status(member, "PUSHING_TO_DLCS", image_count=len(images))
def __push_images_to_dlcs(member, image_paths):
__update_status(member, "PUSHING_TO_DLCS", image_count=len(image_paths))
composite_id = member.json_data.get("compositeId")
customer = member.collection.customer
space = member.json_data["space"]
return s3_client.put_images(images, member.id, composite_id, customer, space)
return s3_client.put_images(image_paths, member.id, composite_id, customer, space)


def __build_dlcs_requests(member, dlcs_uris):
Expand Down

0 comments on commit 3b7ac0d

Please sign in to comment.