From 09ea7240690f42480c4527ed6db5401ad94942a0 Mon Sep 17 00:00:00 2001 From: dvince Date: Thu, 11 Jul 2024 16:12:42 -0400 Subject: [PATCH 1/3] Using correct endpoint to upload a csv file to hmi-server --- src/askem_beaker/contexts/dataset/context.py | 9 ++--- .../python3/hmi_create_csv_dataset.py | 39 +++++++++++++++++++ src/askem_beaker/contexts/pyciemss/context.py | 6 ++- 3 files changed, 47 insertions(+), 7 deletions(-) create mode 100644 src/askem_beaker/contexts/dataset/procedures/python3/hmi_create_csv_dataset.py diff --git a/src/askem_beaker/contexts/dataset/context.py b/src/askem_beaker/contexts/dataset/context.py index b71333e..401636c 100644 --- a/src/askem_beaker/contexts/dataset/context.py +++ b/src/askem_beaker/contexts/dataset/context.py @@ -246,15 +246,14 @@ async def save_dataset_request(self, message): logger.error(f"new dataset: {pprint.pformat(create_req.json())}") new_dataset["id"] = new_dataset_id - new_dataset_url = f"{dataservice_url}/datasets/{new_dataset_id}" - data_url_req = requests.get(f"{new_dataset_url}/upload-url?filename={filename}", auth=self.auth.requests_auth()) - data_url = data_url_req.json().get('url', None) code = self.get_code( - "df_save_as", + "df_create_csv_dataset", { "var_name": var_name, - "data_url": data_url, + "id": new_dataset_id, + "auth": self.get_auth(), + "filename": filename } ) df_response = await self.execute(code) diff --git a/src/askem_beaker/contexts/dataset/procedures/python3/hmi_create_csv_dataset.py b/src/askem_beaker/contexts/dataset/procedures/python3/hmi_create_csv_dataset.py new file mode 100644 index 0000000..5014000 --- /dev/null +++ b/src/askem_beaker/contexts/dataset/procedures/python3/hmi_create_csv_dataset.py @@ -0,0 +1,39 @@ +import os +import requests +import tempfile + +from json import JSONDecodeError + +with tempfile.TemporaryFile() as temp_csv_file: + {{ var_name|default("df") }}.to_csv(temp_csv_file, index=False, header=True) + temp_csv_file.seek(0) + # Get the HMI_SERVER endpoint and auth token from the environment variable + hmi_server = os.getenv('HMI_SERVER_URL') + + # Define the id and filename dynamically + id = "{{id}}" + filename = "{{filename}}" + + # Prepare the request payload + payload = {'id': id, 'filename': filename} + files = {'file': temp_csv_file} + + # Set the headers with the content type + # headers = {} + + # Make the HTTP PUT request to upload the file bytes + url = f'{hmi_server}/datasets/{id}/upload-csv' + response = requests.put(url, data=payload, files=files, auth={{auth}}) + + # Check the response status code + if response.status_code < 300: + try: + message = response.json() + except JSONDecodeError: + message = f'File uploaded successfully with status code {response.status_code}.' + else: + message = f'File upload failed with status code {response.status_code}.' + if response.text: + message += f' Response message: {response.text}' + + message diff --git a/src/askem_beaker/contexts/pyciemss/context.py b/src/askem_beaker/contexts/pyciemss/context.py index c850f3f..8bed857 100644 --- a/src/askem_beaker/contexts/pyciemss/context.py +++ b/src/askem_beaker/contexts/pyciemss/context.py @@ -138,9 +138,11 @@ async def save_results_to_hmi(self, message): data_url_req = requests.get(f"{dataset_url}/upload-url?filename=result.csv", auth=auth) data_url = data_url_req.json().get('url', None) code = self.get_code( - "df_save_as", + "df_create_csv_dataset", { - "data_url": data_url, + "id": dataset_id, + "auth": self.get_auth(), + "filename": "result.csv" } ) kernel_response = await self.execute(code) # TODO: Check error From 2bd1465104d7e458ea0f24980f424873810b2060 Mon Sep 17 00:00:00 2001 From: Brandon Rose Date: Mon, 15 Jul 2024 12:27:37 -0500 Subject: [PATCH 2/3] patches for dataset saving --- src/askem_beaker/contexts/dataset/context.py | 5 +++-- .../dataset/procedures/python3/hmi_create_csv_dataset.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/askem_beaker/contexts/dataset/context.py b/src/askem_beaker/contexts/dataset/context.py index 401636c..fe34a62 100644 --- a/src/askem_beaker/contexts/dataset/context.py +++ b/src/askem_beaker/contexts/dataset/context.py @@ -248,11 +248,12 @@ async def save_dataset_request(self, message): new_dataset["id"] = new_dataset_id code = self.get_code( - "df_create_csv_dataset", + "hmi_create_csv_dataset", { "var_name": var_name, "id": new_dataset_id, - "auth": self.get_auth(), + "username": self.auth.username, + "password": self.auth.password, "filename": filename } ) diff --git a/src/askem_beaker/contexts/dataset/procedures/python3/hmi_create_csv_dataset.py b/src/askem_beaker/contexts/dataset/procedures/python3/hmi_create_csv_dataset.py index 5014000..fb6b66e 100644 --- a/src/askem_beaker/contexts/dataset/procedures/python3/hmi_create_csv_dataset.py +++ b/src/askem_beaker/contexts/dataset/procedures/python3/hmi_create_csv_dataset.py @@ -23,7 +23,7 @@ # Make the HTTP PUT request to upload the file bytes url = f'{hmi_server}/datasets/{id}/upload-csv' - response = requests.put(url, data=payload, files=files, auth={{auth}}) + response = requests.put(url, data=payload, files=files, auth=("{{username}}", "{{password}}")) # Check the response status code if response.status_code < 300: From 3973d37f8ac28c4a37ec8d7f2ff2ecfb2c274ffc Mon Sep 17 00:00:00 2001 From: dvince Date: Mon, 15 Jul 2024 13:59:15 -0400 Subject: [PATCH 3/3] Get the URL safely --- src/askem_beaker/contexts/dataset/context.py | 3 ++- .../dataset/procedures/python3/hmi_create_csv_dataset.py | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/askem_beaker/contexts/dataset/context.py b/src/askem_beaker/contexts/dataset/context.py index fe34a62..41a59b2 100644 --- a/src/askem_beaker/contexts/dataset/context.py +++ b/src/askem_beaker/contexts/dataset/context.py @@ -254,7 +254,8 @@ async def save_dataset_request(self, message): "id": new_dataset_id, "username": self.auth.username, "password": self.auth.password, - "filename": filename + "filename": filename, + "dataservice_url": dataservice_url } ) df_response = await self.execute(code) diff --git a/src/askem_beaker/contexts/dataset/procedures/python3/hmi_create_csv_dataset.py b/src/askem_beaker/contexts/dataset/procedures/python3/hmi_create_csv_dataset.py index fb6b66e..2b49406 100644 --- a/src/askem_beaker/contexts/dataset/procedures/python3/hmi_create_csv_dataset.py +++ b/src/askem_beaker/contexts/dataset/procedures/python3/hmi_create_csv_dataset.py @@ -7,8 +7,8 @@ with tempfile.TemporaryFile() as temp_csv_file: {{ var_name|default("df") }}.to_csv(temp_csv_file, index=False, header=True) temp_csv_file.seek(0) - # Get the HMI_SERVER endpoint and auth token from the environment variable - hmi_server = os.getenv('HMI_SERVER_URL') + # Set the HMI_SERVER endpoint + hmi_server = "{{dataservice_url}}" # Define the id and filename dynamically id = "{{id}}"