From d47fd4bb068a5904a8b9ed4ed6dff7beddf497e2 Mon Sep 17 00:00:00 2001 From: Muhammad Jarir Kanji Date: Thu, 22 Feb 2024 11:35:25 +0500 Subject: [PATCH] Also populate AWS credentials file in-process because SkyPilot doesn't work with env vars. --- dagster_skypilot/assets.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/dagster_skypilot/assets.py b/dagster_skypilot/assets.py index 23b1cf0..9ed29fd 100644 --- a/dagster_skypilot/assets.py +++ b/dagster_skypilot/assets.py @@ -10,16 +10,25 @@ @asset(group_name="ai") def skypilot_model(context: AssetExecutionContext) -> None: # Set up the API credentials by reading from the deployment env vars - key_file = Path.home() / ".lambda_cloud" / "lambda_keys" + lambda_key_file = Path.home() / ".lambda_cloud" / "lambda_keys" + aws_key_file = Path.home() / ".aws" / "credentials" # Don't overwrite local keys, but always populate them dynamically in # Dagster Cloud if not DEPLOYMENT_TYPE == "local": - key_file.touch(exist_ok=True) + lambda_key_file.touch(exist_ok=True) + aws_key_file.touch(exist_ok=True) - with key_file.open("w") as f: + with lambda_key_file.open("w") as f: f.write("api_key = {}".format(os.getenv("LAMBDA_LABS_API_KEY"))) + with aws_key_file.open("w") as f: + f.write( + "[default]\n" + f"aws_access_key_id = {os.getenv('AWS_ACCESS_KEY_ID')}\n" + f"aws_secret_access_key = {os.getenv('AWS_SECRET_ACCESS_KEY')}\n" + ) + # The setup command. setup = r""" set -e # Exit if any command failed.