Skip to content

Commit

Permalink
Also populate AWS credentials file in-process because SkyPilot doesn'…
Browse files Browse the repository at this point in the history
…t work with env vars.
  • Loading branch information
mjkanji committed Feb 22, 2024
1 parent 6c27433 commit d47fd4b
Showing 1 changed file with 12 additions and 3 deletions.
15 changes: 12 additions & 3 deletions dagster_skypilot/assets.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,16 +10,25 @@
@asset(group_name="ai")
def skypilot_model(context: AssetExecutionContext) -> None:
# Set up the API credentials by reading from the deployment env vars
key_file = Path.home() / ".lambda_cloud" / "lambda_keys"
lambda_key_file = Path.home() / ".lambda_cloud" / "lambda_keys"
aws_key_file = Path.home() / ".aws" / "credentials"

# Don't overwrite local keys, but always populate them dynamically in
# Dagster Cloud
if not DEPLOYMENT_TYPE == "local":
key_file.touch(exist_ok=True)
lambda_key_file.touch(exist_ok=True)
aws_key_file.touch(exist_ok=True)

with key_file.open("w") as f:
with lambda_key_file.open("w") as f:
f.write("api_key = {}".format(os.getenv("LAMBDA_LABS_API_KEY")))

with aws_key_file.open("w") as f:
f.write(
"[default]\n"
f"aws_access_key_id = {os.getenv('AWS_ACCESS_KEY_ID')}\n"
f"aws_secret_access_key = {os.getenv('AWS_SECRET_ACCESS_KEY')}\n"
)

# The setup command.
setup = r"""
set -e # Exit if any command failed.
Expand Down

0 comments on commit d47fd4b

Please sign in to comment.