diff --git a/.github/workflows/cleanup.yaml b/.github/workflows/cleanup.yaml index c39310d367..37b7580d58 100644 --- a/.github/workflows/cleanup.yaml +++ b/.github/workflows/cleanup.yaml @@ -1,30 +1,31 @@ -name: Environment cleanup - -on: - workflow_dispatch: - schedule: - - cron: "0 10 * * *" - -jobs: - tests: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - persist-credentials: false - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3.11" - - name: Install dependencies - run: | - python -m pip install --upgrade pip hatch - python -m hatch env create e2e - - name: Run cleaunp - env: - TERM: unknown - SNOWFLAKE_CONNECTIONS_INTEGRATION_HOST: ${{ secrets.SNOWFLAKE_HOST }} - SNOWFLAKE_CONNECTIONS_INTEGRATION_USER: ${{ secrets.SNOWFLAKE_USER }} - SNOWFLAKE_CONNECTIONS_INTEGRATION_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }} - SNOWFLAKE_CONNECTIONS_INTEGRATION_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }} - run: python -m hatch run e2e:cleanup +#name: Environment cleanup +# +#on: +# workflow_dispatch: +# schedule: +# - cron: "0 10 * * *" +# +#jobs: +# tests: +# runs-on: ubuntu-latest +# steps: +# - uses: actions/checkout@v4 +# with: +# persist-credentials: false +# - name: Set up Python +# uses: actions/setup-python@v5 +# with: +# python-version: "3.11" +# - name: Install dependencies +# run: | +# python -m pip install --upgrade pip hatch +# python -m hatch env create e2e +# - name: Run cleaunp +# env: +# TERM: unknown +# SNOWFLAKE_CONNECTIONS_INTEGRATION_AUTHENTICATOR: SNOWFLAKE_JWT +# SNOWFLAKE_CONNECTIONS_INTEGRATION_HOST: ${{ secrets.SNOWFLAKE_HOST }} +# SNOWFLAKE_CONNECTIONS_INTEGRATION_USER: ${{ secrets.SNOWFLAKE_USER }} +# SNOWFLAKE_CONNECTIONS_INTEGRATION_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }} +# SNOWFLAKE_CONNECTIONS_INTEGRATION_PRIVATE_KEY_PATH: ${{ secrets.SNOWFLAKE_PRIVATE_KEY_PATH }} +# run: python -m hatch run e2e:cleanup diff --git a/.github/workflows/test_cli_action.yaml b/.github/workflows/test_cli_action.yaml index b40166b4cf..9f8df3337e 100644 --- a/.github/workflows/test_cli_action.yaml +++ b/.github/workflows/test_cli_action.yaml @@ -1,27 +1,28 @@ -name: "CLI Action testing" - -on: - schedule: - - cron: "0 0 * * *" - workflow_dispatch: - -jobs: - version: - name: "Check Snowflake CLI version" - runs-on: ubuntu-latest - steps: - - name: Checkout repo - uses: actions/checkout@v4 - with: - persist-credentials: false - - uses: Snowflake-Labs/snowflake-cli-action@v1 - with: - cli-version: "latest" - default-config-file-path: "tests_integration/config/connection_configs.toml" - - name: Test connection - env: - TERM: unknown - SNOWFLAKE_CONNECTIONS_INTEGRATION_USER: ${{ secrets.SNOWFLAKE_USER }} - SNOWFLAKE_CONNECTIONS_INTEGRATION_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }} - SNOWFLAKE_CONNECTIONS_INTEGRATION_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }} - run: snow connection test -c integration | grep Status +#name: "CLI Action testing" +# +#on: +# schedule: +# - cron: "0 0 * * *" +# workflow_dispatch: +# +#jobs: +# version: +# name: "Check Snowflake CLI version" +# runs-on: ubuntu-latest +# steps: +# - name: Checkout repo +# uses: actions/checkout@v4 +# with: +# persist-credentials: false +# - uses: Snowflake-Labs/snowflake-cli-action@v1 +# with: +# cli-version: "latest" +# default-config-file-path: "tests_integration/config/connection_configs.toml" +# - name: Test connection +# env: +# TERM: unknown +# SNOWFLAKE_CONNECTIONS_INTEGRATION_AUTHENTICATOR: SNOWFLAKE_JWT +# SNOWFLAKE_CONNECTIONS_INTEGRATION_USER: ${{ secrets.SNOWFLAKE_USER }} +# SNOWFLAKE_CONNECTIONS_INTEGRATION_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }} +# SNOWFLAKE_CONNECTIONS_INTEGRATION_PRIVATE_KEY_PATH: ${{ secrets.SNOWFLAKE_PRIVATE_KEY_PATH }} +# run: snow connection test -c integration | grep Status diff --git a/.github/workflows/test_e2e.yaml b/.github/workflows/test_e2e.yaml index 7d881cb323..b9bad8231e 100644 --- a/.github/workflows/test_e2e.yaml +++ b/.github/workflows/test_e2e.yaml @@ -1,127 +1,129 @@ -name: E2E testing - -on: - pull_request: - branches: - - "*" - push: - tags: - - "v*" # Push events to matching v*, i.e. v1.0, v20.15.10 - branches: - - main - repository_dispatch: - types: [ok-to-test-command] - schedule: - - cron: "0 8 * * *" - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - define-matrix: - uses: ./.github/workflows/matrix.yaml - - e2e-trusted: - needs: define-matrix - strategy: - fail-fast: true - matrix: - os: ${{ fromJSON(needs.define-matrix.outputs.os) }} - python-version: ${{ fromJSON(needs.define-matrix.outputs.python) }} - runs-on: ${{ matrix.os }} - if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository - steps: - - uses: actions/checkout@v4 - with: - persist-credentials: false - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - python -m pip install --upgrade pip hatch - python -m hatch env create e2e - - name: Run end to end tests - env: - TERM: unknown - SNOWFLAKE_CONNECTIONS_INTEGRATION_HOST: ${{ secrets.SNOWFLAKE_HOST }} - SNOWFLAKE_CONNECTIONS_INTEGRATION_USER: ${{ secrets.SNOWFLAKE_USER }} - SNOWFLAKE_CONNECTIONS_INTEGRATION_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }} - SNOWFLAKE_CONNECTIONS_INTEGRATION_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }} - run: python -m hatch run e2e:test - - # Repo owner has commented /ok-to-test on a (fork-based) pull request - e2e-fork: - needs: define-matrix - strategy: - fail-fast: true - matrix: - os: ${{ fromJSON(needs.define-matrix.outputs.os) }} - python-version: ${{ fromJSON(needs.define-matrix.outputs.python) }} - runs-on: ${{ matrix.os }} - permissions: - pull-requests: write - checks: write - if: | - github.event_name == 'repository_dispatch' && - github.event.client_payload.slash_command.args.named.sha != '' && - contains( - github.event.client_payload.pull_request.head.sha, - github.event.client_payload.slash_command.args.named.sha - ) - steps: - - uses: actions/checkout@v4 - with: - persist-credentials: false - ref: 'refs/pull/${{ github.event.client_payload.pull_request.number }}/merge' - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - python -m pip install --upgrade pip hatch - python -m hatch env create e2e - - name: Run end to end tests - env: - TERM: unknown - SNOWFLAKE_CONNECTIONS_INTEGRATION_HOST: ${{ secrets.SNOWFLAKE_HOST }} - SNOWFLAKE_CONNECTIONS_INTEGRATION_USER: ${{ secrets.SNOWFLAKE_USER }} - SNOWFLAKE_CONNECTIONS_INTEGRATION_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }} - SNOWFLAKE_CONNECTIONS_INTEGRATION_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }} - run: python -m hatch run e2e:test - # Update check run called "e2e-fork" - - uses: actions/github-script@v7 - id: update-check-run - if: ${{ always() }} - env: - number: ${{ github.event.client_payload.pull_request.number }} - job: ${{ github.job }} - # Conveniently, job.status maps to https://developer.github.com/v3/checks/runs/#update-a-check-run - conclusion: ${{ job.status }} - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - script: | - const { data: pull } = await github.rest.pulls.get({ - ...context.repo, - pull_number: process.env.number - }); - const ref = pull.head.sha; - - const { data: checks } = await github.rest.checks.listForRef({ - ...context.repo, - ref - }); - - const check = checks.check_runs.filter(c => c.name === process.env.job); - - const { data: result } = await github.rest.checks.update({ - ...context.repo, - check_run_id: check[0].id, - status: 'completed', - conclusion: process.env.conclusion - }); - - return result; +#name: E2E testing +# +#on: +# pull_request: +# branches: +# - "*" +# push: +# tags: +# - "v*" # Push events to matching v*, i.e. v1.0, v20.15.10 +# branches: +# - main +# repository_dispatch: +# types: [ok-to-test-command] +# schedule: +# - cron: "0 8 * * *" +# +#concurrency: +# group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} +# cancel-in-progress: true +# +#jobs: +# define-matrix: +# uses: ./.github/workflows/matrix.yaml +# +# e2e-trusted: +# needs: define-matrix +# strategy: +# fail-fast: true +# matrix: +# os: ${{ fromJSON(needs.define-matrix.outputs.os) }} +# python-version: ${{ fromJSON(needs.define-matrix.outputs.python) }} +# runs-on: ${{ matrix.os }} +# if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository +# steps: +# - uses: actions/checkout@v4 +# with: +# persist-credentials: false +# - name: Set up Python +# uses: actions/setup-python@v5 +# with: +# python-version: ${{ matrix.python-version }} +# - name: Install dependencies +# run: | +# python -m pip install --upgrade pip hatch +# python -m hatch env create e2e +# - name: Run end to end tests +# env: +# TERM: unknown +# SNOWFLAKE_CONNECTIONS_INTEGRATION_AUTHENTICATOR: SNOWFLAKE_JWT +# SNOWFLAKE_CONNECTIONS_INTEGRATION_HOST: ${{ secrets.SNOWFLAKE_HOST }} +# SNOWFLAKE_CONNECTIONS_INTEGRATION_USER: ${{ secrets.SNOWFLAKE_USER }} +# SNOWFLAKE_CONNECTIONS_INTEGRATION_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }} +# SNOWFLAKE_CONNECTIONS_INTEGRATION_PRIVATE_KEY_PATH: ${{ secrets.SNOWFLAKE_PRIVATE_KEY_PATH }} +# run: python -m hatch run e2e:test +# +# # Repo owner has commented /ok-to-test on a (fork-based) pull request +# e2e-fork: +# needs: define-matrix +# strategy: +# fail-fast: true +# matrix: +# os: ${{ fromJSON(needs.define-matrix.outputs.os) }} +# python-version: ${{ fromJSON(needs.define-matrix.outputs.python) }} +# runs-on: ${{ matrix.os }} +# permissions: +# pull-requests: write +# checks: write +# if: | +# github.event_name == 'repository_dispatch' && +# github.event.client_payload.slash_command.args.named.sha != '' && +# contains( +# github.event.client_payload.pull_request.head.sha, +# github.event.client_payload.slash_command.args.named.sha +# ) +# steps: +# - uses: actions/checkout@v4 +# with: +# persist-credentials: false +# ref: 'refs/pull/${{ github.event.client_payload.pull_request.number }}/merge' +# - name: Set up Python +# uses: actions/setup-python@v5 +# with: +# python-version: ${{ matrix.python-version }} +# - name: Install dependencies +# run: | +# python -m pip install --upgrade pip hatch +# python -m hatch env create e2e +# - name: Run end to end tests +# env: +# TERM: unknown +# SNOWFLAKE_CONNECTIONS_INTEGRATION_AUTHENTICATOR: SNOWFLAKE_JWT +# SNOWFLAKE_CONNECTIONS_INTEGRATION_HOST: ${{ secrets.SNOWFLAKE_HOST }} +# SNOWFLAKE_CONNECTIONS_INTEGRATION_USER: ${{ secrets.SNOWFLAKE_USER }} +# SNOWFLAKE_CONNECTIONS_INTEGRATION_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }} +# SNOWFLAKE_CONNECTIONS_INTEGRATION_PRIVATE_KEY_PATH: ${{ secrets.SNOWFLAKE_PRIVATE_KEY_PATH }} +# run: python -m hatch run e2e:test +# # Update check run called "e2e-fork" +# - uses: actions/github-script@v7 +# id: update-check-run +# if: ${{ always() }} +# env: +# number: ${{ github.event.client_payload.pull_request.number }} +# job: ${{ github.job }} +# # Conveniently, job.status maps to https://developer.github.com/v3/checks/runs/#update-a-check-run +# conclusion: ${{ job.status }} +# with: +# github-token: ${{ secrets.GITHUB_TOKEN }} +# script: | +# const { data: pull } = await github.rest.pulls.get({ +# ...context.repo, +# pull_number: process.env.number +# }); +# const ref = pull.head.sha; +# +# const { data: checks } = await github.rest.checks.listForRef({ +# ...context.repo, +# ref +# }); +# +# const check = checks.check_runs.filter(c => c.name === process.env.job); +# +# const { data: result } = await github.rest.checks.update({ +# ...context.repo, +# check_run_id: check[0].id, +# status: 'completed', +# conclusion: process.env.conclusion +# }); +# +# return result; diff --git a/.github/workflows/test_integration.yaml b/.github/workflows/test_integration.yaml index 8bfa7b6229..865cf038f8 100644 --- a/.github/workflows/test_integration.yaml +++ b/.github/workflows/test_integration.yaml @@ -29,8 +29,8 @@ jobs: strategy: fail-fast: true matrix: - os: ${{ fromJSON(needs.define-matrix.outputs.os) }} - python-version: ${{ fromJSON(needs.define-matrix.outputs.python) }} + os: [ubuntu-latest] + python-version: [3.11] runs-on: ${{ matrix.os }} if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository steps: @@ -45,13 +45,22 @@ jobs: run: | python -m pip install --upgrade pip hatch python -m hatch env create integration + - name: Set workspace parent directory + run: echo "PARENT_DIR=$(dirname "${{ github.workspace }}")" >> $GITHUB_ENV - name: Run integration tests env: TERM: unknown + SNOWFLAKE_CONNECTIONS_INTEGRATION_AUTHENTICATOR: SNOWFLAKE_JWT SNOWFLAKE_CONNECTIONS_INTEGRATION_USER: ${{ secrets.SNOWFLAKE_USER }} - SNOWFLAKE_CONNECTIONS_INTEGRATION_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }} SNOWFLAKE_CONNECTIONS_INTEGRATION_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }} - run: python -m hatch run integration:test + SNOWFLAKE_CONNECTIONS_INTEGRATION_PRIVATE_KEY_PATH: ${{ env.PARENT_DIR }}/.ssh/key.p8 + SNOWFLAKE_PRIVATE_KEY: ${{ secrets.SNOWFLAKE_PRIVATE_KEY }} + + run: | + mkdir ${{ env.PARENT_DIR }}/.ssh + echo "${SNOWFLAKE_PRIVATE_KEY}" > ${{ env.PARENT_DIR }}/.ssh/key.p8 + sudo chmod 600 ${{ env.PARENT_DIR }}/.ssh/key.p8 + python -m hatch run integration:test # Repo owner has commented /ok-to-test on a (fork-based) pull request @@ -89,9 +98,10 @@ jobs: - name: Run integration tests env: TERM: unknown + SNOWFLAKE_CONNECTIONS_INTEGRATION_AUTHENTICATOR: SNOWFLAKE_JWT SNOWFLAKE_CONNECTIONS_INTEGRATION_USER: ${{ secrets.SNOWFLAKE_USER }} - SNOWFLAKE_CONNECTIONS_INTEGRATION_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }} SNOWFLAKE_CONNECTIONS_INTEGRATION_ACCOUNT: ${{ secrets.SNOWFLAKE_ACCOUNT }} + SNOWFLAKE_CONNECTIONS_INTEGRATION_PRIVATE_KEY_PATH: ${{ secrets.SNOWFLAKE_PRIVATE_KEY_PATH }} run: python -m hatch run integration:test # Update check run called "integration-fork" - uses: actions/github-script@v7 diff --git a/scripts/cleanup.py b/scripts/cleanup.py index 72faceeb1a..d571e08be6 100644 --- a/scripts/cleanup.py +++ b/scripts/cleanup.py @@ -52,9 +52,12 @@ def remove_resources(single: str, plural: str, known_instances: t.List[str], rol role = "INTEGRATION_TESTS" session = Session.builder.configs( { + "authenticator": "SNOWFLAKE_JWT", "account": os.getenv("SNOWFLAKE_CONNECTIONS_INTEGRATION_ACCOUNT"), "user": os.getenv("SNOWFLAKE_CONNECTIONS_INTEGRATION_USER"), - "password": os.getenv("SNOWFLAKE_CONNECTIONS_INTEGRATION_PASSWORD"), + "private_key_path": os.getenv( + "SNOWFLAKE_CONNECTIONS_INTEGRATION_PRIVATE_KEY_PATH" + ), "database": "SNOWCLI_DB", "role": role, } diff --git a/src/snowflake/cli/app/snow_connector.py b/src/snowflake/cli/app/snow_connector.py index 3fe072baa7..ddda5cbb8c 100644 --- a/src/snowflake/cli/app/snow_connector.py +++ b/src/snowflake/cli/app/snow_connector.py @@ -97,7 +97,7 @@ def connect_to_snowflake( k: v for k, v in connection_parameters.items() if v is not None } - connection_parameters = _update_connection_details_with_private_key( + connection_parameters = update_connection_details_with_private_key( connection_parameters ) @@ -163,7 +163,7 @@ def _raise_errors_related_to_session_token( ) -def _update_connection_details_with_private_key(connection_parameters: Dict): +def update_connection_details_with_private_key(connection_parameters: Dict): if "private_key_path" in connection_parameters: if connection_parameters.get("authenticator") == "SNOWFLAKE_JWT": private_key = _load_pem_to_der(connection_parameters["private_key_path"]) @@ -189,13 +189,6 @@ def _load_pem_to_der(private_key_path: str) -> bytes: Given a private key file path (in PEM format), decode key data into DER format """ - from cryptography.hazmat.backends import default_backend - from cryptography.hazmat.primitives.serialization import ( - Encoding, - NoEncryption, - PrivateFormat, - load_pem_private_key, - ) with SecurePath(private_key_path).open( "rb", read_file_limit_mb=DEFAULT_SIZE_LIMIT_MB @@ -222,6 +215,18 @@ def _load_pem_to_der(private_key_path: str) -> bytes: if private_key_pem.startswith(UNENCRYPTED_PKCS8_PK_HEADER): private_key_passphrase = None + return prepare_private_key(private_key_pem, private_key_passphrase) + + +def prepare_private_key(private_key_pem, private_key_passphrase=None): + from cryptography.hazmat.backends import default_backend + from cryptography.hazmat.primitives.serialization import ( + Encoding, + NoEncryption, + PrivateFormat, + load_pem_private_key, + ) + private_key = load_pem_private_key( private_key_pem, ( @@ -231,7 +236,6 @@ def _load_pem_to_der(private_key_path: str) -> bytes: ), default_backend(), ) - return private_key.private_bytes( encoding=Encoding.DER, format=PrivateFormat.PKCS8, diff --git a/tests_integration/snowflake_connector.py b/tests_integration/snowflake_connector.py index feafdb119b..0020083721 100644 --- a/tests_integration/snowflake_connector.py +++ b/tests_integration/snowflake_connector.py @@ -22,6 +22,7 @@ import pytest from snowflake import connector from snowflake.cli.api.exceptions import EnvironmentVariableNotFoundError +from snowflake.cli.app.snow_connector import update_connection_details_with_private_key _ENV_PARAMETER_PREFIX = "SNOWFLAKE_CONNECTIONS_INTEGRATION" SCHEMA_ENV_PARAMETER = f"{_ENV_PARAMETER_PREFIX}_SCHEMA" @@ -89,14 +90,21 @@ def test_role(snowflake_session): def snowflake_session(): config = { "application": "INTEGRATION_TEST", + "authenticator": "SNOWFLAKE_JWT", "account": _get_from_env("ACCOUNT"), "user": _get_from_env("USER"), - "password": _get_from_env("PASSWORD"), + "password": _get_from_env("PASSWORD", allow_none=True), + "private_key_path": _get_from_env("PRIVATE_KEY_PATH", allow_none=True), "host": _get_from_env("HOST", allow_none=True), "warehouse": _get_from_env("WAREHOUSE", allow_none=True), "role": _get_from_env("ROLE", allow_none=True), } + if config["password"] is None and config["private_key_path"] is None: + raise RuntimeError( + "Environment variable SNOWFLAKE_CONNECTIONS_INTEGRATION_PASSWORD or SNOWFLAKE_CONNECTIONS_INTEGRATION_PRIVATE_KEY_PATH is needed." + ) config = {k: v for k, v in config.items() if v is not None} + update_connection_details_with_private_key(config) connection = connector.connect(**config) yield connection connection.close() diff --git a/tests_integration/test_temporary_connection.py b/tests_integration/test_temporary_connection.py index 0e6ac31581..a928b0cd51 100644 --- a/tests_integration/test_temporary_connection.py +++ b/tests_integration/test_temporary_connection.py @@ -27,8 +27,8 @@ "SNOWFLAKE_CONNECTIONS_INTEGRATION_USER": os.environ.get( "SNOWFLAKE_CONNECTIONS_INTEGRATION_USER", None ), - "SNOWFLAKE_PASSWORD": os.environ.get( - "SNOWFLAKE_CONNECTIONS_INTEGRATION_PASSWORD" + "SNOWFLAKE_CONNECTIONS_INTEGRATION_PRIVATE_KEY_PATH": os.environ.get( + "SNOWFLAKE_CONNECTIONS_INTEGRATION_PRIVATE_KEY_PATH", None ), }, clear=True, @@ -41,10 +41,14 @@ def test_temporary_connection(runner, snapshot): "-q", "select 1", "--temporary-connection", + "--authenticator", + "SNOWFLAKE_JWT", "--account", os.environ["SNOWFLAKE_CONNECTIONS_INTEGRATION_ACCOUNT"], "--user", os.environ["SNOWFLAKE_CONNECTIONS_INTEGRATION_USER"], + "--private-key-path", + os.environ["SNOWFLAKE_CONNECTIONS_INTEGRATION_PRIVATE_KEY_PATH"], ] ) assert result.exit_code == 0