From 99adc821c34785eccfec3c5b3a6af9420bdd12b8 Mon Sep 17 00:00:00 2001 From: Oscar Salazar Date: Tue, 11 Jun 2024 13:24:19 -0600 Subject: [PATCH 1/4] SNOW-1465711: Add version_initializer callback (#1) * SNOW-1465711: Add version_initializer callback * Update setup.sql * Update setup.sql --- spcs-three-tier/app/manifest.yml | 3 +++ spcs-three-tier/app/setup.sql | 17 +++++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/spcs-three-tier/app/manifest.yml b/spcs-three-tier/app/manifest.yml index 6eb610d..4f0f146 100644 --- a/spcs-three-tier/app/manifest.yml +++ b/spcs-three-tier/app/manifest.yml @@ -19,6 +19,9 @@ artifacts: - /spcs_app/napp/img_repo/eap_backend - /spcs_app/napp/img_repo/eap_router +lifecycle_callbacks: + version_initializer: v1.init + privileges: - BIND SERVICE ENDPOINT: description: "Ability to create ingress URLs." diff --git a/spcs-three-tier/app/setup.sql b/spcs-three-tier/app/setup.sql index dda3f74..433b80e 100644 --- a/spcs-three-tier/app/setup.sql +++ b/spcs-three-tier/app/setup.sql @@ -49,6 +49,23 @@ $$; GRANT USAGE ON PROCEDURE v1.get_configuration(STRING) TO APPLICATION ROLE app_admin; +-- The version initializer callback is executed after a successful installation, upgrade, or downgrade of an application object. +-- In case the application fails to upgrade, the version initializer of the previous (successful) version will be executed so you +-- can clean up application state that may have been modified during the failed upgrade. +CREATE OR REPLACE PROCEDURE v1.init() +RETURNS STRING +LANGUAGE SQL +EXECUTE AS OWNER +AS +$$ +BEGIN + ALTER SERVICE IF EXISTS app_public.frontend FROM SPECIFICATION_FILE='frontend.yaml'; + ALTER SERVICE IF EXISTS app_public.backend FROM SPECIFICATION_FILE='backend.yaml'; + RETURN 'init complete'; +END $$; + +GRANT USAGE ON PROCEDURE v1.init() TO APPLICATION ROLE app_admin; + CREATE OR REPLACE PROCEDURE v1.start_backend(pool_name VARCHAR) RETURNS string LANGUAGE sql From bf088ca538e7acf3efd90a2985b5618451462060 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Alfaro=20V=C3=ADquez?= <127549539+sfc-gh-ralfaroviquez@users.noreply.github.com> Date: Mon, 17 Jun 2024 10:05:18 -0600 Subject: [PATCH 2/4] SNOW-1344601-datamapping-repo-update (#2) minor changes to keep up with quickstart --- data-mapping/app/manifest.yml | 1 + data-mapping/prepare_data.sh | 18 +++++++++++++++--- ...age-script.sql => setup_package_script.sql} | 0 data-mapping/snowflake.yml | 2 +- 4 files changed, 17 insertions(+), 4 deletions(-) rename data-mapping/scripts/{setup-package-script.sql => setup_package_script.sql} (100%) diff --git a/data-mapping/app/manifest.yml b/data-mapping/app/manifest.yml index afa02f5..e34e45e 100644 --- a/data-mapping/app/manifest.yml +++ b/data-mapping/app/manifest.yml @@ -1,5 +1,6 @@ manifest_version: 1 artifacts: + readme: README.md setup_script: setup_script.sql default_streamlit: ui."Dashboard" diff --git a/data-mapping/prepare_data.sh b/data-mapping/prepare_data.sh index a85d951..5dd9af2 100755 --- a/data-mapping/prepare_data.sh +++ b/data-mapping/prepare_data.sh @@ -1,7 +1,11 @@ +# Create a database and a schema to hold the data to lookup. snow sql -q " CREATE DATABASE IF NOT EXISTS IP2LOCATION; CREATE SCHEMA IF NOT EXISTS IP2LOCATION; " +# Create the table to host the data. +# Create a file format for the file +# Create a stage so we can upload the file snow sql -q " CREATE TABLE IF NOT EXISTS LITEDB11 ( @@ -27,22 +31,30 @@ COMPRESSION = AUTO; CREATE STAGE IF NOT EXISTS IP2LOCATION.IP2LOCATION.LOCATION_DATA_STAGE file_format = LOCATION_CSV;" --database ip2location --schema ip2location +# Copy the csv files from your local machine to the stage we created previously snow stage copy /USER_PATH_HERE/IP2LOCATION-LITE-DB11.CSV @location_data_stage --database ip2location --schema ip2location +# Copy the csv file from the stage to load the table snow sql -q " copy into litedb11 from @location_data_stage files = ('IP2LOCATION-LITE-DB11.CSV') ;" --database ip2location --schema ip2location +# Simple query test to ensure the table is correctly filled. snow sql -q "SELECT COUNT(*) FROM LITEDB11; SELECT * FROM LITEDB11 LIMIT 10;" --database ip2location --schema ip2location +# Create test database and schema. snow sql -q "CREATE DATABASE IF NOT EXISTS TEST_IPLOCATION; -CREATE SCHEMA IF NOT EXISTS TEST_IPLOCATION; +CREATE SCHEMA IF NOT EXISTS TEST_IPLOCATION;" +# Create test table to insert some values +snow sql -q " CREATE OR REPLACE TABLE TEST_IPLOCATION.TEST_IPLOCATION.TEST_DATA ( IP VARCHAR(16), IP_DATA VARIANT -); +);" -INSERT INTO TEST_IPLOCATION.TEST_IPLOCATION.TEST_DATA(IP) VALUES('73.153.199.206'),('8.8.8.8');" +# Insert testing values to use later on +snow sql -q " +INSERT INTO TEST_IPLOCATION.TEST_IPLOCATION.TEST_DATA(IP) VALUES('73.153.199.206'),('8.8.8.8');" \ No newline at end of file diff --git a/data-mapping/scripts/setup-package-script.sql b/data-mapping/scripts/setup_package_script.sql similarity index 100% rename from data-mapping/scripts/setup-package-script.sql rename to data-mapping/scripts/setup_package_script.sql diff --git a/data-mapping/snowflake.yml b/data-mapping/snowflake.yml index 8ddd289..057502b 100644 --- a/data-mapping/snowflake.yml +++ b/data-mapping/snowflake.yml @@ -7,4 +7,4 @@ native_app: dest: ./ package: scripts: - - scripts/setup-package-script.sql + - scripts/setup_package_script.sql From eaf35b387d2376dd12e2dea060500434d15c8515 Mon Sep 17 00:00:00 2001 From: Oscar Salazar Date: Wed, 19 Jun 2024 09:55:59 -0600 Subject: [PATCH 3/4] SNOW-1482571: Add CI pipeline for apps unit testing (#3) * Create python-app.yml * change yml name * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * add changed files to the pipeline * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Create test.sh * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Delete test.sh * Update ci.yml * Update customers.sql * Update customers.sql * Update ci.yml * apply code suggestions --- .github/workflows/ci.yml | 103 +++++++++++++++++++++++++++++++++++++++ shared_python_ci_env.yml | 15 ++++++ 2 files changed, 118 insertions(+) create mode 100644 .github/workflows/ci.yml create mode 100644 shared_python_ci_env.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..e0198be --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,103 @@ +name: native-app-examples + +on: + pull_request: + types: + - opened + - edited + - labeled + - unlabeled + - synchronize + +permissions: + contents: read + +jobs: + build: + runs-on: ubuntu-latest + strategy: + matrix: + include: + - environment-file: shared_python_ci_env.yml + defaults: + run: + shell: bash -l {0} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: ${{ github.event_name == 'pull_request' && 2 || 0 }} + - name: Set up Python 3.10 + uses: actions/setup-python@v3 + with: + python-version: "3.10" + - name: Get changed files + id: changed-files + run: | + if ${{ github.event_name == 'pull_request' }}; then + echo "changed_files=$(git diff --name-only -r HEAD^1 HEAD | xargs)" >> $GITHUB_OUTPUT + else + echo "changed_files=$(git diff --name-only ${{ github.event.before }} ${{ github.event.after }} | xargs)" >> $GITHUB_OUTPUT + fi + - name: Determine tests to run + uses: actions/github-script@v7 + id: tests_to_run + env: + CHANGED_FILES: ${{ steps.changed-files.outputs.changed_files }} + with: + script: | + const { CHANGED_FILES } = process.env; + const fs = require('fs'); + const path = require('path'); + + function getPytestPaths(dir, callback) + { + const files = fs.readdirSync(dir, { withFileTypes: true }); + for (const file of files) + { + if (file.isDirectory()) + { + getPytestPaths(path.join(dir, file.name), callback); + } + else + { + extension = path.extname(file.name); + + if (extension == '.py') + { + callback(dir); + } + } + } + } + + const paths = new Set(CHANGED_FILES.split(" ") + .map(x => x.substring(0, x.indexOf("/") + 1)) + .filter(x => x.length > 0 && !x.startsWith('.'))); + + const pytestPaths = new Set() + for (const rootPath of paths) + { + getPytestPaths(rootPath, x => pytestPaths.add(x)) + } + + core.setOutput('pytestPaths', [...pytestPaths].join(' ')); + core.setOutput('pytestArgs', [...paths].join(' ')); + + - name: Setup test environment + uses: conda-incubator/setup-miniconda@v2 + with: + environment-file: ${{ matrix.environment-file }} + - name: Install dependencies + run: | + printf "[pytest]\npythonpath=${{ steps.tests_to_run.outputs.pytestPaths }}" > pytest.ini + python -m pip install pytest + - name: Run tests + run: | + args=${{ steps.tests_to_run.outputs.pytestArgs }} + if [ -z "${args}" ]; then + echo “Nothing to test” + else + pytest $args + fi + + \ No newline at end of file diff --git a/shared_python_ci_env.yml b/shared_python_ci_env.yml new file mode 100644 index 0000000..4de72b7 --- /dev/null +++ b/shared_python_ci_env.yml @@ -0,0 +1,15 @@ +# This file is used to install packages for local testing +name: native-apps-examples-testing +channels: + - snowflake +dependencies: + - python=3.8 + - pip + - pip: + - git+https://github.com/snowflakedb/snowflake-telemetry-python.git + - snowflake-native-apps-permission-stub + - snowflake-snowpark-python>=1.15.0 + - snowflake-cli-labs>=2.0.0 + - pytest + - streamlit>=1.26.0 + From 9693c6fbad05fec9309095f661c37c3d7eadfe46 Mon Sep 17 00:00:00 2001 From: Oscar Salazar Date: Wed, 19 Jun 2024 15:54:12 -0600 Subject: [PATCH 4/4] Validate python path for pytest is not empty as well (#7) * Validate python path for pytest is not empty as well * Update ci.yml * Update ci.yml * Update ci.yml --- .github/workflows/ci.yml | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e0198be..1d0594f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -75,13 +75,24 @@ jobs: .filter(x => x.length > 0 && !x.startsWith('.'))); const pytestPaths = new Set() + const pytestArgs = new Set() for (const rootPath of paths) - { - getPytestPaths(rootPath, x => pytestPaths.add(x)) + { + let subFoldersWithPythonFiles = 0; + getPytestPaths(rootPath, x => + { + pytestPaths.add(x); + subFoldersWithPythonFiles++ + }) + + if (subFoldersWithPythonFiles > 0) + { + pytestArgs.add(rootPath) + } } core.setOutput('pytestPaths', [...pytestPaths].join(' ')); - core.setOutput('pytestArgs', [...paths].join(' ')); + core.setOutput('pytestArgs', [...pytestArgs].join(' ')); - name: Setup test environment uses: conda-incubator/setup-miniconda@v2 @@ -94,7 +105,8 @@ jobs: - name: Run tests run: | args=${{ steps.tests_to_run.outputs.pytestArgs }} - if [ -z "${args}" ]; then + pythonpath=${{ steps.tests_to_run.outputs.pytestPaths }} + if [ -z "${args}" ] || [ -z "${pythonpath}" ]; then echo “Nothing to test” else pytest $args