From 8bdd146ef61030dc541dbe197069baf3501138e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ricardo=20Alfaro=20V=C3=ADquez?= <127549539+sfc-gh-ralfaroviquez@users.noreply.github.com> Date: Thu, 4 Jul 2024 12:09:22 -0600 Subject: [PATCH 1/3] SNOW-1449796 Snowflake Cortex Example (#5) * cortex initial commit * cortex initial commit missing file * draft for cortex, missing db integrations and several improvements * minor improvements, using real dataset * using cortex from ml library, adding chat, adding test sample * testing attempts * fix test * Update dashboard.py * tests fixed, added instructions in the readme * trying to fix ci runtest flow * adding cortex lib to ci env * adding snowflake-cortex project to readme index * Update README.md Co-authored-by: Cam Gorrie * Update snowflake-cortex/README.md Co-authored-by: Cam Gorrie * Update snowflake-cortex/README.md Co-authored-by: Cam Gorrie * addressing comments * changing name to dataset columns and db, fixing some wrong commands, deleting version from manifest * repeated dot deletion * changing name from dashboard to ui * added logic for ungiven permissions * fixing tests and adding more * pushing to fix path error * attempt to fix the path, deleting init.py --------- Co-authored-by: Oscar Salazar Co-authored-by: Cam Gorrie --- .github/workflows/ci.yml | 4 +- README.md | 1 + object-level-references/app/manifest.yml | 2 +- shared_python_ci_env.yml | 1 + snowflake-cortex/.gitignore | 4 + snowflake-cortex/README.md | 29 ++++++ snowflake-cortex/app/README.md | 3 + snowflake-cortex/app/manifest.yml | 12 +++ snowflake-cortex/app/setup_script.sql | 21 ++++ snowflake-cortex/local_test_env.yml | 15 +++ snowflake-cortex/prepare/provider_data.sql | 25 +++++ snowflake-cortex/pytest.ini | 2 + snowflake-cortex/scripts/shared_content.sql | 16 +++ snowflake-cortex/snowflake.yml | 12 +++ snowflake-cortex/src/ui/cortexCaller.py | 14 +++ snowflake-cortex/src/ui/dashboard.py | 47 +++++++++ snowflake-cortex/src/ui/environment.yml | 9 ++ snowflake-cortex/test/test_cortex.py | 20 ++++ snowflake-cortex/test/test_ui.py | 102 ++++++++++++++++++++ 19 files changed, 336 insertions(+), 3 deletions(-) create mode 100644 snowflake-cortex/.gitignore create mode 100644 snowflake-cortex/README.md create mode 100644 snowflake-cortex/app/README.md create mode 100644 snowflake-cortex/app/manifest.yml create mode 100644 snowflake-cortex/app/setup_script.sql create mode 100644 snowflake-cortex/local_test_env.yml create mode 100644 snowflake-cortex/prepare/provider_data.sql create mode 100644 snowflake-cortex/pytest.ini create mode 100644 snowflake-cortex/scripts/shared_content.sql create mode 100644 snowflake-cortex/snowflake.yml create mode 100644 snowflake-cortex/src/ui/cortexCaller.py create mode 100644 snowflake-cortex/src/ui/dashboard.py create mode 100644 snowflake-cortex/src/ui/environment.yml create mode 100644 snowflake-cortex/test/test_cortex.py create mode 100644 snowflake-cortex/test/test_ui.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1d0594f..5bfb31f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -104,8 +104,8 @@ jobs: python -m pip install pytest - name: Run tests run: | - args=${{ steps.tests_to_run.outputs.pytestArgs }} - pythonpath=${{ steps.tests_to_run.outputs.pytestPaths }} + args="${{ steps.tests_to_run.outputs.pytestArgs }}" + pythonpath="${{ steps.tests_to_run.outputs.pytestPaths }}" if [ -z "${args}" ] || [ -z "${pythonpath}" ]; then echo “Nothing to test” else diff --git a/README.md b/README.md index 9e8aa01..f9c0d29 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,7 @@ Some applications require other account-level setup before they can be properly | [Object-level References](./object-level-references/) | A simple dashboard to show how to interact with the object-level references and bindings. | | [Reference Usage](./reference-usage/) | How to share a provider table with a native application whose data is replicated to any consumer in the data cloud. | | [SPCS Three-tier](./spcs-three-tier/) | A simple three-tiered web app that can be deployed in Snowpark Container Services. It queries the TPC-H 100 data set and returns the top sales clerks. | +| [Snowflake Cortex](./snowflake-cortex/) | A simple example on how to implement the Cortex Complete and to make it interact with user data. | | [Tasks and Streams](./tasks-streams/) | How to execute a task and visualize changes using streams within a native application. | ## Contributing diff --git a/object-level-references/app/manifest.yml b/object-level-references/app/manifest.yml index 20b0e7c..d102fbd 100644 --- a/object-level-references/app/manifest.yml +++ b/object-level-references/app/manifest.yml @@ -9,7 +9,7 @@ version: artifacts: setup_script: setup_script.sql readme: README.md - default_streamlit: core.dashboard + default_streamlit: core.ui extension_code: true references: diff --git a/shared_python_ci_env.yml b/shared_python_ci_env.yml index 4de72b7..40348bd 100644 --- a/shared_python_ci_env.yml +++ b/shared_python_ci_env.yml @@ -12,4 +12,5 @@ dependencies: - snowflake-cli-labs>=2.0.0 - pytest - streamlit>=1.26.0 + - snowflake-ml-python diff --git a/snowflake-cortex/.gitignore b/snowflake-cortex/.gitignore new file mode 100644 index 0000000..d6aef68 --- /dev/null +++ b/snowflake-cortex/.gitignore @@ -0,0 +1,4 @@ +snowflake.local.yml +output/ +**/__pycache__/ +**/.pytest_cache/ diff --git a/snowflake-cortex/README.md b/snowflake-cortex/README.md new file mode 100644 index 0000000..7a13efe --- /dev/null +++ b/snowflake-cortex/README.md @@ -0,0 +1,29 @@ +# Snowflake Cortex + +This simple Native App shows how to use Cortex Complete and make it interact with user data. + +For this use case, the dataset used is rather small: only 10 entries. This is because the language model used in the Cortex function restricts input data size. You can change the language model used to a bigger / different one, according to your needs. +For more information about it please visit **[this page](https://docs.snowflake.com/en/user-guide/snowflake-cortex/llm-functions#cost-considerations)**. + +## Data preparation + +To run this example first execute this command, that is going to create a database and table with information about songs charts: +```sh +snow sql -f 'prepare/provider_data.sql' +``` +## App execution + +Then run `snow app run` on your terminal. + +## App and Data Deletion +To delete the database and the app run + +```sh +snow sql -q 'DROP DATABASE SONGS_CORTEX_DB;' +snow app teardown +``` + +## Further reading + +For more information about the different ways to use snowflake AI capabilities visit this page: +**[Snowflake AI and ML documentation](https://docs.snowflake.com/en/guides-overview-ai-features)** \ No newline at end of file diff --git a/snowflake-cortex/app/README.md b/snowflake-cortex/app/README.md new file mode 100644 index 0000000..db6bf24 --- /dev/null +++ b/snowflake-cortex/app/README.md @@ -0,0 +1,3 @@ +# Snowflake Native App - Snowflake Cortex + +This is a sample Snowflake Native App that shows the use of Snowflake Cortex inside Native Apps. \ No newline at end of file diff --git a/snowflake-cortex/app/manifest.yml b/snowflake-cortex/app/manifest.yml new file mode 100644 index 0000000..152c382 --- /dev/null +++ b/snowflake-cortex/app/manifest.yml @@ -0,0 +1,12 @@ +# For more information on creating manifest, go to https://docs.snowflake.com/en/developer-guide/native-apps/creating-manifest +manifest_version: 1 + +artifacts: + setup_script: setup_script.sql + readme: README.md + default_streamlit: core.ui + extension_code: true + +privileges: + - IMPORTED PRIVILEGES ON SNOWFLAKE DB: + description: "Imported privileges to use cortex DB" diff --git a/snowflake-cortex/app/setup_script.sql b/snowflake-cortex/app/setup_script.sql new file mode 100644 index 0000000..e057788 --- /dev/null +++ b/snowflake-cortex/app/setup_script.sql @@ -0,0 +1,21 @@ +-- This is the setup script that runs while installing a Snowflake Native App in a consumer account. +-- For more information on how to create setup file, visit https://docs.snowflake.com/en/developer-guide/native-apps/creating-setup-script + +-- A general guideline to building this script looks like: +-- 1. Create application roles +CREATE APPLICATION ROLE IF NOT EXISTS app_public; + +-- 2. Create a versioned schema to hold those UDFs/Stored Procedures +CREATE OR ALTER VERSIONED SCHEMA core; +GRANT USAGE ON SCHEMA core TO APPLICATION ROLE app_public; + +-- 3. Create a streamlit object using the code you wrote in you wrote in src/module-ui, as shown below. +-- The `from` value is derived from the stage path described in snowflake.yml +CREATE STREAMLIT core.ui + FROM '/streamlit/' + MAIN_FILE = 'dashboard.py'; + +-- 4. Grant appropriate privileges over these objects to your application roles. +GRANT USAGE ON STREAMLIT core.ui TO APPLICATION ROLE app_public; + +-- A detailed explanation can be found at https://docs.snowflake.com/en/developer-guide/native-apps/adding-streamlit \ No newline at end of file diff --git a/snowflake-cortex/local_test_env.yml b/snowflake-cortex/local_test_env.yml new file mode 100644 index 0000000..1987c2f --- /dev/null +++ b/snowflake-cortex/local_test_env.yml @@ -0,0 +1,15 @@ +# This file is used to install packages for local testing +name: snowflake-cortex-testing +channels: + - snowflake +dependencies: + - python=3.8 + - pip + - pip: + - snowflake-native-apps-permission-stub + - snowflake-snowpark-python>=1.15.0 + - snowflake-cli-labs>=2.0.0 + - pytest + - streamlit>=1.26.0 + - snowflake-ml-python + diff --git a/snowflake-cortex/prepare/provider_data.sql b/snowflake-cortex/prepare/provider_data.sql new file mode 100644 index 0000000..06c2b5e --- /dev/null +++ b/snowflake-cortex/prepare/provider_data.sql @@ -0,0 +1,25 @@ +USE ROLE ACCOUNTADMIN; +CREATE DATABASE IF NOT EXISTS SONGS_CORTEX_DB; +CREATE SCHEMA IF NOT EXISTS SONGS_CORTEX_DB.SONGS_CORTEX_SCHEMA; +CREATE OR REPLACE TABLE SONGS_CORTEX_DB.SONGS_CORTEX_SCHEMA.SONGS_PROVIDER_DATA( + track_name VARCHAR, + artists_name VARCHAR, + artist_count INTEGER, + released_year INTEGER, + released_month INTEGER, + released_day INTEGER, + in_song_charts INTEGER, + song_streams INTEGER, + danceability_percentage INTEGER +); +INSERT INTO SONGS_CORTEX_DB.SONGS_CORTEX_SCHEMA.SONGS_PROVIDER_DATA VALUES +('Seven \(feat. Latto\) \(Explicit Ver.\)','Latto, Jung Kook',2,2023,7,14,147,141381703,80), +('LALA','Myke Towers',1,2023,3,23,48,133716286,71), +('vampire','Olivia Rodrigo',1,2023,6,30,113,140003974,51), +('Cruel Summer','Taylor Swift',1,2019,8,23,100,800840817,55), +('WHERE SHE GOES','Bad Bunny',1,2023,5,18,50,303236322,65), +('Sprinter','Dave, Central Cee',2,2023,6,1,91,183706234,92), +('Ella Baila Sola','Eslabon Armado, Peso Pluma',2,2023,3,16,50,725980112,67), +('Columbia','Quevedo',1,2023,7,7,43,58149378,67), +('fukumean','Gunna',1,2023,5,15,83,95217315,85), +('La Bebe - Remix','Peso Pluma, Yng Lvcas',2,2023,3,17,44,553634067,81); \ No newline at end of file diff --git a/snowflake-cortex/pytest.ini b/snowflake-cortex/pytest.ini new file mode 100644 index 0000000..7296d72 --- /dev/null +++ b/snowflake-cortex/pytest.ini @@ -0,0 +1,2 @@ +[pytest] +pythonpath=src/ui diff --git a/snowflake-cortex/scripts/shared_content.sql b/snowflake-cortex/scripts/shared_content.sql new file mode 100644 index 0000000..bcb7b32 --- /dev/null +++ b/snowflake-cortex/scripts/shared_content.sql @@ -0,0 +1,16 @@ +-- Grant usage to a external database from the application. +create schema if not exists {{ package_name }}.PACKAGE_SHARED; +use schema {{ package_name }}.PACKAGE_SHARED; + +grant reference_usage on database SONGS_CORTEX_DB + to share in application package {{ package_name }}; + +-- Create a view that references the provider table. +-- The view is going to be shared by the package to the application. +create view if not exists PACKAGE_SHARED.PROVIDER_SONGS_VIEW + as select * from SONGS_CORTEX_DB.SONGS_CORTEX_SCHEMA.SONGS_PROVIDER_DATA; + +grant usage on schema PACKAGE_SHARED + to share in application package {{ package_name }}; +grant select on view PACKAGE_SHARED.PROVIDER_SONGS_VIEW + to share in application package {{ package_name }}; \ No newline at end of file diff --git a/snowflake-cortex/snowflake.yml b/snowflake-cortex/snowflake.yml new file mode 100644 index 0000000..8a68a9d --- /dev/null +++ b/snowflake-cortex/snowflake.yml @@ -0,0 +1,12 @@ +definition_version: 1 +native_app: + name: snowflake_cortex + source_stage: app_src.stage + artifacts: + - src: app/* + dest: ./ + - src: src/ui/* + dest: streamlit/ + package: + scripts: + - scripts/shared_content.sql \ No newline at end of file diff --git a/snowflake-cortex/src/ui/cortexCaller.py b/snowflake-cortex/src/ui/cortexCaller.py new file mode 100644 index 0000000..fd4780a --- /dev/null +++ b/snowflake-cortex/src/ui/cortexCaller.py @@ -0,0 +1,14 @@ +from snowflake.cortex import Complete + +class CortexCaller: + + def call_complete(_self,input_with_table): + response = Complete('llama2-70b-chat', input_with_table) + return response + + def call_cortex(_self, df, input: str): + sample_df = df.to_string() + input_with_table = f"""The following data is a table that contains songs information it is wrapped in this tags {sample_df} {input}""" + response = _self.call_complete(input_with_table) + return response + \ No newline at end of file diff --git a/snowflake-cortex/src/ui/dashboard.py b/snowflake-cortex/src/ui/dashboard.py new file mode 100644 index 0000000..2f2e44c --- /dev/null +++ b/snowflake-cortex/src/ui/dashboard.py @@ -0,0 +1,47 @@ +import streamlit as st +from snowflake.snowpark import Session +from cortexCaller import CortexCaller +import snowflake.permissions as permission + +class Dashboard: + + def __init__(self, session: Session, cortex_caller: CortexCaller = CortexCaller()) -> None: + self.session = session + self.cortex_caller = cortex_caller + + def setup(self): + st.header("Privileges setup") + st.caption(""" + Follow the instructions below to set up your application. + Once you have completed the steps, you will be able to continue to the main example. + """) + privilege = 'IMPORTED PRIVILEGES ON SNOWFLAKE DB' + if not permission.get_held_account_privileges([privilege]): + st.button(f"Grant import privileges on snowflake DB ↗", on_click=permission.request_account_privileges, args=[[privilege]], key='IMPORTED PRIVILEGES ON SNOWFLAKE DB') + else: + st.session_state.privileges_granted = True + st.rerun() + + def run_streamlit(self): + if not permission.get_held_account_privileges(['IMPORTED PRIVILEGES ON SNOWFLAKE DB']): + del st.session_state.privileges_granted + st.rerun() + + st.header('Snowflake Cortex Example') + st.subheader("Simple app showing how cortex can answer questions using the following song's ranking table") + table_df = self.session.table("PACKAGE_SHARED.PROVIDER_SONGS_VIEW").to_pandas() + st.write(table_df) + + cortex_chat = st.container(height=200) + cortex_input = st.chat_input("What do you want to ask to cortex about the table from above?", key="cortex_input") + if cortex_input and not cortex_input.isspace(): + cortex_chat.chat_message("user").write(cortex_input) + cortex_fn = self.cortex_caller.call_cortex(table_df, cortex_input) + cortex_chat.chat_message("assistant").write(cortex_fn) + + +if __name__ == '__main__': + if 'privileges_granted' not in st.session_state: + Dashboard(Session.builder.getOrCreate()).setup() + else: + Dashboard(Session.builder.getOrCreate()).run_streamlit() diff --git a/snowflake-cortex/src/ui/environment.yml b/snowflake-cortex/src/ui/environment.yml new file mode 100644 index 0000000..d1a0cbe --- /dev/null +++ b/snowflake-cortex/src/ui/environment.yml @@ -0,0 +1,9 @@ +# This file is used to install packages used by the Streamlit App. +# For more details, refer to https://docs.snowflake.com/en/developer-guide/streamlit/create-streamlit-sql#label-streamlit-install-packages-manual + +channels: +- snowflake +dependencies: +- streamlit=1.31.0 +- snowflake-ml-python +- snowflake-native-apps-permission \ No newline at end of file diff --git a/snowflake-cortex/test/test_cortex.py b/snowflake-cortex/test/test_cortex.py new file mode 100644 index 0000000..3507393 --- /dev/null +++ b/snowflake-cortex/test/test_cortex.py @@ -0,0 +1,20 @@ +import unittest +from unittest.mock import patch, MagicMock +import pandas as pd +import pytest as pytest +from snowflake import cortex +from snowflake.cortex import Complete + +class TestDataExporter(unittest.TestCase): + + def test_cortex_response(self): + from cortexCaller import CortexCaller + + mock_df = pd.DataFrame([{"TestColumn": "TestValue"}]) + mock_input = 'Test input from user' + cortexCaller = CortexCaller() + cortexCaller.call_complete = MagicMock(name='call_complete') + cortexCaller.call_complete.return_value = 'test answer from cortex' + response = cortexCaller.call_cortex(mock_df, mock_input) + cortexCaller.call_complete.assert_called_once() + assert response == cortexCaller.call_complete.return_value \ No newline at end of file diff --git a/snowflake-cortex/test/test_ui.py b/snowflake-cortex/test/test_ui.py new file mode 100644 index 0000000..8604732 --- /dev/null +++ b/snowflake-cortex/test/test_ui.py @@ -0,0 +1,102 @@ +from cortexCaller import CortexCaller +from streamlit.testing.v1 import AppTest +from snowflake.snowpark import Session +from unittest.mock import patch, MagicMock +import pytest as pytest + +@pytest.fixture(autouse=True) +def session(): + session = Session.builder.config('local_testing', True).create() + yield session + session.close() + +@pytest.fixture(autouse=True) +def cortex_caller(): + cortex_caller = CortexCaller() + yield cortex_caller + + +@patch('snowflake.permissions.request_account_privileges') +def test_setup(request_account_privileges: MagicMock, session, cortex_caller): + + def script(session, cortex_caller): + from dashboard import Dashboard + dashboard = Dashboard(session, cortex_caller) + return dashboard.setup() + + with patch('snowflake.permissions.request_account_privileges', return_value=True) as request_account_privileges: + at = AppTest.from_function(script, kwargs={ "session": session, "cortex_caller": cortex_caller }).run() + assert at.header[0].value == 'Privileges setup' + assert at.caption[0].value == 'Follow the instructions below to set up your application.\nOnce you have completed the steps, you will be able to continue to the main example.' + assert at.button[0].label == 'Grant import privileges on snowflake DB ↗' + assert 'privileges_granted' not in at.session_state + at.button(key='IMPORTED PRIVILEGES ON SNOWFLAKE DB').click().run() + request_account_privileges.assert_called_once_with(['IMPORTED PRIVILEGES ON SNOWFLAKE DB']) + + +@patch('snowflake.permissions.get_held_account_privileges') +def test_not_granted_privileges(get_held_account_privileges: MagicMock, session, cortex_caller): + + get_held_account_privileges.return_value = False + + at = AppTest.from_file("../src/ui/dashboard.py").run() + assert at.header[0].value == 'Privileges setup' + assert at.caption[0].value == 'Follow the instructions below to set up your application.\nOnce you have completed the steps, you will be able to continue to the main example.' + assert at.button[0].label == 'Grant import privileges on snowflake DB ↗' + +@patch('cortexCaller.CortexCaller.call_cortex') +@patch('snowflake.snowpark.session.Session.table') +@patch('snowflake.permissions.get_held_account_privileges') +def test_cortex_call_mock(get_held_account_privileges: MagicMock, table: MagicMock, call_cortex: MagicMock, session, cortex_caller): + + get_held_account_privileges.return_value = True + + def script(session, cortex_caller): + from dashboard import Dashboard + dashboard = Dashboard(session, cortex_caller) + return dashboard.run_streamlit() + + table.return_value=session.create_dataframe([{"TestColumn": "TestValue"}]) + call_cortex.return_value='Mocked response from cortex' + at = AppTest.from_function(script, kwargs={ "session": session, "cortex_caller": cortex_caller }).run() + + at.chat_input(key='cortex_input').set_value('text for testing purposes').run() + call_cortex.assert_called_once() + assert at.chat_message[1].name == 'assistant' + +@patch('cortexCaller.CortexCaller.call_cortex') +@patch('snowflake.snowpark.session.Session.table') +@patch('snowflake.permissions.get_held_account_privileges') +def test_input_showed_in_chat(get_held_account_privileges: MagicMock, table: MagicMock, call_cortex: MagicMock, session, cortex_caller): + + get_held_account_privileges.return_value = True + + def script(session, cortex_caller): + from dashboard import Dashboard + dashboard = Dashboard(session, cortex_caller) + return dashboard.run_streamlit() + + table.return_value=session.create_dataframe([{"TestColumn": "TestValue"}]) + at = AppTest.from_function(script, kwargs={ "session": session, "cortex_caller": cortex_caller }).run() + at.chat_input(key='cortex_input').set_value('text for testing purposes').run() + assert at.chat_message[0].name == 'user' + +@patch('cortexCaller.CortexCaller.call_cortex') +@patch('snowflake.snowpark.session.Session.table') +@patch('snowflake.permissions.get_held_account_privileges') +def test_input_empty(get_held_account_privileges: MagicMock, table: MagicMock, call_cortex: MagicMock, session, cortex_caller): + + get_held_account_privileges.return_value = True + + def script(session, cortex_caller): + from dashboard import Dashboard + dashboard = Dashboard(session, cortex_caller) + return dashboard.run_streamlit() + + table.return_value=session.create_dataframe([{"TestColumn": "TestValue"}]) + at = AppTest.from_function(script, kwargs={ "session": session, "cortex_caller": cortex_caller }).run() + at.chat_input(key='cortex_input').set_value(' ').run() + assert not at.chat_message + + at.chat_input(key='cortex_input').set_value('').run() + assert not at.chat_message \ No newline at end of file From d3167e2e1b8203e9c9e4246fa91cd8400494cecf Mon Sep 17 00:00:00 2001 From: Cam Gorrie Date: Thu, 4 Jul 2024 16:30:14 -0400 Subject: [PATCH 2/3] Fix cortex validation error (#11) fix cortex validation error --- snowflake-cortex/app/setup_script.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/snowflake-cortex/app/setup_script.sql b/snowflake-cortex/app/setup_script.sql index e057788..6d88521 100644 --- a/snowflake-cortex/app/setup_script.sql +++ b/snowflake-cortex/app/setup_script.sql @@ -11,7 +11,7 @@ GRANT USAGE ON SCHEMA core TO APPLICATION ROLE app_public; -- 3. Create a streamlit object using the code you wrote in you wrote in src/module-ui, as shown below. -- The `from` value is derived from the stage path described in snowflake.yml -CREATE STREAMLIT core.ui +CREATE OR REPLACE STREAMLIT core.ui FROM '/streamlit/' MAIN_FILE = 'dashboard.py'; From c7c986c3a96bbca6e7b6fa19fa287cdf18d8efb9 Mon Sep 17 00:00:00 2001 From: Oscar Salazar Date: Thu, 4 Jul 2024 15:15:26 -0600 Subject: [PATCH 3/3] SNOW-1449798: Adding E2E sample for NA + Hybrid Tables (#10) * SNOW-1449798: Adding E2E sample for NA + Hybrid Tables * Update README.md * Update ci.yml * Update ui.py * Update hybrid-tables/app/setup_script.sql Co-authored-by: Cam Gorrie * Update ui.py * Update README.md --------- Co-authored-by: Cam Gorrie --- hybrid-tables/.gitignore | 2 + hybrid-tables/README.md | 70 ++++++++++++++++++++++++ hybrid-tables/app/README.md | 20 +++++++ hybrid-tables/app/manifest.yml | 11 ++++ hybrid-tables/app/setup_script.sql | 43 +++++++++++++++ hybrid-tables/local_test_env.yml | 12 ++++ hybrid-tables/pytest.ini | 2 + hybrid-tables/python/src/environment.yml | 8 +++ hybrid-tables/python/src/ui.py | 30 ++++++++++ hybrid-tables/python/test/test_ui.py | 40 ++++++++++++++ hybrid-tables/snowflake.yml | 11 ++++ 11 files changed, 249 insertions(+) create mode 100644 hybrid-tables/.gitignore create mode 100644 hybrid-tables/README.md create mode 100644 hybrid-tables/app/README.md create mode 100644 hybrid-tables/app/manifest.yml create mode 100644 hybrid-tables/app/setup_script.sql create mode 100644 hybrid-tables/local_test_env.yml create mode 100644 hybrid-tables/pytest.ini create mode 100644 hybrid-tables/python/src/environment.yml create mode 100644 hybrid-tables/python/src/ui.py create mode 100644 hybrid-tables/python/test/test_ui.py create mode 100644 hybrid-tables/snowflake.yml diff --git a/hybrid-tables/.gitignore b/hybrid-tables/.gitignore new file mode 100644 index 0000000..9162cdf --- /dev/null +++ b/hybrid-tables/.gitignore @@ -0,0 +1,2 @@ +snowflake.local.yml +output/** diff --git a/hybrid-tables/README.md b/hybrid-tables/README.md new file mode 100644 index 0000000..9a1ddb5 --- /dev/null +++ b/hybrid-tables/README.md @@ -0,0 +1,70 @@ +# Hybrid Tables + +This Snowflake Native Application sample demonstrates how to use hybrid tables when user needs to enforce a primary key constraint. + +The `internal.dictionary` table simulates the behavior of a `Dictionary/Hash Table` data structure by allowing users add just key/value pairs that don't exist yet in the table. + +```sql +CREATE HYBRID TABLE IF NOT EXISTS internal.dictionary +( + key VARCHAR, + value VARCHAR, + CONSTRAINT pkey PRIMARY KEY (key) +); +``` + +To add new values in the `dictionary` table, there is a Stored Procedure that returns an `OK` status if the columns where added successfully or a json with the error if the new row could not be added. + +```sql +CREATE OR REPLACE PROCEDURE core.add_key_value(KEY VARCHAR, VALUE VARCHAR) +RETURNS VARIANT +LANGUAGE SQL +AS +$$ +BEGIN + INSERT INTO internal.dictionary VALUES (:KEY, :VALUE); + RETURN OBJECT_CONSTRUCT('STATUS', 'OK'); + EXCEPTION + WHEN STATEMENT_ERROR THEN + RETURN OBJECT_CONSTRUCT('STATUS', 'FAILED', + 'SQLCODE', SQLCODE, + 'SQLERRM', SQLERRM, + 'SQLSTATE', SQLSTATE); +END; +$$; +``` + +## Development + +### Setting up / Updating the Environment + +Run the following command to create or update Conda environment. This includes tools like Snowflake CLI and testing packages: + +```sh +conda env update -f local_test_env.yml +``` +To activate the environment, run the following command: + +```sh +conda activate hybrid-tables-testing +``` + +### Automated Testing + +With the conda environment activated, you can test the app as follows: + +```sh +pytest +``` + +### Manual Testing / Deployment to Snowflake + +You can deploy the application in dev mode as follows: + +```sh +snow app run +``` + +## Additional Resources + +- [Hybrid Tables](https://docs.snowflake.com/en/user-guide/tables-hybrid) \ No newline at end of file diff --git a/hybrid-tables/app/README.md b/hybrid-tables/app/README.md new file mode 100644 index 0000000..7463413 --- /dev/null +++ b/hybrid-tables/app/README.md @@ -0,0 +1,20 @@ +## Welcome to Hybrid Tables + Native Apps! + +In this Snowflake Native App, you will be able to explore the usage of `Hybrid Tables` within Native Apps. + +For more information about a Snowflake Native App, please read the [official Snowflake documentation](https://docs.snowflake.com/en/developer-guide/native-apps/native-apps-about) which goes in depth about many additional functionalities of this framework. + +## Using the application after installation +To interact with the application after it has successfully installed in your account, switch to the application owner role first. + +### Calling a stored procedure + +``` +CALL ..; +``` + +### Calling a function + +``` +SELECT ..; +``` diff --git a/hybrid-tables/app/manifest.yml b/hybrid-tables/app/manifest.yml new file mode 100644 index 0000000..03f1186 --- /dev/null +++ b/hybrid-tables/app/manifest.yml @@ -0,0 +1,11 @@ +# This is a manifest.yml file, a required component of creating a Snowflake Native App. +# This file defines properties required by the application package, including the location of the setup script and version definitions. +# Refer to https://docs.snowflake.com/en/developer-guide/native-apps/creating-manifest for a detailed understanding of this file. + +manifest_version: 1 + +artifacts: + setup_script: setup_script.sql + default_streamlit: core.ui + extension_code: true + readme: README.md diff --git a/hybrid-tables/app/setup_script.sql b/hybrid-tables/app/setup_script.sql new file mode 100644 index 0000000..b103a6d --- /dev/null +++ b/hybrid-tables/app/setup_script.sql @@ -0,0 +1,43 @@ +-- This is the setup script that runs while installing a Snowflake Native App in a consumer account. +-- To write this script, you can familiarize yourself with some of the following concepts: +-- Application Roles +-- Versioned Schemas +-- UDFs/Procs +-- Extension Code +-- Refer to https://docs.snowflake.com/en/developer-guide/native-apps/creating-setup-script for a detailed understanding of this file. + +CREATE APPLICATION ROLE IF NOT EXISTS app_public; +CREATE OR ALTER VERSIONED SCHEMA core; +CREATE SCHEMA IF NOT EXISTS internal; +GRANT USAGE ON SCHEMA core TO APPLICATION ROLE app_public; +GRANT USAGE ON SCHEMA internal TO APPLICATION ROLE app_public; + +CREATE HYBRID TABLE IF NOT EXISTS internal.dictionary +( + key VARCHAR, + value VARCHAR, + CONSTRAINT pkey PRIMARY KEY (key) +); + +CREATE OR REPLACE PROCEDURE core.add_key_value(KEY VARCHAR, VALUE VARCHAR) +RETURNS VARIANT +LANGUAGE SQL +AS +$$ +BEGIN + INSERT INTO internal.dictionary VALUES (:KEY, :VALUE); + RETURN OBJECT_CONSTRUCT('STATUS', 'OK'); + EXCEPTION + WHEN STATEMENT_ERROR THEN + RETURN OBJECT_CONSTRUCT('STATUS', 'FAILED', + 'SQLCODE', SQLCODE, + 'SQLERRM', SQLERRM, + 'SQLSTATE', SQLSTATE); +END; +$$; + +CREATE OR REPLACE STREAMLIT core.ui + FROM '/streamlit/' + MAIN_FILE = 'ui.py'; + +GRANT USAGE ON STREAMLIT core.ui TO APPLICATION ROLE app_public; \ No newline at end of file diff --git a/hybrid-tables/local_test_env.yml b/hybrid-tables/local_test_env.yml new file mode 100644 index 0000000..8bffc01 --- /dev/null +++ b/hybrid-tables/local_test_env.yml @@ -0,0 +1,12 @@ +# This file is used to install packages for local testing +name: hybrid-tables-testing +channels: + - snowflake +dependencies: + - python=3.8 + - pip + - pip: + - snowflake-snowpark-python>=1.15.0 + - pytest + - streamlit>=1.26.0 + diff --git a/hybrid-tables/pytest.ini b/hybrid-tables/pytest.ini new file mode 100644 index 0000000..044efc2 --- /dev/null +++ b/hybrid-tables/pytest.ini @@ -0,0 +1,2 @@ +[pytest] +pythonpath=python/src diff --git a/hybrid-tables/python/src/environment.yml b/hybrid-tables/python/src/environment.yml new file mode 100644 index 0000000..faef235 --- /dev/null +++ b/hybrid-tables/python/src/environment.yml @@ -0,0 +1,8 @@ +# This file is used to install packages used by the Streamlit App. +# For more details, refer to https://docs.snowflake.com/en/developer-guide/streamlit/create-streamlit-sql#label-streamlit-install-packages-manual + +channels: +- snowflake +dependencies: +- streamlit=1.26.0 +- snowflake-native-apps-permission diff --git a/hybrid-tables/python/src/ui.py b/hybrid-tables/python/src/ui.py new file mode 100644 index 0000000..c283759 --- /dev/null +++ b/hybrid-tables/python/src/ui.py @@ -0,0 +1,30 @@ +from snowflake.snowpark import Session +import json +import streamlit as st + +class UI: + def __init__(self, session: Session) -> None: + self.session = session + + def run(self): + with st.form('key_value_form', clear_on_submit=True): + col1, col2 = st.columns(2) + key = col1.text_input('Add a key', key='Key') + value = col2.text_input('Add a value for the key', key='Value') + if st.form_submit_button('Add'): + if key == '' or value == '': + st.error("Key-value pairs should not be empty") + else: + self.add(key, value) + + st.dataframe(self.session.table('internal.dictionary').to_pandas(), use_container_width=True) + + def add(self, key: str, value: str): + result = json.loads(self.session.call('core.add_key_value', key, value)) + if "SQLCODE" in result: + error = f"Primary key violation on Hybrid Table. **{key}** already exists." if result["SQLCODE"] == 200001 else result["SQLERRM"] + st.error(error, icon="🚨") + +if __name__ == '__main__': + ui = UI(Session.builder.getOrCreate()) + ui.run() \ No newline at end of file diff --git a/hybrid-tables/python/test/test_ui.py b/hybrid-tables/python/test/test_ui.py new file mode 100644 index 0000000..42ae0ab --- /dev/null +++ b/hybrid-tables/python/test/test_ui.py @@ -0,0 +1,40 @@ +import pytest +from unittest.mock import patch, MagicMock +from snowflake.snowpark import Session +from streamlit.testing.v1 import AppTest +from ui import UI + +@pytest.fixture() +def session(): + session = Session.builder.config('local_testing', True).create() + yield session + session.close() + +@patch('snowflake.snowpark.session.Session.table') +def test_run(table: MagicMock, session): + # arrange + def script(session): + from ui import UI + sut = UI(session) + return sut.run() + + table.return_value = session.create_dataframe([{"key": "mykey", "value": "myvalue"}]) + + # act + at = AppTest.from_function(script, kwargs={ "session": session }).run() + + # assert + assert len(at.dataframe[0].value.index) == 1 + +@patch('snowflake.snowpark.session.Session.call') +def test_add(call: MagicMock, session): + # arrange + call.return_value = '{ "STATUS": "OK" }' + sut = UI(session) + + # act + sut.add('', '') + + # assert + call.assert_called_once() + \ No newline at end of file diff --git a/hybrid-tables/snowflake.yml b/hybrid-tables/snowflake.yml new file mode 100644 index 0000000..f85655e --- /dev/null +++ b/hybrid-tables/snowflake.yml @@ -0,0 +1,11 @@ +# This is a project definition file, a required component if you intend to use Snowflake CLI in a project directory such as this template. + +definition_version: 1 +native_app: + name: hybrid_tables + source_stage: app_src.stage + artifacts: + - src: app/* + dest: ./ + - src: python/src/* + dest: streamlit/