diff --git a/src/snowflake/cli/plugins/nativeapp/codegen/compiler.py b/src/snowflake/cli/plugins/nativeapp/codegen/compiler.py index 4eae91638d..aa6c5bf6ae 100644 --- a/src/snowflake/cli/plugins/nativeapp/codegen/compiler.py +++ b/src/snowflake/cli/plugins/nativeapp/codegen/compiler.py @@ -3,6 +3,7 @@ from pathlib import Path from typing import Dict, Optional +from snowflake.cli.api.console import cli_console as cc from snowflake.cli.api.project.schemas.native_app.native_app import NativeApp from snowflake.cli.api.project.schemas.native_app.path_mapping import ( PathMapping, @@ -54,19 +55,28 @@ def compile_artifacts(self): Go through every artifact object in the project definition of a native app, and execute processors in order of specification for each of the artifact object. May have side-effects on the filesystem by either directly editing source files or the deploy root. """ + should_proceed = False for artifact in self.artifacts: - for processor in artifact.processors: - artifact_processor = self._try_create_processor( - processor_mapping=processor, - ) - if artifact_processor is None: - raise UnsupportedArtifactProcessorError( - processor_name=processor.name - ) - else: - artifact_processor.process( - artifact_to_process=artifact, processor_mapping=processor + if artifact.processors: + should_proceed = True + break + if not should_proceed: + return + + with cc.phase("Invoking artifact processors"): + for artifact in self.artifacts: + for processor in artifact.processors: + artifact_processor = self._try_create_processor( + processor_mapping=processor, ) + if artifact_processor is None: + raise UnsupportedArtifactProcessorError( + processor_name=processor.name + ) + else: + artifact_processor.process( + artifact_to_process=artifact, processor_mapping=processor + ) def _try_create_processor( self, diff --git a/src/snowflake/cli/plugins/nativeapp/codegen/snowpark/python_processor.py b/src/snowflake/cli/plugins/nativeapp/codegen/snowpark/python_processor.py index 0fe3e0735a..4fe1ee9d5f 100644 --- a/src/snowflake/cli/plugins/nativeapp/codegen/snowpark/python_processor.py +++ b/src/snowflake/cli/plugins/nativeapp/codegen/snowpark/python_processor.py @@ -1,7 +1,6 @@ from __future__ import annotations import json -import pprint import re from pathlib import Path from textwrap import dedent @@ -173,7 +172,7 @@ def process( artifact_to_process: PathMapping, processor_mapping: Optional[ProcessorMapping], **kwargs, - ) -> str: # String output is temporary until we have better e2e testing mechanism + ) -> None: """ Collects code annotations from Snowpark python files containing extension functions and augments the existing setup script with generated SQL that registers these functions. @@ -201,22 +200,16 @@ def process( continue relative_py_file = py_file.relative_to(bundle_map.deploy_root()) - cc.message( - "-- Generating Snowpark annotation SQL code for {}".format( - relative_py_file - ) - ) - cc.message(create_stmt) - collected_output.append(f"-- {relative_py_file}") - collected_output.append(create_stmt) grant_statements = generate_grant_sql_ddl_statements(extension_fn) if grant_statements is not None: - cc.message(grant_statements) collected_output.append(grant_statements) with open(sql_file, "a") as file: - file.write("\n") + file.write( + f"-- Generated by the Snowflake CLI from {relative_py_file}\n" + ) + file.write(f"-- DO NOT EDIT\n") file.write(create_stmt) if grant_statements is not None: file.write("\n") @@ -231,8 +224,6 @@ def process( generated_root=self.generated_root, ) - return "\n".join(collected_output) - def _normalize_imports( self, extension_fn: NativeAppExtensionFunction, @@ -308,6 +299,11 @@ def collect_extension_functions( for src_file, dest_file in bundle_map.all_mappings( absolute=True, expand_directories=True, predicate=_is_python_file_artifact ): + cc.step( + "Processing Snowpark annotations from {}".format( + dest_file.relative_to(bundle_map.deploy_root()) + ) + ) collected_extension_function_json = _execute_in_sandbox( py_file=str(dest_file.resolve()), deploy_root=self.deploy_root, @@ -333,10 +329,6 @@ def collect_extension_functions( cc.warning("Invalid extension function definition") if collected_extension_functions: - cc.message(f"This is the file path in deploy root: {dest_file}\n") - cc.message("This is the list of collected extension functions:") - cc.message(pprint.pformat(collected_extension_functions)) - collected_extension_fns_by_path[ dest_file ] = collected_extension_functions @@ -479,7 +471,7 @@ def edit_setup_script_with_exec_imm_sql( sql_file_relative_path = sql_file.relative_to( deploy_root ) # Path on stage, without the leading slash - file.write(f"\nEXECUTE IMMEDIATE FROM '/{sql_file_relative_path}';") + file.write(f"EXECUTE IMMEDIATE FROM '/{sql_file_relative_path}';\n") # Find the setup script in the deploy root. setup_file_path = find_setup_script_file(deploy_root=deploy_root) @@ -493,4 +485,5 @@ def edit_setup_script_with_exec_imm_sql( generated_file_relative_path = generated_file_path.relative_to(deploy_root) with open(setup_file_path, "w", encoding="utf-8") as file: file.write(code) - file.write(f"\nEXECUTE IMMEDIATE FROM '/{generated_file_relative_path}';\n") + file.write(f"\nEXECUTE IMMEDIATE FROM '/{generated_file_relative_path}';") + file.write(f"\n") diff --git a/tests/nativeapp/codegen/snowpark/__snapshots__/test_python_processor.ambr b/tests/nativeapp/codegen/snowpark/__snapshots__/test_python_processor.ambr index 4216860bfd..bee57cb660 100644 --- a/tests/nativeapp/codegen/snowpark/__snapshots__/test_python_processor.ambr +++ b/tests/nativeapp/codegen/snowpark/__snapshots__/test_python_processor.ambr @@ -1,381 +1,240 @@ # serializer version: 1 # name: test_edit_setup_script_with_exec_imm_sql ''' + d output/deploy + d output/deploy/__generated + f output/deploy/__generated/__generated.sql + f output/deploy/__generated/dummy.sql + d output/deploy/__generated/moduleB + f output/deploy/__generated/moduleB/dummy.sql + f output/deploy/manifest.yml + d output/deploy/moduleA + d output/deploy/moduleA/moduleC + f output/deploy/moduleA/moduleC/setup.sql + ''' +# --- +# name: test_edit_setup_script_with_exec_imm_sql.1 + ''' + ===== Contents of: output/deploy/__generated/__generated.sql ===== EXECUTE IMMEDIATE FROM '/__generated/moduleB/dummy.sql'; EXECUTE IMMEDIATE FROM '/__generated/dummy.sql'; ''' # --- -# name: test_edit_setup_script_with_exec_imm_sql.1 +# name: test_edit_setup_script_with_exec_imm_sql.2 ''' - create application role app_public; - EXECUTE IMMEDIATE FROM '/__generated/__generated.sql'; - + ===== Contents of: output/deploy/__generated/dummy.sql ===== + #this is a file ''' # --- -# name: test_edit_setup_script_with_exec_imm_sql_noop - '#some text' +# name: test_edit_setup_script_with_exec_imm_sql.3 + ''' + ===== Contents of: output/deploy/__generated/moduleB/dummy.sql ===== + #this is a file + ''' # --- -# name: test_edit_setup_script_with_exec_imm_sql_symlink +# name: test_edit_setup_script_with_exec_imm_sql.4 ''' + ===== Contents of: output/deploy/manifest.yml ===== + manifest_version: 1 + + artifacts: + setup_script: moduleA/moduleC/setup.sql - EXECUTE IMMEDIATE FROM '/__generated/moduleB/dummy.sql'; - EXECUTE IMMEDIATE FROM '/__generated/dummy.sql'; ''' # --- -# name: test_edit_setup_script_with_exec_imm_sql_symlink.1 +# name: test_edit_setup_script_with_exec_imm_sql.5 ''' - create application role admin; + ===== Contents of: output/deploy/moduleA/moduleC/setup.sql ===== + create application role app_public; EXECUTE IMMEDIATE FROM '/__generated/__generated.sql'; ''' # --- -# name: test_edit_setup_script_with_exec_imm_sql_symlink.2 - 'create application role admin;' -# --- -# name: test_generate_create_sql_ddl_statements_w_all_entries +# name: test_edit_setup_script_with_exec_imm_sql_noop ''' - CREATE OR REPLACE - PROCEDURE DATA.my_function(first int DEFAULT 42) - RETURNS int - LANGUAGE PYTHON - RUNTIME_VERSION=3.11 - IMPORTS=('/path/to/import1.py', '/path/to/import2.zip') - PACKAGES=('package_one==1.0.2', 'package_two') - EXTERNAL_ACCESS_INTEGRATIONS=('integration_one', 'integration_two') - SECRETS=('key1'=secret_one, 'key2'=integration_two) - HANDLER='my_function_handler' - EXECUTE AS OWNER; - + d output/deploy + d output/deploy/__generated + f output/deploy/__generated/__generated.sql + f output/deploy/manifest.yml + d output/deploy/moduleA + d output/deploy/moduleA/moduleC + d output/deploy/moduleA/moduleC/setup.sql ''' # --- -# name: test_generate_create_sql_ddl_statements_w_select_entries +# name: test_edit_setup_script_with_exec_imm_sql_noop.1 ''' - CREATE OR REPLACE - PROCEDURE my_function(first int DEFAULT 42) - RETURNS int - LANGUAGE PYTHON - RUNTIME_VERSION=3.11 - HANDLER='my_function_handler' - EXECUTE AS OWNER; - + ===== Contents of: output/deploy/__generated/__generated.sql ===== + #some text ''' # --- -# name: test_generate_grant_sql_ddl_statements +# name: test_edit_setup_script_with_exec_imm_sql_noop.2 ''' - GRANT USAGE ON PROCEDURE DATA.my_function(int) - TO APPLICATION ROLE APP_ADMIN; - GRANT USAGE ON PROCEDURE DATA.my_function(int) - TO APPLICATION ROLE APP_VIEWER; + ===== Contents of: output/deploy/manifest.yml ===== + manifest_version: 1 + + artifacts: + setup_script: moduleA/moduleC/setup.sql + ''' # --- -# name: test_package_normalization[package_decl0] +# name: test_edit_setup_script_with_exec_imm_sql_symlink ''' - -- stagepath/main.py - CREATE OR REPLACE - PROCEDURE DATA.my_function(first int DEFAULT 42) - RETURNS int - LANGUAGE PYTHON - RUNTIME_VERSION=3.11 - IMPORTS=('/path/to/import1.py', '/path/to/import2.zip', '/stagepath/main.py') - PACKAGES=('snowflake-snowpark-python') - EXTERNAL_ACCESS_INTEGRATIONS=('integration_one', 'integration_two') - SECRETS=('key1'=secret_one, 'key2'=integration_two) - HANDLER='main.my_function_handler' - EXECUTE AS OWNER; - - GRANT USAGE ON PROCEDURE DATA.my_function(int) - TO APPLICATION ROLE APP_ADMIN; - GRANT USAGE ON PROCEDURE DATA.my_function(int) - TO APPLICATION ROLE APP_VIEWER; + d output/deploy + d output/deploy/__generated + f output/deploy/__generated/__generated.sql + f output/deploy/manifest.yml + f output/deploy/setup.sql ''' # --- -# name: test_package_normalization[package_decl10] +# name: test_edit_setup_script_with_exec_imm_sql_symlink.1 ''' - -- stagepath/main.py - CREATE OR REPLACE - PROCEDURE DATA.my_function(first int DEFAULT 42) - RETURNS int - LANGUAGE PYTHON - RUNTIME_VERSION=3.11 - IMPORTS=('/path/to/import1.py', '/path/to/import2.zip', '/stagepath/main.py') - PACKAGES=('snowflake-snowpark-python===0.15.0-rc1') - EXTERNAL_ACCESS_INTEGRATIONS=('integration_one', 'integration_two') - SECRETS=('key1'=secret_one, 'key2'=integration_two) - HANDLER='main.my_function_handler' - EXECUTE AS OWNER; + ===== Contents of: output/deploy/__generated/__generated.sql ===== - GRANT USAGE ON PROCEDURE DATA.my_function(int) - TO APPLICATION ROLE APP_ADMIN; - GRANT USAGE ON PROCEDURE DATA.my_function(int) - TO APPLICATION ROLE APP_VIEWER; + EXECUTE IMMEDIATE FROM '/__generated/moduleB/dummy.sql'; + EXECUTE IMMEDIATE FROM '/__generated/dummy.sql'; ''' # --- -# name: test_package_normalization[package_decl11] +# name: test_edit_setup_script_with_exec_imm_sql_symlink.2 ''' - -- stagepath/main.py - CREATE OR REPLACE - PROCEDURE DATA.my_function(first int DEFAULT 42) - RETURNS int - LANGUAGE PYTHON - RUNTIME_VERSION=3.11 - IMPORTS=('/path/to/import1.py', '/path/to/import2.zip', '/stagepath/main.py') - PACKAGES=('snowflake-snowpark-python===0.15.0-rc1') - EXTERNAL_ACCESS_INTEGRATIONS=('integration_one', 'integration_two') - SECRETS=('key1'=secret_one, 'key2'=integration_two) - HANDLER='main.my_function_handler' - EXECUTE AS OWNER; + ===== Contents of: output/deploy/manifest.yml ===== + manifest_version: 1 + + artifacts: + setup_script: setup.sql - GRANT USAGE ON PROCEDURE DATA.my_function(int) - TO APPLICATION ROLE APP_ADMIN; - GRANT USAGE ON PROCEDURE DATA.my_function(int) - TO APPLICATION ROLE APP_VIEWER; ''' # --- -# name: test_package_normalization[package_decl12] +# name: test_edit_setup_script_with_exec_imm_sql_symlink.3 ''' - -- stagepath/main.py - CREATE OR REPLACE - PROCEDURE DATA.my_function(first int DEFAULT 42) - RETURNS int - LANGUAGE PYTHON - RUNTIME_VERSION=3.11 - IMPORTS=('/path/to/import1.py', '/path/to/import2.zip', '/stagepath/main.py') - PACKAGES=('snowflake-snowpark-python<<0.15.0', 'snowflake-snowpark-python') - EXTERNAL_ACCESS_INTEGRATIONS=('integration_one', 'integration_two') - SECRETS=('key1'=secret_one, 'key2'=integration_two) - HANDLER='main.my_function_handler' - EXECUTE AS OWNER; + ===== Contents of: output/deploy/setup.sql ===== + create application role admin; + EXECUTE IMMEDIATE FROM '/__generated/__generated.sql'; - GRANT USAGE ON PROCEDURE DATA.my_function(int) - TO APPLICATION ROLE APP_ADMIN; - GRANT USAGE ON PROCEDURE DATA.my_function(int) - TO APPLICATION ROLE APP_VIEWER; ''' # --- -# name: test_package_normalization[package_decl13] +# name: test_generate_create_sql_ddl_statements_w_all_entries ''' - -- stagepath/main.py CREATE OR REPLACE PROCEDURE DATA.my_function(first int DEFAULT 42) RETURNS int LANGUAGE PYTHON RUNTIME_VERSION=3.11 - IMPORTS=('/path/to/import1.py', '/path/to/import2.zip', '/stagepath/main.py') - PACKAGES=('snowflake-snowpark-python2', 'snowflake-snowpark-python') + IMPORTS=('/path/to/import1.py', '/path/to/import2.zip') + PACKAGES=('package_one==1.0.2', 'package_two') EXTERNAL_ACCESS_INTEGRATIONS=('integration_one', 'integration_two') SECRETS=('key1'=secret_one, 'key2'=integration_two) - HANDLER='main.my_function_handler' + HANDLER='my_function_handler' EXECUTE AS OWNER; - GRANT USAGE ON PROCEDURE DATA.my_function(int) - TO APPLICATION ROLE APP_ADMIN; - GRANT USAGE ON PROCEDURE DATA.my_function(int) - TO APPLICATION ROLE APP_VIEWER; ''' # --- -# name: test_package_normalization[package_decl14] +# name: test_generate_create_sql_ddl_statements_w_select_entries ''' - -- stagepath/main.py CREATE OR REPLACE - PROCEDURE DATA.my_function(first int DEFAULT 42) + PROCEDURE my_function(first int DEFAULT 42) RETURNS int LANGUAGE PYTHON RUNTIME_VERSION=3.11 - IMPORTS=('/path/to/import1.py', '/path/to/import2.zip', '/stagepath/main.py') - PACKAGES=('snowflake-snowpark-python == 0.15.0') - EXTERNAL_ACCESS_INTEGRATIONS=('integration_one', 'integration_two') - SECRETS=('key1'=secret_one, 'key2'=integration_two) - HANDLER='main.my_function_handler' + HANDLER='my_function_handler' EXECUTE AS OWNER; - GRANT USAGE ON PROCEDURE DATA.my_function(int) - TO APPLICATION ROLE APP_ADMIN; - GRANT USAGE ON PROCEDURE DATA.my_function(int) - TO APPLICATION ROLE APP_VIEWER; ''' # --- -# name: test_package_normalization[package_decl1] +# name: test_generate_grant_sql_ddl_statements ''' - -- stagepath/main.py - CREATE OR REPLACE - PROCEDURE DATA.my_function(first int DEFAULT 42) - RETURNS int - LANGUAGE PYTHON - RUNTIME_VERSION=3.11 - IMPORTS=('/path/to/import1.py', '/path/to/import2.zip', '/stagepath/main.py') - PACKAGES=('snowflake-snowpark-python') - EXTERNAL_ACCESS_INTEGRATIONS=('integration_one', 'integration_two') - SECRETS=('key1'=secret_one, 'key2'=integration_two) - HANDLER='main.my_function_handler' - EXECUTE AS OWNER; - GRANT USAGE ON PROCEDURE DATA.my_function(int) TO APPLICATION ROLE APP_ADMIN; GRANT USAGE ON PROCEDURE DATA.my_function(int) TO APPLICATION ROLE APP_VIEWER; ''' # --- -# name: test_package_normalization[package_decl2] +# name: test_process_no_collected_functions ''' - -- stagepath/main.py - CREATE OR REPLACE - PROCEDURE DATA.my_function(first int DEFAULT 42) - RETURNS int - LANGUAGE PYTHON - RUNTIME_VERSION=3.11 - IMPORTS=('/path/to/import1.py', '/path/to/import2.zip', '/stagepath/main.py') - PACKAGES=('other-package', 'snowflake-snowpark-python') - EXTERNAL_ACCESS_INTEGRATIONS=('integration_one', 'integration_two') - SECRETS=('key1'=secret_one, 'key2'=integration_two) - HANDLER='main.my_function_handler' - EXECUTE AS OWNER; - - GRANT USAGE ON PROCEDURE DATA.my_function(int) - TO APPLICATION ROLE APP_ADMIN; - GRANT USAGE ON PROCEDURE DATA.my_function(int) - TO APPLICATION ROLE APP_VIEWER; + d output/deploy + f output/deploy/manifest.yml + d output/deploy/moduleA + d output/deploy/moduleA/moduleC + f output/deploy/moduleA/moduleC/setup.sql + d output/deploy/stagepath + f output/deploy/stagepath/data.py + d output/deploy/stagepath/extra_import1.zip + d output/deploy/stagepath/extra_import2.zip + f output/deploy/stagepath/main.py ''' # --- -# name: test_package_normalization[package_decl3] +# name: test_process_no_collected_functions.1 ''' - -- stagepath/main.py - CREATE OR REPLACE - PROCEDURE DATA.my_function(first int DEFAULT 42) - RETURNS int - LANGUAGE PYTHON - RUNTIME_VERSION=3.11 - IMPORTS=('/path/to/import1.py', '/path/to/import2.zip', '/stagepath/main.py') - PACKAGES=('snowflake-snowpark-python==0.15.0') - EXTERNAL_ACCESS_INTEGRATIONS=('integration_one', 'integration_two') - SECRETS=('key1'=secret_one, 'key2'=integration_two) - HANDLER='main.my_function_handler' - EXECUTE AS OWNER; + ===== Contents of: output/deploy/manifest.yml ===== + manifest_version: 1 + + artifacts: + setup_script: moduleA/moduleC/setup.sql - GRANT USAGE ON PROCEDURE DATA.my_function(int) - TO APPLICATION ROLE APP_ADMIN; - GRANT USAGE ON PROCEDURE DATA.my_function(int) - TO APPLICATION ROLE APP_VIEWER; ''' # --- -# name: test_package_normalization[package_decl4] +# name: test_process_no_collected_functions.2 ''' - -- stagepath/main.py - CREATE OR REPLACE - PROCEDURE DATA.my_function(first int DEFAULT 42) - RETURNS int - LANGUAGE PYTHON - RUNTIME_VERSION=3.11 - IMPORTS=('/path/to/import1.py', '/path/to/import2.zip', '/stagepath/main.py') - PACKAGES=('snowflake-snowpark-python==0.15.0') - EXTERNAL_ACCESS_INTEGRATIONS=('integration_one', 'integration_two') - SECRETS=('key1'=secret_one, 'key2'=integration_two) - HANDLER='main.my_function_handler' - EXECUTE AS OWNER; - - GRANT USAGE ON PROCEDURE DATA.my_function(int) - TO APPLICATION ROLE APP_ADMIN; - GRANT USAGE ON PROCEDURE DATA.my_function(int) - TO APPLICATION ROLE APP_VIEWER; + ===== Contents of: output/deploy/moduleA/moduleC/setup.sql ===== + create application role app_public; ''' # --- -# name: test_package_normalization[package_decl5] +# name: test_process_no_collected_functions.3 ''' - -- stagepath/main.py - CREATE OR REPLACE - PROCEDURE DATA.my_function(first int DEFAULT 42) - RETURNS int - LANGUAGE PYTHON - RUNTIME_VERSION=3.11 - IMPORTS=('/path/to/import1.py', '/path/to/import2.zip', '/stagepath/main.py') - PACKAGES=('snowflake-snowpark-python<0.16.0') - EXTERNAL_ACCESS_INTEGRATIONS=('integration_one', 'integration_two') - SECRETS=('key1'=secret_one, 'key2'=integration_two) - HANDLER='main.my_function_handler' - EXECUTE AS OWNER; + ===== Contents of: output/deploy/stagepath/data.py ===== + # this is a file - GRANT USAGE ON PROCEDURE DATA.my_function(int) - TO APPLICATION ROLE APP_ADMIN; - GRANT USAGE ON PROCEDURE DATA.my_function(int) - TO APPLICATION ROLE APP_VIEWER; ''' # --- -# name: test_package_normalization[package_decl6] +# name: test_process_no_collected_functions.4 ''' - -- stagepath/main.py - CREATE OR REPLACE - PROCEDURE DATA.my_function(first int DEFAULT 42) - RETURNS int - LANGUAGE PYTHON - RUNTIME_VERSION=3.11 - IMPORTS=('/path/to/import1.py', '/path/to/import2.zip', '/stagepath/main.py') - PACKAGES=('snowflake-snowpark-python<=0.17.0') - EXTERNAL_ACCESS_INTEGRATIONS=('integration_one', 'integration_two') - SECRETS=('key1'=secret_one, 'key2'=integration_two) - HANDLER='main.my_function_handler' - EXECUTE AS OWNER; + ===== Contents of: output/deploy/stagepath/main.py ===== + # this is a file - GRANT USAGE ON PROCEDURE DATA.my_function(int) - TO APPLICATION ROLE APP_ADMIN; - GRANT USAGE ON PROCEDURE DATA.my_function(int) - TO APPLICATION ROLE APP_VIEWER; ''' # --- -# name: test_package_normalization[package_decl7] +# name: test_process_with_collected_functions ''' - -- stagepath/main.py - CREATE OR REPLACE - PROCEDURE DATA.my_function(first int DEFAULT 42) - RETURNS int - LANGUAGE PYTHON - RUNTIME_VERSION=3.11 - IMPORTS=('/path/to/import1.py', '/path/to/import2.zip', '/stagepath/main.py') - PACKAGES=('snowflake-snowpark-python>=0.14.0') - EXTERNAL_ACCESS_INTEGRATIONS=('integration_one', 'integration_two') - SECRETS=('key1'=secret_one, 'key2'=integration_two) - HANDLER='main.my_function_handler' - EXECUTE AS OWNER; - - GRANT USAGE ON PROCEDURE DATA.my_function(int) - TO APPLICATION ROLE APP_ADMIN; - GRANT USAGE ON PROCEDURE DATA.my_function(int) - TO APPLICATION ROLE APP_VIEWER; + d output/deploy + d output/deploy/__generated + f output/deploy/__generated/__generated.sql + d output/deploy/__generated/stagepath + f output/deploy/__generated/stagepath/data.sql + f output/deploy/__generated/stagepath/main.sql + f output/deploy/manifest.yml + d output/deploy/moduleA + d output/deploy/moduleA/moduleC + f output/deploy/moduleA/moduleC/setup.sql + d output/deploy/stagepath + f output/deploy/stagepath/data.py + d output/deploy/stagepath/extra_import1.zip + d output/deploy/stagepath/extra_import2.zip + f output/deploy/stagepath/main.py ''' # --- -# name: test_package_normalization[package_decl8] +# name: test_process_with_collected_functions.1 ''' - -- stagepath/main.py - CREATE OR REPLACE - PROCEDURE DATA.my_function(first int DEFAULT 42) - RETURNS int - LANGUAGE PYTHON - RUNTIME_VERSION=3.11 - IMPORTS=('/path/to/import1.py', '/path/to/import2.zip', '/stagepath/main.py') - PACKAGES=('snowflake-snowpark-python>0.13.0') - EXTERNAL_ACCESS_INTEGRATIONS=('integration_one', 'integration_two') - SECRETS=('key1'=secret_one, 'key2'=integration_two) - HANDLER='main.my_function_handler' - EXECUTE AS OWNER; + ===== Contents of: output/deploy/__generated/__generated.sql ===== - GRANT USAGE ON PROCEDURE DATA.my_function(int) - TO APPLICATION ROLE APP_ADMIN; - GRANT USAGE ON PROCEDURE DATA.my_function(int) - TO APPLICATION ROLE APP_VIEWER; + EXECUTE IMMEDIATE FROM '/__generated/stagepath/main.sql'; + EXECUTE IMMEDIATE FROM '/__generated/stagepath/data.sql'; ''' # --- -# name: test_package_normalization[package_decl9] +# name: test_process_with_collected_functions.2 ''' - -- stagepath/main.py + ===== Contents of: output/deploy/__generated/stagepath/data.sql ===== + -- Generated by the Snowflake CLI from stagepath/data.py + -- DO NOT EDIT CREATE OR REPLACE PROCEDURE DATA.my_function(first int DEFAULT 42) RETURNS int LANGUAGE PYTHON RUNTIME_VERSION=3.11 - IMPORTS=('/path/to/import1.py', '/path/to/import2.zip', '/stagepath/main.py') - PACKAGES=('snowflake-snowpark-python===0.15.0') + IMPORTS=('/', '/stagepath/data.py', '/stagepath/extra_import1.zip', '/stagepath/some_dir_str', '/stagepath/withslash.py', '@dummy_stage_str') + PACKAGES=('package_one==1.0.2', 'package_two', 'snowflake-snowpark-python') EXTERNAL_ACCESS_INTEGRATIONS=('integration_one', 'integration_two') SECRETS=('key1'=secret_one, 'key2'=integration_two) - HANDLER='main.my_function_handler' + HANDLER='data.my_function_handler' EXECUTE AS OWNER; GRANT USAGE ON PROCEDURE DATA.my_function(int) @@ -384,12 +243,11 @@ TO APPLICATION ROLE APP_VIEWER; ''' # --- -# name: test_process_no_collected_functions - '' -# --- -# name: test_process_with_collected_functions +# name: test_process_with_collected_functions.3 ''' - -- stagepath/main.py + ===== Contents of: output/deploy/__generated/stagepath/main.sql ===== + -- Generated by the Snowflake CLI from stagepath/main.py + -- DO NOT EDIT CREATE OR REPLACE PROCEDURE DATA.my_function(first int DEFAULT 42) RETURNS int @@ -402,82 +260,41 @@ HANDLER='main.my_function_handler' EXECUTE AS OWNER; - GRANT USAGE ON PROCEDURE DATA.my_function(int) - TO APPLICATION ROLE APP_ADMIN; - GRANT USAGE ON PROCEDURE DATA.my_function(int) - TO APPLICATION ROLE APP_VIEWER; - -- stagepath/data.py - CREATE OR REPLACE - PROCEDURE DATA.my_function(first int DEFAULT 42) - RETURNS int - LANGUAGE PYTHON - RUNTIME_VERSION=3.11 - IMPORTS=('/', '/stagepath/data.py', '/stagepath/extra_import1.zip', '/stagepath/some_dir_str', '/stagepath/withslash.py', '@dummy_stage_str') - PACKAGES=('package_one==1.0.2', 'package_two', 'snowflake-snowpark-python') - EXTERNAL_ACCESS_INTEGRATIONS=('integration_one', 'integration_two') - SECRETS=('key1'=secret_one, 'key2'=integration_two) - HANDLER='data.my_function_handler' - EXECUTE AS OWNER; - GRANT USAGE ON PROCEDURE DATA.my_function(int) TO APPLICATION ROLE APP_ADMIN; GRANT USAGE ON PROCEDURE DATA.my_function(int) TO APPLICATION ROLE APP_VIEWER; ''' # --- -# name: test_process_with_collected_functions.1 +# name: test_process_with_collected_functions.4 ''' + ===== Contents of: output/deploy/manifest.yml ===== + manifest_version: 1 + + artifacts: + setup_script: moduleA/moduleC/setup.sql - EXECUTE IMMEDIATE FROM '/__generated/stagepath/main.sql'; - EXECUTE IMMEDIATE FROM '/__generated/stagepath/data.sql'; ''' # --- -# name: test_process_with_collected_functions.2 +# name: test_process_with_collected_functions.5 ''' + ===== Contents of: output/deploy/moduleA/moduleC/setup.sql ===== create application role app_public; EXECUTE IMMEDIATE FROM '/__generated/__generated.sql'; ''' # --- -# name: test_process_with_collected_functions.3 +# name: test_process_with_collected_functions.6 ''' + ===== Contents of: output/deploy/stagepath/data.py ===== + # this is a file - CREATE OR REPLACE - PROCEDURE DATA.my_function(first int DEFAULT 42) - RETURNS int - LANGUAGE PYTHON - RUNTIME_VERSION=3.11 - IMPORTS=('/path/to/import1.py', '/path/to/import2.zip', '/stagepath/main.py') - PACKAGES=('package_one==1.0.2', 'package_two', 'snowflake-snowpark-python') - EXTERNAL_ACCESS_INTEGRATIONS=('integration_one', 'integration_two') - SECRETS=('key1'=secret_one, 'key2'=integration_two) - HANDLER='main.my_function_handler' - EXECUTE AS OWNER; - - GRANT USAGE ON PROCEDURE DATA.my_function(int) - TO APPLICATION ROLE APP_ADMIN; - GRANT USAGE ON PROCEDURE DATA.my_function(int) - TO APPLICATION ROLE APP_VIEWER; ''' # --- -# name: test_process_with_collected_functions.4 +# name: test_process_with_collected_functions.7 ''' + ===== Contents of: output/deploy/stagepath/main.py ===== + # this is a file - CREATE OR REPLACE - PROCEDURE DATA.my_function(first int DEFAULT 42) - RETURNS int - LANGUAGE PYTHON - RUNTIME_VERSION=3.11 - IMPORTS=('/', '/stagepath/data.py', '/stagepath/extra_import1.zip', '/stagepath/some_dir_str', '/stagepath/withslash.py', '@dummy_stage_str') - PACKAGES=('package_one==1.0.2', 'package_two', 'snowflake-snowpark-python') - EXTERNAL_ACCESS_INTEGRATIONS=('integration_one', 'integration_two') - SECRETS=('key1'=secret_one, 'key2'=integration_two) - HANDLER='data.my_function_handler' - EXECUTE AS OWNER; - - GRANT USAGE ON PROCEDURE DATA.my_function(int) - TO APPLICATION ROLE APP_ADMIN; - GRANT USAGE ON PROCEDURE DATA.my_function(int) - TO APPLICATION ROLE APP_VIEWER; ''' # --- diff --git a/tests/nativeapp/codegen/snowpark/test_python_processor.py b/tests/nativeapp/codegen/snowpark/test_python_processor.py index 83163de39c..af7a72c906 100644 --- a/tests/nativeapp/codegen/snowpark/test_python_processor.py +++ b/tests/nativeapp/codegen/snowpark/test_python_processor.py @@ -1,9 +1,12 @@ from __future__ import annotations +import contextlib import copy +import os import subprocess from pathlib import Path from textwrap import dedent +from typing import Iterator, List, Set from unittest import mock import pytest @@ -25,6 +28,56 @@ PROJECT_ROOT = Path("/path/to/project") + +# contextlib.chdir isn't available before Python 3.11, so this is an alternative for older versions +@contextlib.contextmanager +def in_cwd(path: Path) -> Iterator[None]: + old_cwd = os.getcwd() + os.chdir(str(path)) + try: + yield + except Exception: + pass + + os.chdir(old_cwd) + + +def stringify(p: Path): + if p.is_dir(): + return f"d {p}" + else: + return f"f {p}" + + +def all_paths_under_dir(root: Path) -> List[Path]: + assert root.is_dir() + + paths: Set[Path] = set() + for subdir, dirs, files in os.walk(root): + subdir_path = Path(subdir) + paths.add(subdir_path) + for d in dirs: + paths.add(subdir_path / d) + for f in files: + paths.add(subdir_path / f) + + return sorted(paths) + + +def assert_dir_snapshot(root: Path, snapshot) -> None: + all_paths = all_paths_under_dir(root) + + # Verify the contents of the directory matches expectations + assert "\n".join([stringify(p) for p in all_paths]) == snapshot + + # Verify that each file under the directory matches expectations + for path in all_paths: + if path.is_file(): + snapshot_contents = f"===== Contents of: {path} =====\n" + snapshot_contents += path.read_text(encoding="utf-8") + assert snapshot_contents == snapshot + + # -------------------------------------------------------- # ------------- _determine_virtual_env ------------------- # -------------------------------------------------------- @@ -194,27 +247,21 @@ def test_edit_setup_script_with_exec_imm_sql(snapshot): } with temp_local_dir(dir_structure=dir_structure) as local_path: - deploy_root = Path(local_path, "output", "deploy") - generated_root = Path(deploy_root, "__generated") - collected_sql_files = [ - Path(generated_root, "moduleB", "dummy.sql"), - Path(generated_root, "dummy.sql"), - ] - - edit_setup_script_with_exec_imm_sql( - collected_sql_files=collected_sql_files, - deploy_root=deploy_root, - generated_root=generated_root, - ) + with in_cwd(local_path): + deploy_root = Path(local_path, "output", "deploy") + generated_root = Path(deploy_root, "__generated") + collected_sql_files = [ + Path(generated_root, "moduleB", "dummy.sql"), + Path(generated_root, "dummy.sql"), + ] - main_file = Path(deploy_root, "__generated", "__generated.sql") - assert main_file.is_file() - with open(main_file, "r") as f: - assert f.read() == snapshot + edit_setup_script_with_exec_imm_sql( + collected_sql_files=collected_sql_files, + deploy_root=deploy_root, + generated_root=generated_root, + ) - setup_file = Path(deploy_root, "moduleA", "moduleC", "setup.sql") - with open(setup_file, "r") as f: - assert f.read() == snapshot + assert_dir_snapshot(deploy_root.relative_to(local_path), snapshot) def test_edit_setup_script_with_exec_imm_sql_noop(snapshot): @@ -233,22 +280,18 @@ def test_edit_setup_script_with_exec_imm_sql_noop(snapshot): } with temp_local_dir(dir_structure=dir_structure) as local_path: - deploy_root = Path(local_path, "output", "deploy") - collected_sql_files = [ - Path(deploy_root, "__generated", "dummy.sql"), - ] - edit_setup_script_with_exec_imm_sql( - collected_sql_files=collected_sql_files, - deploy_root=deploy_root, - generated_root=Path(deploy_root, "__generated"), - ) - main_file = Path(deploy_root, "__generated", "__generated.sql") - assert main_file.is_file() - with open(main_file, "r") as f: - assert f.read() == snapshot + with in_cwd(local_path): + deploy_root = Path(local_path, "output", "deploy") + collected_sql_files = [ + Path(deploy_root, "__generated", "dummy.sql"), + ] + edit_setup_script_with_exec_imm_sql( + collected_sql_files=collected_sql_files, + deploy_root=deploy_root, + generated_root=Path(deploy_root, "__generated"), + ) - dummy_file = Path(deploy_root, "__generated", "dummy.sql") - assert not dummy_file.exists() + assert_dir_snapshot(deploy_root.relative_to(local_path), snapshot) def test_edit_setup_script_with_exec_imm_sql_symlink(snapshot): @@ -266,35 +309,24 @@ def test_edit_setup_script_with_exec_imm_sql_symlink(snapshot): } with temp_local_dir(dir_structure=dir_structure) as local_path: - deploy_root = Path(local_path, "output", "deploy") - - deploy_root_setup_script = Path(deploy_root, "setup.sql") - deploy_root_setup_script.symlink_to(Path(local_path, "setup.sql")) - - generated_root = Path(deploy_root, "__generated") - collected_sql_files = [ - Path(generated_root, "moduleB", "dummy.sql"), - Path(generated_root, "dummy.sql"), - ] - edit_setup_script_with_exec_imm_sql( - collected_sql_files=collected_sql_files, - deploy_root=deploy_root, - generated_root=Path(deploy_root, "__generated"), - ) - - main_file = Path(deploy_root, "__generated", "__generated.sql") - assert main_file.is_file() - with open(main_file, "r") as f: - assert f.read() == snapshot + with in_cwd(local_path): + deploy_root = Path(local_path, "output", "deploy") - # Should not be a symlink anymore - assert not deploy_root_setup_script.is_symlink() + deploy_root_setup_script = Path(deploy_root, "setup.sql") + deploy_root_setup_script.symlink_to(Path(local_path, "setup.sql")) - with open(deploy_root_setup_script, "r") as f: - assert f.read() == snapshot + generated_root = Path(deploy_root, "__generated") + collected_sql_files = [ + Path(generated_root, "moduleB", "dummy.sql"), + Path(generated_root, "dummy.sql"), + ] + edit_setup_script_with_exec_imm_sql( + collected_sql_files=collected_sql_files, + deploy_root=deploy_root, + generated_root=Path(deploy_root, "__generated"), + ) - with open(Path(local_path, "setup.sql"), "r") as f: - assert f.read() == snapshot + assert_dir_snapshot(deploy_root.relative_to(local_path), snapshot) # -------------------------------------------------------- @@ -329,25 +361,25 @@ def test_process_no_collected_functions( mock_sandbox, native_app_project_instance, snapshot ): with temp_local_dir(minimal_dir_structure) as local_path: - native_app_project_instance.native_app.artifacts = [ - {"src": "a/b/c/*.py", "dest": "stagepath/", "processors": ["SNOWPARK"]} - ] - mock_sandbox.side_effect = [None, []] - deploy_root = Path(local_path, "output/deploy") - generated_root = Path(deploy_root, "__generated") - output = SnowparkAnnotationProcessor( - project_definition=native_app_project_instance, - project_root=local_path, - deploy_root=deploy_root, - generated_root=generated_root, - ).process( - artifact_to_process=native_app_project_instance.native_app.artifacts[0], - processor_mapping=ProcessorMapping(name="SNOWPARK"), - write_to_sql=False, # For testing - ) - assert output == snapshot - assert not Path(generated_root, "stagepath/main.sql").exists() - assert not Path(generated_root, "stagepath/data.sql").exists() + with in_cwd(local_path): + native_app_project_instance.native_app.artifacts = [ + {"src": "a/b/c/*.py", "dest": "stagepath/", "processors": ["SNOWPARK"]} + ] + mock_sandbox.side_effect = [None, []] + deploy_root = Path(local_path, "output/deploy") + generated_root = Path(deploy_root, "__generated") + SnowparkAnnotationProcessor( + project_definition=native_app_project_instance, + project_root=local_path, + deploy_root=deploy_root, + generated_root=generated_root, + ).process( + artifact_to_process=native_app_project_instance.native_app.artifacts[0], + processor_mapping=ProcessorMapping(name="SNOWPARK"), + write_to_sql=False, # For testing + ) + + assert_dir_snapshot(deploy_root.relative_to(local_path), snapshot) @mock.patch( @@ -361,63 +393,44 @@ def test_process_with_collected_functions( ): with temp_local_dir(minimal_dir_structure) as local_path: - imports_variation = copy.deepcopy(native_app_extension_function_raw_data) - imports_variation["imports"] = [ - "@dummy_stage_str", - "/", - "stagepath/extra_import1.zip", - "stagepath/some_dir_str", - "/stagepath/withslash.py", - "stagepath/data.py", - ] - processor_mapping = ProcessorMapping( - name="snowpark", - properties={"env": {"type": "conda", "name": "snowpark-dev"}}, - ) - native_app_project_instance.native_app.artifacts = [ - { - "src": "a/b/c/*.py", - "dest": "stagepath/", - "processors": [processor_mapping], - } - ] - mock_sandbox.side_effect = [ - [native_app_extension_function_raw_data], - [imports_variation], - ] - deploy_root = Path(local_path, "output/deploy") - generated_root = Path(deploy_root, "__generated") - output = SnowparkAnnotationProcessor( - project_definition=native_app_project_instance, - project_root=local_path, - deploy_root=deploy_root, - generated_root=generated_root, - ).process( - artifact_to_process=native_app_project_instance.native_app.artifacts[0], - processor_mapping=processor_mapping, - ) - assert output == snapshot - - main_file = Path(deploy_root, "__generated", "__generated.sql") - assert main_file.is_file() - with open(main_file, "r") as f: - assert f.read() == snapshot - - # Should have execute imm - with open( - Path(local_path, "output/deploy/moduleA/moduleC/setup.sql"), "r" - ) as f: - assert f.read() == snapshot - - with open( - Path(local_path, "output/deploy/__generated/stagepath/main.sql"), "r" - ) as f: - assert f.read() == snapshot - - with open( - Path(local_path, "output/deploy/__generated/stagepath/data.sql"), "r" - ) as f: - assert f.read() == snapshot + with in_cwd(local_path): + imports_variation = copy.deepcopy(native_app_extension_function_raw_data) + imports_variation["imports"] = [ + "@dummy_stage_str", + "/", + "stagepath/extra_import1.zip", + "stagepath/some_dir_str", + "/stagepath/withslash.py", + "stagepath/data.py", + ] + processor_mapping = ProcessorMapping( + name="snowpark", + properties={"env": {"type": "conda", "name": "snowpark-dev"}}, + ) + native_app_project_instance.native_app.artifacts = [ + { + "src": "a/b/c/*.py", + "dest": "stagepath/", + "processors": [processor_mapping], + } + ] + mock_sandbox.side_effect = [ + [native_app_extension_function_raw_data], + [imports_variation], + ] + deploy_root = Path(local_path, "output/deploy") + generated_root = Path(deploy_root, "__generated") + SnowparkAnnotationProcessor( + project_definition=native_app_project_instance, + project_root=local_path, + deploy_root=deploy_root, + generated_root=generated_root, + ).process( + artifact_to_process=native_app_project_instance.native_app.artifacts[0], + processor_mapping=processor_mapping, + ) + + assert_dir_snapshot(deploy_root.relative_to(local_path), snapshot) @pytest.mark.parametrize( @@ -452,28 +465,31 @@ def test_package_normalization( ): with temp_local_dir(minimal_dir_structure) as local_path: - processor_mapping = ProcessorMapping( - name="snowpark", - ) - native_app_project_instance.native_app.artifacts = [ - { - "src": "a/b/c/main.py", - "dest": "stagepath/", - "processors": [processor_mapping], - } - ] - native_app_extension_function_raw_data["packages"] = package_decl - mock_sandbox.side_effect = [[native_app_extension_function_raw_data]] - deploy_root = Path(local_path, "output/deploy") - generated_root = Path(deploy_root, "__generated") - output = SnowparkAnnotationProcessor( - project_definition=native_app_project_instance, - project_root=local_path, - deploy_root=deploy_root, - generated_root=generated_root, - ).process( - artifact_to_process=native_app_project_instance.native_app.artifacts[0], - processor_mapping=processor_mapping, - ) - - assert output == snapshot + with in_cwd(local_path): + processor_mapping = ProcessorMapping( + name="snowpark", + ) + native_app_project_instance.native_app.artifacts = [ + { + "src": "a/b/c/main.py", + "dest": "stagepath/", + "processors": [processor_mapping], + } + ] + native_app_extension_function_raw_data["packages"] = package_decl + mock_sandbox.side_effect = [[native_app_extension_function_raw_data]] + deploy_root = Path(local_path, "output/deploy") + generated_root = Path(deploy_root, "__generated") + SnowparkAnnotationProcessor( + project_definition=native_app_project_instance, + project_root=local_path, + deploy_root=deploy_root, + generated_root=generated_root, + ).process( + artifact_to_process=native_app_project_instance.native_app.artifacts[0], + processor_mapping=processor_mapping, + ) + + dest_file = generated_root / "stagepath" / "main.py" + assert dest_file.is_file() + assert dest_file.read_text(encoding="utf-8") == snapshot