Skip to content

Commit

Permalink
Add bundle root for generation (#1262)
Browse files Browse the repository at this point in the history
  • Loading branch information
sfc-gh-bdufour authored Jun 28, 2024
1 parent 2def92c commit 913d26d
Show file tree
Hide file tree
Showing 12 changed files with 492 additions and 199 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,12 @@ class NativeApp(UpdatableModel):
artifacts: List[Union[PathMapping, str]] = Field(
title="List of file source and destination pairs to add to the deploy root",
)
bundle_root: Optional[str] = Field(
title="Folder at the root of your project where artifacts necessary to perform the bundle step are stored.",
default="output/bundle/",
)
deploy_root: Optional[str] = Field(
title="Folder at the root of your project where the build step copies the artifacts.",
title="Folder at the root of your project where the bundle step copies the artifacts.",
default="output/deploy/",
)
generated_root: Optional[str] = Field(
Expand Down
14 changes: 3 additions & 11 deletions src/snowflake/cli/plugins/nativeapp/codegen/artifact_processor.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,15 +15,14 @@
from __future__ import annotations

from abc import ABC, abstractmethod
from pathlib import Path
from typing import Optional

from click import ClickException
from snowflake.cli.api.project.schemas.native_app.native_app import NativeApp
from snowflake.cli.api.project.schemas.native_app.path_mapping import (
PathMapping,
ProcessorMapping,
)
from snowflake.cli.plugins.nativeapp.project_model import NativeAppProjectModel


class UnsupportedArtifactProcessorError(ClickException):
Expand All @@ -36,18 +35,11 @@ def __init__(self, processor_name: str):


class ArtifactProcessor(ABC):
@abstractmethod
def __init__(
self,
project_definition: NativeApp,
project_root: Path,
deploy_root: Path,
generated_root: Path,
**kwargs,
na_project: NativeAppProjectModel,
) -> None:
assert project_root.is_absolute()
assert deploy_root.is_absolute()
assert generated_root.is_absolute()
self._na_project = na_project

@abstractmethod
def process(
Expand Down
30 changes: 6 additions & 24 deletions src/snowflake/cli/plugins/nativeapp/codegen/compiler.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,23 +14,20 @@

from __future__ import annotations

from pathlib import Path
from typing import Dict, Optional

from snowflake.cli.api.console import cli_console as cc
from snowflake.cli.api.project.schemas.native_app.native_app import NativeApp
from snowflake.cli.api.project.schemas.native_app.path_mapping import (
PathMapping,
ProcessorMapping,
)
from snowflake.cli.plugins.nativeapp.artifacts import resolve_without_follow
from snowflake.cli.plugins.nativeapp.codegen.artifact_processor import (
ArtifactProcessor,
UnsupportedArtifactProcessorError,
)
from snowflake.cli.plugins.nativeapp.codegen.snowpark.python_processor import (
SnowparkAnnotationProcessor,
)
from snowflake.cli.plugins.nativeapp.project_model import NativeAppProjectModel

SNOWPARK_PROCESSOR = "snowpark"

Expand All @@ -46,21 +43,9 @@ class NativeAppCompiler:

def __init__(
self,
project_definition: NativeApp,
project_root: Path,
deploy_root: Path,
generated_root: Path,
na_project: NativeAppProjectModel,
):
self.project_definition = project_definition
self.project_root = project_root
self.deploy_root = deploy_root
self.generated_root = generated_root

self.artifacts = [
artifact
for artifact in project_definition.artifacts
if isinstance(artifact, PathMapping)
]
self._na_project = na_project
# dictionary of all processors created and shared between different artifact objects.
self.cached_processors: Dict[str, ArtifactProcessor] = {}

Expand All @@ -70,15 +55,15 @@ def compile_artifacts(self):
May have side-effects on the filesystem by either directly editing source files or the deploy root.
"""
should_proceed = False
for artifact in self.artifacts:
for artifact in self._na_project.artifacts:
if artifact.processors:
should_proceed = True
break
if not should_proceed:
return

with cc.phase("Invoking artifact processors"):
for artifact in self.artifacts:
for artifact in self._na_project.artifacts:
for processor in artifact.processors:
artifact_processor = self._try_create_processor(
processor_mapping=processor,
Expand Down Expand Up @@ -107,10 +92,7 @@ def _try_create_processor(
return curr_processor
else:
curr_processor = SnowparkAnnotationProcessor(
project_definition=self.project_definition,
project_root=resolve_without_follow(self.project_root),
deploy_root=resolve_without_follow(self.deploy_root),
generated_root=resolve_without_follow(self.generated_root),
na_project=self._na_project,
)
self.cached_processors[SNOWPARK_PROCESSOR] = curr_processor
return curr_processor
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@
from click import ClickException
from snowflake.cli.api.console import cli_console as cc
from snowflake.cli.api.project.errors import SchemaValidationError
from snowflake.cli.api.project.schemas.native_app.native_app import NativeApp
from snowflake.cli.api.project.schemas.native_app.path_mapping import (
PathMapping,
ProcessorMapping,
Expand Down Expand Up @@ -52,6 +51,7 @@
ExtensionFunctionTypeEnum,
NativeAppExtensionFunction,
)
from snowflake.cli.plugins.nativeapp.project_model import NativeAppProjectModel
from snowflake.cli.plugins.stage.diff import to_stage_path

DEFAULT_TIMEOUT = 30
Expand Down Expand Up @@ -163,25 +163,18 @@ class SnowparkAnnotationProcessor(ArtifactProcessor):

def __init__(
self,
project_definition: NativeApp,
project_root: Path,
deploy_root: Path,
generated_root: Path,
na_project: NativeAppProjectModel,
):
super().__init__(
project_definition=project_definition,
project_root=project_root,
deploy_root=deploy_root,
generated_root=generated_root,
)
self.project_definition = project_definition
self.project_root = project_root
self.deploy_root = deploy_root
self.generated_root = generated_root
super().__init__(na_project=na_project)

if self.generated_root.exists():
assert self._na_project.bundle_root.is_absolute()
assert self._na_project.deploy_root.is_absolute()
assert self._na_project.generated_root.is_absolute()
assert self._na_project.project_root.is_absolute()

if self._na_project.generated_root.exists():
raise ClickException(
f"Path {self.generated_root} already exists. Please choose a different name for your generated directory in the project definition file."
f"Path {self._na_project.generated_root} already exists. Please choose a different name for your generated directory in the project definition file."
)

def process(
Expand All @@ -196,7 +189,8 @@ def process(
"""

bundle_map = BundleMap(
project_root=self.project_root, deploy_root=self.deploy_root
project_root=self._na_project.project_root,
deploy_root=self._na_project.deploy_root,
)
bundle_map.add(artifact_to_process)

Expand Down Expand Up @@ -242,7 +236,7 @@ def process(
edit_setup_script_with_exec_imm_sql(
collected_sql_files=collected_sql_files,
deploy_root=bundle_map.deploy_root(),
generated_root=self.generated_root,
generated_root=self._na_project.generated_root,
)

def _normalize_imports(
Expand Down Expand Up @@ -323,7 +317,7 @@ def collect_extension_functions(
self, bundle_map: BundleMap, processor_mapping: Optional[ProcessorMapping]
) -> Dict[Path, List[NativeAppExtensionFunction]]:
kwargs = (
_determine_virtual_env(self.project_root, processor_mapping)
_determine_virtual_env(self._na_project.project_root, processor_mapping)
if processor_mapping is not None
else {}
)
Expand All @@ -342,7 +336,7 @@ def collect_extension_functions(
)
collected_extension_function_json = _execute_in_sandbox(
py_file=str(dest_file.resolve()),
deploy_root=self.deploy_root,
deploy_root=self._na_project.deploy_root,
kwargs=kwargs,
)

Expand Down Expand Up @@ -373,8 +367,10 @@ def generate_new_sql_file_name(self, py_file: Path) -> Path:
"""
Generates a SQL filename for the generated root from the python file, and creates its parent directories.
"""
relative_py_file = py_file.relative_to(self.deploy_root)
sql_file = Path(self.generated_root, relative_py_file.with_suffix(".sql"))
relative_py_file = py_file.relative_to(self._na_project.deploy_root)
sql_file = Path(
self._na_project.generated_root, relative_py_file.with_suffix(".sql")
)
if sql_file.exists():
cc.warning(
f"""\
Expand Down
Loading

0 comments on commit 913d26d

Please sign in to comment.