Merge pull request #2 from ccpgames/feature/file_embed

Version 0.2.0 - Embedding Extension & Base64 Filter update

CHANGELOG.md

@@ -6,10 +6,58 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 
-## [0.1.0] - UNRELEASED
+## [0.2.0] - 2024-05-28
+
+Embedding Extension & Base64 Filter update
+
+### Added
+
+- The `EmbedExtension` to the Jinja2 environment used that enables using the 
+  `{% embed 'my_file.txt' %}` to embed the requested file "as-is" (so not 
+  parsed as a template or anything like that) into templates wholesale with 
+  these optional arguments:
+  - `indent` (default: 0) to indent each line of the embedded file by a certain
+    number of spaces, e.g. for embedding into YAML files, where indent is 
+    syntactically important
+  - `alviss` (default: `False`) to run the embedded file though the Alviss 
+    "render_loader" before embedding the results
+  - `env` (default: `False`) to make the Alviss parser render `${__ENV__:...}` 
+    expressions
+  - `fidelius` (default: `False`) to make the Alviss parser render 
+    `${__FID__:...}` expressions
+- The main use case here was considered things like creating Kubernetes File 
+  Config maps where e.g. config file content is embedded wholesale into the 
+  manifest's YAML
+- The `base64` filter to the Jinja2 environment used to quicly encode context 
+  variables as base64 encoded strings
+  - The main use case here was considered things like rendering of Kubernetes 
+    Secret manifests where keys need to be base64 encoded  
+- A bunch of "proper" unittests for the new embedding and filter functions
+
+### Changed
+
+- The `FileTemplate` now uses the Jinja2 environment's `FileSystemLoader`, 
+  pre-seeded with search paths of the current working directory (`os.getcwd()`) 
+  as well as the root directory of the script being executed (`sys.argv[0]`) 
+  plus whatever paths are defined in the `STENCIL_TEMPLATE_PATH` environment 
+  variable, if any (multiple paths seperated by `;`)
+
+
+## [0.1.0] - 2024-05-24
 
 Initial release
 
 ### Added
 
-- Everything!
+- The project in its entirety
+- Template types:
+  - `FileTemplate`
+  - `StringTemplate`
+- Context types:
+  - `DictContext`
+  - `KeyWordArgumentContext`
+  - `AlvissContext`
+- Renderer Types
+  - `StringRenderer`
+  - `StdOutRenderer`
+  - `FileRenderer`

ccpstencil/__init__.py

@@ -1,4 +1,4 @@
-__version__ = '0.1.0'
+__version__ = '0.2.0'
 
 __author__ = 'Thordur Matthiasson <[email protected]>'
 __license__ = 'MIT License'

ccpstencil/context/_alviss.py

@@ -5,13 +5,14 @@
 from ccpstencil.structs import *
 from alviss import quickloader
 from ccptools.tpu import iters
+from pathlib import Path
 import logging
 log = logging.getLogger(__file__)
 
 
 class AlvissContext(IContext):
-    def __init__(self, file_name: str, **kwargs):
-        self._file_name = file_name
+    def __init__(self, file_name: Union[str, Path], **kwargs):
+        self._file_name = str(file_name)
         self._data = quickloader.autoload(file_name)
         self._update_map: Dict[Tuple[str], Any] = {}
 

ccpstencil/jinjaext/extensions/__init__.py

@@ -0,0 +1 @@
+from ._embed import *

ccpstencil/jinjaext/extensions/_embed.py

@@ -0,0 +1,52 @@
+__all__ = [
+    'EmbedExtension',
+]
+
+import os
+from jinja2 import nodes, TemplateSyntaxError
+from jinja2.ext import Extension
+from ccptools.structs import *
+
+class EmbedExtension(Extension):
+    tags = {'embed'}
+
+    def __init__(self, environment):
+        super().__init__(environment)
+        environment.extend(stencil_renderer=None)
+
+    def parse(self, parser):
+        lineno = next(parser.stream).lineno
+        args = [parser.parse_expression()]
+        source_file = parser.filename
+        kwargs = [nodes.Keyword('source_file', nodes.Const(source_file))]
+        while parser.stream.skip_if('comma'):
+            key = parser.stream.expect('name').value
+            parser.stream.expect('assign')
+            value = parser.parse_expression()
+            kwargs.append(nodes.Keyword(key, value))
+
+        return nodes.CallBlock(self.call_method('_render_embed', args, kwargs), [], [], []).set_lineno(lineno)
+
+    def _render_embed(self, file_path, source_file: Optional[str] = None, indent: int = 0,
+                      alviss: bool = False, env: bool = False,  caller=None, **kwargs):
+        # file_path = os.path.abspath(file_path)
+
+        c = caller()
+
+        caller_source = c.strip()
+
+        # Check if file_path is a variable in the context
+        if hasattr(file_path, '__call__'):
+            file_path = file_path()
+
+        content = self.environment.stencil_renderer.get_embed(file_path, alviss=alviss, env=env)
+
+        # Detect the current line's indentation level
+        if indent:
+            indent_str = ' '*indent
+        else:
+            indent_str = ''
+
+        joiner = f'{indent_str}'
+
+        return f'{indent_str}{joiner.join(content.splitlines(True))}\n'

ccpstencil/jinjaext/filters/__init__.py

@@ -0,0 +1 @@
+from ._base64 import *

ccpstencil/jinjaext/filters/_base64.py

@@ -0,0 +1,9 @@
+__all__ = [
+    'base64',
+]
+
+import base64 as _base64
+
+
+def base64(value: str) -> str:
+    return _base64.encodebytes(value.encode('utf-8')).decode('utf-8').strip()

ccpstencil/renderer/_base.py

@@ -2,9 +2,13 @@
     '_BaseRenderer',
 ]
 
+import os
+import sys
+from pathlib import Path
+from ccptools.tpu import strimp
 from ccpstencil.structs import *
-
-import jinja2
+from jinja2.ext import Extension
+from alviss import quickloader
 
 import logging
 log = logging.getLogger(__file__)
@@ -27,14 +31,52 @@ def __init__(self, context: Optional[IContext] = None, template: Optional[ITempl
             self.template = template
         if kwargs:
             log.warning(f'Unrecognized kwargs for {self.__class__.__name__}: {kwargs}')
+        self._search_paths: List[Path] = []
+        self._load_search_paths()
         self._env: jinja2.Environment = self._make_environment()
+        self._load_filters()
 
     def _make_environment(self) -> jinja2.Environment:
-        return jinja2.Environment(
+        env = jinja2.Environment(
             lstrip_blocks=True,
             trim_blocks=True,
-            undefined=jinja2.ChainableUndefined
+            keep_trailing_newline=True,
+            undefined=jinja2.ChainableUndefined,
+            extensions=self._get_extensions(),
+            loader=jinja2.FileSystemLoader([str(p) for p in self._search_paths])
         )
+        env.stencil_renderer = self
+        return env
+
+    def _get_extensions(self) -> List[Type[Extension]]:
+        buffer_list = []
+        extension_module = strimp.get_module('ccpstencil.jinjaext.extensions')
+        for name, item in extension_module.__dict__.items():
+            if name.startswith('_'):
+                continue
+            if issubclass(item, Extension):
+                buffer_list.append(item)
+        return buffer_list
+
+    def _load_filters(self):
+        filter_module = strimp.get_module('ccpstencil.jinjaext.filters')
+        for name, item in filter_module.__dict__.items():
+            if name.startswith('_'):
+                continue
+            if isinstance(item, Callable):
+                self.jinja_environment.filters[name] = item
+
+    def _load_search_paths(self):
+        self._search_paths.append(Path(os.getcwd()).absolute())  # Current Working Directory!
+        script_path = Path(sys.argv[0]).parent.absolute()
+        if script_path not in self._search_paths:
+            self._search_paths.append(script_path)
+        stp = os.environ.get('STENCIL_TEMPLATE_PATH', None)  # Extra paths! :D
+        if stp:
+            for tp in stp.split(';'):
+                tpp = Path(tp).absolute()
+                if tpp not in self._search_paths:
+                    self._search_paths.append(tpp)
 
     def _pre_flight(self):
         if not self.template:
@@ -95,3 +137,41 @@ def render(self):
     def jinja_environment(self) -> jinja2.Environment:
         return self._env
 
+    def is_template_loadable(self, template_name: str) -> bool:
+        try:
+            self.jinja_environment.get_template(template_name)
+            return True
+        except jinja2.TemplateNotFound:
+            return False
+
+    def get_embed(self, file_path: str, source_file: Optional[str] = None,
+                  alviss: bool = False, env: bool = False, fidelius: bool = False) -> str:
+        as_path = Path(file_path)
+        if as_path.is_absolute():
+            if not as_path.exists():
+                raise EmbedFileNotFound(f'Embed file not found via absolute path: {file_path}')
+            return self._get_embed(file_path, alviss=alviss, env=env, fidelius=fidelius)
+
+        search_paths = []
+
+        if source_file:
+            search_paths.append(Path(source_file).absolute().parent)
+
+        search_paths += self._search_paths
+
+        for p in search_paths:
+            f = p / file_path
+            if f.exists():
+                return self._get_embed(str(f.absolute()), alviss=alviss, env=env, fidelius=fidelius)
+
+        raise EmbedFileNotFound(f'Embed file not found in any search path: {file_path}')
+
+    def _get_embed(self, abs_file_path: str, alviss: bool = False, env: bool = False, fidelius: bool = False) -> str:
+        if alviss:
+            return quickloader.render_load(abs_file_path,
+                                           skip_env_loading=not env,
+                                           skip_fidelius=not fidelius)
+
+        else:
+            with open(abs_file_path, 'r', newline=None) as fin:
+                return fin.read()

ccpstencil/shortcuts/__init__.py

@@ -0,0 +1 @@
+from ._render_stencil import *

ccpstencil/shortcuts/_render_stencil.py

@@ -0,0 +1,17 @@
+__all__ = [
+    'render_stencil',
+]
+
+
+from ccpstencil.stencils import *
+from ccpstencil.utils import *
+from ccpstencil.structs import *
+
+
+def render_stencil(template: T_PATH, context: Union[Dict, T_PATH]) -> str:
+    renderer = StringRenderer()
+    template = guess_template_by_argument(template, renderer)
+    context = guess_context_by_argument(context)
+    renderer.template = template
+    renderer.context = context
+    return renderer.render()

ccpstencil/stencils.py

@@ -1,3 +1,6 @@
 from .context import *
 from .template import *
 from .renderer import *
+from .shortcuts import *
+
+

ccpstencil/structs/__init__.py

@@ -1,3 +1,4 @@
 from ._base import *
+from ._aliases import *
 from .interfaces import *
 from ._errors import *

ccpstencil/structs/_aliases.py

@@ -0,0 +1,7 @@
+__all__ = [
+    'T_PATH',
+]
+from typing import Union
+from pathlib import Path
+
+T_PATH = Union[str, Path]

ccpstencil/structs/_errors.py

@@ -11,6 +11,7 @@
     'InvalidContextTypeForRendererError',
     'InvalidTemplateTypeForRendererError',
     'OutputFileExistsError',
+    'EmbedFileNotFound',
 ]
 
 
@@ -48,3 +49,7 @@ class InvalidTemplateTypeForRendererError(RenderError, TypeError):
 
 class OutputFileExistsError(RenderError, FileExistsError):
     pass
+
+
+class EmbedFileNotFound(RenderError, FileNotFoundError):
+    pass

ccpstencil/structs/interfaces/_renderer.py

@@ -40,3 +40,11 @@ def render(self):
     @abc.abstractmethod
     def jinja_environment(self) -> jinja2.Environment:
         pass
+
+    @abc.abstractmethod
+    def is_template_loadable(self, template_name: str) -> bool:
+        pass
+
+    @abc.abstractmethod
+    def get_embed(self, file_path: str, source_file: Optional[str] = None, alviss: bool = False, env: bool = False) -> str:
+        pass

ccpstencil/template/_file.py

@@ -5,19 +5,30 @@
 from ccpstencil.structs import *
 from pathlib import Path
 
-from ._string import *
+from ._base import *
 
 
-class FileTemplate(StringTemplate):
-    def __init__(self, file_path: Union[str, Path], **kwargs):
-        if isinstance(file_path, str):
-            file_path = Path(file_path)
-        self._file_path: Path = file_path
-        super().__init__(template_string=self._read_file(), **kwargs)
+class FileTemplate(_BaseTemplate):
+    def __init__(self, file_path: T_PATH, **kwargs):
+        super().__init__(**kwargs)
+        self._file_path: T_PATH = file_path
 
     def _read_file(self) -> str:
-        if not self._file_path.exists():
-            raise TemplateNotFoundError(f'File {self._file_path} does not exist')
+        if isinstance(self._file_path, str):
+            as_path = Path(self._file_path)
+        else:
+            as_path = self._file_path
+        if not as_path.exists():
+            raise TemplateNotFoundError(f'File {as_path} does not exist')
 
-        with open(self._file_path, 'r') as fin:
+        with open(as_path, 'r') as fin:
             return fin.read()
+
+    def get_jinja_template(self) -> jinja2.Template:
+        if isinstance(self._file_path, str):
+            try:
+                return self.renderer.jinja_environment.get_template(self._file_path)
+            except jinja2.exceptions.TemplateNotFound:
+                pass
+        template_string = self._read_file()
+        return self.renderer.jinja_environment.from_string(template_string)

ccpstencil/utils/__init__.py

@@ -0,0 +1 @@
+from ._guessers import *