Merge pull request #545 from jkloetzke/new-include-syntax

New include syntax

doc/manual/configuration.rst

@@ -508,19 +508,31 @@ build time. If both the keyword with the build time language suffix and without
 a suffix are present then the keyword with the build language suffix takes
 precedence.
 
-The script is subject to file inclusion with the ``$<<path>>`` and
-``$<'path'>`` syntax. The files are included relative to the current recipe.
-The given ``path`` might be a shell globbing pattern. If multiple files are
-matched by ``path`` the files are sorted by name and then concatenated. The
-``$<<path>>`` syntax imports the file(s) as is and replaces the escape pattern
-with a (possibly temporary) file name which has the same content. Similar to
-that, the ``$<'path'>`` syntax includes the file(s) inline as a quoted string.
-In any case, the strings are fully quoted and *not* subject to any parameter
-substitution.
-
-.. note::
-   When including files as quoted strings (``$<'path'>`` syntax), they have to
-   be UTF-8 encoded.
+The script is subject to file inclusion with the ``$<<path>>``, ``$<@path@>``
+and ``$<'path'>`` syntax. The files are included relative to the current
+recipe.  The given ``path`` might be a shell globbing pattern. If multiple
+files are matched by ``path``, the files are sorted by name before being
+processed. Matching no file leads to an error. Depending on the particular
+syntax, the file(s) are included in different ways:
+
+``$<<path>>``
+    This syntax concatenates the file(s) and replaces the escape pattern with a
+    (possibly temporary) file name which has all the content. The script will
+    always see a single file name.
+
+``$<@path@>``
+    For each matched file, the script will see a (possibly temporary) file with
+    its content. The order of files is still sorted by the original file name.
+    Like the ``$<<path>>`` syntax, the file names are not predictable at
+    runtime and do not resemble the original file names.
+
+``$<'path'>``
+    This syntax concatenates the file(s) and inserts the result as string
+    literal. The strings are fully quoted and *not* subject to any parameter
+    substitution.
+
+    .. note::
+       When including files as quoted strings, they have to be UTF-8 encoded.
 
 The scripts of any classes that are inherited which define
 a script for the same step are joined in front of this script in the order the

pym/bob/input.py

@@ -1733,8 +1733,8 @@ def __init__(self, scriptLanguage, fileLoader, baseDir, varBase, sourceName):
         self.__pattern = re.compile(r"""
             \$<(?:
                 (?P<escaped>\$)     |
-                (?P<named>[<'][^'>]+)['>]>  |
-                (?P<braced>[<'][^'>]+)['>]> |
+                (?P<named>[<'@][^'>@]+)[@'>]>  |
+                (?P<braced>[<'@][^'>@]+)[@'>]> |
                 (?P<invalid>)
             )
             """, re.VERBOSE)

pym/bob/languages.py

@@ -7,6 +7,7 @@
 from .errors import ParseError
 from .utils import escapePwsh, quotePwsh, isWindows, asHexStr, getBashPath
 from .utils import joinScripts, sliceString
+from abc import ABC, abstractmethod
 from base64 import b64encode
 from enum import Enum
 from glob import glob
@@ -147,7 +148,7 @@ class ScriptLanguage(Enum):
     PWSH = 'PowerShell'
 
 
-class IncludeResolver:
+class IncludeResolver(ABC):
     def __init__(self, fileLoader, baseDir, origText, sourceName, varBase):
         self.fileLoader = fileLoader
         self.baseDir = baseDir
@@ -168,25 +169,33 @@ def __getitem__(self, item):
                 content.append(self.fileLoader(path))
         except OSError as e:
             raise ParseError("Error including '"+item+"': " + str(e))
-        content = b''.join(content)
+        allContent = b''.join(content)
 
-        self.__incDigests.append(hashlib.sha1(content).digest().hex())
+        self.__incDigests.append(hashlib.sha1(allContent).digest().hex())
         if mode == '<':
-            ret = self._includeFile(content)
+            ret = self._includeFile(allContent)
+        elif mode == '@':
+            ret = self._includeFiles(content)
         else:
             assert mode == "'"
-            ret = self._includeLiteral(content)
+            ret = self._includeLiteral(allContent)
 
         return ret
 
+    @abstractmethod
     def _includeFile(self, content):
-        raise NotImplementedError()
+        pass
 
+    def _includeFiles(self, contentList):
+        return " ".join((self._includeFile(content) for content in contentList))
+
+    @abstractmethod
     def _includeLiteral(self, content):
-        raise NotImplementedError()
+        pass
 
+    @abstractmethod
     def _resolveContent(self, result):
-        raise NotImplementedError()
+        pass
 
     def resolve(self, result):
         return (self._resolveContent(result), "\n".join(self.__incDigests))

test/black-box/include/output/many-1.txt

@@ -0,0 +1 @@
+foo

test/black-box/include/output/many-2.txt

@@ -0,0 +1 @@
+bar

test/black-box/include/output/many-3.txt

@@ -0,0 +1 @@
+baz

test/black-box/include/recipes/many/01-foo.txt

@@ -0,0 +1 @@
+foo

test/black-box/include/recipes/many/02-bar.txt

@@ -0,0 +1 @@
+bar

test/black-box/include/recipes/many/03-baz.txt

@@ -0,0 +1 @@
+baz

test/black-box/include/recipes/root.yaml

@@ -4,3 +4,8 @@ packageScript: |
     echo $<'inline.txt'> > result.txt
     cat $<<file.txt>> >> result.txt
 
+    i=1
+    for f in $<@many/*.txt@> ; do
+        cp "$f" "many-$i.txt"
+        : $((i++))
+    done