Add tests for generated modules and fix bugs

.circleci/config.yml

@@ -2,63 +2,46 @@ version: 2
 jobs:
   build:
     docker:
-      - image: ubuntu:latest
-    environment:
-      DEBIAN_FRONTEND: noninteractive
+      - image: nutti/fake_bpy_module_ci:v1.0
     branches:
       only:
         - master
         - ci_testing
     steps:
       - checkout
-      - run:
-          name: Get required packages
-          command: apt-get update && apt-get install -y git python3 python3-pip python-sphinx curl unzip blender pandoc zip
-      - run:
-          name: Install pip package dependency
-          command: pip3 install -r requirements.txt
-      - run:
-          name: Install pip package 'pathlib'
-          command: pip3 install pathlib
-      - run:
-          name: Download Blender Source
-          command: git clone git://git.blender.org/blender.git
-      - run:
-          name: Download all Blender
-          command: bash tools/utils/download_blender.sh all blender-bin
-      - run:
-          name: Make build directory
-          command: mkdir build
       - run:
           name: Generate pip Packages (Blender 2.78)
-          command: bash tools/pip_package/build_pip_package.sh release 2.78 blender blender-bin/blender-v278-bin
+          command: bash tools/pip_package/build_pip_package.sh release 2.78 ../blender ../blender-bin/blender-v278-bin
       - run:
           name: Generate pip Packages (Blender 2.78a)
-          command: bash tools/pip_package/build_pip_package.sh release 2.78a blender blender-bin/blender-v278a-bin
+          command: bash tools/pip_package/build_pip_package.sh release 2.78a ../blender ../blender-bin/blender-v278a-bin
       - run:
           name: Generate pip Packages (Blender 2.78b)
-          command: bash tools/pip_package/build_pip_package.sh release 2.78b blender blender-bin/blender-v278b-bin
+          command: bash tools/pip_package/build_pip_package.sh release 2.78b ../blender ../blender-bin/blender-v278b-bin
       - run:
           name: Generate pip Packages (Blender 2.78c)
-          command: bash tools/pip_package/build_pip_package.sh release 2.78c blender blender-bin/blender-v278c-bin
+          command: bash tools/pip_package/build_pip_package.sh release 2.78c ../blender ../blender-bin/blender-v278c-bin
       - run:
           name: Generate pip Packages (Blender 2.79)
-          command: bash tools/pip_package/build_pip_package.sh release 2.79 blender blender-bin/blender-v279-bin
+          command: bash tools/pip_package/build_pip_package.sh release 2.79 ../blender ../blender-bin/blender-v279-bin
       - run:
           name: Generate pip Packages (Blender 2.79a)
-          command: bash tools/pip_package/build_pip_package.sh release 2.79a blender blender-bin/blender-v279a-bin
+          command: bash tools/pip_package/build_pip_package.sh release 2.79a ../blender ../blender-bin/blender-v279a-bin
       - run:
           name: Generate pip Packages (Blender 2.79b)
-          command: bash tools/pip_package/build_pip_package.sh release 2.79b blender blender-bin/blender-v279b-bin
+          command: bash tools/pip_package/build_pip_package.sh release 2.79b ../blender ../blender-bin/blender-v279b-bin
       - run:
           name: Generate pip Packages (Blender 2.80)
-          command: bash tools/pip_package/build_pip_package.sh release 2.80 blender blender-bin/blender-v280-bin
+          command: bash tools/pip_package/build_pip_package.sh release 2.80 ../blender ../blender-bin/blender-v280-bin
       - run:
           name: Compress All Generated Packages
           command: tar cvfz all.tar.gz release/* && mv all.tar.gz release
       - run:
           name: Compress All Raw Modules
           command: tar cvfz all.tar.gz raw_modules/* && mv all.tar.gz raw_modules
+      - run:
+          name: Test Generated Modules
+          command: bash tests/run_tests.sh raw_modules
       - store_artifacts:
           path: release
           destination: pip_packages

src/fake_bpy_module/analyzer.py

@@ -268,13 +268,36 @@ def _parse_desc_content(self, elm, result):
             elif child.tag == "field_list":
                 self._parse_field_list(child, result)
 
+    def _parse_parameter_body_text(self, text):
+        sp = [re.sub(" ", "", p) for p in text.split(",")]
+
+        params = []
+        parenthese_level = 0
+        next_param = ""
+        for p in sp:
+            parenthese_level += p.count("(") - p.count(")")
+            if parenthese_level < 0:
+                raise ValueError("Parenthese Level must be >= 0, but {}. (text: {}, filename: {})"
+                                 .format(parenthese_level, text, self.filename))
+
+            next_param += p + ","
+            if parenthese_level == 0:
+                params.append(next_param[:-1])  # strip last ","
+                next_param = ""
+
+        if parenthese_level != 0:
+            raise ValueError("Parenthese Level must be == 0, but {}. (text: {}, filename: {})"
+                             .format(parenthese_level, text, self.filename))
+
+        return params
+
     def _get_parameters(self, elm):
         result = []
+        param_bodies = []
         for child in list(elm):
             if child.tag == "desc_parameter":
-                sp = child.text.split(",")
-                for s in sp:
-                    result.append(re.sub(" ", "", s))
+                param_bodies.append(child.text)
+        result.extend(self._parse_parameter_body_text(",".join(param_bodies)))
         return result
 
     def analyze(self, elm) -> 'FunctionInfo':
@@ -287,28 +310,44 @@ def analyze(self, elm) -> 'FunctionInfo':
                 fullname = child.get("fullname")
                 text = child.find("desc_name").text
                 lp = text.find("(")
-                rp = text.find(")")
-                if lp == -1:
+                rp = text.rfind(")")
+                if (lp == -1) or (rp == -1):
                     output_log(
                         LOG_LEVEL_NOTICE,
-                        "'(' and ')' are not found (text={0})".format(text)
+                        "'(' or ')' are not found (text={}, filename={})".format(text, self.filename)
                     )
 
                     # get name data
                     name = text
 
+                    # TODO: freestyle.shader.SmoothingShader.__init__() matches this case.
+                    parenthese_index = name.find("(")
+                    if parenthese_index != -1:
+                        name = name[:parenthese_index]
+                        output_log(
+                            LOG_LEVEL_WARN,
+                            "Function name has parenthesis. But this should be fixed at Blender itself. (name: {}, filename: {})"
+                            .format(name, self.filename)
+                        )
+
                     # get parameters
                     params = []
                     c = child.find("desc_parameterlist")
                     if c is not None:
                         params = self._get_parameters(c)
                 else:
+                    if lp == -1:
+                        raise ValueError("( is not found. (text: {}, filename: {})"
+                                         .format(text, self.filename))
+                    if rp == -1:
+                        raise ValueError(") is not found. (text: {}, filename: {})"
+                                         .format(text, self.filename))
+
                     # get name data
                     name = text[0:lp]
 
                     # get parameters
-                    params = [re.sub(" ", "", p)
-                              for p in text[lp + 1:rp].split(",")]
+                    params = self._parse_parameter_body_text(text[lp + 1:rp])
 
                 self.info.set_name(name)
                 self.info.add_parameters(params)
@@ -319,7 +358,8 @@ def analyze(self, elm) -> 'FunctionInfo':
                         output_log(
                             LOG_LEVEL_NOTICE,
                             "fullname does not match text "
-                            "(fullname={0}, text={1})".format(fullname, name)
+                            "(fullname={}, text={}, filename={})"
+                            .format(fullname, name, self.filename)
                         )
 
                 # get module/class data

src/fake_bpy_module/common.py

@@ -169,7 +169,7 @@ def to_string(self) -> str:
                    .format(MODIFIER_DATA_TYPE_TO_TYPING[self._modifier],
                                    self._data_types[0])
         else:
-            return "{}[typeing.Union[{}]]"\
+            return "{}[typing.Union[{}]]"\
                    .format(MODIFIER_DATA_TYPE_TO_TYPING[self._modifier],
                            ", ".join(self._data_types))
 
@@ -516,6 +516,11 @@ def set_parameter(self, idx: int, param: str):
         self._parameters[idx] = param
 
     def add_parameter(self, param: str):
+        if param in self._parameters:
+            output_log(LOG_LEVEL_WARN,
+                       "Parameter {} is already registered in ({}), so skip to add this parameter."
+                       .format(param, " | ".join(self._parameters)))
+            return
         self._parameters.append(param)
 
     def add_parameters(self, params: List[str]):
@@ -809,7 +814,8 @@ def from_dict(self, data: dict, method: str='NONE'):
                             update_m.from_dict(m, 'UPDATE')
                             break
                     else:
-                        raise RuntimeError("{} is not found".format(m["name"]))
+                        raise RuntimeError("Method '{}' is not found at class '{}.{}'"
+                                           .format(m["name"], self._module, self._name))
             else:
                 raise RuntimeError("Unsupported method: {}".format(method))
 
@@ -1032,7 +1038,7 @@ def get_refined_data_type(self, data_type: 'DataType', module_name: str) -> 'Dat
                     return ModifierDataType(modifier)
                 return CustomDataType(dtype, modifier)
             else:
-                return UnknownDataType()
+                return ModifierDataType(modifier)
 
     def get_base_name(self, data_type: str) -> str:
         if data_type is None:

src/gen.py

@@ -24,11 +24,17 @@ def _modify_post_process(self, result: 'fbm.AnalysisResult'):
             for info in section.info_list:
                 if info.type() == "function":
                     for i, p in enumerate(info.parameters()):
-                        info.set_parameter(i, p.replace("=noise.types.STDPERLIN", "=types.STDPERLIN"))
+                        converted = p
+                        converted = converted.replace("=noise.types.STDPERLIN", "=types.STDPERLIN")
+                        converted = converted.replace("=noise.distance_metrics.DISTANCE", "=None")
+                        info.set_parameter(i, converted)
                 elif info.type() == "class":
                     for m in info.methods():
                         for i, p in enumerate(m.parameters()):
-                            m.set_parameter(i, p.replace("=noise.types.STDPERLIN", "=types.STDPERLIN"))
+                            converted = p
+                            converted = converted.replace("=noise.types.STDPERLIN", "=types.STDPERLIN")
+                            converted = converted.replace("=noise.distance_metrics.DISTANCE", "=None")
+                            m.set_parameter(i, converted)
 
 
 class BglAnalyzer(fbm.AnalyzerWithModFile):
@@ -40,6 +46,32 @@ def _modify_post_process(self, result: 'fbm.AnalysisResult'):
                         info.set_module("bgl")
 
 
+class FreestyleAnalyzer(fbm.AnalyzerWithModFile):
+    def _modify_post_process(self, result: 'fbm.AnalysisResult'):
+        for section in result.section_info:
+            for info in section.info_list:
+                if info.type() == "function":
+                    for i, p in enumerate(info.parameters()):
+                        info.set_parameter(i, p.replace("=IntegrationType.MEAN", "=None"))
+                elif info.type() == "class":
+                    for m in info.methods():
+                        for i, p in enumerate(m.parameters()):
+                            m.set_parameter(i, p.replace("=IntegrationType.MEAN", "=None"))
+
+
+class BpyAnalyzer(fbm.AnalyzerWithModFile):
+    def _modify_post_process(self, result: 'fbm.AnalysisResult'):
+        for section in result.section_info:
+            for info in section.info_list:
+                if info.type() == "function":
+                    for i, p in enumerate(info.parameters()):
+                        info.set_parameter(i, p.replace("=bpy.context.scene.frame_current", "=None"))
+                elif info.type() == "class":
+                    for m in info.methods():
+                        for i, p in enumerate(m.parameters()):
+                            m.set_parameter(i, p.replace("=bpy.context.scene.frame_current", "=None"))
+
+
 def make_bpy_rule(config: 'fbm.PackageGeneratorConfig') -> 'fbm.PackageGenerationRule':
     all_files = glob.glob(INPUT_DIR + "/bpy*.xml")
     excludes_files = glob.glob(INPUT_DIR + "/bpy_extras*.xml")
@@ -50,7 +82,7 @@ def make_bpy_rule(config: 'fbm.PackageGeneratorConfig') -> 'fbm.PackageGeneratio
     ]
     if config.mod_version == "2.80":
         mod_files.append("{}/mods/2.80/analyzer/bpy.json".format(MOD_FILES_DIR).replace("\\", "/"))
-    return fbm.PackageGenerationRule("bpy", files, fbm.AnalyzerWithModFile(mod_files), fbm.BaseGenerator())
+    return fbm.PackageGenerationRule("bpy", files, BpyAnalyzer(mod_files), fbm.BaseGenerator())
 
 
 def make_bgl_rule(config: 'fbm.PackageGeneratorConfig') -> 'fbm.PackageGenerationRule':
@@ -82,7 +114,7 @@ def make_gpu_rule(config: 'fbm.PackageGeneratorConfig') -> 'fbm.PackageGeneratio
     return fbm.PackageGenerationRule("gpu", files, fbm.AnalyzerWithModFile(mod_files), fbm.BaseGenerator())
 
 
-def make_gpu_extra_rule(config: 'fbm.PackageGeneratorConfig') -> 'fbm.PackageGenerationRule':
+def make_gpu_extras_rule(config: 'fbm.PackageGeneratorConfig') -> 'fbm.PackageGenerationRule':
     all_files = glob.glob(INPUT_DIR + "/bpy_extras*.xml")
     excludes_files = glob.glob(INPUT_DIR + "/bpy_extras*.xml")
     files = list(set(all_files) - set(excludes_files))
@@ -94,13 +126,13 @@ def make_freestyle_rule(config: 'fbm.PackageGeneratorConfig') -> 'fbm.PackageGen
     mod_files = [
         "{}/mods/common/analyzer/freestyle.json".format(MOD_FILES_DIR).replace("\\", "/"),
     ]
-    return fbm.PackageGenerationRule("freestyle", files, fbm.AnalyzerWithModFile(mod_files), fbm.BaseGenerator())
+    return fbm.PackageGenerationRule("freestyle", files, FreestyleAnalyzer(mod_files), fbm.BaseGenerator())
 
 
-def make_bpy_extra_rule(config: 'fbm.PackageGeneratorConfig') -> 'fbm.PackageGenerationRule':
+def make_bpy_extras_rule(config: 'fbm.PackageGeneratorConfig') -> 'fbm.PackageGenerationRule':
     files = glob.glob(INPUT_DIR + "/bpy_extras*.xml")
     mod_files = [
-        "{}/mods/common/analyzer/bpy_extra.json".format(MOD_FILES_DIR).replace("\\", "/"),
+        "{}/mods/common/analyzer/bpy_extras.json".format(MOD_FILES_DIR).replace("\\", "/"),
     ]
     return fbm.PackageGenerationRule("bpy_extras", files, fbm.AnalyzerWithModFile(mod_files), fbm.BaseGenerator())
 
@@ -175,9 +207,9 @@ def main():
     pkg_generator.add_rule(make_blf_rule(config))
     pkg_generator.add_rule(make_mathutils_rule(config))
     pkg_generator.add_rule(make_gpu_rule(config))
-    pkg_generator.add_rule(make_gpu_extra_rule(config))
+    pkg_generator.add_rule(make_gpu_extras_rule(config))
     pkg_generator.add_rule(make_freestyle_rule(config))
-    pkg_generator.add_rule(make_bpy_extra_rule(config))
+    pkg_generator.add_rule(make_bpy_extras_rule(config))
     pkg_generator.add_rule(make_aud_rule(config))
     pkg_generator.add_rule(make_bmesh_rule(config))
     pkg_generator.generate()

src/gen_module.sh

@@ -24,7 +24,7 @@ cd ${source_dir}
 git fetch --prune
 git checkout master
 git pull origin master
-git checkout ${branch_name}
+git checkout -f ${branch_name}
 git pull origin ${branch_name}
 
 # generate .rst documents

tests/python/fake_bpy_module_test/__init__.py

@@ -0,0 +1,10 @@
+from . import bpy_test
+from . import bgl_test
+from . import blf_test
+from . import mathutils_test
+from . import gpu_test
+from . import gpu_extras_test
+from . import freestyle_test
+from . import bpy_extras_test
+from . import aud_test
+from . import bmesh_test

tests/python/fake_bpy_module_test/aud_test.py

@@ -0,0 +1,17 @@
+import sys
+
+from . import common
+
+
+class AudTest(common.FakeBpyModuleTestBase):
+
+    module_name = "aud"
+
+    def setUp(self):
+        import aud
+
+    def tearDown(self):
+        sys.modules.pop(self.module_name)
+
+    def test_nothing(self):
+        print("[Test] Nothing")

tests/python/fake_bpy_module_test/bgl_test.py

@@ -0,0 +1,17 @@
+import sys
+
+from . import common
+
+
+class BglTest(common.FakeBpyModuleTestBase):
+
+    module_name = "bgl"
+
+    def setUp(self):
+        import bgl
+
+    def tearDown(self):
+        sys.modules.pop(self.module_name)
+
+    def test_nothing(self):
+        print("[Test] Nothing")

tests/python/fake_bpy_module_test/blf_test.py

@@ -0,0 +1,17 @@
+import sys
+
+from . import common
+
+
+class BlfTest(common.FakeBpyModuleTestBase):
+
+    module_name = "blf"
+
+    def setUp(self):
+        import blf
+
+    def tearDown(self):
+        sys.modules.pop(self.module_name)
+
+    def test_nothing(self):
+        print("[Test] Nothing")