Merge pull request #350 from jhonabreul/bug-push-files-filters

Improve file filters for pushing projects

lean/components/cloud/push_manager.py

@@ -97,13 +97,13 @@ def _push_project(self, project_path: Path, organization_id: str, suggested_rena
         :param organization_id: the id of the organization to push the project to
         :param suggested_rename_path: the path to move the project to.
         """
-        
+
         project_name = project_path.relative_to(Path.cwd()).as_posix()
 
         potential_new_name = project_name
         if suggested_rename_path and suggested_rename_path != project_path:
             potential_new_name = suggested_rename_path.relative_to(Path.cwd()).as_posix()
-            
+
 
         project_config = self._project_config_manager.get_project_config(project_path)
         cloud_id = project_config.get("cloud-id")
@@ -154,17 +154,13 @@ def _get_files(self, project: Path) -> List[Dict[str, str]]:
         :param project: the local project to push the files of
         """
         paths = self._project_manager.get_source_files(project)
-        files = []
-
-        for path in paths:
-            relative_path = path.relative_to(project).as_posix()
-            if "bin/" in relative_path and "obj/" in relative_path and ".ipynb_checkpoints/" in relative_path:
-                continue
-
-            files.append({
-                'name': relative_path,
+        files = [
+            {
+                'name': path.relative_to(project).as_posix(),
                 'content': path.read_text(encoding="utf-8")
-            })
+            }
+            for path in paths
+        ]
 
         return files
 

lean/components/util/project_manager.py

@@ -123,12 +123,15 @@ def get_source_files(self, directory: Path) -> List[Path]:
 
         for obj in directory.iterdir():
             if obj.is_dir():
-                if obj.name in ["bin", "obj", ".ipynb_checkpoints", "backtests", "live", "optimizations"]:
+                if (obj.name in ["bin", "obj", ".ipynb_checkpoints", "backtests", "live", "optimizations"] or
+                        obj.name.startswith(".") or
+                        # ignore python virtual environments
+                        (obj / "pyvenv.cfg").is_file()):
                     continue
 
                 source_files.extend(self.get_source_files(obj))
 
-            if obj.suffix not in [".py", ".cs", ".ipynb"]:
+            if obj.suffix not in [".py", ".cs", ".ipynb", ".css", ".html"]:
                 continue
 
             source_files.append(obj)

tests/components/util/test_project_manager.py

@@ -138,7 +138,9 @@ def test_get_source_files_returns_all_source_files() -> None:
     assert sorted(files_to_sync) == sorted(files)
 
 
-@pytest.mark.parametrize("directory", ["bin", "obj", ".ipynb_checkpoints", "backtests", "live", "optimizations"])
+@pytest.mark.parametrize(
+    "directory",
+    ["bin", "obj", ".ipynb_checkpoints", "backtests", "live", "optimizations", ".hidden"])
 def test_get_source_files_ignores_generated_source_files(directory: str) -> None:
     project_path = Path.cwd() / "My Project"
     project_path.mkdir()
@@ -154,6 +156,44 @@ def test_get_source_files_ignores_generated_source_files(directory: str) -> None
     assert files_to_sync == [files[0]]
 
 
+@pytest.mark.parametrize("filename", ["file.csv", "file.xlsx", "file.java"])
+def test_get_source_files_ignores_unsuported_extension_files(filename: str) -> None:
+    project_path = Path.cwd() / "My Project"
+    project_path.mkdir()
+
+    files = [
+        project_path / "main.py",
+        project_path / "main.cs",
+        project_path / "main.ipynb",
+        project_path / "main.css",
+        project_path / "main.html",
+        project_path / filename]
+    for file in files:
+        file.parent.mkdir(parents=True, exist_ok=True)
+        file.touch()
+
+    project_manager = _create_project_manager()
+    files_to_sync = project_manager.get_source_files(project_path)
+
+    assert files_to_sync == files[:-1]
+
+
+@pytest.mark.parametrize("directory", ["venv", "pyvenv", "my_venv"])
+def test_get_source_files_ignores_python_virtual_environments(directory: str) -> None:
+    project_path = Path.cwd() / "My Project"
+    project_path.mkdir()
+
+    files = [project_path / "main.py", project_path / directory / "pyvenv.cfg"]
+    for file in files:
+        file.parent.mkdir(parents=True, exist_ok=True)
+        file.touch()
+
+    project_manager = _create_project_manager()
+    files_to_sync = project_manager.get_source_files(project_path)
+
+    assert files_to_sync == [files[0]]
+
+
 def test_update_last_modified_time_updates_file_properties() -> None:
     local_file = Path.cwd() / "file.txt"
     local_file.touch()
@@ -190,7 +230,9 @@ def test_copy_code_copies_source_files_to_output_directory() -> None:
         assert (output_dir / file).read_text(encoding="utf-8") == file.name
 
 
-@pytest.mark.parametrize("directory", ["bin", "obj", ".ipynb_checkpoints", "backtests", "live", "optimizations"])
+@pytest.mark.parametrize(
+    "directory",
+    ["bin", "obj", ".ipynb_checkpoints", "backtests", "live", "optimizations", ".hidden"])
 def test_copy_code_ignores_generated_source_files(directory: str) -> None:
     project_path = Path.cwd() / "My Project"
     project_path.mkdir()