Get additional docker configs in local deployment commands.

This accepts only device_requests for now.

lean/commands/backtest.py

@@ -11,7 +11,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-
+import json
 from pathlib import Path
 from typing import List, Optional, Tuple
 from click import command, option, argument, Choice
@@ -289,6 +289,10 @@ def _select_organization() -> QCMinimalOrganization:
               type=(str, str),
               multiple=True,
               hidden=True)
+@option("--extra-docker-config",
+              type=str,
+              default="{}",
+              hidden=True)
 @option("--no-update",
               is_flag=True,
               default=False,
@@ -307,6 +311,7 @@ def backtest(project: Path,
              backtest_name: str,
              addon_module: Optional[List[str]],
              extra_config: Optional[Tuple[str, str]],
+             extra_docker_config: Optional[str],
              no_update: bool,
              **kwargs) -> None:
     """Backtest a project locally using Docker.
@@ -407,4 +412,5 @@ def backtest(project: Path,
                          engine_image,
                          debugging_method,
                          release,
-                         detach)
+                         detach,
+                         json.loads(extra_docker_config))

lean/commands/live/deploy.py

@@ -11,6 +11,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import json
 from pathlib import Path
 from typing import Any, Dict, List, Optional, Tuple
 from click import option, argument, Choice
@@ -255,6 +256,10 @@ def _get_default_value(key: str) -> Optional[Any]:
               type=(str, str),
               multiple=True,
               hidden=True)
+@option("--extra-docker-config",
+              type=str,
+              default="{}",
+              hidden=True)
 @option("--no-update",
               is_flag=True,
               default=False,
@@ -275,6 +280,7 @@ def deploy(project: Path,
            show_secrets: bool,
            addon_module: Optional[List[str]],
            extra_config: Optional[Tuple[str, str]],
+           extra_docker_config: Optional[str],
            no_update: bool,
            **kwargs) -> None:
     """Start live trading a project locally using Docker.
@@ -430,4 +436,4 @@ def deploy(project: Path,
             raise RuntimeError(f"InteractiveBrokers is currently not supported for ARM hosts")
 
     lean_runner = container.lean_runner
-    lean_runner.run_lean(lean_config, environment_name, algorithm_file, output, engine_image, None, release, detach)
+    lean_runner.run_lean(lean_config, environment_name, algorithm_file, output, engine_image, None, release, detach, json.loads(extra_docker_config))

lean/commands/optimize.py

@@ -11,13 +11,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import json
 from pathlib import Path
 from typing import Optional, List, Tuple
 from datetime import datetime, timedelta
 
 from click import command, argument, option, Choice, IntRange
 
 from lean.click import LeanCommand, PathParameter, ensure_options
+from lean.components.docker.lean_runner import LeanRunner
 from lean.constants import DEFAULT_ENGINE_IMAGE
 from lean.container import container
 from lean.models.api import QCParameter, QCBacktest
@@ -119,6 +121,10 @@ def get_filename_timestamp(path: Path) -> datetime:
               type=(str, str),
               multiple=True,
               hidden=True)
+@option("--extra-docker-config",
+              type=str,
+              default="{}",
+              hidden=True)
 @option("--no-update",
               is_flag=True,
               default=False,
@@ -139,6 +145,7 @@ def optimize(project: Path,
              max_concurrent_backtests: Optional[int],
              addon_module: Optional[List[str]],
              extra_config: Optional[Tuple[str, str]],
+             extra_docker_config: Optional[str],
              no_update: bool) -> None:
     """Optimize a project's parameters locally using Docker.
 
@@ -308,6 +315,9 @@ def optimize(project: Path,
     )
     container.update_manager.pull_docker_image_if_necessary(engine_image, update, no_update)
 
+    # Add known additional run options from the extra docker config
+    LeanRunner.parse_extra_docker_config(run_options, json.loads(extra_docker_config))
+
     project_manager.copy_code(algorithm_file.parent, output / "code")
 
     success = container.docker_manager.run_image(engine_image, **run_options)

lean/commands/research.py

@@ -11,10 +11,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import json
 from pathlib import Path
 from typing import Optional, Tuple
 from click import command, argument, option, Choice
 from lean.click import LeanCommand, PathParameter
+from lean.components.docker.lean_runner import LeanRunner
 from lean.constants import DEFAULT_RESEARCH_IMAGE, LEAN_ROOT_PATH
 from lean.container import container
 from lean.models.data_providers import QuantConnectDataProvider, all_data_providers
@@ -65,6 +67,10 @@ def _check_docker_output(chunk: str, port: int) -> None:
               type=(str, str),
               multiple=True,
               hidden=True)
+@option("--extra-docker-config",
+              type=str,
+              default="{}",
+              hidden=True)
 @option("--no-update",
               is_flag=True,
               default=False,
@@ -79,6 +85,7 @@ def research(project: Path,
              image: Optional[str],
              update: bool,
              extra_config: Optional[Tuple[str, str]],
+             extra_docker_config: Optional[str],
              no_update: bool,
              **kwargs) -> None:
     """Run a Jupyter Lab environment locally using Docker.
@@ -116,7 +123,7 @@ def research(project: Path,
     # Set extra config
     for key, value in extra_config:
         lean_config[key] = value
-        
+
     run_options = lean_runner.get_basic_docker_config(lean_config,
                                                       algorithm_file,
                                                       temp_manager.create_temporary_directory(),
@@ -160,6 +167,9 @@ def research(project: Path,
     # Run the script that starts Jupyter Lab when all set up has been done
     run_options["commands"].append("./start.sh")
 
+    # Add known additional run options from the extra docker config
+    LeanRunner.parse_extra_docker_config(run_options, json.loads(extra_docker_config))
+
     project_config_manager = container.project_config_manager
     cli_config_manager = container.cli_config_manager
 

lean/components/docker/lean_runner.py

@@ -13,6 +13,8 @@
 
 from pathlib import Path
 from typing import Any, Dict, Optional, List
+import docker.types
+
 from lean.components.cloud.module_manager import ModuleManager
 from lean.components.config.lean_config_manager import LeanConfigManager
 from lean.components.config.output_config_manager import OutputConfigManager
@@ -70,7 +72,8 @@ def run_lean(self,
                  image: DockerImage,
                  debugging_method: Optional[DebuggingMethod],
                  release: bool,
-                 detach: bool) -> None:
+                 detach: bool,
+                 extra_docker_config: Optional[Dict[str, Any]] = None) -> None:
         """Runs the LEAN engine locally in Docker.
 
         Raises an error if something goes wrong.
@@ -83,6 +86,7 @@ def run_lean(self,
         :param debugging_method: the debugging method if debugging needs to be enabled, None if not
         :param release: whether C# projects should be compiled in release configuration instead of debug
         :param detach: whether LEAN should run in a detached container
+        :param extra_docker_config: additional docker configurations
         """
         project_dir = algorithm_file.parent
 
@@ -95,6 +99,9 @@ def run_lean(self,
                                                    release,
                                                    detach)
 
+        # Add known additional run options from the extra docker config
+        self.parse_extra_docker_config(run_options, extra_docker_config)
+
         # Set up PTVSD debugging
         if debugging_method == DebuggingMethod.PTVSD:
             run_options["ports"]["5678"] = "5678"
@@ -762,3 +769,11 @@ def mount_project_and_library_directories(self, project_dir: Path, run_options:
                 "bind": "/Library",
                 "mode": "rw"
             }
+
+    @staticmethod
+    def parse_extra_docker_config(run_options: Dict[str, Any], extra_docker_config: Optional[Dict[str, Any]]) -> None:
+        # Add known additional run options from the extra docker config.
+        # For now, only device_requests is supported
+        if extra_docker_config is not None and "device_requests" in extra_docker_config:
+            run_options["device_requests"] = [docker.types.DeviceRequest(**device_request)
+                                              for device_request in extra_docker_config["device_requests"]]

tests/commands/test_backtest.py

@@ -57,7 +57,8 @@ def test_backtest_calls_lean_runner_with_correct_algorithm_file() -> None:
                                                  ENGINE_IMAGE,
                                                  None,
                                                  False,
-                                                 False)
+                                                 False,
+                                                 {})
 
 
 def test_backtest_calls_lean_runner_with_default_output_directory() -> None:
@@ -88,7 +89,8 @@ def test_backtest_calls_lean_runner_with_custom_output_directory() -> None:
                                                  ENGINE_IMAGE,
                                                  None,
                                                  False,
-                                                 False)
+                                                 False,
+                                                 {})
 
 
 def test_backtest_calls_lean_runner_with_release_mode() -> None:
@@ -105,7 +107,8 @@ def test_backtest_calls_lean_runner_with_release_mode() -> None:
                                                  ENGINE_IMAGE,
                                                  None,
                                                  True,
-                                                 False)
+                                                 False,
+                                                 {})
 
 
 def test_backtest_calls_lean_runner_with_detach() -> None:
@@ -122,7 +125,8 @@ def test_backtest_calls_lean_runner_with_detach() -> None:
                                                  ENGINE_IMAGE,
                                                  None,
                                                  False,
-                                                 True)
+                                                 True,
+                                                 {})
 
 
 def test_backtest_aborts_when_project_does_not_exist() -> None:
@@ -163,7 +167,8 @@ def test_backtest_forces_update_when_update_option_given() -> None:
                                                  ENGINE_IMAGE,
                                                  None,
                                                  False,
-                                                 False)
+                                                 False,
+                                                 {})
 
 
 def test_backtest_passes_custom_image_to_lean_runner_when_set_in_config() -> None:
@@ -182,7 +187,8 @@ def test_backtest_passes_custom_image_to_lean_runner_when_set_in_config() -> Non
                                                  DockerImage(name="custom/lean", tag="123"),
                                                  None,
                                                  False,
-                                                 False)
+                                                 False,
+                                                 {})
 
 
 def test_backtest_passes_custom_image_to_lean_runner_when_given_as_option() -> None:
@@ -201,7 +207,8 @@ def test_backtest_passes_custom_image_to_lean_runner_when_given_as_option() -> N
                                                  DockerImage(name="custom/lean", tag="456"),
                                                  None,
                                                  False,
-                                                 False)
+                                                 False,
+                                                 {})
 
 
 @pytest.mark.parametrize("python_venv", ["Custom-venv",
@@ -289,7 +296,8 @@ def test_backtest_passes_correct_debugging_method_to_lean_runner(value: str, deb
                                                  ENGINE_IMAGE,
                                                  debugging_method,
                                                  False,
-                                                 False)
+                                                 False,
+                                                 {})
 
 
 def test_backtest_auto_updates_outdated_python_pycharm_debug_config() -> None:
@@ -649,3 +657,23 @@ def test_backtest_adds_python_libraries_path_to_lean_config() -> None:
     expected_library_path = (Path("/") / library_path.relative_to(lean_cli_root_dir)).as_posix()
 
     assert expected_library_path in lean_config.get('python-additional-paths')
+
+
+def test_backtest_calls_lean_runner_with_extra_docker_config() -> None:
+    create_fake_lean_cli_directory()
+
+    result = CliRunner().invoke(lean, ["backtest", "Python Project",
+                                       "--extra-docker-config",
+                                       '{"device_requests": [{"count": -1, "capabilities": [["compute"]]}]}'])
+
+    assert result.exit_code == 0
+
+    container.lean_runner.run_lean.assert_called_once_with(mock.ANY,
+                                                           "backtesting",
+                                                           Path("Python Project/main.py").resolve(),
+                                                           mock.ANY,
+                                                           ENGINE_IMAGE,
+                                                           None,
+                                                           False,
+                                                           False,
+                                                           {"device_requests": [{"count": -1, "capabilities": [["compute"]]}]})