Add support for different datafeed in cloud (#359)

* Fix bug and add unit test

After debugging the logs from the strategies deployed in the cloud, it
was found the configuration name used by the API to use QC + IB data
feed. Therefore, only the method which returned the data handler name
was changed.

* Modify get_price_data_handler() to rely on json

* Solve bugs

* Make get_price_data_handler() more generic

* Fix bug

* update version of modules json file

* Update README.md

* Add support for interactive mode

* Add _configure_data_feed() method

README.md

@@ -269,12 +269,16 @@ Options:
                                   Weekly restart UTC time (hh:mm:ss). Each week on Sunday your algorithm is restarted at
                                   this time, and will require 2FA verification. This is required by Interactive Brokers.
                                   Use this option explicitly to override the default value.
-  --ib-data-feed BOOLEAN          Whether the Interactive Brokers price data feed must be used instead of the
-                                  QuantConnect price data feed
+  --ib-data-feed [QuantConnect|Interactive Brokers|QuantConnect + InteractiveBrokers]
+                                  The data feed to use. These are the available ones: Interactive Brokers price data
+                                  feed, QuantConnect price data feed or QuantConnect + InteractiveBrokers price data feed.
   --tradier-account-id TEXT       Your Tradier account id
   --tradier-access-token TEXT     Your Tradier access token
   --tradier-environment [live|paper]
                                   Whether the developer sandbox should be used
+  --tradier-data-feed [QuantConnect|Tradier Brokerage]
+                                  The data feed to use. These are the available ones: QuantConnect price data feed or
+                                  Tradier Brokerage data feed.
   --oanda-account-id TEXT         Your OANDA account id
   --oanda-access-token TEXT       Your OANDA API token
   --oanda-environment [Practice|Trade]

lean/commands/cloud/live/deploy.py

@@ -25,7 +25,7 @@
 from lean.models.brokerages.cloud.cloud_brokerage import CloudBrokerage
 from lean.models.configuration import InternalInputUserInput
 from lean.models.click_options import options_from_json, get_configs_for_options
-from lean.models.brokerages.cloud import all_cloud_brokerages
+from lean.models.brokerages.cloud import all_cloud_brokerages, cloud_brokerage_data_feeds
 from lean.commands.cloud.live.live import live
 from lean.components.util.live_utils import get_last_portfolio_cash_holdings, configure_initial_cash_balance, configure_initial_holdings,\
                                             _configure_initial_cash_interactively, _configure_initial_holdings_interactively
@@ -113,6 +113,20 @@ def _configure_brokerage(lean_config: Dict[str, Any], logger: Logger, user_provi
                                                                              user_provided_options,
                                                                              hide_input=not show_secrets)
 
+def _configure_data_feed(brokerage: CloudBrokerage, logger: Logger) -> None:
+    """Configures the data feed to use based on the brokerage given.
+
+    :param brokerage: the cloud brokerage
+    :param logger: the logger to use
+    """
+    if len(cloud_brokerage_data_feeds[brokerage]) != 0:
+        data_feed_selected = logger.prompt_list("Select a data feed", [
+            Option(id=data_feed, label=data_feed) for data_feed in cloud_brokerage_data_feeds[brokerage]
+        ], multiple=False)
+        data_feed_property_name = [name for name in brokerage.get_required_properties([InternalInputUserInput]) if ("data-feed" in name)]
+        data_feed_property_name = data_feed_property_name[0] if len(data_feed_property_name) != 0 else ""
+        brokerage.update_value_for_given_config(data_feed_property_name, data_feed_selected)
+
 
 def _configure_live_node(logger: Logger, api_client: APIClient, cloud_project: QCProject) -> QCNode:
     """Interactively configures the live node to use.
@@ -251,7 +265,7 @@ def deploy(project: str,
         ensure_options(essential_properties)
         essential_properties_value = {brokerage_instance.convert_variable_to_lean_key(prop) : kwargs[prop] for prop in essential_properties}
         brokerage_instance.update_configs(essential_properties_value)
-        # now required properties can be fetched as per data provider from esssential properties
+        # now required properties can be fetched as per data provider from essential properties
         required_properties = [brokerage_instance.convert_lean_key_to_variable(prop) for prop in brokerage_instance.get_required_properties([InternalInputUserInput])]
         ensure_options(required_properties)
         required_properties_value = {brokerage_instance.convert_variable_to_lean_key(prop) : kwargs[prop] for prop in required_properties}
@@ -308,6 +322,7 @@ def deploy(project: str,
     else:
         lean_config = container.lean_config_manager.get_lean_config()
         brokerage_instance = _configure_brokerage(lean_config, logger, kwargs, show_secrets=show_secrets)
+        _configure_data_feed(brokerage_instance, logger)
         live_node = _configure_live_node(logger, api_client, cloud_project)
         notify_order_events, notify_insights, notify_methods = _configure_notifications(logger)
         auto_restart = _configure_auto_restart(logger)

lean/models/__init__.py

@@ -17,7 +17,7 @@
 from time import time
 
 json_modules = {}
-file_name = "modules-1.11.json"
+file_name = "modules-1.12.json"
 directory = Path(__file__).parent
 file_path = directory.parent / file_name
 

lean/models/brokerages/cloud/__init__.py

@@ -13,13 +13,28 @@
 
 from lean.models.brokerages.cloud.cloud_brokerage import CloudBrokerage
 from lean.models import json_modules
-from typing import List
+from typing import Dict, Type, List
+from lean.models.brokerages.local.data_feed import DataFeed
 
 all_cloud_brokerages: List[CloudBrokerage] = []
+all_cloud_data_feeds: List[DataFeed] = []
+cloud_brokerage_data_feeds: Dict[Type[CloudBrokerage],
+                                 List[Type[DataFeed]]] = {}
 
 for json_module in json_modules:
     if "cloud-brokerage" in json_module["type"]:
         all_cloud_brokerages.append(CloudBrokerage(json_module))
+    if "data-queue-handler" in json_module["type"]:
+        all_cloud_data_feeds.append((DataFeed(json_module)))
+
+for cloud_brokerage in all_cloud_brokerages:
+    data_feed_property_found = False
+    for x in cloud_brokerage.get_all_input_configs():
+        if "data-feed" in x.__getattribute__("_id"):
+            data_feed_property_found = True
+            cloud_brokerage_data_feeds[cloud_brokerage] = x.__getattribute__("_choices")
+    if not data_feed_property_found:
+        cloud_brokerage_data_feeds[cloud_brokerage] = []
 
 [PaperTradingBrokerage] = [
     cloud_brokerage for cloud_brokerage in all_cloud_brokerages if cloud_brokerage._id == "QuantConnectBrokerage"]

lean/models/brokerages/cloud/cloud_brokerage.py

@@ -79,6 +79,12 @@ def get_price_data_handler(self) -> str:
         :return: the value to assign to the "dataHandler" property of the live/create API endpoint
         """
         # TODO: Handle this case with json conditions
-        if self.get_name() == "Interactive Brokers":
-            return "InteractiveBrokersHandler" if self.get_config_value_from_name("ib-data-feed") else "QuantConnectHandler"
+        property_name = [name for name in self.get_required_properties([InternalInputUserInput]) if ("data-feed" in name)]
+        property_name = property_name[0] if len(property_name) != 0 else ""
+        brokerage_name = self.get_name().replace(" ", "")
+        if property_name != "":
+            if "QuantConnect +" in self.get_config_value_from_name(property_name):
+                return "quantconnecthandler+" + brokerage_name.lower() + "handler"
+            else:
+                return self.get_config_value_from_name(property_name).replace(" ", "") + "Handler"
         return "QuantConnectHandler"

tests/commands/cloud/live/test_cloud_live_commands.py

@@ -94,6 +94,49 @@ def test_cloud_live_deploy() -> None:
                                                   mock.ANY,
                                                   mock.ANY)
 
+def test_cloud_live_deploy_with_ib_using_hybrid_datafeed() -> None:
+    create_fake_lean_cli_directory()
+
+    api_client = mock.Mock()
+    api_client.nodes.get_all.return_value = create_qc_nodes()
+    api_client.get.return_value = {'portfolio': {"cash": {}}, 'live': []}
+    container.api_client = api_client
+
+    cloud_project_manager = mock.Mock()
+    container.cloud_project_manager = cloud_project_manager
+
+    cloud_runner = mock.Mock()
+    container.cloud_runner = cloud_runner
+
+    result = CliRunner().invoke(lean, ["cloud", "live", "Python Project", "--brokerage", "Interactive Brokers", "--node", "live",
+                                       "--auto-restart", "yes", "--notify-order-events", "no", "--notify-insights", "no",
+                                       "--ib-data-feed", "QuantConnect + InteractiveBrokers", "--ib-user-name", "test_user",
+                                       "--ib-account", "DU2366417", "--ib-password", "test_password"])
+
+    assert result.exit_code == 0
+    assert "Data provider: quantconnecthandler+interactivebrokershandler" in result.output.split("\n")
+
+def test_cloud_live_deploy_with_tradier_using_tradier_datafeed() -> None:
+    create_fake_lean_cli_directory()
+
+    api_client = mock.Mock()
+    api_client.nodes.get_all.return_value = create_qc_nodes()
+    container.api_client = api_client
+
+    cloud_project_manager = mock.Mock()
+    container.cloud_project_manager = cloud_project_manager
+
+    cloud_runner = mock.Mock()
+    container.cloud_runner = cloud_runner
+
+    result = CliRunner().invoke(lean, ["cloud", "live", "Python Project", "--brokerage", "Tradier", "--node", "live",
+                                       "--auto-restart", "yes", "--notify-order-events", "no", "--notify-insights", "no",
+                                       "--tradier-data-feed", "Tradier Brokerage", "--tradier-account-id", "123",
+                                       "--tradier-access-token", "456", "--tradier-environment", "paper"])
+
+    assert result.exit_code == 0
+    assert "Data provider: TradierBrokerage" in result.output.split("\n")
+
 @pytest.mark.parametrize("notice_method,configs", [("emails", "customAddress:customSubject"),
                                              ("emails", "customAddress1:customSubject1,customAddress2:customSubject2"),
                                              ("webhooks", "customAddress:header1=value1"),
@@ -287,7 +330,9 @@ def test_cloud_live_deploy_with_live_holdings(brokerage: str, holdings: str) ->
     if brokerage == "Trading Technologies":
         options.extend(["--live-cash-balance", "USD:100"])
     elif brokerage == "Interactive Brokers":
-        options.extend(["--ib-data-feed", "no"])
+        options.extend(["--ib-data-feed", "QuantConnect"])
+    elif brokerage == "Tradier":
+        options.extend(["--tradier-data-feed", "QuantConnect"])
 
     result = CliRunner().invoke(lean, ["cloud", "live", "Python Project", "--brokerage", brokerage, "--live-holdings", holdings,
                                        "--node", "live", "--auto-restart", "yes", "--notify-order-events", "no",